eric: comparison src/eric7/CodeFormatting/BlackFormattingDialog.py

-:c44b7aa4cf71
+:0e9d2c4e379e
 Module implementing a dialog showing the code formatting progress and the result.
 """
 import copy
 import datetime
+import multiprocessing
 import pathlib
 from dataclasses import dataclass
 import black
 from . import BlackUtilities
 from .BlackDiffWidget import BlackDiffWidget
 from .BlackFormattingAction import BlackFormattingAction
 import Utilities
+import Preferences
 class BlackFormattingDialog(QDialog, Ui_BlackFormattingDialog):
 """
 Class implementing a dialog showing the code formatting progress and the result.
 self.statisticsGroup.setVisible(False)
 self.__statistics = BlackStatistics()
-self.__report = BlackReport(self)
-self.__report.check = action is BlackFormattingAction.Check
-self.__report.diff = action is BlackFormattingAction.Diff
-self.__report.result.connect(self.__handleBlackFormattingResult)
 self.__config = copy.deepcopy(configuration)
+self.__config["__action__"] = action  # needed by the workers
 self.__project = project
 self.__action = action
 self.__cancelled = False
 self.__diffDialog = None
 self.buttonBox.button(QDialogButtonBox.StandardButton.Cancel).setDefault(True)
 self.show()
 QCoreApplication.processEvents()
-self.__files = self.__filterFiles(filesList)
+files = self.__filterFiles(filesList)
-self.__formatFiles()
+if len(files) > 1:
+self.__formatManyFiles(files)
+elif len(files) == 1:
+self.__formatOneFile(files[0])
 def __filterFiles(self, filesList):
 """
 Private method to filter the given list of files according the
 configuration parameters.
 for file in filesList:
 file = Utilities.fromNativeSeparators(file)
 for filterRegExp in filterRegExps:
 filterMatch = filterRegExp.search(file)
 if filterMatch and filterMatch.group(0):
-self.__report.path_ignored(file)
+self.__handleBlackFormattingResult("ignored", file, "")
 break
 else:
 files.append(file)
 return files
 """
 Private method to update the statistics about the recent formatting run and
 make them visible.
 """
 self.reformattedLabel.setText(
-self.tr("reformatted")
+self.tr("Reformatted:")
 if self.__action is BlackFormattingAction.Format
-else self.tr("would reformat")
+else self.tr("Would Reformat:")
 )
 total = self.progressBar.maximum()
-processed = total - self.__statistics.ignoreCount
 self.totalCountLabel.setText("{0:n}".format(total))
 self.excludedCountLabel.setText("{0:n}".format(self.__statistics.ignoreCount))
 self.failuresCountLabel.setText("{0:n}".format(self.__statistics.failureCount))
-self.processedCountLabel.setText("{0:n}".format(processed))
+self.processedCountLabel.setText(
+"{0:n}".format(self.__statistics.processedCount)
+)
 self.reformattedCountLabel.setText(
 "{0:n}".format(self.__statistics.changeCount)
 )
 self.unchangedCountLabel.setText("{0:n}".format(self.__statistics.sameCount))
 elif dataType == "diff":
 if self.__diffDialog is None:
 self.__diffDialog = BlackDiffWidget()
 self.__diffDialog.showDiff(item.data(0, BlackFormattingDialog.DataRole))
-def __formatFiles(self):
+def __formatManyFiles(self, files):
+"""
+Private method to format the list of files according the configuration using
+multiple processes in parallel.
+@param files list of files to be processed
+@type list of str
+"""
+maxProcesses = Preferences.getUI("BackgroundServiceProcesses")
+if maxProcesses == 0:
+# determine based on CPU count
+try:
+NumberOfProcesses = multiprocessing.cpu_count()
+if NumberOfProcesses >= 1:
+NumberOfProcesses -= 1
+except NotImplementedError:
+NumberOfProcesses = 1
+else:
+NumberOfProcesses = maxProcesses
+# Create queues
+taskQueue = multiprocessing.Queue()
+doneQueue = multiprocessing.Queue()
+# Submit tasks
+for file in files:
+relSrc = self.__project.getRelativePath(str(file)) if self.__project else ""
+taskQueue.put((file, relSrc))
+# Start worker processes
+workers = [
+multiprocessing.Process(
+target=self.formattingWorkerTask,
+args=(taskQueue, doneQueue, self.__config),
+)
+for _ in range(NumberOfProcesses)
+]
+for worker in workers:
+worker.start()
+# Get the results from the worker tasks
+for _ in range(len(files)):
+result = doneQueue.get()
+self.__handleBlackFormattingResult(
+result.status, result.filename, result.data
+)
+if self.__cancelled:
+break
+if self.__cancelled:
+for worker in workers:
+worker.terminate()
+else:
+# Tell child processes to stop
+for _ in range(NumberOfProcesses):
+taskQueue.put("STOP")
+for worker in workers:
+worker.join()
+worker.close()
+taskQueue.close()
+doneQueue.close()
+self.__finish()
+@staticmethod
+def formattingWorkerTask(inputQueue, outputQueue, config):
+"""
+Static method acting as the parallel worker for the formatting task.
+@param inputQueue input queue
+@type multiprocessing.Queue
+@param outputQueue output queue
+@type multiprocessing.Queue
+@param config dictionary containing the configuration parameters
+@type dict
+"""
+report = BlackMultiprocessingReport(outputQueue)
+report.check = config["__action__"] is BlackFormattingAction.Check
+report.diff = config["__action__"] is BlackFormattingAction.Diff
+versions = (
+{black.TargetVersion[target.upper()] for target in config["target-version"]}
+if config["target-version"]
+else set()
+)
+mode = black.Mode(
+target_versions=versions,
+line_length=int(config["line-length"]),
+string_normalization=not config["skip-string-normalization"],
+magic_trailing_comma=not config["skip-magic-trailing-comma"],
+)
+if config["__action__"] is BlackFormattingAction.Diff:
+for file, relSrc in iter(inputQueue.get, "STOP"):
+BlackFormattingDialog.__diffFormatFile(
+pathlib.Path(file),
+fast=False,
+mode=mode,
+report=report,
+relSrc=relSrc,
+)
+else:
+writeBack = black.WriteBack.from_configuration(
+check=config["__action__"] is BlackFormattingAction.Check,
+diff=config["__action__"] is BlackFormattingAction.Diff,
+)
+for file, _relSrc in iter(inputQueue.get, "STOP"):
+black.reformat_one(
+pathlib.Path(file),
+fast=False,
+write_back=writeBack,
+mode=mode,
+report=report,
+)
+def __formatOneFile(self, file):
 """
 Private method to format the list of files according the configuration.
-"""
+@param file name of the file to be processed
+@type str
+"""
+report = BlackReport(self)
+report.check = self.__config["__action__"] is BlackFormattingAction.Check
+report.diff = self.__config["__action__"] is BlackFormattingAction.Diff
+report.result.connect(self.__handleBlackFormattingResult)
 writeBack = black.WriteBack.from_configuration(
-check=self.__action is BlackFormattingAction.Check,
+check=self.__config["__action__"] is BlackFormattingAction.Check,
-diff=self.__action is BlackFormattingAction.Diff,
+diff=self.__config["__action__"] is BlackFormattingAction.Diff,
 )
 versions = (
 {
 black.TargetVersion[target.upper()]
 line_length=int(self.__config["line-length"]),
 string_normalization=not self.__config["skip-string-normalization"],
 magic_trailing_comma=not self.__config["skip-magic-trailing-comma"],
 )
-for file in self.__files:
+if self.__action is BlackFormattingAction.Diff:
-if self.__action is BlackFormattingAction.Diff:
+relSrc = self.__project.getRelativePath(str(file)) if self.__project else ""
 self.__diffFormatFile(
-pathlib.Path(file), fast=False, mode=mode, report=self.__report
+pathlib.Path(file), fast=False, mode=mode, report=report, relSrc=relSrc
 )
 else:
 black.reformat_one(
 pathlib.Path(file),
 fast=False,
 write_back=writeBack,
 mode=mode,
-report=self.__report,
+report=report,
 )
-if self.__cancelled:
-break
 self.__finish()
-def __diffFormatFile(self, src, fast, mode, report):
+@staticmethod
-"""
+def __diffFormatFile(src, fast, mode, report, relSrc=""):
-Private method to check, if the given files need to be reformatted, and generate
+"""
+Static method to check, if the given files need to be reformatted, and generate
 a unified diff.
 @param src path of file to be checked
 @type pathlib.Path
 @param fast flag indicating fast operation
 @type bool
 @param mode code formatting options
 @type black.Mode
 @param report reference to the report object
 @type BlackReport
+@param relSrc name of the file relative to the project (defaults to "")
+@type str (optional)
 """
 then = datetime.datetime.utcfromtimestamp(src.stat().st_mtime)
 with open(src, "rb") as buf:
 srcContents, _, _ = black.decode_bytes(buf.read())
 try:
 dstContents = black.format_file_contents(srcContents, fast=fast, mode=mode)
 except black.NothingChanged:
 report.done(src, black.Changed.NO)
 return
-fileName = str(src)
+fileName = relSrc if bool(relSrc) else str(src)
-if self.__project:
-fileName = self.__project.getRelativePath(fileName)
 now = datetime.datetime.utcnow()
 srcName = f"{fileName}\t{then} +0000"
 dstName = f"{fileName}\t{now} +0000"
 diffContents = black.diff(srcContents, dstContents, srcName, dstName)
 else:
 statusMsg = self.tr("invalid status ({0})").format(status)
 self.__statistics.failureCount += 1
 isError = True
+if status != "ignored":
+self.__statistics.processedCount += 1
 if self.__project:
 filename = self.__project.getRelativePath(filename)
 itm = QTreeWidgetItem(self.resultsList, [statusMsg, filename])
 if data:
 ignoreCount: int = 0
 changeCount: int = 0
 sameCount: int = 0
 failureCount: int = 0
+processedCount: int = 0
 class BlackReport(QObject, black.Report):
 """
 Class extending the black Report to work with our dialog.
 'failed' or 'ignored'), the file name and data related to the result
 """
 result = pyqtSignal(str, str, str)
-def __init__(self, dialog):
+def __init__(self, parent=None):
 """
 Constructor
-@param dialog reference to the result dialog
+@param parent reference to the parent object (defaults to None
-@type QDialog
+@type QObject (optional)
 """
-QObject.__init__(self, dialog)
+QObject.__init__(self, parent)
 black.Report.__init__(self)
-self.ignored_count = 0
-self.__dialog = dialog
 def done(self, src, changed, diff=""):
 """
 Public method to handle the end of a reformat.
 @type pathlib.Path or str
 @param message ignore message (default to "")
 @type str (optional)
 """
 self.result.emit("ignored", str(src), "")
+@dataclass
+class BlackMultiprocessingResult:
+"""
+Class containing the reformatting result data.
+This class is used when reformatting multiple files in parallel using processes.
+"""
+status: str = ""
+filename: str = ""
+data: str = ""
+class BlackMultiprocessingReport(black.Report):
+"""
+Class extending the black Report to work with multiprocessing.
+"""
+def __init__(self, resultQueue):
+"""
+Constructor
+@param resultQueue reference to the queue to put the results into
+@type multiprocessing.Queue
+"""
+super().__init__()
+self.__queue = resultQueue
+def done(self, src, changed, diff=""):
+"""
+Public method to handle the end of a reformat.
+@param src name of the processed file
+@type pathlib.Path
+@param changed change status
+@type black.Changed
+@param diff unified diff of potential changes (defaults to "")
+@type str
+"""
+if changed is black.Changed.YES:
+status = "changed"
+elif changed is black.Changed.NO:
+status = "unchanged"
+elif changed is black.Changed.CACHED:
+status = "unmodified"
+self.__queue.put(
+BlackMultiprocessingResult(status=status, filename=str(src), data=diff)
+)
+def failed(self, src, message):
+"""
+Public method to handle a reformat failure.
+@param src name of the processed file
+@type pathlib.Path
+@param message error message
+@type str
+"""
+self.__queue.put(
+BlackMultiprocessingResult(status="failed", filename=str(src), data=message)
+)
+def path_ignored(self, src, message=""):
+"""
+Public method handling an ignored path.
+@param src name of the processed file
+@type pathlib.Path or str
+@param message ignore message (default to "")
+@type str (optional)
+"""
+self.__queue.put(
+BlackMultiprocessingResult(status="ignored", filename=str(src), data="")
+)

Mercurial Repositories > eric / file comparison

comparison: src/eric7/CodeFormatting/BlackFormattingDialog.py

src/eric7/CodeFormatting/BlackFormattingDialog.py