Tue, 10 Dec 2024 15:46:34 +0100
Updated copyright for 2025.
7335 | 1 | # -*- coding: utf-8 -*- |
2 | ||
11090
f5f5f5803935
Updated copyright for 2025.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
10439
diff
changeset
|
3 | # Copyright (c) 2019 - 2025 Detlev Offenbach <detlev@die-offenbachs.de> |
7335 | 4 | # |
5 | ||
6 | """ | |
7 | Module implementing the syntax check for JSON. | |
8 | """ | |
9 | ||
9482
a2bc06a54d9d
Corrected/acknowledged some bad import style and removed some obsolete code.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9473
diff
changeset
|
10 | import json |
9473
3f23dbf37dbe
Resorted the import statements using isort.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9292
diff
changeset
|
11 | import multiprocessing |
7335 | 12 | import queue |
13 | ||
14 | ||
15 | def initService(): | |
16 | """ | |
17 | Initialize the service and return the entry point. | |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
18 | |
7335 | 19 | @return the entry point for the background client |
10341
3fdffd9cc21d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
10111
diff
changeset
|
20 | @rtype function |
7335 | 21 | """ |
22 | return jsonSyntaxCheck | |
23 | ||
24 | ||
25 | def initBatchService(): | |
26 | """ | |
27 | Initialize the batch service and return the entry point. | |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
28 | |
7335 | 29 | @return the entry point for the background client |
10341
3fdffd9cc21d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
10111
diff
changeset
|
30 | @rtype function |
7335 | 31 | """ |
32 | return jsonSyntaxBatchCheck | |
33 | ||
34 | ||
35 | def jsonSyntaxCheck(file, codestring): | |
36 | """ | |
37 | Function to check a JSON source file for syntax errors. | |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
38 | |
7335 | 39 | @param file source filename |
40 | @type str | |
41 | @param codestring string containing the code to check | |
42 | @type str | |
10341
3fdffd9cc21d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
10111
diff
changeset
|
43 | @return list of dictionaries with the key 'error' which contain a tuple with |
3fdffd9cc21d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
10111
diff
changeset
|
44 | details about the syntax error. Each tuple contains the file name, line |
3fdffd9cc21d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
10111
diff
changeset
|
45 | number, column, code string and the error message. |
3fdffd9cc21d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
10111
diff
changeset
|
46 | @rtype list of dict |
7335 | 47 | """ |
48 | return __jsonSyntaxCheck(file, codestring) | |
49 | ||
50 | ||
51 | def jsonSyntaxBatchCheck(argumentsList, send, fx, cancelled, maxProcesses=0): | |
52 | """ | |
53 | Module function to check syntax for a batch of files. | |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
54 | |
10341
3fdffd9cc21d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
10111
diff
changeset
|
55 | @param argumentsList list of arguments tuples as given for jsonSyntaxCheck |
7335 | 56 | @type list |
57 | @param send reference to send function | |
10341
3fdffd9cc21d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
10111
diff
changeset
|
58 | @type function |
7335 | 59 | @param fx registered service name |
60 | @type str | |
61 | @param cancelled reference to function checking for a cancellation | |
10341
3fdffd9cc21d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
10111
diff
changeset
|
62 | @type function |
7335 | 63 | @param maxProcesses number of processes to be used |
64 | @type int | |
65 | """ | |
66 | if maxProcesses == 0: | |
67 | # determine based on CPU count | |
68 | try: | |
69 | NumberOfProcesses = multiprocessing.cpu_count() | |
70 | if NumberOfProcesses >= 1: | |
71 | NumberOfProcesses -= 1 | |
72 | except NotImplementedError: | |
73 | NumberOfProcesses = 1 | |
74 | else: | |
75 | NumberOfProcesses = maxProcesses | |
76 | ||
77 | # Create queues | |
78 | taskQueue = multiprocessing.Queue() | |
79 | doneQueue = multiprocessing.Queue() | |
80 | ||
9289
ba49c41e8f63
Did some optimizations in the multiprocessing code.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9284
diff
changeset
|
81 | # Submit tasks (initially two times the number of processes) |
ba49c41e8f63
Did some optimizations in the multiprocessing code.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9284
diff
changeset
|
82 | tasks = len(argumentsList) |
9292
a5c8a0213fe3
Fixed an issue in the multiprocessing usage causing a traceback when then number of tasks is smaller than the number of worker processes.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9289
diff
changeset
|
83 | initialTasks = min(2 * NumberOfProcesses, tasks) |
9289
ba49c41e8f63
Did some optimizations in the multiprocessing code.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9284
diff
changeset
|
84 | for _ in range(initialTasks): |
ba49c41e8f63
Did some optimizations in the multiprocessing code.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9284
diff
changeset
|
85 | taskQueue.put(argumentsList.pop(0)) |
7335 | 86 | |
87 | # Start worker processes | |
8650
100726f55a9a
Changed the 'multiprocessing.Process()' code of the background batch services to (hopefully) cure the slow down when used multiple times.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
8312
diff
changeset
|
88 | workers = [ |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
89 | multiprocessing.Process(target=workerTask, args=(taskQueue, doneQueue)) |
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
90 | for _ in range(NumberOfProcesses) |
8650
100726f55a9a
Changed the 'multiprocessing.Process()' code of the background batch services to (hopefully) cure the slow down when used multiple times.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
8312
diff
changeset
|
91 | ] |
100726f55a9a
Changed the 'multiprocessing.Process()' code of the background batch services to (hopefully) cure the slow down when used multiple times.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
8312
diff
changeset
|
92 | for worker in workers: |
100726f55a9a
Changed the 'multiprocessing.Process()' code of the background batch services to (hopefully) cure the slow down when used multiple times.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
8312
diff
changeset
|
93 | worker.start() |
7335 | 94 | |
95 | # Get and send results | |
9289
ba49c41e8f63
Did some optimizations in the multiprocessing code.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9284
diff
changeset
|
96 | for _ in range(tasks): |
7335 | 97 | resultSent = False |
98 | wasCancelled = False | |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
99 | |
7335 | 100 | while not resultSent: |
101 | try: | |
102 | # get result (waiting max. 3 seconds and send it to frontend | |
103 | filename, result = doneQueue.get() | |
104 | send(fx, filename, result) | |
105 | resultSent = True | |
106 | except queue.Empty: | |
107 | # ignore empty queue, just carry on | |
108 | if cancelled(): | |
109 | wasCancelled = True | |
110 | break | |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
111 | |
7335 | 112 | if wasCancelled or cancelled(): |
113 | # just exit the loop ignoring the results of queued tasks | |
114 | break | |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
115 | |
9289
ba49c41e8f63
Did some optimizations in the multiprocessing code.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9284
diff
changeset
|
116 | if argumentsList: |
ba49c41e8f63
Did some optimizations in the multiprocessing code.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9284
diff
changeset
|
117 | taskQueue.put(argumentsList.pop(0)) |
7335 | 118 | |
119 | # Tell child processes to stop | |
120 | for _ in range(NumberOfProcesses): | |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
121 | taskQueue.put("STOP") |
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
122 | |
8650
100726f55a9a
Changed the 'multiprocessing.Process()' code of the background batch services to (hopefully) cure the slow down when used multiple times.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
8312
diff
changeset
|
123 | for worker in workers: |
100726f55a9a
Changed the 'multiprocessing.Process()' code of the background batch services to (hopefully) cure the slow down when used multiple times.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
8312
diff
changeset
|
124 | worker.join() |
100726f55a9a
Changed the 'multiprocessing.Process()' code of the background batch services to (hopefully) cure the slow down when used multiple times.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
8312
diff
changeset
|
125 | worker.close() |
9284
3b3a4f659782
"Blacked" the sources.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9279
diff
changeset
|
126 | |
9279
e252f827aaa7
Added code to explicitly close the queues to/from the workers at the end of a batch check.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9221
diff
changeset
|
127 | taskQueue.close() |
e252f827aaa7
Added code to explicitly close the queues to/from the workers at the end of a batch check.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9221
diff
changeset
|
128 | doneQueue.close() |
7335 | 129 | |
130 | ||
8650
100726f55a9a
Changed the 'multiprocessing.Process()' code of the background batch services to (hopefully) cure the slow down when used multiple times.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
8312
diff
changeset
|
131 | def workerTask(inputQueue, outputQueue): |
7335 | 132 | """ |
133 | Module function acting as the parallel worker for the syntax check. | |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
134 | |
7335 | 135 | @param inputQueue input queue |
136 | @type multiprocessing.Queue | |
137 | @param outputQueue output queue | |
138 | @type multiprocessing.Queue | |
139 | """ | |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
140 | for filename, args in iter(inputQueue.get, "STOP"): |
7335 | 141 | source = args[0] |
142 | result = __jsonSyntaxCheck(filename, source) | |
143 | outputQueue.put((filename, result)) | |
144 | ||
145 | ||
146 | def __jsonSyntaxCheck(file, codestring): | |
147 | """ | |
10111
049fbbd2253d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9653
diff
changeset
|
148 | Function to check a JSON source file for syntax errors. |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
149 | |
7335 | 150 | @param file source filename |
151 | @type str | |
152 | @param codestring string containing the code to check | |
153 | @type str | |
10341
3fdffd9cc21d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
10111
diff
changeset
|
154 | @return list of dictionaries with the key 'error' which contain a tuple with |
3fdffd9cc21d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
10111
diff
changeset
|
155 | details about the syntax error. Each tuple contains the file name, line |
3fdffd9cc21d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
10111
diff
changeset
|
156 | number, column, code string and the error message. |
3fdffd9cc21d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
10111
diff
changeset
|
157 | @rtype list of dict |
7335 | 158 | """ |
10111
049fbbd2253d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9653
diff
changeset
|
159 | if codestring: |
049fbbd2253d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9653
diff
changeset
|
160 | try: |
049fbbd2253d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9653
diff
changeset
|
161 | json.loads(codestring) |
049fbbd2253d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9653
diff
changeset
|
162 | except json.JSONDecodeError as exc: |
049fbbd2253d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9653
diff
changeset
|
163 | line = exc.lineno |
049fbbd2253d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9653
diff
changeset
|
164 | column = exc.colno |
049fbbd2253d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9653
diff
changeset
|
165 | error = exc.msg |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
166 | |
10111
049fbbd2253d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9653
diff
changeset
|
167 | cline = min(len(codestring.splitlines()), int(line)) - 1 |
049fbbd2253d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9653
diff
changeset
|
168 | code = codestring.splitlines()[cline] |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
169 | |
10111
049fbbd2253d
Syntax Checker
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9653
diff
changeset
|
170 | return [{"error": (file, line, column, code, error)}] |
9221
bf71ee032bb4
Reformatted the source code using the 'Black' utility.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
9209
diff
changeset
|
171 | |
7335 | 172 | return [{}] |