eric6/DebugClients/Python/coverage/multiproc.py

Sun, 14 Apr 2019 15:09:21 +0200

author
Detlev Offenbach <detlev@die-offenbachs.de>
date
Sun, 14 Apr 2019 15:09:21 +0200
changeset 6942
2602857055c5
parent 6219
DebugClients/Python/coverage/multiproc.py@d6c795b5ce33
child 7275
9a25fe1fab84
permissions
-rw-r--r--

Major restructuring of the source tree to get prepared for a setup.py based installation.

6219
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
1 # Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
2 # For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
3
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
4 """Monkey-patching to add multiprocessing support for coverage.py"""
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
5
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
6 import multiprocessing
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
7 import multiprocessing.process
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
8 import os
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
9 import sys
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
10
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
11 from coverage.misc import contract
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
12
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
13 # An attribute that will be set on the module to indicate that it has been
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
14 # monkey-patched.
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
15 PATCHED_MARKER = "_coverage$patched"
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
16
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
17 # The environment variable that specifies the rcfile for subprocesses.
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
18 COVERAGE_RCFILE_ENV = "_COVERAGE_RCFILE"
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
19
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
20
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
21 if sys.version_info >= (3, 4):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
22 OriginalProcess = multiprocessing.process.BaseProcess
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
23 else:
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
24 OriginalProcess = multiprocessing.Process
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
25
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
26 original_bootstrap = OriginalProcess._bootstrap
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
27
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
28 class ProcessWithCoverage(OriginalProcess):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
29 """A replacement for multiprocess.Process that starts coverage."""
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
30
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
31 def _bootstrap(self):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
32 """Wrapper around _bootstrap to start coverage."""
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
33 from coverage import Coverage # avoid circular import
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
34 rcfile = os.environ[COVERAGE_RCFILE_ENV]
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
35 cov = Coverage(data_suffix=True, config_file=rcfile)
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
36 cov.start()
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
37 debug = cov.debug
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
38 try:
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
39 if debug.should("multiproc"):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
40 debug.write("Calling multiprocessing bootstrap")
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
41 return original_bootstrap(self)
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
42 finally:
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
43 if debug.should("multiproc"):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
44 debug.write("Finished multiprocessing bootstrap")
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
45 cov.stop()
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
46 cov.save()
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
47 if debug.should("multiproc"):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
48 debug.write("Saved multiprocessing data")
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
49
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
50 class Stowaway(object):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
51 """An object to pickle, so when it is unpickled, it can apply the monkey-patch."""
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
52 def __init__(self, rcfile):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
53 self.rcfile = rcfile
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
54
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
55 def __getstate__(self):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
56 return {'rcfile': self.rcfile}
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
57
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
58 def __setstate__(self, state):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
59 patch_multiprocessing(state['rcfile'])
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
60
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
61
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
62 @contract(rcfile=str)
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
63 def patch_multiprocessing(rcfile):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
64 """Monkey-patch the multiprocessing module.
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
65
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
66 This enables coverage measurement of processes started by multiprocessing.
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
67 This involves aggressive monkey-patching.
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
68
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
69 `rcfile` is the path to the rcfile being used.
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
70
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
71 """
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
72
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
73 if hasattr(multiprocessing, PATCHED_MARKER):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
74 return
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
75
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
76 if sys.version_info >= (3, 4):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
77 OriginalProcess._bootstrap = ProcessWithCoverage._bootstrap
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
78 else:
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
79 multiprocessing.Process = ProcessWithCoverage
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
80
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
81 # Set the value in ProcessWithCoverage that will be pickled into the child
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
82 # process.
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
83 os.environ[COVERAGE_RCFILE_ENV] = rcfile
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
84
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
85 # When spawning processes rather than forking them, we have no state in the
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
86 # new process. We sneak in there with a Stowaway: we stuff one of our own
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
87 # objects into the data that gets pickled and sent to the sub-process. When
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
88 # the Stowaway is unpickled, it's __setstate__ method is called, which
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
89 # re-applies the monkey-patch.
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
90 # Windows only spawns, so this is needed to keep Windows working.
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
91 try:
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
92 from multiprocessing import spawn
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
93 original_get_preparation_data = spawn.get_preparation_data
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
94 except (ImportError, AttributeError):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
95 pass
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
96 else:
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
97 def get_preparation_data_with_stowaway(name):
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
98 """Get the original preparation data, and also insert our stowaway."""
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
99 d = original_get_preparation_data(name)
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
100 d['stowaway'] = Stowaway(rcfile)
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
101 return d
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
102
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
103 spawn.get_preparation_data = get_preparation_data_with_stowaway
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
104
d6c795b5ce33 Third Party, coverage: updated coverage.py to 4.5.1.
Detlev Offenbach <detlev@die-offenbachs.de>
parents:
diff changeset
105 setattr(multiprocessing, PATCHED_MARKER, True)

eric ide

mercurial