Mercurial Repositories > eric / file comparison

comparison: src/eric7/DebugClients/Python/coverage/sqldata.py

src/eric7/DebugClients/Python/coverage/sqldata.py

branch
eric7
changeset 9209
b99e7fd55fd3
parent 9099
0e511e0e94a3
child 9252
32dd11232e06
equal deleted inserted replaced
9208:3fc8dfeb6ebe 9209:b99e7fd55fd3
1 # Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
2 # For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt
3
4 """SQLite coverage data."""
5
6 import collections
7 import datetime
8 import functools
9 import glob
10 import itertools
11 import os
12 import random
13 import re
14 import socket
15 import sqlite3
16 import sys
17 import textwrap
18 import threading
19 import zlib
20
21 from coverage.debug import NoDebugging, SimpleReprMixin, clipped_repr
22 from coverage.exceptions import CoverageException, DataError
23 from coverage.files import PathAliases
24 from coverage.misc import contract, file_be_gone, isolate_module
25 from coverage.numbits import numbits_to_nums, numbits_union, nums_to_numbits
26 from coverage.version import __version__
27
28 os = isolate_module(os)
29
30 # If you change the schema, increment the SCHEMA_VERSION, and update the
31 # docs in docs/dbschema.rst by running "make cogdoc".
32
33 SCHEMA_VERSION = 7
34
35 # Schema versions:
36 # 1: Released in 5.0a2
37 # 2: Added contexts in 5.0a3.
38 # 3: Replaced line table with line_map table.
39 # 4: Changed line_map.bitmap to line_map.numbits.
40 # 5: Added foreign key declarations.
41 # 6: Key-value in meta.
42 # 7: line_map -> line_bits
43
44 SCHEMA = """\
45 CREATE TABLE coverage_schema (
46 -- One row, to record the version of the schema in this db.
47 version integer
48 );
49
50 CREATE TABLE meta (
51 -- Key-value pairs, to record metadata about the data
52 key text,
53 value text,
54 unique (key)
55 -- Keys:
56 -- 'has_arcs' boolean -- Is this data recording branches?
57 -- 'sys_argv' text -- The coverage command line that recorded the data.
58 -- 'version' text -- The version of coverage.py that made the file.
59 -- 'when' text -- Datetime when the file was created.
60 );
61
62 CREATE TABLE file (
63 -- A row per file measured.
64 id integer primary key,
65 path text,
66 unique (path)
67 );
68
69 CREATE TABLE context (
70 -- A row per context measured.
71 id integer primary key,
72 context text,
73 unique (context)
74 );
75
76 CREATE TABLE line_bits (
77 -- If recording lines, a row per context per file executed.
78 -- All of the line numbers for that file/context are in one numbits.
79 file_id integer, -- foreign key to `file`.
80 context_id integer, -- foreign key to `context`.
81 numbits blob, -- see the numbits functions in coverage.numbits
82 foreign key (file_id) references file (id),
83 foreign key (context_id) references context (id),
84 unique (file_id, context_id)
85 );
86
87 CREATE TABLE arc (
88 -- If recording branches, a row per context per from/to line transition executed.
89 file_id integer, -- foreign key to `file`.
90 context_id integer, -- foreign key to `context`.
91 fromno integer, -- line number jumped from.
92 tono integer, -- line number jumped to.
93 foreign key (file_id) references file (id),
94 foreign key (context_id) references context (id),
95 unique (file_id, context_id, fromno, tono)
96 );
97
98 CREATE TABLE tracer (
99 -- A row per file indicating the tracer used for that file.
100 file_id integer primary key,
101 tracer text,
102 foreign key (file_id) references file (id)
103 );
104 """
105
106 class CoverageData(SimpleReprMixin):
107 """Manages collected coverage data, including file storage.
108
109 This class is the public supported API to the data that coverage.py
110 collects during program execution. It includes information about what code
111 was executed. It does not include information from the analysis phase, to
112 determine what lines could have been executed, or what lines were not
113 executed.
114
115 .. note::
116
117 The data file is currently a SQLite database file, with a
118 :ref:`documented schema <dbschema>`. The schema is subject to change
119 though, so be careful about querying it directly. Use this API if you
120 can to isolate yourself from changes.
121
122 There are a number of kinds of data that can be collected:
123
124 * **lines**: the line numbers of source lines that were executed.
125 These are always available.
126
127 * **arcs**: pairs of source and destination line numbers for transitions
128 between source lines. These are only available if branch coverage was
129 used.
130
131 * **file tracer names**: the module names of the file tracer plugins that
132 handled each file in the data.
133
134 Lines, arcs, and file tracer names are stored for each source file. File
135 names in this API are case-sensitive, even on platforms with
136 case-insensitive file systems.
137
138 A data file either stores lines, or arcs, but not both.
139
140 A data file is associated with the data when the :class:`CoverageData`
141 is created, using the parameters `basename`, `suffix`, and `no_disk`. The
142 base name can be queried with :meth:`base_filename`, and the actual file
143 name being used is available from :meth:`data_filename`.
144
145 To read an existing coverage.py data file, use :meth:`read`. You can then
146 access the line, arc, or file tracer data with :meth:`lines`, :meth:`arcs`,
147 or :meth:`file_tracer`.
148
149 The :meth:`has_arcs` method indicates whether arc data is available. You
150 can get a set of the files in the data with :meth:`measured_files`. As
151 with most Python containers, you can determine if there is any data at all
152 by using this object as a boolean value.
153
154 The contexts for each line in a file can be read with
155 :meth:`contexts_by_lineno`.
156
157 To limit querying to certain contexts, use :meth:`set_query_context` or
158 :meth:`set_query_contexts`. These will narrow the focus of subsequent
159 :meth:`lines`, :meth:`arcs`, and :meth:`contexts_by_lineno` calls. The set
160 of all measured context names can be retrieved with
161 :meth:`measured_contexts`.
162
163 Most data files will be created by coverage.py itself, but you can use
164 methods here to create data files if you like. The :meth:`add_lines`,
165 :meth:`add_arcs`, and :meth:`add_file_tracers` methods add data, in ways
166 that are convenient for coverage.py.
167
168 To record data for contexts, use :meth:`set_context` to set a context to
169 be used for subsequent :meth:`add_lines` and :meth:`add_arcs` calls.
170
171 To add a source file without any measured data, use :meth:`touch_file`,
172 or :meth:`touch_files` for a list of such files.
173
174 Write the data to its file with :meth:`write`.
175
176 You can clear the data in memory with :meth:`erase`. Two data collections
177 can be combined by using :meth:`update` on one :class:`CoverageData`,
178 passing it the other.
179
180 Data in a :class:`CoverageData` can be serialized and deserialized with
181 :meth:`dumps` and :meth:`loads`.
182
183 The methods used during the coverage.py collection phase
184 (:meth:`add_lines`, :meth:`add_arcs`, :meth:`set_context`, and
185 :meth:`add_file_tracers`) are thread-safe. Other methods may not be.
186
187 """
188
189 def __init__(self, basename=None, suffix=None, no_disk=False, warn=None, debug=None):
190 """Create a :class:`CoverageData` object to hold coverage-measured data.
191
192 Arguments:
193 basename (str): the base name of the data file, defaulting to
194 ".coverage". This can be a path to a file in another directory.
195 suffix (str or bool): has the same meaning as the `data_suffix`
196 argument to :class:`coverage.Coverage`.
197 no_disk (bool): if True, keep all data in memory, and don't
198 write any disk file.
199 warn: a warning callback function, accepting a warning message
200 argument.
201 debug: a `DebugControl` object (optional)
202
203 """
204 self._no_disk = no_disk
205 self._basename = os.path.abspath(basename or ".coverage")
206 self._suffix = suffix
207 self._warn = warn
208 self._debug = debug or NoDebugging()
209
210 self._choose_filename()
211 self._file_map = {}
212 # Maps thread ids to SqliteDb objects.
213 self._dbs = {}
214 self._pid = os.getpid()
215 # Synchronize the operations used during collection.
216 self._lock = threading.RLock()
217
218 # Are we in sync with the data file?
219 self._have_used = False
220
221 self._has_lines = False
222 self._has_arcs = False
223
224 self._current_context = None
225 self._current_context_id = None
226 self._query_context_ids = None
227
228 def _locked(method): # pylint: disable=no-self-argument
229 """A decorator for methods that should hold self._lock."""
230 @functools.wraps(method)
231 def _wrapped(self, *args, **kwargs):
232 if self._debug.should("lock"):
233 self._debug.write(f"Locking {self._lock!r} for {method.__name__}")
234 with self._lock:
235 if self._debug.should("lock"):
236 self._debug.write(f"Locked {self._lock!r} for {method.__name__}")
237 # pylint: disable=not-callable
238 return method(self, *args, **kwargs)
239 return _wrapped
240
241 def _choose_filename(self):
242 """Set self._filename based on inited attributes."""
243 if self._no_disk:
244 self._filename = ":memory:"
245 else:
246 self._filename = self._basename
247 suffix = filename_suffix(self._suffix)
248 if suffix:
249 self._filename += "." + suffix
250
251 def _reset(self):
252 """Reset our attributes."""
253 if not self._no_disk:
254 for db in self._dbs.values():
255 db.close()
256 self._dbs = {}
257 self._file_map = {}
258 self._have_used = False
259 self._current_context_id = None
260
261 def _open_db(self):
262 """Open an existing db file, and read its metadata."""
263 if self._debug.should("dataio"):
264 self._debug.write(f"Opening data file {self._filename!r}")
265 self._dbs[threading.get_ident()] = SqliteDb(self._filename, self._debug)
266 self._read_db()
267
268 def _read_db(self):
269 """Read the metadata from a database so that we are ready to use it."""
270 with self._dbs[threading.get_ident()] as db:
271 try:
272 schema_version, = db.execute_one("select version from coverage_schema")
273 except Exception as exc:
274 if "no such table: coverage_schema" in str(exc):
275 self._init_db(db)
276 else:
277 raise DataError(
278 "Data file {!r} doesn't seem to be a coverage data file: {}".format(
279 self._filename, exc
280 )
281 ) from exc
282 else:
283 if schema_version != SCHEMA_VERSION:
284 raise DataError(
285 "Couldn't use data file {!r}: wrong schema: {} instead of {}".format(
286 self._filename, schema_version, SCHEMA_VERSION
287 )
288 )
289
290 for row in db.execute("select value from meta where key = 'has_arcs'"):
291 self._has_arcs = bool(int(row[0]))
292 self._has_lines = not self._has_arcs
293
294 for file_id, path in db.execute("select id, path from file"):
295 self._file_map[path] = file_id
296
297 def _init_db(self, db):
298 """Write the initial contents of the database."""
299 if self._debug.should("dataio"):
300 self._debug.write(f"Initing data file {self._filename!r}")
301 db.executescript(SCHEMA)
302 db.execute("insert into coverage_schema (version) values (?)", (SCHEMA_VERSION,))
303 db.executemany(
304 "insert or ignore into meta (key, value) values (?, ?)",
305 [
306 ("sys_argv", str(getattr(sys, "argv", None))),
307 ("version", __version__),
308 ("when", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")),
309 ]
310 )
311
312 def _connect(self):
313 """Get the SqliteDb object to use."""
314 if threading.get_ident() not in self._dbs:
315 self._open_db()
316 return self._dbs[threading.get_ident()]
317
318 def __bool__(self):
319 if (threading.get_ident() not in self._dbs and not os.path.exists(self._filename)):
320 return False
321 try:
322 with self._connect() as con:
323 rows = con.execute("select * from file limit 1")
324 return bool(list(rows))
325 except CoverageException:
326 return False
327
328 @contract(returns="bytes")
329 def dumps(self):
330 """Serialize the current data to a byte string.
331
332 The format of the serialized data is not documented. It is only
333 suitable for use with :meth:`loads` in the same version of
334 coverage.py.
335
336 Note that this serialization is not what gets stored in coverage data
337 files. This method is meant to produce bytes that can be transmitted
338 elsewhere and then deserialized with :meth:`loads`.
339
340 Returns:
341 A byte string of serialized data.
342
343 .. versionadded:: 5.0
344
345 """
346 if self._debug.should("dataio"):
347 self._debug.write(f"Dumping data from data file {self._filename!r}")
348 with self._connect() as con:
349 script = con.dump()
350 return b"z" + zlib.compress(script.encode("utf-8"))
351
352 @contract(data="bytes")
353 def loads(self, data):
354 """Deserialize data from :meth:`dumps`.
355
356 Use with a newly-created empty :class:`CoverageData` object. It's
357 undefined what happens if the object already has data in it.
358
359 Note that this is not for reading data from a coverage data file. It
360 is only for use on data you produced with :meth:`dumps`.
361
362 Arguments:
363 data: A byte string of serialized data produced by :meth:`dumps`.
364
365 .. versionadded:: 5.0
366
367 """
368 if self._debug.should("dataio"):
369 self._debug.write(f"Loading data into data file {self._filename!r}")
370 if data[:1] != b"z":
371 raise DataError(
372 f"Unrecognized serialization: {data[:40]!r} (head of {len(data)} bytes)"
373 )
374 script = zlib.decompress(data[1:]).decode("utf-8")
375 self._dbs[threading.get_ident()] = db = SqliteDb(self._filename, self._debug)
376 with db:
377 db.executescript(script)
378 self._read_db()
379 self._have_used = True
380
381 def _file_id(self, filename, add=False):
382 """Get the file id for `filename`.
383
384 If filename is not in the database yet, add it if `add` is True.
385 If `add` is not True, return None.
386 """
387 if filename not in self._file_map:
388 if add:
389 with self._connect() as con:
390 self._file_map[filename] = con.execute_for_rowid(
391 "insert or replace into file (path) values (?)",
392 (filename,)
393 )
394 return self._file_map.get(filename)
395
396 def _context_id(self, context):
397 """Get the id for a context."""
398 assert context is not None
399 self._start_using()
400 with self._connect() as con:
401 row = con.execute_one("select id from context where context = ?", (context,))
402 if row is not None:
403 return row[0]
404 else:
405 return None
406
407 @_locked
408 def set_context(self, context):
409 """Set the current context for future :meth:`add_lines` etc.
410
411 `context` is a str, the name of the context to use for the next data
412 additions. The context persists until the next :meth:`set_context`.
413
414 .. versionadded:: 5.0
415
416 """
417 if self._debug.should("dataop"):
418 self._debug.write(f"Setting context: {context!r}")
419 self._current_context = context
420 self._current_context_id = None
421
422 def _set_context_id(self):
423 """Use the _current_context to set _current_context_id."""
424 context = self._current_context or ""
425 context_id = self._context_id(context)
426 if context_id is not None:
427 self._current_context_id = context_id
428 else:
429 with self._connect() as con:
430 self._current_context_id = con.execute_for_rowid(
431 "insert into context (context) values (?)",
432 (context,)
433 )
434
435 def base_filename(self):
436 """The base filename for storing data.
437
438 .. versionadded:: 5.0
439
440 """
441 return self._basename
442
443 def data_filename(self):
444 """Where is the data stored?
445
446 .. versionadded:: 5.0
447
448 """
449 return self._filename
450
451 @_locked
452 def add_lines(self, line_data):
453 """Add measured line data.
454
455 `line_data` is a dictionary mapping file names to iterables of ints::
456
457 { filename: { line1, line2, ... }, ...}
458
459 """
460 if self._debug.should("dataop"):
461 self._debug.write("Adding lines: %d files, %d lines total" % (
462 len(line_data), sum(len(lines) for lines in line_data.values())
463 ))
464 self._start_using()
465 self._choose_lines_or_arcs(lines=True)
466 if not line_data:
467 return
468 with self._connect() as con:
469 self._set_context_id()
470 for filename, linenos in line_data.items():
471 linemap = nums_to_numbits(linenos)
472 file_id = self._file_id(filename, add=True)
473 query = "select numbits from line_bits where file_id = ? and context_id = ?"
474 existing = list(con.execute(query, (file_id, self._current_context_id)))
475 if existing:
476 linemap = numbits_union(linemap, existing[0][0])
477
478 con.execute(
479 "insert or replace into line_bits " +
480 " (file_id, context_id, numbits) values (?, ?, ?)",
481 (file_id, self._current_context_id, linemap),
482 )
483
484 @_locked
485 def add_arcs(self, arc_data):
486 """Add measured arc data.
487
488 `arc_data` is a dictionary mapping file names to iterables of pairs of
489 ints::
490
491 { filename: { (l1,l2), (l1,l2), ... }, ...}
492
493 """
494 if self._debug.should("dataop"):
495 self._debug.write("Adding arcs: %d files, %d arcs total" % (
496 len(arc_data), sum(len(arcs) for arcs in arc_data.values())
497 ))
498 self._start_using()
499 self._choose_lines_or_arcs(arcs=True)
500 if not arc_data:
501 return
502 with self._connect() as con:
503 self._set_context_id()
504 for filename, arcs in arc_data.items():
505 file_id = self._file_id(filename, add=True)
506 data = [(file_id, self._current_context_id, fromno, tono) for fromno, tono in arcs]
507 con.executemany(
508 "insert or ignore into arc " +
509 "(file_id, context_id, fromno, tono) values (?, ?, ?, ?)",
510 data,
511 )
512
513 def _choose_lines_or_arcs(self, lines=False, arcs=False):
514 """Force the data file to choose between lines and arcs."""
515 assert lines or arcs
516 assert not (lines and arcs)
517 if lines and self._has_arcs:
518 if self._debug.should("dataop"):
519 self._debug.write("Error: Can't add line measurements to existing branch data")
520 raise DataError("Can't add line measurements to existing branch data")
521 if arcs and self._has_lines:
522 if self._debug.should("dataop"):
523 self._debug.write("Error: Can't add branch measurements to existing line data")
524 raise DataError("Can't add branch measurements to existing line data")
525 if not self._has_arcs and not self._has_lines:
526 self._has_lines = lines
527 self._has_arcs = arcs
528 with self._connect() as con:
529 con.execute(
530 "insert or ignore into meta (key, value) values (?, ?)",
531 ("has_arcs", str(int(arcs)))
532 )
533
534 @_locked
535 def add_file_tracers(self, file_tracers):
536 """Add per-file plugin information.
537
538 `file_tracers` is { filename: plugin_name, ... }
539
540 """
541 if self._debug.should("dataop"):
542 self._debug.write("Adding file tracers: %d files" % (len(file_tracers),))
543 if not file_tracers:
544 return
545 self._start_using()
546 with self._connect() as con:
547 for filename, plugin_name in file_tracers.items():
548 file_id = self._file_id(filename)
549 if file_id is None:
550 raise DataError(
551 f"Can't add file tracer data for unmeasured file '{filename}'"
552 )
553
554 existing_plugin = self.file_tracer(filename)
555 if existing_plugin:
556 if existing_plugin != plugin_name:
557 raise DataError(
558 "Conflicting file tracer name for '{}': {!r} vs {!r}".format(
559 filename, existing_plugin, plugin_name,
560 )
561 )
562 elif plugin_name:
563 con.execute(
564 "insert into tracer (file_id, tracer) values (?, ?)",
565 (file_id, plugin_name)
566 )
567
568 def touch_file(self, filename, plugin_name=""):
569 """Ensure that `filename` appears in the data, empty if needed.
570
571 `plugin_name` is the name of the plugin responsible for this file. It is used
572 to associate the right filereporter, etc.
573 """
574 self.touch_files([filename], plugin_name)
575
576 def touch_files(self, filenames, plugin_name=""):
577 """Ensure that `filenames` appear in the data, empty if needed.
578
579 `plugin_name` is the name of the plugin responsible for these files. It is used
580 to associate the right filereporter, etc.
581 """
582 if self._debug.should("dataop"):
583 self._debug.write(f"Touching {filenames!r}")
584 self._start_using()
585 with self._connect(): # Use this to get one transaction.
586 if not self._has_arcs and not self._has_lines:
587 raise DataError("Can't touch files in an empty CoverageData")
588
589 for filename in filenames:
590 self._file_id(filename, add=True)
591 if plugin_name:
592 # Set the tracer for this file
593 self.add_file_tracers({filename: plugin_name})
594
595 def update(self, other_data, aliases=None):
596 """Update this data with data from several other :class:`CoverageData` instances.
597
598 If `aliases` is provided, it's a `PathAliases` object that is used to
599 re-map paths to match the local machine's.
600 """
601 if self._debug.should("dataop"):
602 self._debug.write("Updating with data from {!r}".format(
603 getattr(other_data, "_filename", "???"),
604 ))
605 if self._has_lines and other_data._has_arcs:
606 raise DataError("Can't combine arc data with line data")
607 if self._has_arcs and other_data._has_lines:
608 raise DataError("Can't combine line data with arc data")
609
610 aliases = aliases or PathAliases()
611
612 # Force the database we're writing to to exist before we start nesting
613 # contexts.
614 self._start_using()
615
616 # Collector for all arcs, lines and tracers
617 other_data.read()
618 with other_data._connect() as con:
619 # Get files data.
620 cur = con.execute("select path from file")
621 files = {path: aliases.map(path) for (path,) in cur}
622 cur.close()
623
624 # Get contexts data.
625 cur = con.execute("select context from context")
626 contexts = [context for (context,) in cur]
627 cur.close()
628
629 # Get arc data.
630 cur = con.execute(
631 "select file.path, context.context, arc.fromno, arc.tono " +
632 "from arc " +
633 "inner join file on file.id = arc.file_id " +
634 "inner join context on context.id = arc.context_id"
635 )
636 arcs = [(files[path], context, fromno, tono) for (path, context, fromno, tono) in cur]
637 cur.close()
638
639 # Get line data.
640 cur = con.execute(
641 "select file.path, context.context, line_bits.numbits " +
642 "from line_bits " +
643 "inner join file on file.id = line_bits.file_id " +
644 "inner join context on context.id = line_bits.context_id"
645 )
646 lines = {(files[path], context): numbits for (path, context, numbits) in cur}
647 cur.close()
648
649 # Get tracer data.
650 cur = con.execute(
651 "select file.path, tracer " +
652 "from tracer " +
653 "inner join file on file.id = tracer.file_id"
654 )
655 tracers = {files[path]: tracer for (path, tracer) in cur}
656 cur.close()
657
658 with self._connect() as con:
659 con.con.isolation_level = "IMMEDIATE"
660
661 # Get all tracers in the DB. Files not in the tracers are assumed
662 # to have an empty string tracer. Since Sqlite does not support
663 # full outer joins, we have to make two queries to fill the
664 # dictionary.
665 this_tracers = {path: "" for path, in con.execute("select path from file")}
666 this_tracers.update({
667 aliases.map(path): tracer
668 for path, tracer in con.execute(
669 "select file.path, tracer from tracer " +
670 "inner join file on file.id = tracer.file_id"
671 )
672 })
673
674 # Create all file and context rows in the DB.
675 con.executemany(
676 "insert or ignore into file (path) values (?)",
677 ((file,) for file in files.values())
678 )
679 file_ids = {
680 path: id
681 for id, path in con.execute("select id, path from file")
682 }
683 self._file_map.update(file_ids)
684 con.executemany(
685 "insert or ignore into context (context) values (?)",
686 ((context,) for context in contexts)
687 )
688 context_ids = {
689 context: id
690 for id, context in con.execute("select id, context from context")
691 }
692
693 # Prepare tracers and fail, if a conflict is found.
694 # tracer_paths is used to ensure consistency over the tracer data
695 # and tracer_map tracks the tracers to be inserted.
696 tracer_map = {}
697 for path in files.values():
698 this_tracer = this_tracers.get(path)
699 other_tracer = tracers.get(path, "")
700 # If there is no tracer, there is always the None tracer.
701 if this_tracer is not None and this_tracer != other_tracer:
702 raise DataError(
703 "Conflicting file tracer name for '{}': {!r} vs {!r}".format(
704 path, this_tracer, other_tracer
705 )
706 )
707 tracer_map[path] = other_tracer
708
709 # Prepare arc and line rows to be inserted by converting the file
710 # and context strings with integer ids. Then use the efficient
711 # `executemany()` to insert all rows at once.
712 arc_rows = (
713 (file_ids[file], context_ids[context], fromno, tono)
714 for file, context, fromno, tono in arcs
715 )
716
717 # Get line data.
718 cur = con.execute(
719 "select file.path, context.context, line_bits.numbits " +
720 "from line_bits " +
721 "inner join file on file.id = line_bits.file_id " +
722 "inner join context on context.id = line_bits.context_id"
723 )
724 for path, context, numbits in cur:
725 key = (aliases.map(path), context)
726 if key in lines:
727 numbits = numbits_union(lines[key], numbits)
728 lines[key] = numbits
729 cur.close()
730
731 if arcs:
732 self._choose_lines_or_arcs(arcs=True)
733
734 # Write the combined data.
735 con.executemany(
736 "insert or ignore into arc " +
737 "(file_id, context_id, fromno, tono) values (?, ?, ?, ?)",
738 arc_rows
739 )
740
741 if lines:
742 self._choose_lines_or_arcs(lines=True)
743 con.execute("delete from line_bits")
744 con.executemany(
745 "insert into line_bits " +
746 "(file_id, context_id, numbits) values (?, ?, ?)",
747 [
748 (file_ids[file], context_ids[context], numbits)
749 for (file, context), numbits in lines.items()
750 ]
751 )
752 con.executemany(
753 "insert or ignore into tracer (file_id, tracer) values (?, ?)",
754 ((file_ids[filename], tracer) for filename, tracer in tracer_map.items())
755 )
756
757 if not self._no_disk:
758 # Update all internal cache data.
759 self._reset()
760 self.read()
761
762 def erase(self, parallel=False):
763 """Erase the data in this object.
764
765 If `parallel` is true, then also deletes data files created from the
766 basename by parallel-mode.
767
768 """
769 self._reset()
770 if self._no_disk:
771 return
772 if self._debug.should("dataio"):
773 self._debug.write(f"Erasing data file {self._filename!r}")
774 file_be_gone(self._filename)
775 if parallel:
776 data_dir, local = os.path.split(self._filename)
777 localdot = local + ".*"
778 pattern = os.path.join(os.path.abspath(data_dir), localdot)
779 for filename in glob.glob(pattern):
780 if self._debug.should("dataio"):
781 self._debug.write(f"Erasing parallel data file {filename!r}")
782 file_be_gone(filename)
783
784 def read(self):
785 """Start using an existing data file."""
786 if os.path.exists(self._filename):
787 with self._connect():
788 self._have_used = True
789
790 def write(self):
791 """Ensure the data is written to the data file."""
792 pass
793
794 def _start_using(self):
795 """Call this before using the database at all."""
796 if self._pid != os.getpid():
797 # Looks like we forked! Have to start a new data file.
798 self._reset()
799 self._choose_filename()
800 self._pid = os.getpid()
801 if not self._have_used:
802 self.erase()
803 self._have_used = True
804
805 def has_arcs(self):
806 """Does the database have arcs (True) or lines (False)."""
807 return bool(self._has_arcs)
808
809 def measured_files(self):
810 """A set of all files that had been measured."""
811 return set(self._file_map)
812
813 def measured_contexts(self):
814 """A set of all contexts that have been measured.
815
816 .. versionadded:: 5.0
817
818 """
819 self._start_using()
820 with self._connect() as con:
821 contexts = {row[0] for row in con.execute("select distinct(context) from context")}
822 return contexts
823
824 def file_tracer(self, filename):
825 """Get the plugin name of the file tracer for a file.
826
827 Returns the name of the plugin that handles this file. If the file was
828 measured, but didn't use a plugin, then "" is returned. If the file
829 was not measured, then None is returned.
830
831 """
832 self._start_using()
833 with self._connect() as con:
834 file_id = self._file_id(filename)
835 if file_id is None:
836 return None
837 row = con.execute_one("select tracer from tracer where file_id = ?", (file_id,))
838 if row is not None:
839 return row[0] or ""
840 return "" # File was measured, but no tracer associated.
841
842 def set_query_context(self, context):
843 """Set a context for subsequent querying.
844
845 The next :meth:`lines`, :meth:`arcs`, or :meth:`contexts_by_lineno`
846 calls will be limited to only one context. `context` is a string which
847 must match a context exactly. If it does not, no exception is raised,
848 but queries will return no data.
849
850 .. versionadded:: 5.0
851
852 """
853 self._start_using()
854 with self._connect() as con:
855 cur = con.execute("select id from context where context = ?", (context,))
856 self._query_context_ids = [row[0] for row in cur.fetchall()]
857
858 def set_query_contexts(self, contexts):
859 """Set a number of contexts for subsequent querying.
860
861 The next :meth:`lines`, :meth:`arcs`, or :meth:`contexts_by_lineno`
862 calls will be limited to the specified contexts. `contexts` is a list
863 of Python regular expressions. Contexts will be matched using
864 :func:`re.search <python:re.search>`. Data will be included in query
865 results if they are part of any of the contexts matched.
866
867 .. versionadded:: 5.0
868
869 """
870 self._start_using()
871 if contexts:
872 with self._connect() as con:
873 context_clause = " or ".join(["context regexp ?"] * len(contexts))
874 cur = con.execute("select id from context where " + context_clause, contexts)
875 self._query_context_ids = [row[0] for row in cur.fetchall()]
876 else:
877 self._query_context_ids = None
878
879 def lines(self, filename):
880 """Get the list of lines executed for a source file.
881
882 If the file was not measured, returns None. A file might be measured,
883 and have no lines executed, in which case an empty list is returned.
884
885 If the file was executed, returns a list of integers, the line numbers
886 executed in the file. The list is in no particular order.
887
888 """
889 self._start_using()
890 if self.has_arcs():
891 arcs = self.arcs(filename)
892 if arcs is not None:
893 all_lines = itertools.chain.from_iterable(arcs)
894 return list({l for l in all_lines if l > 0})
895
896 with self._connect() as con:
897 file_id = self._file_id(filename)
898 if file_id is None:
899 return None
900 else:
901 query = "select numbits from line_bits where file_id = ?"
902 data = [file_id]
903 if self._query_context_ids is not None:
904 ids_array = ", ".join("?" * len(self._query_context_ids))
905 query += " and context_id in (" + ids_array + ")"
906 data += self._query_context_ids
907 bitmaps = list(con.execute(query, data))
908 nums = set()
909 for row in bitmaps:
910 nums.update(numbits_to_nums(row[0]))
911 return list(nums)
912
913 def arcs(self, filename):
914 """Get the list of arcs executed for a file.
915
916 If the file was not measured, returns None. A file might be measured,
917 and have no arcs executed, in which case an empty list is returned.
918
919 If the file was executed, returns a list of 2-tuples of integers. Each
920 pair is a starting line number and an ending line number for a
921 transition from one line to another. The list is in no particular
922 order.
923
924 Negative numbers have special meaning. If the starting line number is
925 -N, it represents an entry to the code object that starts at line N.
926 If the ending ling number is -N, it's an exit from the code object that
927 starts at line N.
928
929 """
930 self._start_using()
931 with self._connect() as con:
932 file_id = self._file_id(filename)
933 if file_id is None:
934 return None
935 else:
936 query = "select distinct fromno, tono from arc where file_id = ?"
937 data = [file_id]
938 if self._query_context_ids is not None:
939 ids_array = ", ".join("?" * len(self._query_context_ids))
940 query += " and context_id in (" + ids_array + ")"
941 data += self._query_context_ids
942 arcs = con.execute(query, data)
943 return list(arcs)
944
945 def contexts_by_lineno(self, filename):
946 """Get the contexts for each line in a file.
947
948 Returns:
949 A dict mapping line numbers to a list of context names.
950
951 .. versionadded:: 5.0
952
953 """
954 self._start_using()
955 with self._connect() as con:
956 file_id = self._file_id(filename)
957 if file_id is None:
958 return {}
959
960 lineno_contexts_map = collections.defaultdict(set)
961 if self.has_arcs():
962 query = (
963 "select arc.fromno, arc.tono, context.context " +
964 "from arc, context " +
965 "where arc.file_id = ? and arc.context_id = context.id"
966 )
967 data = [file_id]
968 if self._query_context_ids is not None:
969 ids_array = ", ".join("?" * len(self._query_context_ids))
970 query += " and arc.context_id in (" + ids_array + ")"
971 data += self._query_context_ids
972 for fromno, tono, context in con.execute(query, data):
973 if fromno > 0:
974 lineno_contexts_map[fromno].add(context)
975 if tono > 0:
976 lineno_contexts_map[tono].add(context)
977 else:
978 query = (
979 "select l.numbits, c.context from line_bits l, context c " +
980 "where l.context_id = c.id " +
981 "and file_id = ?"
982 )
983 data = [file_id]
984 if self._query_context_ids is not None:
985 ids_array = ", ".join("?" * len(self._query_context_ids))
986 query += " and l.context_id in (" + ids_array + ")"
987 data += self._query_context_ids
988 for numbits, context in con.execute(query, data):
989 for lineno in numbits_to_nums(numbits):
990 lineno_contexts_map[lineno].add(context)
991
992 return {lineno: list(contexts) for lineno, contexts in lineno_contexts_map.items()}
993
994 @classmethod
995 def sys_info(cls):
996 """Our information for `Coverage.sys_info`.
997
998 Returns a list of (key, value) pairs.
999
1000 """
1001 with SqliteDb(":memory:", debug=NoDebugging()) as db:
1002 temp_store = [row[0] for row in db.execute("pragma temp_store")]
1003 copts = [row[0] for row in db.execute("pragma compile_options")]
1004 copts = textwrap.wrap(", ".join(copts), width=75)
1005
1006 return [
1007 ("sqlite3_version", sqlite3.version),
1008 ("sqlite3_sqlite_version", sqlite3.sqlite_version),
1009 ("sqlite3_temp_store", temp_store),
1010 ("sqlite3_compile_options", copts),
1011 ]
1012
1013
1014 def filename_suffix(suffix):
1015 """Compute a filename suffix for a data file.
1016
1017 If `suffix` is a string or None, simply return it. If `suffix` is True,
1018 then build a suffix incorporating the hostname, process id, and a random
1019 number.
1020
1021 Returns a string or None.
1022
1023 """
1024 if suffix is True:
1025 # If data_suffix was a simple true value, then make a suffix with
1026 # plenty of distinguishing information. We do this here in
1027 # `save()` at the last minute so that the pid will be correct even
1028 # if the process forks.
1029 dice = random.Random(os.urandom(8)).randint(0, 999999)
1030 suffix = "%s.%s.%06d" % (socket.gethostname(), os.getpid(), dice)
1031 return suffix
1032
1033
1034 class SqliteDb(SimpleReprMixin):
1035 """A simple abstraction over a SQLite database.
1036
1037 Use as a context manager, then you can use it like a
1038 :class:`python:sqlite3.Connection` object::
1039
1040 with SqliteDb(filename, debug_control) as db:
1041 db.execute("insert into schema (version) values (?)", (SCHEMA_VERSION,))
1042
1043 """
1044 def __init__(self, filename, debug):
1045 self.debug = debug
1046 self.filename = filename
1047 self.nest = 0
1048 self.con = None
1049
1050 def _connect(self):
1051 """Connect to the db and do universal initialization."""
1052 if self.con is not None:
1053 return
1054
1055 # It can happen that Python switches threads while the tracer writes
1056 # data. The second thread will also try to write to the data,
1057 # effectively causing a nested context. However, given the idempotent
1058 # nature of the tracer operations, sharing a connection among threads
1059 # is not a problem.
1060 if self.debug.should("sql"):
1061 self.debug.write(f"Connecting to {self.filename!r}")
1062 try:
1063 self.con = sqlite3.connect(self.filename, check_same_thread=False)
1064 except sqlite3.Error as exc:
1065 raise DataError(f"Couldn't use data file {self.filename!r}: {exc}") from exc
1066
1067 self.con.create_function("REGEXP", 2, _regexp)
1068
1069 # This pragma makes writing faster. It disables rollbacks, but we never need them.
1070 # PyPy needs the .close() calls here, or sqlite gets twisted up:
1071 # https://bitbucket.org/pypy/pypy/issues/2872/default-isolation-mode-is-different-on
1072 self.execute("pragma journal_mode=off").close()
1073 # This pragma makes writing faster.
1074 self.execute("pragma synchronous=off").close()
1075
1076 def close(self):
1077 """If needed, close the connection."""
1078 if self.con is not None and self.filename != ":memory:":
1079 self.con.close()
1080 self.con = None
1081
1082 def __enter__(self):
1083 if self.nest == 0:
1084 self._connect()
1085 self.con.__enter__()
1086 self.nest += 1
1087 return self
1088
1089 def __exit__(self, exc_type, exc_value, traceback):
1090 self.nest -= 1
1091 if self.nest == 0:
1092 try:
1093 self.con.__exit__(exc_type, exc_value, traceback)
1094 self.close()
1095 except Exception as exc:
1096 if self.debug.should("sql"):
1097 self.debug.write(f"EXCEPTION from __exit__: {exc}")
1098 raise DataError(f"Couldn't end data file {self.filename!r}: {exc}") from exc
1099
1100 def execute(self, sql, parameters=()):
1101 """Same as :meth:`python:sqlite3.Connection.execute`."""
1102 if self.debug.should("sql"):
1103 tail = f" with {parameters!r}" if parameters else ""
1104 self.debug.write(f"Executing {sql!r}{tail}")
1105 try:
1106 try:
1107 return self.con.execute(sql, parameters)
1108 except Exception:
1109 # In some cases, an error might happen that isn't really an
1110 # error. Try again immediately.
1111 # https://github.com/nedbat/coveragepy/issues/1010
1112 return self.con.execute(sql, parameters)
1113 except sqlite3.Error as exc:
1114 msg = str(exc)
1115 try:
1116 # `execute` is the first thing we do with the database, so try
1117 # hard to provide useful hints if something goes wrong now.
1118 with open(self.filename, "rb") as bad_file:
1119 cov4_sig = b"!coverage.py: This is a private format"
1120 if bad_file.read(len(cov4_sig)) == cov4_sig:
1121 msg = (
1122 "Looks like a coverage 4.x data file. " +
1123 "Are you mixing versions of coverage?"
1124 )
1125 except Exception: # pragma: cant happen
1126 pass
1127 if self.debug.should("sql"):
1128 self.debug.write(f"EXCEPTION from execute: {msg}")
1129 raise DataError(f"Couldn't use data file {self.filename!r}: {msg}") from exc
1130
1131 def execute_for_rowid(self, sql, parameters=()):
1132 """Like execute, but returns the lastrowid."""
1133 con = self.execute(sql, parameters)
1134 rowid = con.lastrowid
1135 if self.debug.should("sqldata"):
1136 self.debug.write(f"Row id result: {rowid!r}")
1137 return rowid
1138
1139 def execute_one(self, sql, parameters=()):
1140 """Execute a statement and return the one row that results.
1141
1142 This is like execute(sql, parameters).fetchone(), except it is
1143 correct in reading the entire result set. This will raise an
1144 exception if more than one row results.
1145
1146 Returns a row, or None if there were no rows.
1147 """
1148 rows = list(self.execute(sql, parameters))
1149 if len(rows) == 0:
1150 return None
1151 elif len(rows) == 1:
1152 return rows[0]
1153 else:
1154 raise AssertionError(f"SQL {sql!r} shouldn't return {len(rows)} rows")
1155
1156 def executemany(self, sql, data):
1157 """Same as :meth:`python:sqlite3.Connection.executemany`."""
1158 if self.debug.should("sql"):
1159 data = list(data)
1160 final = ":" if self.debug.should("sqldata") else ""
1161 self.debug.write(f"Executing many {sql!r} with {len(data)} rows{final}")
1162 if self.debug.should("sqldata"):
1163 for i, row in enumerate(data):
1164 self.debug.write(f"{i:4d}: {row!r}")
1165 try:
1166 return self.con.executemany(sql, data)
1167 except Exception: # pragma: cant happen
1168 # In some cases, an error might happen that isn't really an
1169 # error. Try again immediately.
1170 # https://github.com/nedbat/coveragepy/issues/1010
1171 return self.con.executemany(sql, data)
1172
1173 def executescript(self, script):
1174 """Same as :meth:`python:sqlite3.Connection.executescript`."""
1175 if self.debug.should("sql"):
1176 self.debug.write("Executing script with {} chars: {}".format(
1177 len(script), clipped_repr(script, 100),
1178 ))
1179 self.con.executescript(script)
1180
1181 def dump(self):
1182 """Return a multi-line string, the SQL dump of the database."""
1183 return "\n".join(self.con.iterdump())
1184
1185
1186 def _regexp(text, pattern):
1187 """A regexp function for SQLite."""
1188 return re.search(text, pattern) is not None

Mercurial Repository: eric

eric ide

mercurial