From c24e594796b860531521be0190fc2f922c092c0e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sun, 15 Jul 2018 11:59:33 -0400 Subject: CoverageData now also handles file operations --- coverage/data.py | 272 ++++++++++++++++++++++++++----------------------------- 1 file changed, 127 insertions(+), 145 deletions(-) (limited to 'coverage/data.py') diff --git a/coverage/data.py b/coverage/data.py index 9f2d1308..6d30e2ba 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -57,8 +57,7 @@ class CoverageData(object): names in this API are case-sensitive, even on platforms with case-insensitive file systems. - To read a coverage.py data file, use :meth:`read_file`, or - :meth:`read_fileobj` if you have an already-opened file. You can then + To read a coverage.py data file, use :meth:`read_file`. You can then access the line, arc, or file tracer data with :meth:`lines`, :meth:`arcs`, or :meth:`file_tracer`. Run information is available with :meth:`run_infos`. @@ -78,8 +77,7 @@ class CoverageData(object): To add a file without any measured data, use :meth:`touch_file`. - You write to a named file with :meth:`write_file`, or to an already opened - file with :meth:`write_fileobj`. + You write to a named file with :meth:`write_file`. You can clear the data in memory with :meth:`erase`. Two data collections can be combined by using :meth:`update` on one :class:`CoverageData`, @@ -112,13 +110,19 @@ class CoverageData(object): # line data is easily recovered from the arcs: it is all the first elements # of the pairs that are greater than zero. - def __init__(self, debug=None): + def __init__(self, basename=None, warn=None, debug=None): """Create a CoverageData. + `warn` is the warning function to use. + + `basename` is the name of the file to use for storing data. + `debug` is a `DebugControl` object for writing debug messages. """ + self._warn = warn self._debug = debug + self.filename = os.path.abspath(basename or ".coverage") # A map from canonical Python source file name to a dictionary in # which there's an entry for each line number that has been @@ -262,7 +266,12 @@ class CoverageData(object): __bool__ = __nonzero__ - def read_fileobj(self, file_obj): + def read(self): + """Read the coverage data.""" + if os.path.exists(self.filename): + self.read_file(self.filename) + + def _read_fileobj(self, file_obj): """Read the coverage data from the given file object. Should only be used on an empty CoverageData object. @@ -290,7 +299,7 @@ class CoverageData(object): self._debug.write("Reading data from %r" % (filename,)) try: with self._open_for_reading(filename) as f: - self.read_fileobj(f) + self._read_fileobj(f) except Exception as exc: raise CoverageException( "Couldn't read data from '%s': %s: %s" % ( @@ -438,7 +447,34 @@ class CoverageData(object): self._validate() - def write_fileobj(self, file_obj): + def write(self, suffix=None): + """Write the collected coverage data to a file. + + `suffix` is a suffix to append to the base file name. This can be used + for multiple or parallel execution, so that many coverage data files + can exist simultaneously. A dot will be used to join the base name and + the suffix. + + """ + filename = self.filename + if suffix is True: + # If data_suffix was a simple true value, then make a suffix with + # plenty of distinguishing information. We do this here in + # `save()` at the last minute so that the pid will be correct even + # if the process forks. + extra = "" + if _TEST_NAME_FILE: # pragma: debugging + with open(_TEST_NAME_FILE) as f: + test_name = f.read() + extra = "." + test_name + dice = random.Random(os.urandom(8)).randint(0, 999999) + suffix = "%s%s.%s.%06d" % (socket.gethostname(), extra, os.getpid(), dice) + + if suffix: + filename += "." + suffix + self.write_file(filename) + + def _write_fileobj(self, file_obj): """Write the coverage data to `file_obj`.""" # Create the file data. @@ -465,16 +501,33 @@ class CoverageData(object): if self._debug and self._debug.should('dataio'): self._debug.write("Writing data to %r" % (filename,)) with open(filename, 'w') as fdata: - self.write_fileobj(fdata) + self._write_fileobj(fdata) + + def erase(self, parallel=False): + """Erase the data in this object. + + If `parallel` is true, then also deletes data files created from the + basename by parallel-mode. - def erase(self): - """Erase the data in this object.""" + """ self._lines = None self._arcs = None self._file_tracers = {} self._runs = [] self._validate() + if self._debug and self._debug.should('dataio'): + self._debug.write("Erasing data file %r" % (self.filename,)) + file_be_gone(self.filename) + if parallel: + data_dir, local = os.path.split(self.filename) + localdot = local + '.*' + pattern = os.path.join(os.path.abspath(data_dir), localdot) + for filename in glob.glob(pattern): + if self._debug and self._debug.should('dataio'): + self._debug.write("Erasing parallel data file %r" % (filename,)) + file_be_gone(filename) + def update(self, other_data, aliases=None): """Update this data with data from another `CoverageData`. @@ -535,6 +588,69 @@ class CoverageData(object): self._validate() + def combine_parallel_data(self, aliases=None, data_paths=None, strict=False): + """Combine a number of data files together. + + Treat `self.filename` as a file prefix, and combine the data from all + of the data files starting with that prefix plus a dot. + + If `aliases` is provided, it's a `PathAliases` object that is used to + re-map paths to match the local machine's. + + If `data_paths` is provided, it is a list of directories or files to + combine. Directories are searched for files that start with + `self.filename` plus dot as a prefix, and those files are combined. + + If `data_paths` is not provided, then the directory portion of + `self.filename` is used as the directory to search for data files. + + Every data file found and combined is then deleted from disk. If a file + cannot be read, a warning will be issued, and the file will not be + deleted. + + If `strict` is true, and no files are found to combine, an error is + raised. + + """ + # Because of the os.path.abspath in the constructor, data_dir will + # never be an empty string. + data_dir, local = os.path.split(self.filename) + localdot = local + '.*' + + data_paths = data_paths or [data_dir] + files_to_combine = [] + for p in data_paths: + if os.path.isfile(p): + files_to_combine.append(os.path.abspath(p)) + elif os.path.isdir(p): + pattern = os.path.join(os.path.abspath(p), localdot) + files_to_combine.extend(glob.glob(pattern)) + else: + raise CoverageException("Couldn't combine from non-existent path '%s'" % (p,)) + + if strict and not files_to_combine: + raise CoverageException("No data to combine") + + files_combined = 0 + for f in files_to_combine: + new_data = CoverageData(debug=self._debug) + try: + new_data.read_file(f) + except CoverageException as exc: + if self._warn: + # The CoverageException has the file name in it, so just + # use the message as the warning. + self._warn(str(exc)) + else: + self.update(new_data, aliases=aliases) + files_combined += 1 + if self._debug and self._debug.should('dataio'): + self._debug.write("Deleting combined data file %r" % (f,)) + file_be_gone(f) + + if strict and not files_combined: + raise CoverageException("No usable data files") + ## ## Miscellaneous ## @@ -609,140 +725,6 @@ class CoverageData(object): return self._arcs is not None -class CoverageDataFiles(object): - """Manage the use of coverage data files.""" - - def __init__(self, basename=None, warn=None, debug=None): - """Create a CoverageDataFiles to manage data files. - - `warn` is the warning function to use. - - `basename` is the name of the file to use for storing data. - - `debug` is a `DebugControl` object for writing debug messages. - - """ - self.warn = warn - self.debug = debug - - # Construct the file name that will be used for data storage. - self.filename = os.path.abspath(basename or ".coverage") - - def erase(self, parallel=False): - """Erase the data from the file storage. - - If `parallel` is true, then also deletes data files created from the - basename by parallel-mode. - - """ - if self.debug and self.debug.should('dataio'): - self.debug.write("Erasing data file %r" % (self.filename,)) - file_be_gone(self.filename) - if parallel: - data_dir, local = os.path.split(self.filename) - localdot = local + '.*' - pattern = os.path.join(os.path.abspath(data_dir), localdot) - for filename in glob.glob(pattern): - if self.debug and self.debug.should('dataio'): - self.debug.write("Erasing parallel data file %r" % (filename,)) - file_be_gone(filename) - - def read(self, data): - """Read the coverage data.""" - if os.path.exists(self.filename): - data.read_file(self.filename) - - def write(self, data, suffix=None): - """Write the collected coverage data to a file. - - `suffix` is a suffix to append to the base file name. This can be used - for multiple or parallel execution, so that many coverage data files - can exist simultaneously. A dot will be used to join the base name and - the suffix. - - """ - filename = self.filename - if suffix is True: - # If data_suffix was a simple true value, then make a suffix with - # plenty of distinguishing information. We do this here in - # `save()` at the last minute so that the pid will be correct even - # if the process forks. - extra = "" - if _TEST_NAME_FILE: # pragma: debugging - with open(_TEST_NAME_FILE) as f: - test_name = f.read() - extra = "." + test_name - dice = random.Random(os.urandom(8)).randint(0, 999999) - suffix = "%s%s.%s.%06d" % (socket.gethostname(), extra, os.getpid(), dice) - - if suffix: - filename += "." + suffix - data.write_file(filename) - - def combine_parallel_data(self, data, aliases=None, data_paths=None, strict=False): - """Combine a number of data files together. - - Treat `self.filename` as a file prefix, and combine the data from all - of the data files starting with that prefix plus a dot. - - If `aliases` is provided, it's a `PathAliases` object that is used to - re-map paths to match the local machine's. - - If `data_paths` is provided, it is a list of directories or files to - combine. Directories are searched for files that start with - `self.filename` plus dot as a prefix, and those files are combined. - - If `data_paths` is not provided, then the directory portion of - `self.filename` is used as the directory to search for data files. - - Every data file found and combined is then deleted from disk. If a file - cannot be read, a warning will be issued, and the file will not be - deleted. - - If `strict` is true, and no files are found to combine, an error is - raised. - - """ - # Because of the os.path.abspath in the constructor, data_dir will - # never be an empty string. - data_dir, local = os.path.split(self.filename) - localdot = local + '.*' - - data_paths = data_paths or [data_dir] - files_to_combine = [] - for p in data_paths: - if os.path.isfile(p): - files_to_combine.append(os.path.abspath(p)) - elif os.path.isdir(p): - pattern = os.path.join(os.path.abspath(p), localdot) - files_to_combine.extend(glob.glob(pattern)) - else: - raise CoverageException("Couldn't combine from non-existent path '%s'" % (p,)) - - if strict and not files_to_combine: - raise CoverageException("No data to combine") - - files_combined = 0 - for f in files_to_combine: - new_data = CoverageData(debug=self.debug) - try: - new_data.read_file(f) - except CoverageException as exc: - if self.warn: - # The CoverageException has the file name in it, so just - # use the message as the warning. - self.warn(str(exc)) - else: - data.update(new_data, aliases=aliases) - files_combined += 1 - if self.debug and self.debug.should('dataio'): - self.debug.write("Deleting combined data file %r" % (f,)) - file_be_gone(f) - - if strict and not files_combined: - raise CoverageException("No usable data files") - - def canonicalize_json_data(data): """Canonicalize our JSON data so it can be compared.""" for fname, lines in iitems(data.get('lines', {})): -- cgit v1.2.1 From 7d71b1e052b2adead8c43bbc320582eab4938221 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sun, 15 Jul 2018 16:26:48 -0400 Subject: Make file operations implicit on constructed filename --- coverage/data.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) (limited to 'coverage/data.py') diff --git a/coverage/data.py b/coverage/data.py index 6d30e2ba..23e612a1 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -57,7 +57,10 @@ class CoverageData(object): names in this API are case-sensitive, even on platforms with case-insensitive file systems. - To read a coverage.py data file, use :meth:`read_file`. You can then + A data file is associated with the data when the :class:`CoverageData` + is created. + + To read a coverage.py data file, use :meth:`read`. You can then access the line, arc, or file tracer data with :meth:`lines`, :meth:`arcs`, or :meth:`file_tracer`. Run information is available with :meth:`run_infos`. @@ -68,16 +71,15 @@ class CoverageData(object): most Python containers, you can determine if there is any data at all by using this object as a boolean value. - Most data files will be created by coverage.py itself, but you can use methods here to create data files if you like. The :meth:`add_lines`, :meth:`add_arcs`, and :meth:`add_file_tracers` methods add data, in ways that are convenient for coverage.py. The :meth:`add_run_info` method adds key-value pairs to the run information. - To add a file without any measured data, use :meth:`touch_file`. + To add a source file without any measured data, use :meth:`touch_file`. - You write to a named file with :meth:`write_file`. + Write the data to its file with :meth:`write`. You can clear the data in memory with :meth:`erase`. Two data collections can be combined by using :meth:`update` on one :class:`CoverageData`, @@ -267,9 +269,13 @@ class CoverageData(object): __bool__ = __nonzero__ def read(self): - """Read the coverage data.""" + """Read the coverage data. + + It is fine for the file to not exist, in which case no data is read. + + """ if os.path.exists(self.filename): - self.read_file(self.filename) + self._read_file(self.filename) def _read_fileobj(self, file_obj): """Read the coverage data from the given file object. @@ -293,7 +299,7 @@ class CoverageData(object): self._validate() - def read_file(self, filename): + def _read_file(self, filename): """Read the coverage data from `filename` into this object.""" if self._debug and self._debug.should('dataio'): self._debug.write("Reading data from %r" % (filename,)) @@ -472,7 +478,7 @@ class CoverageData(object): if suffix: filename += "." + suffix - self.write_file(filename) + self._write_file(filename) def _write_fileobj(self, file_obj): """Write the coverage data to `file_obj`.""" @@ -496,7 +502,7 @@ class CoverageData(object): file_obj.write(self._GO_AWAY) json.dump(file_data, file_obj, separators=(',', ':')) - def write_file(self, filename): + def _write_file(self, filename): """Write the coverage data to `filename`.""" if self._debug and self._debug.should('dataio'): self._debug.write("Writing data to %r" % (filename,)) @@ -635,7 +641,7 @@ class CoverageData(object): for f in files_to_combine: new_data = CoverageData(debug=self._debug) try: - new_data.read_file(f) + new_data._read_file(f) except CoverageException as exc: if self._warn: # The CoverageException has the file name in it, so just -- cgit v1.2.1 From e301a01b772cfab9f567724e01df33e862d3b72f Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sun, 29 Jul 2018 18:46:02 -0400 Subject: WIP WIP WIP --- coverage/data.py | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) (limited to 'coverage/data.py') diff --git a/coverage/data.py b/coverage/data.py index 23e612a1..afb12df4 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -12,6 +12,7 @@ import os.path import random import re import socket +import sqlite3 from coverage import env from coverage.backward import iitems, string_class @@ -731,6 +732,126 @@ class CoverageData(object): return self._arcs is not None +SCHEMA = """ +create table schema ( + version integer +); + +insert into schema (version) values (1); + +create table file ( + id integer primary key, + path text, + tracer text, + unique(path) +); + +create table line ( + file_id integer, + lineno integer, + unique(file_id, lineno) +); +""" + +def _create_db(filename): + con = sqlite3.connect(filename) + with con: + for stmt in SCHEMA.split(';'): + con.execute(stmt.strip()) + con.close() + + +class CoverageDataSqlite(object): + def __init__(self, basename=None, warn=None, debug=None): + self.filename = os.path.abspath(basename or ".coverage") + self._warn = warn + self._debug = debug + + self._file_map = {} + self._db = None + + def _reset(self): + self._file_map = {} + if self._db is not None: + self._db.close() + self._db = None + + def _connect(self): + if self._db is None: + if not os.path.exists(self.filename): + if self._debug and self._debug.should('dataio'): + self._debug.write("Creating data file %r" % (self.filename,)) + _create_db(self.filename) + self._db = sqlite3.connect(self.filename) + for path, id in self._db.execute("select path, id from file"): + self._file_map[path] = id + return self._db + + def _file_id(self, filename): + if filename not in self._file_map: + with self._connect() as con: + cur = con.cursor() + cur.execute("insert into file (path) values (?)", (filename,)) + self._file_map[filename] = cur.lastrowid + return self._file_map[filename] + + def add_lines(self, line_data): + """Add measured line data. + + `line_data` is a dictionary mapping file names to dictionaries:: + + { filename: { lineno: None, ... }, ...} + + """ + with self._connect() as con: + for filename, linenos in iitems(line_data): + file_id = self._file_id(filename) + for lineno in linenos: + con.execute( + "insert or ignore into line (file_id, lineno) values (?, ?)", + (file_id, lineno), + ) + + def add_file_tracers(self, file_tracers): + """Add per-file plugin information. + + `file_tracers` is { filename: plugin_name, ... } + + """ + with self._connect() as con: + for filename, tracer in iitems(file_tracers): + con.execute( + "insert into file (path, tracer) values (?, ?) on duplicate key update", + (filename, tracer), + ) + + def erase(self, parallel=False): + """Erase the data in this object. + + If `parallel` is true, then also deletes data files created from the + basename by parallel-mode. + + """ + self._reset() + if self._debug and self._debug.should('dataio'): + self._debug.write("Erasing data file %r" % (self.filename,)) + file_be_gone(self.filename) + if parallel: + data_dir, local = os.path.split(self.filename) + localdot = local + '.*' + pattern = os.path.join(os.path.abspath(data_dir), localdot) + for filename in glob.glob(pattern): + if self._debug and self._debug.should('dataio'): + self._debug.write("Erasing parallel data file %r" % (filename,)) + file_be_gone(filename) + + def write(self, suffix=None): + """Write the collected coverage data to a file.""" + pass + +CoverageData = CoverageDataSqlite + + def canonicalize_json_data(data): """Canonicalize our JSON data so it can be compared.""" for fname, lines in iitems(data.get('lines', {})): -- cgit v1.2.1 From e7f8cd3804245104657e41b548a431801f6c1cee Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 31 Jul 2018 06:09:00 -0400 Subject: Move sqlite into sqldata.py --- coverage/data.py | 119 +------------------------------------------------------ 1 file changed, 1 insertion(+), 118 deletions(-) (limited to 'coverage/data.py') diff --git a/coverage/data.py b/coverage/data.py index afb12df4..eda1a341 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -12,7 +12,6 @@ import os.path import random import re import socket -import sqlite3 from coverage import env from coverage.backward import iitems, string_class @@ -732,123 +731,7 @@ class CoverageData(object): return self._arcs is not None -SCHEMA = """ -create table schema ( - version integer -); - -insert into schema (version) values (1); - -create table file ( - id integer primary key, - path text, - tracer text, - unique(path) -); - -create table line ( - file_id integer, - lineno integer, - unique(file_id, lineno) -); -""" - -def _create_db(filename): - con = sqlite3.connect(filename) - with con: - for stmt in SCHEMA.split(';'): - con.execute(stmt.strip()) - con.close() - - -class CoverageDataSqlite(object): - def __init__(self, basename=None, warn=None, debug=None): - self.filename = os.path.abspath(basename or ".coverage") - self._warn = warn - self._debug = debug - - self._file_map = {} - self._db = None - - def _reset(self): - self._file_map = {} - if self._db is not None: - self._db.close() - self._db = None - - def _connect(self): - if self._db is None: - if not os.path.exists(self.filename): - if self._debug and self._debug.should('dataio'): - self._debug.write("Creating data file %r" % (self.filename,)) - _create_db(self.filename) - self._db = sqlite3.connect(self.filename) - for path, id in self._db.execute("select path, id from file"): - self._file_map[path] = id - return self._db - - def _file_id(self, filename): - if filename not in self._file_map: - with self._connect() as con: - cur = con.cursor() - cur.execute("insert into file (path) values (?)", (filename,)) - self._file_map[filename] = cur.lastrowid - return self._file_map[filename] - - def add_lines(self, line_data): - """Add measured line data. - - `line_data` is a dictionary mapping file names to dictionaries:: - - { filename: { lineno: None, ... }, ...} - - """ - with self._connect() as con: - for filename, linenos in iitems(line_data): - file_id = self._file_id(filename) - for lineno in linenos: - con.execute( - "insert or ignore into line (file_id, lineno) values (?, ?)", - (file_id, lineno), - ) - - def add_file_tracers(self, file_tracers): - """Add per-file plugin information. - - `file_tracers` is { filename: plugin_name, ... } - - """ - with self._connect() as con: - for filename, tracer in iitems(file_tracers): - con.execute( - "insert into file (path, tracer) values (?, ?) on duplicate key update", - (filename, tracer), - ) - - def erase(self, parallel=False): - """Erase the data in this object. - - If `parallel` is true, then also deletes data files created from the - basename by parallel-mode. - - """ - self._reset() - if self._debug and self._debug.should('dataio'): - self._debug.write("Erasing data file %r" % (self.filename,)) - file_be_gone(self.filename) - if parallel: - data_dir, local = os.path.split(self.filename) - localdot = local + '.*' - pattern = os.path.join(os.path.abspath(data_dir), localdot) - for filename in glob.glob(pattern): - if self._debug and self._debug.should('dataio'): - self._debug.write("Erasing parallel data file %r" % (filename,)) - file_be_gone(filename) - - def write(self, suffix=None): - """Write the collected coverage data to a file.""" - pass - +from coverage.sqldata import CoverageDataSqlite CoverageData = CoverageDataSqlite -- cgit v1.2.1 From b457052020ec90fdba964ff8bd5abe6d92032e6b Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 3 Aug 2018 07:44:56 -0400 Subject: Make writing data faster --- coverage/data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'coverage/data.py') diff --git a/coverage/data.py b/coverage/data.py index eda1a341..e9243166 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -22,7 +22,7 @@ from coverage.misc import CoverageException, file_be_gone, isolate_module os = isolate_module(os) -class CoverageData(object): +class CoverageJsonData(object): """Manages collected coverage data, including file storage. This class is the public supported API to the data coverage.py collects @@ -731,8 +731,8 @@ class CoverageData(object): return self._arcs is not None -from coverage.sqldata import CoverageDataSqlite -CoverageData = CoverageDataSqlite +from coverage.sqldata import CoverageSqliteData +CoverageData = CoverageSqliteData def canonicalize_json_data(data): -- cgit v1.2.1 From 2f0d57856550ef7ad248e4e6127700bdabb91e7d Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sat, 4 Aug 2018 07:36:13 -0400 Subject: Pull combine_parallel_data out of CoverageData --- coverage/data.py | 134 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 69 insertions(+), 65 deletions(-) (limited to 'coverage/data.py') diff --git a/coverage/data.py b/coverage/data.py index e9243166..0b3b640b 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -594,69 +594,6 @@ class CoverageJsonData(object): self._validate() - def combine_parallel_data(self, aliases=None, data_paths=None, strict=False): - """Combine a number of data files together. - - Treat `self.filename` as a file prefix, and combine the data from all - of the data files starting with that prefix plus a dot. - - If `aliases` is provided, it's a `PathAliases` object that is used to - re-map paths to match the local machine's. - - If `data_paths` is provided, it is a list of directories or files to - combine. Directories are searched for files that start with - `self.filename` plus dot as a prefix, and those files are combined. - - If `data_paths` is not provided, then the directory portion of - `self.filename` is used as the directory to search for data files. - - Every data file found and combined is then deleted from disk. If a file - cannot be read, a warning will be issued, and the file will not be - deleted. - - If `strict` is true, and no files are found to combine, an error is - raised. - - """ - # Because of the os.path.abspath in the constructor, data_dir will - # never be an empty string. - data_dir, local = os.path.split(self.filename) - localdot = local + '.*' - - data_paths = data_paths or [data_dir] - files_to_combine = [] - for p in data_paths: - if os.path.isfile(p): - files_to_combine.append(os.path.abspath(p)) - elif os.path.isdir(p): - pattern = os.path.join(os.path.abspath(p), localdot) - files_to_combine.extend(glob.glob(pattern)) - else: - raise CoverageException("Couldn't combine from non-existent path '%s'" % (p,)) - - if strict and not files_to_combine: - raise CoverageException("No data to combine") - - files_combined = 0 - for f in files_to_combine: - new_data = CoverageData(debug=self._debug) - try: - new_data._read_file(f) - except CoverageException as exc: - if self._warn: - # The CoverageException has the file name in it, so just - # use the message as the warning. - self._warn(str(exc)) - else: - self.update(new_data, aliases=aliases) - files_combined += 1 - if self._debug and self._debug.should('dataio'): - self._debug.write("Deleting combined data file %r" % (f,)) - file_be_gone(f) - - if strict and not files_combined: - raise CoverageException("No usable data files") - ## ## Miscellaneous ## @@ -731,9 +668,76 @@ class CoverageJsonData(object): return self._arcs is not None -from coverage.sqldata import CoverageSqliteData -CoverageData = CoverageSqliteData +which = os.environ.get("COV_STORAGE", "json") +if which == "json": + CoverageData = CoverageJsonData +elif which == "sql": + from coverage.sqldata import CoverageSqliteData + CoverageData = CoverageSqliteData + +def combine_parallel_data(data, aliases=None, data_paths=None, strict=False): + """Combine a number of data files together. + + Treat `data.filename` as a file prefix, and combine the data from all + of the data files starting with that prefix plus a dot. + + If `aliases` is provided, it's a `PathAliases` object that is used to + re-map paths to match the local machine's. + + If `data_paths` is provided, it is a list of directories or files to + combine. Directories are searched for files that start with + `data.filename` plus dot as a prefix, and those files are combined. + + If `data_paths` is not provided, then the directory portion of + `data.filename` is used as the directory to search for data files. + + Every data file found and combined is then deleted from disk. If a file + cannot be read, a warning will be issued, and the file will not be + deleted. + + If `strict` is true, and no files are found to combine, an error is + raised. + + """ + # Because of the os.path.abspath in the constructor, data_dir will + # never be an empty string. + data_dir, local = os.path.split(data.filename) + localdot = local + '.*' + + data_paths = data_paths or [data_dir] + files_to_combine = [] + for p in data_paths: + if os.path.isfile(p): + files_to_combine.append(os.path.abspath(p)) + elif os.path.isdir(p): + pattern = os.path.join(os.path.abspath(p), localdot) + files_to_combine.extend(glob.glob(pattern)) + else: + raise CoverageException("Couldn't combine from non-existent path '%s'" % (p,)) + + if strict and not files_to_combine: + raise CoverageException("No data to combine") + + files_combined = 0 + for f in files_to_combine: + try: + new_data = CoverageData(f, debug=data._debug) + new_data.read() + except CoverageException as exc: + if data._warn: + # The CoverageException has the file name in it, so just + # use the message as the warning. + data._warn(str(exc)) + else: + data.update(new_data, aliases=aliases) + files_combined += 1 + if data._debug and data._debug.should('dataio'): + data._debug.write("Deleting combined data file %r" % (f,)) + file_be_gone(f) + + if strict and not files_combined: + raise CoverageException("No usable data files") def canonicalize_json_data(data): """Canonicalize our JSON data so it can be compared.""" -- cgit v1.2.1 From 3335bb8df9226fbb3fb71dca65b7f795ee5c9552 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 9 Aug 2018 21:32:59 -0400 Subject: Keep the env var naming scheme --- coverage/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'coverage/data.py') diff --git a/coverage/data.py b/coverage/data.py index 0b3b640b..db9cd526 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -668,7 +668,7 @@ class CoverageJsonData(object): return self._arcs is not None -which = os.environ.get("COV_STORAGE", "json") +which = os.environ.get("COVERAGE_STORAGE", "json") if which == "json": CoverageData = CoverageJsonData elif which == "sql": -- cgit v1.2.1 From 90bb6a77e02cbac6a23723b5907d5f59d1db1b82 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 10 Aug 2018 16:15:00 -0400 Subject: Move a common method outside the data classes --- coverage/data.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) (limited to 'coverage/data.py') diff --git a/coverage/data.py b/coverage/data.py index db9cd526..9c82ccef 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -641,20 +641,6 @@ class CoverageJsonData(object): for key in val: assert isinstance(key, string_class), "Key in _runs shouldn't be %r" % (key,) - def add_to_hash(self, filename, hasher): - """Contribute `filename`'s data to the `hasher`. - - `hasher` is a `coverage.misc.Hasher` instance to be updated with - the file's data. It should only get the results data, not the run - data. - - """ - if self._has_arcs(): - hasher.update(sorted(self.arcs(filename) or [])) - else: - hasher.update(sorted(self.lines(filename) or [])) - hasher.update(self.file_tracer(filename)) - ## ## Internal ## @@ -676,6 +662,21 @@ elif which == "sql": CoverageData = CoverageSqliteData +def add_data_to_hash(data, filename, hasher): + """Contribute `filename`'s data to the `hasher`. + + `hasher` is a `coverage.misc.Hasher` instance to be updated with + the file's data. It should only get the results data, not the run + data. + + """ + if data.has_arcs(): + hasher.update(sorted(data.arcs(filename) or [])) + else: + hasher.update(sorted(data.lines(filename) or [])) + hasher.update(data.file_tracer(filename)) + + def combine_parallel_data(data, aliases=None, data_paths=None, strict=False): """Combine a number of data files together. -- cgit v1.2.1 From 8562aeb29eddf3349f5c363c1842f9822b18a450 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 10 Aug 2018 16:39:22 -0400 Subject: Move line_counts out of the data classes --- coverage/data.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) (limited to 'coverage/data.py') diff --git a/coverage/data.py b/coverage/data.py index 9c82ccef..44b75439 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -244,25 +244,6 @@ class CoverageJsonData(object): """A list of all files that had been measured.""" return list(self._arcs or self._lines or {}) - def line_counts(self, fullpath=False): - """Return a dict summarizing the line coverage data. - - Keys are based on the file names, and values are the number of executed - lines. If `fullpath` is true, then the keys are the full pathnames of - the files, otherwise they are the basenames of the files. - - Returns a dict mapping file names to counts of lines. - - """ - summ = {} - if fullpath: - filename_fn = lambda f: f - else: - filename_fn = os.path.basename - for filename in self.measured_files(): - summ[filename_fn(filename)] = len(self.lines(filename)) - return summ - def __nonzero__(self): return bool(self._lines or self._arcs) @@ -662,6 +643,26 @@ elif which == "sql": CoverageData = CoverageSqliteData +def line_counts(data, fullpath=False): + """Return a dict summarizing the line coverage data. + + Keys are based on the file names, and values are the number of executed + lines. If `fullpath` is true, then the keys are the full pathnames of + the files, otherwise they are the basenames of the files. + + Returns a dict mapping file names to counts of lines. + + """ + summ = {} + if fullpath: + filename_fn = lambda f: f + else: + filename_fn = os.path.basename + for filename in data.measured_files(): + summ[filename_fn(filename)] = len(data.lines(filename)) + return summ + + def add_data_to_hash(data, filename, hasher): """Contribute `filename`'s data to the `hasher`. -- cgit v1.2.1 From 420c1b10ddeed1da66a2ffb81d7ac2af32939be5 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sat, 11 Aug 2018 07:40:05 -0400 Subject: Implement more --- coverage/data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'coverage/data.py') diff --git a/coverage/data.py b/coverage/data.py index 44b75439..4b8b7eb2 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -635,10 +635,10 @@ class CoverageJsonData(object): return self._arcs is not None -which = os.environ.get("COVERAGE_STORAGE", "json") -if which == "json": +STORAGE = os.environ.get("COVERAGE_STORAGE", "json") +if STORAGE == "json": CoverageData = CoverageJsonData -elif which == "sql": +elif STORAGE == "sql": from coverage.sqldata import CoverageSqliteData CoverageData = CoverageSqliteData -- cgit v1.2.1 From 5997b823da8d60d909e776424d4ba488bb3927ec Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sun, 12 Aug 2018 07:05:33 -0400 Subject: Start moving suffix to constructor --- coverage/data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'coverage/data.py') diff --git a/coverage/data.py b/coverage/data.py index 4b8b7eb2..15d0a273 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -112,7 +112,7 @@ class CoverageJsonData(object): # line data is easily recovered from the arcs: it is all the first elements # of the pairs that are greater than zero. - def __init__(self, basename=None, warn=None, debug=None): + def __init__(self, basename=None, suffix=None, warn=None, debug=None): """Create a CoverageData. `warn` is the warning function to use. @@ -125,6 +125,7 @@ class CoverageJsonData(object): self._warn = warn self._debug = debug self.filename = os.path.abspath(basename or ".coverage") + self.suffix = suffix # A map from canonical Python source file name to a dictionary in # which there's an entry for each line number that has been @@ -434,7 +435,7 @@ class CoverageJsonData(object): self._validate() - def write(self, suffix=None): + def write(self): """Write the collected coverage data to a file. `suffix` is a suffix to append to the base file name. This can be used @@ -444,6 +445,7 @@ class CoverageJsonData(object): """ filename = self.filename + suffix = self.suffix if suffix is True: # If data_suffix was a simple true value, then make a suffix with # plenty of distinguishing information. We do this here in -- cgit v1.2.1 From f087c213dbe2ffb1b4a0661c9d25e67915987a99 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 14 Aug 2018 08:05:57 -0400 Subject: Remove an unused debugging thing --- coverage/data.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'coverage/data.py') diff --git a/coverage/data.py b/coverage/data.py index 15d0a273..5e85fc10 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -15,7 +15,6 @@ import socket from coverage import env from coverage.backward import iitems, string_class -from coverage.debug import _TEST_NAME_FILE from coverage.files import PathAliases from coverage.misc import CoverageException, file_be_gone, isolate_module @@ -451,13 +450,8 @@ class CoverageJsonData(object): # plenty of distinguishing information. We do this here in # `save()` at the last minute so that the pid will be correct even # if the process forks. - extra = "" - if _TEST_NAME_FILE: # pragma: debugging - with open(_TEST_NAME_FILE) as f: - test_name = f.read() - extra = "." + test_name dice = random.Random(os.urandom(8)).randint(0, 999999) - suffix = "%s%s.%s.%06d" % (socket.gethostname(), extra, os.getpid(), dice) + suffix = "%s.%s.%06d" % (socket.gethostname(), os.getpid(), dice) if suffix: filename += "." + suffix -- cgit v1.2.1 From 067d0a60384b5f12cfee622381cfb5905efb8e13 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 14 Aug 2018 20:38:39 -0400 Subject: Use pid-random suffixes for SQL files --- coverage/data.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'coverage/data.py') diff --git a/coverage/data.py b/coverage/data.py index 5e85fc10..aa23e7d4 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -21,6 +21,17 @@ from coverage.misc import CoverageException, file_be_gone, isolate_module os = isolate_module(os) +def filename_suffix(suffix): + if suffix is True: + # If data_suffix was a simple true value, then make a suffix with + # plenty of distinguishing information. We do this here in + # `save()` at the last minute so that the pid will be correct even + # if the process forks. + dice = random.Random(os.urandom(8)).randint(0, 999999) + suffix = "%s.%s.%06d" % (socket.gethostname(), os.getpid(), dice) + return suffix + + class CoverageJsonData(object): """Manages collected coverage data, including file storage. @@ -444,15 +455,7 @@ class CoverageJsonData(object): """ filename = self.filename - suffix = self.suffix - if suffix is True: - # If data_suffix was a simple true value, then make a suffix with - # plenty of distinguishing information. We do this here in - # `save()` at the last minute so that the pid will be correct even - # if the process forks. - dice = random.Random(os.urandom(8)).randint(0, 999999) - suffix = "%s.%s.%06d" % (socket.gethostname(), os.getpid(), dice) - + suffix = filename_suffix(self.suffix) if suffix: filename += "." + suffix self._write_file(filename) -- cgit v1.2.1 From a6097893ac54e6332a7c7b4b3667fc3064d9fb1b Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 23 Aug 2018 08:36:47 -0400 Subject: Make SQLite the default storage --- coverage/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'coverage/data.py') diff --git a/coverage/data.py b/coverage/data.py index aa23e7d4..f03e90ca 100644 --- a/coverage/data.py +++ b/coverage/data.py @@ -634,7 +634,7 @@ class CoverageJsonData(object): return self._arcs is not None -STORAGE = os.environ.get("COVERAGE_STORAGE", "json") +STORAGE = os.environ.get("COVERAGE_STORAGE", "sql") if STORAGE == "json": CoverageData = CoverageJsonData elif STORAGE == "sql": -- cgit v1.2.1