Refactor numbits into their own files

4 files changed, 86 insertions, 63 deletions

diff --git a/coverage/numbits.py b/coverage/numbits.py
new file mode 100644
index 00000000..bc29ed94
--- /dev/null
+++ b/coverage/numbits.py
@@ -0,0 +1,42 @@
+# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
+# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt
+
+"""
+Functions to manipulate packed binary representations of number sets.
+
+To save space, coverage stores sets of line numbers in SQLite using a packed
+binary representation called a numbits.  A numbits is stored as a blob in the
+database.  The exact meaning of the bytes in the blobs should be considered an
+implementation detail that might change in the future.  Use these functions to
+work with those binary blobs of data.
+
+"""
+
+from coverage.backward import bytes_to_ints, binary_bytes, zip_longest
+from coverage.misc import contract
+
+
+@contract(nums='Iterable', returns='bytes')
+def nums_to_numbits(nums):
+    """Convert `nums` (an iterable of ints) into a numbits."""
+    nbytes = max(nums) // 8 + 1
+    b = bytearray(nbytes)
+    for num in nums:
+        b[num//8] |= 1 << num % 8
+    return bytes(b)
+
+@contract(numbits='bytes', returns='list[int]')
+def numbits_to_nums(numbits):
+    """Convert a numbits into a list of numbers."""
+    nums = []
+    for byte_i, byte in enumerate(bytes_to_ints(numbits)):
+        for bit_i in range(8):
+            if (byte & (1 << bit_i)):
+                nums.append(byte_i * 8 + bit_i)
+    return nums
+
+@contract(numbits1='bytes', numbits2='bytes', returns='bytes')
+def merge_numbits(numbits1, numbits2):
+    """Merge two numbits"""
+    byte_pairs = zip_longest(bytes_to_ints(numbits1), bytes_to_ints(numbits2), fillvalue=0)
+    return binary_bytes(b1 | b2 for b1, b2 in byte_pairs)
diff --git a/coverage/sqldata.py b/coverage/sqldata.py
index 0404004d..1856ee32 100644
--- a/coverage/sqldata.py
+++ b/coverage/sqldata.py
@@ -17,13 +17,13 @@ import sqlite3
 import sys
 import zlib
 
-from coverage.backward import get_thread_id, iitems
-from coverage.backward import bytes_to_ints, binary_bytes, zip_longest, to_bytes, to_string
+from coverage.backward import get_thread_id, iitems, to_bytes, to_string
 from coverage.debug import NoDebugging, SimpleReprMixin
 from coverage import env
 from coverage.files import PathAliases
 from coverage.misc import CoverageException, file_be_gone, filename_suffix, isolate_module
 from coverage.misc import contract
+from coverage.numbits import nums_to_numbits, numbits_to_nums, merge_numbits
 
 os = isolate_module(os)
 
@@ -355,12 +355,12 @@ class CoverageData(SimpleReprMixin):
         with self._connect() as con:
             self._set_context_id()
             for filename, linenos in iitems(line_data):
-                linemap = nums_to_bitmap(linenos)
+                linemap = nums_to_numbits(linenos)
                 file_id = self._file_id(filename, add=True)
                 query = "select bitmap from line_map where file_id = ? and context_id = ?"
                 existing = list(con.execute(query, (file_id, self._current_context_id)))
                 if existing:
-                    linemap = merge_bitmaps(linemap, from_blob(existing[0][0]))
+                    linemap = merge_numbits(linemap, from_blob(existing[0][0]))
 
                 con.execute(
                     "insert or replace into line_map (file_id, context_id, bitmap) values (?, ?, ?)",
@@ -583,7 +583,7 @@ class CoverageData(SimpleReprMixin):
                 key = (aliases.map(path), context)
                 bitmap = from_blob(bitmap)
                 if key in lines:
-                    bitmap = merge_bitmaps(lines[key], bitmap)
+                    bitmap = merge_numbits(lines[key], bitmap)
                 lines[key] = bitmap
             cur.close()
 
@@ -727,7 +727,7 @@ class CoverageData(SimpleReprMixin):
                 bitmaps = list(con.execute(query, data))
                 nums = set()
                 for row in bitmaps:
-                    nums.update(bitmap_to_nums(from_blob(row[0])))
+                    nums.update(numbits_to_nums(from_blob(row[0])))
                 return sorted(nums)
 
     def arcs(self, filename, contexts=None):
@@ -784,7 +784,7 @@ class CoverageData(SimpleReprMixin):
                     query += " and l.context_id in (" + ids_array + ")"
                     data += context_ids
                 for bitmap, context in con.execute(query, data):
-                    for lineno in bitmap_to_nums(from_blob(bitmap)):
+                    for lineno in numbits_to_nums(from_blob(bitmap)):
                         lineno_contexts_map[lineno].append(context)
         return lineno_contexts_map
 
@@ -868,29 +868,3 @@ class SqliteDb(SimpleReprMixin):
     def dump(self):
         """Return a multi-line string, the dump of the database."""
         return "\n".join(self.con.iterdump())
-
-
-@contract(nums='Iterable', returns='bytes')
-def nums_to_bitmap(nums):
-    """Convert `nums` (an iterable of ints) into a bitmap."""
-    nbytes = max(nums) // 8 + 1
-    b = bytearray(nbytes)
-    for num in nums:
-        b[num//8] |= 1 << num % 8
-    return bytes(b)
-
-@contract(bitmap='bytes', returns='list[int]')
-def bitmap_to_nums(bitmap):
-    """Convert a bitmap into a list of numbers."""
-    nums = []
-    for byte_i, byte in enumerate(bytes_to_ints(bitmap)):
-        for bit_i in range(8):
-            if (byte & (1 << bit_i)):
-                nums.append(byte_i * 8 + bit_i)
-    return nums
-
-@contract(map1='bytes', map2='bytes', returns='bytes')
-def merge_bitmaps(map1, map2):
-    """Merge two bitmaps"""
-    byte_pairs = zip_longest(bytes_to_ints(map1), bytes_to_ints(map2), fillvalue=0)
-    return binary_bytes(b1 | b2 for b1, b2 in byte_pairs)
diff --git a/tests/test_data.py b/tests/test_data.py
index ad752d65..ff97b330 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -6,7 +6,6 @@
 import glob
 import os
 import os.path
-import random
 import re
 import sqlite3
 import threading
@@ -18,7 +17,6 @@ from coverage.data import add_data_to_hash, line_counts
 from coverage.debug import DebugControlString
 from coverage.files import PathAliases, canonical_filename
 from coverage.misc import CoverageException
-from coverage.sqldata import nums_to_bitmap, bitmap_to_nums, merge_bitmaps
 
 from tests.coveragetest import CoverageTest
 
@@ -835,31 +833,3 @@ class DumpsLoadsTest(DataTestHelpers, CoverageTest):
             )
         with self.assertRaisesRegex(CoverageException, msg):
             covdata.loads(bad_data)
-
-
-class BitmapOpTest(CoverageTest):
-    """Tests of the bitmap operations in sqldata.py."""
-
-    run_in_temp_dir = False
-
-    def numbers(self, r):
-        """Produce a list of numbers from a Random object."""
-        return list(set(r.randint(1, 1000) for _ in range(r.randint(100, 200))))
-
-    def test_conversion(self):
-        r = random.Random(1792)
-        for _ in range(10):
-            nums = self.numbers(r)
-            bitmap = nums_to_bitmap(nums)
-            self.assertEqual(sorted(bitmap_to_nums(bitmap)), sorted(nums))
-
-    def test_merging(self):
-        r = random.Random(314159)
-        for _ in range(10):
-            nums1 = self.numbers(r)
-            nums2 = self.numbers(r)
-            merged = bitmap_to_nums(merge_bitmaps(nums_to_bitmap(nums1), nums_to_bitmap(nums2)))
-            all_nums = set()
-            all_nums.update(nums1)
-            all_nums.update(nums2)
-            self.assertEqual(sorted(all_nums), sorted(merged))
diff --git a/tests/test_numbits.py b/tests/test_numbits.py
new file mode 100644
index 00000000..b54f65df
--- /dev/null
+++ b/tests/test_numbits.py
@@ -0,0 +1,37 @@
+# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
+# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt
+
+"""Tests for coverage.numbits"""
+
+import random
+
+from coverage.numbits import nums_to_numbits, numbits_to_nums, merge_numbits
+
+from tests.coveragetest import CoverageTest
+
+class NumbitsOpTest(CoverageTest):
+    """Tests of the numbits operations in numbits.py."""
+
+    run_in_temp_dir = False
+
+    def numbers(self, r):
+        """Produce a list of numbers from a Random object."""
+        return list(set(r.randint(1, 1000) for _ in range(r.randint(100, 200))))
+
+    def test_conversion(self):
+        r = random.Random(1792)
+        for _ in range(10):
+            nums = self.numbers(r)
+            numbits = nums_to_numbits(nums)
+            self.assertEqual(sorted(numbits_to_nums(numbits)), sorted(nums))
+
+    def test_merging(self):
+        r = random.Random(314159)
+        for _ in range(10):
+            nums1 = self.numbers(r)
+            nums2 = self.numbers(r)
+            merged = numbits_to_nums(merge_numbits(nums_to_numbits(nums1), nums_to_numbits(nums2)))
+            all_nums = set()
+            all_nums.update(nums1)
+            all_nums.update(nums2)
+            self.assertEqual(sorted(all_nums), sorted(merged))