summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.opendev.org>2023-02-09 08:46:13 +0000
committerGerrit Code Review <review@openstack.org>2023-02-09 08:46:13 +0000
commit6994200026f7a1f2721a57943a7c346405551d55 (patch)
tree0dde152210a9d0e21bcecca2aa8228e1a18eaad5
parentb27f6c751a136d5c742574ef48ece695fc147518 (diff)
parenta548da916f5bb3097c42de54b53e54a57853220b (diff)
downloadswift-6994200026f7a1f2721a57943a7c346405551d55.tar.gz
Merge "Remove :memory: from DatabaseBrokers and unittests"
Diffstat
-rw-r--r--swift/common/db.py38
-rw-r--r--swift/container/backend.py11
-rw-r--r--test/unit/__init__.py7
-rw-r--r--test/unit/account/test_auditor.py7
-rw-r--r--test/unit/account/test_backend.py109
-rw-r--r--test/unit/account/test_utils.py13
-rw-r--r--test/unit/common/test_db.py121
-rw-r--r--test/unit/container/test_backend.py233
8 files changed, 345 insertions, 194 deletions
diff --git a/swift/common/db.py b/swift/common/db.py
index d21103191..918baeaa3 100644
--- a/swift/common/db.py
+++ b/swift/common/db.py
@@ -205,7 +205,7 @@ def get_db_connection(path, timeout=30, logger=None, okay_to_create=False):
factory=GreenDBConnection, timeout=timeout)
if QUERY_LOGGING and logger and not six.PY2:
conn.set_trace_callback(logger.debug)
- if path != ':memory:' and not okay_to_create:
+ if not okay_to_create:
# attempt to detect and fail when connect creates the db file
stat = os.stat(path)
if stat.st_size == 0 and stat.st_ctime >= connect_time:
@@ -359,17 +359,13 @@ class DatabaseBroker(object):
:param put_timestamp: internalized timestamp of initial PUT request
:param storage_policy_index: only required for containers
"""
- if self._db_file == ':memory:':
- tmp_db_file = None
- conn = get_db_connection(self._db_file, self.timeout, self.logger)
- else:
- mkdirs(self.db_dir)
- fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir)
- os.close(fd)
- conn = sqlite3.connect(tmp_db_file, check_same_thread=False,
- factory=GreenDBConnection, timeout=0)
- if QUERY_LOGGING and not six.PY2:
- conn.set_trace_callback(self.logger.debug)
+ mkdirs(self.db_dir)
+ fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir)
+ os.close(fd)
+ conn = sqlite3.connect(tmp_db_file, check_same_thread=False,
+ factory=GreenDBConnection, timeout=0)
+ if QUERY_LOGGING and not six.PY2:
+ conn.set_trace_callback(self.logger.debug)
# creating dbs implicitly does a lot of transactions, so we
# pick fast, unsafe options here and do a big fsync at the end.
with closing(conn.cursor()) as cur:
@@ -541,7 +537,7 @@ class DatabaseBroker(object):
def get(self):
"""Use with the "with" statement; returns a database connection."""
if not self.conn:
- if self.db_file != ':memory:' and os.path.exists(self.db_file):
+ if os.path.exists(self.db_file):
try:
self.conn = get_db_connection(self.db_file, self.timeout,
self.logger)
@@ -569,7 +565,7 @@ class DatabaseBroker(object):
def lock(self):
"""Use with the "with" statement; locks a database."""
if not self.conn:
- if self.db_file != ':memory:' and os.path.exists(self.db_file):
+ if os.path.exists(self.db_file):
self.conn = get_db_connection(self.db_file, self.timeout,
self.logger)
else:
@@ -636,7 +632,7 @@ class DatabaseBroker(object):
:returns: True if the DB is considered to be deleted, False otherwise
"""
- if self.db_file != ':memory:' and not os.path.exists(self.db_file):
+ if not os.path.exists(self.db_file):
return True
self._commit_puts_stale_ok()
with self.get() as conn:
@@ -769,8 +765,8 @@ class DatabaseBroker(object):
"""
Put a record into the DB. If the DB has an associated pending file with
space then the record is appended to that file and a commit to the DB
- is deferred. If the DB is in-memory or its pending file is full then
- the record will be committed immediately.
+ is deferred. If its pending file is full then the record will be
+ committed immediately.
:param record: a record to be added to the DB.
:raises DatabaseConnectionError: if the DB file does not exist or if
@@ -778,9 +774,6 @@ class DatabaseBroker(object):
:raises LockTimeout: if a timeout occurs while waiting to take a lock
to write to the pending file.
"""
- if self._db_file == ':memory:':
- self.merge_items([record])
- return
if not os.path.exists(self.db_file):
raise DatabaseConnectionError(self.db_file, "DB doesn't exist")
if self.skip_commits:
@@ -806,8 +799,7 @@ class DatabaseBroker(object):
fp.flush()
def _skip_commit_puts(self):
- return (self._db_file == ':memory:' or self.skip_commits or not
- os.path.exists(self.pending_file))
+ return self.skip_commits or not os.path.exists(self.pending_file)
def _commit_puts(self, item_list=None):
"""
@@ -921,7 +913,7 @@ class DatabaseBroker(object):
within 512k of a boundary, it allocates to the next boundary.
Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after.
"""
- if not DB_PREALLOCATION or self._db_file == ':memory:':
+ if not DB_PREALLOCATION:
return
MB = (1024 * 1024)
diff --git a/swift/container/backend.py b/swift/container/backend.py
index 1130d828c..c1842d9bd 100644
--- a/swift/container/backend.py
+++ b/swift/container/backend.py
@@ -387,10 +387,7 @@ class ContainerBroker(DatabaseBroker):
stale_reads_ok=False, skip_commits=False,
force_db_file=False):
self._init_db_file = db_file
- if db_file == ':memory:':
- base_db_file = db_file
- else:
- base_db_file = make_db_file_path(db_file, None)
+ base_db_file = make_db_file_path(db_file, None)
super(ContainerBroker, self).__init__(
base_db_file, timeout, logger, account, container, pending_timeout,
stale_reads_ok, skip_commits=skip_commits)
@@ -440,8 +437,6 @@ class ContainerBroker(DatabaseBroker):
"""
Returns the current state of on disk db files.
"""
- if self._db_file == ':memory:':
- return UNSHARDED
if not self.db_files:
return NOTFOUND
if len(self.db_files) > 1:
@@ -482,8 +477,6 @@ class ContainerBroker(DatabaseBroker):
"""
Reloads the cached list of valid on disk db files for this broker.
"""
- if self._db_file == ':memory:':
- return
# reset connection so the next access will use the correct DB file
self.conn = None
self._db_files = get_db_files(self._init_db_file)
@@ -890,7 +883,7 @@ class ContainerBroker(DatabaseBroker):
:returns: a tuple, in the form (info, is_deleted) info is a dict as
returned by get_info and is_deleted is a boolean.
"""
- if self.db_file != ':memory:' and not os.path.exists(self.db_file):
+ if not os.path.exists(self.db_file):
return {}, True
info = self.get_info()
return info, self._is_deleted_info(**info)
diff --git a/test/unit/__init__.py b/test/unit/__init__.py
index 4a9dcbef5..51b3ff952 100644
--- a/test/unit/__init__.py
+++ b/test/unit/__init__.py
@@ -42,6 +42,7 @@ import random
import errno
import xattr
from io import BytesIO
+from uuid import uuid4
import six
import six.moves.cPickle as pickle
@@ -1405,3 +1406,9 @@ def group_by_byte(contents):
return [
(char, sum(1 for _ in grp))
for char, grp in itertools.groupby(byte_iter)]
+
+
+def generate_db_path(tempdir, server_type):
+ return os.path.join(
+ tempdir, '%ss' % server_type, 'part', 'suffix', 'hash',
+ '%s-%s.db' % (server_type, uuid4()))
diff --git a/test/unit/account/test_auditor.py b/test/unit/account/test_auditor.py
index d01b2e367..d37485a7b 100644
--- a/test/unit/account/test_auditor.py
+++ b/test/unit/account/test_auditor.py
@@ -53,14 +53,15 @@ class TestAuditorRealBrokerMigration(
self.assertUnmigrated(self.broker)
# run auditor, and validate migration
- conf = {'devices': self.tempdir, 'mount_check': False,
- 'recon_cache_path': self.tempdir}
+ conf = {'devices': self.testdir, 'mount_check': False,
+ 'recon_cache_path': self.testdir}
test_auditor = auditor.AccountAuditor(conf, logger=debug_logger())
test_auditor.run_once()
self.restore_account_broker()
- broker = auditor.AccountBroker(self.db_path)
+ broker = auditor.AccountBroker(self.db_path, account='a')
+ broker.initialize(Timestamp('1').internal, 0)
# go after rows directly to avoid unintentional migration
with broker.get() as conn:
rows = conn.execute('''
diff --git a/test/unit/account/test_backend.py b/test/unit/account/test_backend.py
index 9e6cb55c3..ca8ef30e6 100644
--- a/test/unit/account/test_backend.py
+++ b/test/unit/account/test_backend.py
@@ -17,13 +17,10 @@
from collections import defaultdict
import json
-import unittest
import pickle
import os
from time import sleep, time
from uuid import uuid4
-from tempfile import mkdtemp
-from shutil import rmtree
import sqlite3
import itertools
from contextlib import contextmanager
@@ -46,23 +43,25 @@ from test.unit.common import test_db
@patch_policies
-class TestAccountBroker(unittest.TestCase):
+class TestAccountBroker(test_db.TestDbBase):
"""Tests for AccountBroker"""
def setUp(self):
+ super(TestAccountBroker, self).setUp()
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
def test_creation(self):
# Test AccountBroker.__init__
- broker = AccountBroker(':memory:', account='a')
- self.assertEqual(broker.db_file, ':memory:')
+ db_file = self.get_db_path()
+ broker = AccountBroker(db_file, account='a')
+ self.assertEqual(broker.db_file, db_file)
try:
with broker.get() as conn:
pass
except DatabaseConnectionError as e:
self.assertTrue(hasattr(e, 'path'))
- self.assertEqual(e.path, ':memory:')
+ self.assertEqual(e.path, db_file)
self.assertTrue(hasattr(e, 'msg'))
self.assertEqual(e.msg, "DB doesn't exist")
except Exception as e:
@@ -76,7 +75,7 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(curs.fetchall()[0][0], 1)
def test_initialize_fail(self):
- broker = AccountBroker(':memory:')
+ broker = AccountBroker(self.get_db_path())
with self.assertRaises(ValueError) as cm:
broker.initialize(Timestamp('1').internal)
self.assertEqual(str(cm.exception), 'Attempting to create a new'
@@ -85,7 +84,7 @@ class TestAccountBroker(unittest.TestCase):
def test_exception(self):
# Test AccountBroker throwing a conn away after exception
first_conn = None
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
first_conn = conn
@@ -99,7 +98,7 @@ class TestAccountBroker(unittest.TestCase):
def test_empty(self):
# Test AccountBroker.empty
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
self.assertTrue(broker.empty())
broker.put_container('o', Timestamp.now().internal, 0, 0, 0,
@@ -112,12 +111,12 @@ class TestAccountBroker(unittest.TestCase):
def test_is_status_deleted(self):
# Test AccountBroker.is_status_deleted
- broker1 = AccountBroker(':memory:', account='a')
+ broker1 = AccountBroker(self.get_db_path(), account='a')
broker1.initialize(Timestamp.now().internal)
self.assertFalse(broker1.is_status_deleted())
broker1.delete_db(Timestamp.now().internal)
self.assertTrue(broker1.is_status_deleted())
- broker2 = AccountBroker(':memory:', account='a')
+ broker2 = AccountBroker(self.get_db_path(), account='a')
broker2.initialize(Timestamp.now().internal)
# Set delete_timestamp greater than put_timestamp
broker2.merge_timestamps(
@@ -126,10 +125,12 @@ class TestAccountBroker(unittest.TestCase):
self.assertTrue(broker2.is_status_deleted())
def test_reclaim(self):
- broker = AccountBroker(':memory:', account='test_account')
+ broker = AccountBroker(self.get_db_path(), account='test_account')
broker.initialize(Timestamp('1').internal)
broker.put_container('c', Timestamp.now().internal, 0, 0, 0,
POLICIES.default.idx)
+ # commit pending file into db
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@@ -148,6 +149,7 @@ class TestAccountBroker(unittest.TestCase):
sleep(.00001)
broker.put_container('c', 0, Timestamp.now().internal, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@@ -176,6 +178,7 @@ class TestAccountBroker(unittest.TestCase):
broker.put_container('x', 0, 0, 0, 0, POLICIES.default.idx)
broker.put_container('y', 0, 0, 0, 0, POLICIES.default.idx)
broker.put_container('z', 0, 0, 0, 0, POLICIES.default.idx)
+ broker._commit_puts()
broker.reclaim(Timestamp.now().internal, time())
# Now delete the account
broker.delete_db(Timestamp.now().internal)
@@ -193,7 +196,7 @@ class TestAccountBroker(unittest.TestCase):
random.seed(now)
random.shuffle(container_specs)
policy_indexes = list(p.idx for p in POLICIES)
- broker = AccountBroker(':memory:', account='test_account')
+ broker = AccountBroker(self.get_db_path(), account='test_account')
broker.initialize(Timestamp('1').internal)
for i, container_spec in enumerate(container_specs):
# with container12 before container2 and shuffled ts.internal we
@@ -206,6 +209,9 @@ class TestAccountBroker(unittest.TestCase):
else:
broker.put_container(name, ts.internal, 0, 0, 0, pidx)
+ # commit pending file into db
+ broker._commit_puts()
+
def count_reclaimable(conn, reclaim_age):
return conn.execute(
"SELECT count(*) FROM container "
@@ -251,7 +257,7 @@ class TestAccountBroker(unittest.TestCase):
def test_delete_db_status(self):
start = next(self.ts)
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(start.internal)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], start.internal)
@@ -273,10 +279,12 @@ class TestAccountBroker(unittest.TestCase):
def test_delete_container(self):
# Test AccountBroker.delete_container
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('o', Timestamp.now().internal, 0, 0, 0,
POLICIES.default.idx)
+ # commit pending file into db
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@@ -287,6 +295,7 @@ class TestAccountBroker(unittest.TestCase):
sleep(.00001)
broker.put_container('o', 0, Timestamp.now().internal, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@@ -297,13 +306,15 @@ class TestAccountBroker(unittest.TestCase):
def test_put_container(self):
# Test AccountBroker.put_container
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
# Create initial container
timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
+ # commit pending file into db
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@@ -317,6 +328,7 @@ class TestAccountBroker(unittest.TestCase):
# Reput same event
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@@ -332,6 +344,7 @@ class TestAccountBroker(unittest.TestCase):
timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@@ -346,6 +359,7 @@ class TestAccountBroker(unittest.TestCase):
otimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_container('"{<container \'&\' name>}"', otimestamp, 0, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@@ -360,6 +374,7 @@ class TestAccountBroker(unittest.TestCase):
dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_container('"{<container \'&\' name>}"', 0, dtimestamp, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@@ -378,6 +393,7 @@ class TestAccountBroker(unittest.TestCase):
timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', 0, timestamp, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@@ -393,6 +409,7 @@ class TestAccountBroker(unittest.TestCase):
timestamp = Timestamp.now().internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@@ -405,7 +422,7 @@ class TestAccountBroker(unittest.TestCase):
def test_get_info(self):
# Test AccountBroker.get_info
- broker = AccountBroker(':memory:', account='test1')
+ broker = AccountBroker(self.get_db_path(), account='test1')
broker.initialize(Timestamp('1').internal)
info = broker.get_info()
@@ -452,7 +469,7 @@ class TestAccountBroker(unittest.TestCase):
def test_list_containers_iter(self):
# Test AccountBroker.list_containers_iter
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
for cont1 in range(4):
for cont2 in range(125):
@@ -582,7 +599,7 @@ class TestAccountBroker(unittest.TestCase):
def test_list_objects_iter_order_and_reverse(self):
# Test ContainerBroker.list_objects_iter
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal, 0)
broker.put_container(
@@ -617,7 +634,7 @@ class TestAccountBroker(unittest.TestCase):
def test_list_container_iter_with_reserved_name(self):
# Test ContainerBroker.list_objects_iter
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal, 0)
broker.put_container(
@@ -754,11 +771,13 @@ class TestAccountBroker(unittest.TestCase):
}
failures = []
for expected in expectations:
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal, 0)
for name in expected['containers']:
broker.put_container(name, next(self.ts).internal, 0, 0, 0,
POLICIES.default.idx)
+ # commit pending file into db
+ broker._commit_puts()
params = default_listing_params.copy()
params.update(expected['params'])
listing = list(c[0] for c in broker.list_containers_iter(**params))
@@ -773,7 +792,7 @@ class TestAccountBroker(unittest.TestCase):
def test_double_check_trailing_delimiter(self):
# Test AccountBroker.list_containers_iter for an
# account that has an odd container with a trailing delimiter
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('a', Timestamp.now().internal, 0, 0, 0,
POLICIES.default.idx)
@@ -814,7 +833,7 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual([row[0] for row in listing], ['b-a', 'b-b'])
def test_chexor(self):
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('a', Timestamp(1).internal,
Timestamp(0).internal, 0, 0,
@@ -842,9 +861,9 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(broker.get_info()['hash'], hashc)
def test_merge_items(self):
- broker1 = AccountBroker(':memory:', account='a')
+ broker1 = AccountBroker(self.get_db_path(), account='a')
broker1.initialize(Timestamp('1').internal)
- broker2 = AccountBroker(':memory:', account='a')
+ broker2 = AccountBroker(self.get_db_path(), account='a')
broker2.initialize(Timestamp('1').internal)
broker1.put_container('a', Timestamp(1).internal, 0, 0, 0,
POLICIES.default.idx)
@@ -869,10 +888,10 @@ class TestAccountBroker(unittest.TestCase):
snowman = u'\N{SNOWMAN}'
if six.PY2:
snowman = snowman.encode('utf-8')
- broker1 = AccountBroker(':memory:', account='a')
+ broker1 = AccountBroker(self.get_db_path(), account='a')
broker1.initialize(Timestamp('1').internal, 0)
id1 = broker1.get_info()['id']
- broker2 = AccountBroker(':memory:', account='a')
+ broker2 = AccountBroker(self.get_db_path(), account='a')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_container(snowman, Timestamp(2).internal, 0, 1, 100,
POLICIES.default.idx)
@@ -974,7 +993,7 @@ class TestAccountBroker(unittest.TestCase):
StoragePolicy(2, 'two', False),
StoragePolicy(3, 'three', False)])
def test_get_policy_stats(self):
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal)
# check empty policy_stats
self.assertTrue(broker.empty())
@@ -1041,7 +1060,7 @@ class TestAccountBroker(unittest.TestCase):
@patch_policies([StoragePolicy(0, 'zero', False),
StoragePolicy(1, 'one', True)])
def test_policy_stats_tracking(self):
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(next(self.ts).internal)
# policy 0
@@ -1162,6 +1181,7 @@ def premetadata_create_account_stat_table(self, conn, put_timestamp):
class TestCommonAccountBroker(test_db.TestExampleBroker):
broker_class = AccountBroker
+ server_type = 'account'
def setUp(self):
super(TestCommonAccountBroker, self).setUp()
@@ -1183,13 +1203,14 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker):
"""
def setUp(self):
+ super(TestAccountBroker, self).setUp()
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
self._imported_create_account_stat_table = \
AccountBroker.create_account_stat_table
AccountBroker.create_account_stat_table = \
premetadata_create_account_stat_table
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
exc = None
with broker.get() as conn:
@@ -1202,10 +1223,11 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker):
def tearDown(self):
AccountBroker.create_account_stat_table = \
self._imported_create_account_stat_table
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
conn.execute('SELECT metadata FROM account_stat')
+ super(TestAccountBrokerBeforeMetadata, self).tearDown()
def prespi_create_container_table(self, conn):
@@ -1270,6 +1292,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
"""
def setUp(self):
+ super(TestAccountBrokerBeforeSPI, self).setUp()
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
self._imported_create_container_table = \
@@ -1278,7 +1301,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
prespi_create_container_table
self._imported_initialize = AccountBroker._initialize
AccountBroker._initialize = prespi_AccountBroker_initialize
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
exc = None
with broker.get() as conn:
@@ -1299,10 +1322,11 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
AccountBroker.create_container_table = \
self._imported_create_container_table
AccountBroker._initialize = self._imported_initialize
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
conn.execute('SELECT storage_policy_index FROM container')
+ super(TestAccountBrokerBeforeSPI, self).tearDown()
@with_tempdir
def test_policy_table_migration(self, tempdir):
@@ -1681,7 +1705,7 @@ def pre_track_containers_create_container_table(self, conn):
""" + OLD_POLICY_STAT_TRIGGER_SCRIPT)
-class AccountBrokerPreTrackContainerCountSetup(object):
+class AccountBrokerPreTrackContainerCountSetup(test_db.TestDbBase):
def assertUnmigrated(self, broker):
with broker.get() as conn:
try:
@@ -1696,6 +1720,7 @@ class AccountBrokerPreTrackContainerCountSetup(object):
'trying to select container_count from policy_stat!')
def setUp(self):
+ super(AccountBrokerPreTrackContainerCountSetup, self).setUp()
# use old version of policy_stat
self._imported_create_policy_stat_table = \
AccountBroker.create_policy_stat_table
@@ -1708,17 +1733,16 @@ class AccountBrokerPreTrackContainerCountSetup(object):
AccountBroker.create_container_table = \
pre_track_containers_create_container_table
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
self.assertUnmigrated(broker)
- self.tempdir = mkdtemp()
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
- self.db_path = os.path.join(self.tempdir, 'sda', 'accounts',
+ self.db_path = os.path.join(self.testdir, 'sda', 'accounts',
'0', '0', '0', 'test.db')
- self.broker = AccountBroker(self.db_path, account='a')
+ self.broker = AccountBroker(self.get_db_path(), account='a')
self.broker.initialize(next(self.ts).internal)
# Common sanity-check that our starting, pre-migration state correctly
@@ -1726,14 +1750,13 @@ class AccountBrokerPreTrackContainerCountSetup(object):
self.assertUnmigrated(self.broker)
def tearDown(self):
- rmtree(self.tempdir, ignore_errors=True)
-
self.restore_account_broker()
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.get_db_path(), account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
conn.execute('SELECT container_count FROM policy_stat')
+ super(AccountBrokerPreTrackContainerCountSetup, self).tearDown()
def restore_account_broker(self):
AccountBroker.create_policy_stat_table = \
@@ -1925,7 +1948,7 @@ class TestAccountBrokerBeforePerPolicyContainerTrack(
self.assertEqual(policy_stat['container_count'], 1)
def test_migrate_add_storage_policy_index_fail(self):
- broker = AccountBroker(':memory:', account='a')
+ broker = AccountBroker(self.db_path, account='a')
broker.initialize(Timestamp('1').internal)
with mock.patch.object(
broker, 'create_policy_stat_table',
diff --git a/test/unit/account/test_utils.py b/test/unit/account/test_utils.py
index eb332395a..473b3b053 100644
--- a/test/unit/account/test_utils.py
+++ b/test/unit/account/test_utils.py
@@ -26,6 +26,7 @@ from swift.common.header_key_dict import HeaderKeyDict
from swift.common.request_helpers import get_reserved_name
from test.unit import patch_policies, make_timestamp_iter
+from test.unit.common.test_db import TestDbBase
class TestFakeAccountBroker(unittest.TestCase):
@@ -58,9 +59,11 @@ class TestFakeAccountBroker(unittest.TestCase):
self.assertEqual(broker.get_policy_stats(), {})
-class TestAccountUtils(unittest.TestCase):
+class TestAccountUtils(TestDbBase):
+ server_type = 'account'
def setUp(self):
+ super(TestAccountUtils, self).setUp()
self.ts = make_timestamp_iter()
def test_get_response_headers_fake_broker(self):
@@ -78,7 +81,7 @@ class TestAccountUtils(unittest.TestCase):
self.assertEqual(resp_headers, expected)
def test_get_response_headers_empty_memory_broker(self):
- broker = backend.AccountBroker(':memory:', account='a')
+ broker = backend.AccountBroker(self.db_path, account='a')
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal)
@@ -94,7 +97,7 @@ class TestAccountUtils(unittest.TestCase):
@patch_policies
def test_get_response_headers_with_data(self):
- broker = backend.AccountBroker(':memory:', account='a')
+ broker = backend.AccountBroker(self.db_path, account='a')
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal)
@@ -141,7 +144,7 @@ class TestAccountUtils(unittest.TestCase):
@patch_policies
def test_get_response_headers_with_legacy_data(self):
- broker = backend.AccountBroker(':memory:', account='a')
+ broker = backend.AccountBroker(self.db_path, account='a')
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal)
@@ -213,7 +216,7 @@ class TestAccountUtils(unittest.TestCase):
@patch_policies([StoragePolicy(0, 'zero', is_default=True)])
def test_account_listing_reserved_names(self):
- broker = backend.AccountBroker(':memory:', account='a')
+ broker = backend.AccountBroker(self.db_path, account='a')
put_timestamp = next(self.ts)
now = time.time()
with mock.patch('time.time', new=lambda: now):
diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py
index 75de2488b..a193b876f 100644
--- a/test/unit/common/test_db.py
+++ b/test/unit/common/test_db.py
@@ -46,7 +46,7 @@ from swift.common.utils import normalize_timestamp, mkdirs, Timestamp
from swift.common.exceptions import LockTimeout
from swift.common.swob import HTTPException
-from test.unit import with_tempdir, make_timestamp_iter
+from test.unit import make_timestamp_iter, generate_db_path
class TestHelperFunctions(unittest.TestCase):
@@ -180,10 +180,35 @@ class TestGreenDBConnection(unittest.TestCase):
InterceptConnection.commit.call_count))
-class TestGetDBConnection(unittest.TestCase):
+class TestDbBase(unittest.TestCase):
+ server_type = 'container'
+ testdir = None
+
+ def setUp(self):
+ self.testdir = mkdtemp()
+ self.db_path = self.get_db_path()
+
+ def tearDown(self):
+ rmtree(self.testdir, ignore_errors=True)
+
+ def get_db_path(self):
+ return generate_db_path(self.testdir, self.server_type)
+
+
+class TestGetDBConnection(TestDbBase):
+ def setUp(self):
+ super(TestGetDBConnection, self).setUp()
+ self.db_path = self.init_db_path()
+
+ def init_db_path(self):
+ # Test ContainerBroker.empty
+ db_path = self.get_db_path()
+ broker = ExampleBroker(db_path, account='a')
+ broker.initialize(Timestamp.now().internal, 0)
+ return db_path
def test_normal_case(self):
- conn = get_db_connection(':memory:')
+ conn = get_db_connection(self.db_path)
self.assertTrue(hasattr(conn, 'execute'))
def test_invalid_path(self):
@@ -201,8 +226,8 @@ class TestGetDBConnection(unittest.TestCase):
InterceptCursor.execute = mock_db_cmd
with patch('sqlite3.Cursor', new=InterceptCursor):
- self.assertRaises(Timeout, get_db_connection, ':memory:',
- timeout=0.1)
+ self.assertRaises(Timeout, get_db_connection,
+ self.db_path, timeout=0.1)
self.assertTrue(mock_db_cmd.called)
self.assertEqual(mock_db_cmd.call_args_list,
list((mock_db_cmd.call_args,) *
@@ -319,7 +344,7 @@ class ExampleBroker(DatabaseBroker):
Timestamp(info['put_timestamp']))
-class TestExampleBroker(unittest.TestCase):
+class TestExampleBroker(TestDbBase):
"""
Tests that use the mostly Concrete enough ExampleBroker to exercise some
of the abstract methods on DatabaseBroker.
@@ -327,19 +352,21 @@ class TestExampleBroker(unittest.TestCase):
broker_class = ExampleBroker
policy = 0
+ server_type = 'example'
def setUp(self):
+ super(TestExampleBroker, self).setUp()
self.ts = make_timestamp_iter()
def test_delete_db(self):
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(next(self.ts).internal)
broker.delete_db(next(self.ts).internal)
self.assertTrue(broker.is_deleted())
def test_merge_timestamps_simple_delete(self):
put_timestamp = next(self.ts).internal
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp)
created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp, '0')
@@ -366,7 +393,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_delete_with_objects(self):
put_timestamp = next(self.ts).internal
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp, '0')
@@ -397,7 +424,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_simple_recreate(self):
put_timestamp = next(self.ts).internal
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
virgin_status_changed_at = broker.get_info()['status_changed_at']
created_at = broker.get_info()['created_at']
@@ -426,7 +453,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_recreate_with_objects(self):
put_timestamp = next(self.ts).internal
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
# delete
@@ -460,7 +487,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_update_put_no_status_change(self):
put_timestamp = next(self.ts).internal
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
info = broker.get_info()
orig_status_changed_at = info['status_changed_at']
@@ -473,7 +500,7 @@ class TestExampleBroker(unittest.TestCase):
def test_merge_timestamps_update_delete_no_status_change(self):
put_timestamp = next(self.ts).internal
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp,
@@ -487,19 +514,23 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual(orig_status_changed_at, info['status_changed_at'])
def test_get_max_row(self):
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy))
self.assertEqual(-1, broker.get_max_row())
self.put_item(broker, next(self.ts).internal)
+ # commit pending file into db
+ broker._commit_puts()
self.assertEqual(1, broker.get_max_row())
self.delete_item(broker, next(self.ts).internal)
+ broker._commit_puts()
self.assertEqual(2, broker.get_max_row())
self.put_item(broker, next(self.ts).internal)
+ broker._commit_puts()
self.assertEqual(3, broker.get_max_row())
def test_get_info(self):
- broker = self.broker_class(':memory:', account='test', container='c')
+ broker = self.broker_class(self.db_path, account='test', container='c')
created_at = time.time()
with patch('swift.common.db.time.time', new=lambda: created_at):
broker.initialize(Timestamp(1).internal,
@@ -519,7 +550,7 @@ class TestExampleBroker(unittest.TestCase):
k, info[k], v))
def test_get_raw_metadata(self):
- broker = self.broker_class(':memory:', account='test', container='c')
+ broker = self.broker_class(self.db_path, account='test', container='c')
broker.initialize(Timestamp(0).internal,
storage_policy_index=int(self.policy))
self.assertEqual(broker.metadata, {})
@@ -544,7 +575,7 @@ class TestExampleBroker(unittest.TestCase):
json.dumps(metadata))
def test_put_timestamp(self):
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
orig_put_timestamp = next(self.ts).internal
broker.initialize(orig_put_timestamp,
storage_policy_index=int(self.policy))
@@ -565,7 +596,7 @@ class TestExampleBroker(unittest.TestCase):
newer_put_timestamp)
def test_status_changed_at(self):
- broker = self.broker_class(':memory:', account='test', container='c')
+ broker = self.broker_class(self.db_path, account='test', container='c')
put_timestamp = next(self.ts).internal
created_at = time.time()
with patch('swift.common.db.time.time', new=lambda: created_at):
@@ -591,7 +622,7 @@ class TestExampleBroker(unittest.TestCase):
status_changed_at)
def test_get_syncs(self):
- broker = self.broker_class(':memory:', account='a', container='c')
+ broker = self.broker_class(self.db_path, account='a', container='c')
broker.initialize(Timestamp.now().internal,
storage_policy_index=int(self.policy))
self.assertEqual([], broker.get_syncs())
@@ -604,9 +635,8 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual([{'sync_point': 2, 'remote_id': 'remote2'}],
broker.get_syncs(incoming=False))
- @with_tempdir
- def test_commit_pending(self, tempdir):
- broker = self.broker_class(os.path.join(tempdir, 'test.db'),
+ def test_commit_pending(self):
+ broker = self.broker_class(os.path.join(self.testdir, 'test.db'),
account='a', container='c')
broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy))
@@ -617,12 +647,12 @@ class TestExampleBroker(unittest.TestCase):
info = rows[0]
count_key = '%s_count' % broker.db_contains_type
self.assertEqual(0, info[count_key])
- broker.get_info()
+ # commit pending file into db
+ broker._commit_puts()
self.assertEqual(1, broker.get_info()[count_key])
- @with_tempdir
- def test_maybe_get(self, tempdir):
- broker = self.broker_class(os.path.join(tempdir, 'test.db'),
+ def test_maybe_get(self):
+ broker = self.broker_class(os.path.join(self.testdir, 'test.db'),
account='a', container='c')
broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy))
@@ -640,13 +670,7 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual(broker.conn, conn)
-class TestDatabaseBroker(unittest.TestCase):
-
- def setUp(self):
- self.testdir = mkdtemp()
-
- def tearDown(self):
- rmtree(self.testdir, ignore_errors=1)
+class TestDatabaseBroker(TestDbBase):
def test_DB_PREALLOCATION_setting(self):
u = uuid4().hex
@@ -657,8 +681,8 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertRaises(OSError, b._preallocate)
def test_memory_db_init(self):
- broker = DatabaseBroker(':memory:')
- self.assertEqual(broker.db_file, ':memory:')
+ broker = DatabaseBroker(self.db_path)
+ self.assertEqual(broker.db_file, self.db_path)
self.assertRaises(AttributeError, broker.initialize,
normalize_timestamp('0'))
@@ -687,7 +711,7 @@ class TestDatabaseBroker(unittest.TestCase):
def test_initialize(self):
self.assertRaises(AttributeError,
- DatabaseBroker(':memory:').initialize,
+ DatabaseBroker(self.db_path).initialize,
normalize_timestamp('1'))
stub_dict = {}
@@ -695,7 +719,7 @@ class TestDatabaseBroker(unittest.TestCase):
stub_dict.clear()
stub_dict['args'] = args
stub_dict.update(kwargs)
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.db_path)
broker._initialize = stub
broker.initialize(normalize_timestamp('1'))
self.assertTrue(hasattr(stub_dict['args'][0], 'execute'))
@@ -735,7 +759,7 @@ class TestDatabaseBroker(unittest.TestCase):
def do_test(expected_metadata, delete_meta_whitelist=None):
if not delete_meta_whitelist:
delete_meta_whitelist = []
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.get_db_path())
broker.delete_meta_whitelist = delete_meta_whitelist
broker.db_type = 'test'
broker._initialize = init_stub
@@ -791,13 +815,13 @@ class TestDatabaseBroker(unittest.TestCase):
['x-container-meta-test', 'x-something-else'])
def test_get(self):
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.db_path)
with self.assertRaises(DatabaseConnectionError) as raised, \
broker.get() as conn:
conn.execute('SELECT 1')
self.assertEqual(
str(raised.exception),
- "DB connection error (:memory:, 0):\nDB doesn't exist")
+ "DB connection error (%s, 0):\nDB doesn't exist" % self.db_path)
broker = DatabaseBroker(os.path.join(self.testdir, '1.db'))
with self.assertRaises(DatabaseConnectionError) as raised, \
@@ -947,7 +971,7 @@ class TestDatabaseBroker(unittest.TestCase):
pass
def test_newid(self):
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.db_path)
broker.db_type = 'test'
broker.db_contains_type = 'test'
uuid1 = str(uuid4())
@@ -998,7 +1022,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertEqual(points[0][1], uuid2)
def test_get_items_since(self):
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.db_path)
broker.db_type = 'test'
broker.db_contains_type = 'test'
@@ -1020,7 +1044,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertEqual(broker.get_items_since(999, 2), [])
def test_get_sync(self):
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.db_path)
broker.db_type = 'test'
broker.db_contains_type = 'test'
uuid1 = str(uuid4())
@@ -1060,7 +1084,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertEqual(broker.get_sync(uuid3, incoming=False), 2)
def test_merge_syncs(self):
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.db_path)
def stub(*args, **kwargs):
pass
@@ -1105,7 +1129,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.get_replication_info_tester(metadata=True)
def get_replication_info_tester(self, metadata=False):
- broker = DatabaseBroker(':memory:', account='a')
+ broker = DatabaseBroker(self.db_path, account='a')
broker.db_type = 'test'
broker.db_contains_type = 'test'
broker.db_reclaim_timestamp = 'created_at'
@@ -1428,7 +1452,7 @@ class TestDatabaseBroker(unittest.TestCase):
(dbpath, qpath, hint))
def test_skip_commits(self):
- broker = DatabaseBroker(':memory:')
+ broker = DatabaseBroker(self.db_path)
self.assertTrue(broker._skip_commit_puts())
broker._initialize = MagicMock()
broker.initialize(Timestamp.now())
@@ -1585,7 +1609,7 @@ class TestDatabaseBroker(unittest.TestCase):
self.assertFalse(pending)
-class TestTombstoneReclaimer(unittest.TestCase):
+class TestTombstoneReclaimer(TestDbBase):
def _make_object(self, broker, obj_name, ts, deleted):
if deleted:
broker.delete_test(obj_name, ts.internal)
@@ -1603,7 +1627,8 @@ class TestTombstoneReclaimer(unittest.TestCase):
return self._count_reclaimable(conn, reclaim_age)
def _setup_tombstones(self, reverse_names=True):
- broker = ExampleBroker(':memory:', account='test_account',
+ broker = ExampleBroker(self.db_path,
+ account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
now = time.time()
diff --git a/test/unit/container/test_backend.py b/test/unit/container/test_backend.py
index 7e0a77872..0f5603800 100644
--- a/test/unit/container/test_backend.py
+++ b/test/unit/container/test_backend.py
@@ -53,13 +53,15 @@ from test.unit import (patch_policies, with_tempdir, make_timestamp_iter,
from test.unit.common import test_db
-class TestContainerBroker(unittest.TestCase):
+class TestContainerBroker(test_db.TestDbBase):
"""Tests for ContainerBroker"""
expected_db_tables = {'outgoing_sync', 'incoming_sync', 'object',
'sqlite_sequence', 'policy_stat',
'container_info', 'shard_range'}
+ server_type = 'container'
def setUp(self):
+ super(TestContainerBroker, self).setUp()
self.ts = make_timestamp_iter()
def _assert_shard_ranges(self, broker, expected, include_own=False):
@@ -70,8 +72,9 @@ class TestContainerBroker(unittest.TestCase):
def test_creation(self):
# Test ContainerBroker.__init__
- broker = ContainerBroker(':memory:', account='a', container='c')
- self.assertEqual(broker._db_file, ':memory:')
+ db_file = self.get_db_path()
+ broker = ContainerBroker(db_file, account='a', container='c')
+ self.assertEqual(broker._db_file, db_file)
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
curs = conn.cursor()
@@ -83,6 +86,8 @@ class TestContainerBroker(unittest.TestCase):
# check the update trigger
broker.put_object('blah', Timestamp.now().internal, 0, 'text/plain',
'etag', 0, 0)
+ # commit pending file into db
+ broker._commit_puts()
with broker.get() as conn:
with self.assertRaises(sqlite3.DatabaseError) as cm:
conn.execute('UPDATE object SET name="blah";')
@@ -98,7 +103,7 @@ class TestContainerBroker(unittest.TestCase):
@patch_policies
def test_storage_policy_property(self):
for policy in POLICIES:
- broker = ContainerBroker(':memory:', account='a',
+ broker = ContainerBroker(self.get_db_path(), account='a',
container='policy_%s' % policy.name)
broker.initialize(next(self.ts).internal, policy.idx)
with broker.get() as conn:
@@ -121,7 +126,8 @@ class TestContainerBroker(unittest.TestCase):
# Test ContainerBroker throwing a conn away after
# unhandled exception
first_conn = None
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(),
+ account='a', container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
first_conn = conn
@@ -617,11 +623,14 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual('.shards_a/cc', broker.root_path)
def test_reclaim(self):
- broker = ContainerBroker(':memory:', account='test_account',
+ broker = ContainerBroker(self.get_db_path(),
+ account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('o', Timestamp.now().internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
+ # commit pending file into db
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM object "
@@ -639,6 +648,7 @@ class TestContainerBroker(unittest.TestCase):
"WHERE deleted = 1").fetchone()[0], 0)
sleep(.00001)
broker.delete_object('o', Timestamp.now().internal)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM object "
@@ -672,6 +682,7 @@ class TestContainerBroker(unittest.TestCase):
'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('z', Timestamp.now().internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
+ broker._commit_puts()
# Test before deletion
broker.reclaim(Timestamp.now().internal, time())
broker.delete_db(Timestamp.now().internal)
@@ -688,7 +699,8 @@ class TestContainerBroker(unittest.TestCase):
random.seed(now)
random.shuffle(obj_specs)
policy_indexes = list(p.idx for p in POLICIES)
- broker = ContainerBroker(':memory:', account='test_account',
+ broker = ContainerBroker(self.get_db_path(),
+ account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
for i, obj_spec in enumerate(obj_specs):
@@ -702,6 +714,8 @@ class TestContainerBroker(unittest.TestCase):
else:
broker.put_object(obj_name, ts.internal, 0, 'text/plain',
'etag', storage_policy_index=pidx)
+ # commit pending file into db
+ broker._commit_puts()
def count_reclaimable(conn, reclaim_age):
return conn.execute(
@@ -748,7 +762,8 @@ class TestContainerBroker(unittest.TestCase):
trace[1][2] > trace[2][2])
def test_reclaim_with_duplicate_names(self):
- broker = ContainerBroker(':memory:', account='test_account',
+ broker = ContainerBroker(self.get_db_path(),
+ account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
now = time()
@@ -757,6 +772,8 @@ class TestContainerBroker(unittest.TestCase):
for spidx in range(10):
obj_name = 'object%s' % i
broker.delete_object(obj_name, ages_ago.internal, spidx)
+ # commit pending file into db
+ broker._commit_puts()
reclaim_age = now - (2 * 7 * 24 * 60 * 60)
with broker.get() as conn:
self.assertEqual(conn.execute(
@@ -843,7 +860,8 @@ class TestContainerBroker(unittest.TestCase):
def test_get_info_is_deleted(self):
ts = make_timestamp_iter()
start = next(ts)
- broker = ContainerBroker(':memory:', account='test_account',
+ broker = ContainerBroker(self.get_db_path(),
+ account='test_account',
container='test_container')
# create it
broker.initialize(start.internal, POLICIES.default.idx)
@@ -892,10 +910,13 @@ class TestContainerBroker(unittest.TestCase):
def test_delete_object(self):
# Test ContainerBroker.delete_object
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('o', Timestamp.now().internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
+ # commit pending file into db
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM object "
@@ -905,6 +926,7 @@ class TestContainerBroker(unittest.TestCase):
"WHERE deleted = 1").fetchone()[0], 0)
sleep(.00001)
broker.delete_object('o', Timestamp.now().internal)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM object "
@@ -915,7 +937,8 @@ class TestContainerBroker(unittest.TestCase):
def test_put_object(self):
# Test ContainerBroker.put_object
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
# Create initial object
@@ -923,6 +946,8 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 123,
'application/x-test',
'5af83e3196bf99f440f31f2e1a6c9afe')
+ # commit pending file into db
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -944,6 +969,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 123,
'application/x-test',
'5af83e3196bf99f440f31f2e1a6c9afe')
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -967,6 +993,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 124,
'application/x-test',
'aa0749bacbc79ec65fe206943d8fe449')
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -989,6 +1016,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', otimestamp, 124,
'application/x-test',
'aa0749bacbc79ec65fe206943d8fe449')
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -1010,6 +1038,7 @@ class TestContainerBroker(unittest.TestCase):
dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_object('"{<object \'&\' name>}"', dtimestamp, 0, '', '',
deleted=1)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -1032,6 +1061,7 @@ class TestContainerBroker(unittest.TestCase):
timestamp = Timestamp.now().internal
broker.put_object('"{<object \'&\' name>}"', timestamp, 0, '', '',
deleted=1)
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -1047,6 +1077,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 123,
'application/x-test',
'5af83e3196bf99f440f31f2e1a6c9afe')
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -1095,6 +1126,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('"{<object \'&\' name>}"', timestamp, 456,
'application/x-test3',
'6af83e3196bf99f440f31f2e1a6c9afe')
+ broker._commit_puts()
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM object").fetchone()[0],
@@ -1114,7 +1146,8 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_shard_range_single_record(self):
# Test ContainerBroker.merge_shard_range
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
# Stash these for later
@@ -1419,7 +1452,8 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_shard_ranges_deleted(self):
# Test ContainerBroker.merge_shard_ranges sets deleted attribute
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
# put shard range
broker.merge_shard_ranges(ShardRange('a/o', next(self.ts).internal))
@@ -1452,7 +1486,8 @@ class TestContainerBroker(unittest.TestCase):
'storage_policy_index': '2',
'ctype_timestamp': None,
'meta_timestamp': None}
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
expect = ('obj', '1234567890.12345', 42, 'text/plain', 'hash_test',
'1', '2', None, None)
@@ -1626,7 +1661,8 @@ class TestContainerBroker(unittest.TestCase):
def test_put_object_multiple_encoded_timestamps_using_memory(self):
# Test ContainerBroker.put_object with differing data, content-type
# and metadata timestamps
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
self._test_put_object_multiple_encoded_timestamps(broker)
@with_tempdir
@@ -1963,7 +1999,8 @@ class TestContainerBroker(unittest.TestCase):
def test_put_object_multiple_explicit_timestamps_using_memory(self):
# Test ContainerBroker.put_object with differing data, content-type
# and metadata timestamps passed as explicit args
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
self._test_put_object_multiple_explicit_timestamps(broker)
@with_tempdir
@@ -1979,7 +2016,8 @@ class TestContainerBroker(unittest.TestCase):
# Test container listing reports the most recent of data or metadata
# timestamp as last-modified time
ts = make_timestamp_iter()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(next(ts).internal, 0)
# simple 'single' timestamp case
@@ -2032,7 +2070,7 @@ class TestContainerBroker(unittest.TestCase):
def test_put_misplaced_object_does_not_effect_container_stats(self):
policy = random.choice(list(POLICIES))
ts = make_timestamp_iter()
- broker = ContainerBroker(':memory:',
+ broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker.initialize(next(ts).internal, policy.idx)
# migration tests may not honor policy on initialize
@@ -2059,7 +2097,7 @@ class TestContainerBroker(unittest.TestCase):
def test_has_multiple_policies(self):
policy = random.choice(list(POLICIES))
ts = make_timestamp_iter()
- broker = ContainerBroker(':memory:',
+ broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker.initialize(next(ts).internal, policy.idx)
# migration tests may not honor policy on initialize
@@ -2071,18 +2109,21 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('correct_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=policy.idx)
+ # commit pending file into db
+ broker._commit_puts()
self.assertFalse(broker.has_multiple_policies())
other_policy = [p for p in POLICIES if p is not policy][0]
broker.put_object('wrong_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=other_policy.idx)
+ broker._commit_puts()
self.assertTrue(broker.has_multiple_policies())
@patch_policies
def test_get_policy_info(self):
policy = random.choice(list(POLICIES))
ts = make_timestamp_iter()
- broker = ContainerBroker(':memory:',
+ broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
broker.initialize(next(ts).internal, policy.idx)
# migration tests may not honor policy on initialize
@@ -2099,6 +2140,8 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('correct_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=policy.idx)
+ # commit pending file into db
+ broker._commit_puts()
policy_stats = broker.get_policy_stats()
expected = {policy.idx: {'bytes_used': 123, 'object_count': 1}}
self.assertEqual(policy_stats, expected)
@@ -2109,6 +2152,7 @@ class TestContainerBroker(unittest.TestCase):
broker.put_object('wrong_o', next(ts).internal, 123, 'text/plain',
'5af83e3196bf99f440f31f2e1a6c9afe',
storage_policy_index=other_policy.idx)
+ broker._commit_puts()
policy_stats = broker.get_policy_stats()
expected = {
policy.idx: {'bytes_used': 123, 'object_count': 1},
@@ -2119,7 +2163,7 @@ class TestContainerBroker(unittest.TestCase):
@patch_policies
def test_policy_stat_tracking(self):
ts = make_timestamp_iter()
- broker = ContainerBroker(':memory:',
+ broker = ContainerBroker(self.get_db_path(),
account='a', container='c')
# Note: in subclasses of this TestCase that inherit the
# ContainerBrokerMigrationMixin, passing POLICIES.default.idx here has
@@ -2151,6 +2195,8 @@ class TestContainerBroker(unittest.TestCase):
# track the size of the latest timestamp put for each object
# in each storage policy
stats[policy_index][name] = size
+ # commit pending file into db
+ broker._commit_puts()
policy_stats = broker.get_policy_stats()
if POLICIES.default.idx not in stats:
# unlikely, but check empty default index still in policy stats
@@ -2163,7 +2209,7 @@ class TestContainerBroker(unittest.TestCase):
sum(stats[policy_index].values()))
def test_initialize_container_broker_in_default(self):
- broker = ContainerBroker(':memory:', account='test1',
+ broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2')
# initialize with no storage_policy_index argument
@@ -2202,7 +2248,7 @@ class TestContainerBroker(unittest.TestCase):
def test_get_info(self):
# Test ContainerBroker.get_info
- broker = ContainerBroker(':memory:', account='test1',
+ broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2')
broker.initialize(Timestamp('1').internal, 0)
@@ -2343,7 +2389,7 @@ class TestContainerBroker(unittest.TestCase):
'db_state': 'collapsed'})
def test_set_x_syncs(self):
- broker = ContainerBroker(':memory:', account='test1',
+ broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2')
broker.initialize(Timestamp('1').internal, 0)
@@ -2357,7 +2403,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(info['x_container_sync_point2'], 2)
def test_get_report_info(self):
- broker = ContainerBroker(':memory:', account='test1',
+ broker = ContainerBroker(self.get_db_path(), account='test1',
container='test2')
broker.initialize(Timestamp('1').internal, 0)
@@ -2529,7 +2575,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertFalse(get_rows(broker))
def test_get_objects(self):
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
objects_0 = [{'name': 'obj_0_%d' % i,
'created_at': next(self.ts).normal,
@@ -2577,7 +2624,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(objects_0 + objects_1, actual)
def test_get_objects_since_row(self):
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
obj_names = ['obj%03d' % i for i in range(20)]
timestamps = [next(self.ts) for o in obj_names]
@@ -2627,7 +2675,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter(self):
# Test ContainerBroker.list_objects_iter
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
for obj1 in range(4):
for obj2 in range(125):
@@ -2777,7 +2826,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual([row[0] for row in listing], ['3/0000', '3/0001'])
def test_list_objects_iter_with_reserved_name(self):
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(next(self.ts).internal, 0)
broker.put_object(
@@ -2936,10 +2986,13 @@ class TestContainerBroker(unittest.TestCase):
}
failures = []
for expected in expectations:
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(),
+ account='a', container='c')
broker.initialize(next(ts).internal, 0)
for name in expected['objects']:
broker.put_object(name, next(ts).internal, **obj_create_params)
+ # commit pending file into db
+ broker._commit_puts()
params = default_listing_params.copy()
params.update(expected['params'])
listing = list(o[0] for o in broker.list_objects_iter(**params))
@@ -2954,7 +3007,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter_non_slash(self):
# Test ContainerBroker.list_objects_iter using a
# delimiter that is not a slash
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
for obj1 in range(4):
for obj2 in range(125):
@@ -3071,7 +3125,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter_prefix_delim(self):
# Test ContainerBroker.list_objects_iter
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object(
@@ -3110,7 +3165,8 @@ class TestContainerBroker(unittest.TestCase):
def test_list_objects_iter_order_and_reverse(self):
# Test ContainerBroker.list_objects_iter
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object(
@@ -3152,7 +3208,8 @@ class TestContainerBroker(unittest.TestCase):
def test_double_check_trailing_delimiter(self):
# Test ContainerBroker.list_objects_iter for a
# container that has an odd file with a trailing delimiter
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp.now().internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@@ -3232,7 +3289,8 @@ class TestContainerBroker(unittest.TestCase):
def test_double_check_trailing_delimiter_non_slash(self):
# Test ContainerBroker.list_objects_iter for a
# container that has an odd file with a trailing delimiter
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp.now().internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@@ -3315,7 +3373,8 @@ class TestContainerBroker(unittest.TestCase):
s = s.encode('utf8')
return md5(s, usedforsecurity=False).hexdigest()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp(1).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@@ -3367,7 +3426,8 @@ class TestContainerBroker(unittest.TestCase):
def test_get_items_since(self):
# test DatabaseBroker.get_items_since
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object('a', Timestamp(1).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
@@ -3380,9 +3440,11 @@ class TestContainerBroker(unittest.TestCase):
def test_sync_merging(self):
# exercise the DatabaseBroker sync functions a bit
- broker1 = ContainerBroker(':memory:', account='a', container='c')
+ broker1 = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker1.initialize(Timestamp('1').internal, 0)
- broker2 = ContainerBroker(':memory:', account='a', container='c')
+ broker2 = ContainerBroker(self.get_db_path(),
+ account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
self.assertEqual(broker2.get_sync('12345'), -1)
broker1.merge_syncs([{'sync_point': 3, 'remote_id': '12345'}])
@@ -3390,14 +3452,18 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(broker2.get_sync('12345'), 3)
def test_merge_items(self):
- broker1 = ContainerBroker(':memory:', account='a', container='c')
+ broker1 = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker1.initialize(Timestamp('1').internal, 0)
- broker2 = ContainerBroker(':memory:', account='a', container='c')
+ broker2 = ContainerBroker(self.get_db_path(),
+ account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object('a', Timestamp(1).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ # commit pending file into db
+ broker1._commit_puts()
id = broker1.get_info()['id']
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
@@ -3406,6 +3472,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items]))
broker1.put_object('c', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
@@ -3433,19 +3500,24 @@ class TestContainerBroker(unittest.TestCase):
snowman = u'\N{SNOWMAN}'
if six.PY2:
snowman = snowman.encode('utf-8')
- broker1 = ContainerBroker(':memory:', account='a', container='c')
+ broker1 = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker1.initialize(Timestamp('1').internal, 0)
id = broker1.get_info()['id']
- broker2 = ContainerBroker(':memory:', account='a', container='c')
+ broker2 = ContainerBroker(self.get_db_path(),
+ account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object(snowman, Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ # commit pending file into db
+ broker1._commit_puts()
broker2.merge_items(json.loads(json.dumps(broker1.get_items_since(
broker2.get_sync(id), 1000))), id)
broker1.put_object(snowman, Timestamp(4).internal, 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
+ broker1._commit_puts()
broker2.merge_items(json.loads(json.dumps(broker1.get_items_since(
broker2.get_sync(id), 1000))), id)
items = broker2.get_items_since(-1, 1000)
@@ -3459,19 +3531,24 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_items_overwrite(self):
# test DatabaseBroker.merge_items
- broker1 = ContainerBroker(':memory:', account='a', container='c')
+ broker1 = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker1.initialize(Timestamp('1').internal, 0)
id = broker1.get_info()['id']
- broker2 = ContainerBroker(':memory:', account='a', container='c')
+ broker2 = ContainerBroker(self.get_db_path(),
+ account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object('a', Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ # commit pending file into db
+ broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
broker1.put_object('a', Timestamp(4).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
@@ -3484,19 +3561,24 @@ class TestContainerBroker(unittest.TestCase):
def test_merge_items_post_overwrite_out_of_order(self):
# test DatabaseBroker.merge_items
- broker1 = ContainerBroker(':memory:', account='a', container='c')
+ broker1 = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker1.initialize(Timestamp('1').internal, 0)
id = broker1.get_info()['id']
- broker2 = ContainerBroker(':memory:', account='a', container='c')
+ broker2 = ContainerBroker(self.get_db_path(),
+ account='a', container='c')
broker2.initialize(Timestamp('1').internal, 0)
broker1.put_object('a', Timestamp(2).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker1.put_object('b', Timestamp(3).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ # commit pending file into db
+ broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
broker1.put_object('a', Timestamp(4).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
@@ -3516,6 +3598,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(rec['created_at'], Timestamp(3).internal)
broker1.put_object('b', Timestamp(5).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+ broker1._commit_puts()
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
@@ -3529,7 +3612,8 @@ class TestContainerBroker(unittest.TestCase):
def test_set_storage_policy_index(self):
ts = make_timestamp_iter()
- broker = ContainerBroker(':memory:', account='test_account',
+ broker = ContainerBroker(self.get_db_path(),
+ account='test_account',
container='test_container')
timestamp = next(ts)
broker.initialize(timestamp.internal, 0)
@@ -3586,7 +3670,8 @@ class TestContainerBroker(unittest.TestCase):
def test_set_storage_policy_index_empty(self):
# Putting an object may trigger migrations, so test with a
# never-had-an-object container to make sure we handle it
- broker = ContainerBroker(':memory:', account='test_account',
+ broker = ContainerBroker(self.get_db_path(),
+ account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
info = broker.get_info()
@@ -3597,7 +3682,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(2, info['storage_policy_index'])
def test_reconciler_sync(self):
- broker = ContainerBroker(':memory:', account='test_account',
+ broker = ContainerBroker(self.get_db_path(),
+ account='test_account',
container='test_container')
broker.initialize(Timestamp('1').internal, 0)
self.assertEqual(-1, broker.get_reconciler_sync())
@@ -5339,6 +5425,7 @@ class TestContainerBroker(unittest.TestCase):
class TestCommonContainerBroker(test_db.TestExampleBroker):
broker_class = ContainerBroker
+ server_type = 'container'
def setUp(self):
super(TestCommonContainerBroker, self).setUp()
@@ -5353,7 +5440,7 @@ class TestCommonContainerBroker(test_db.TestExampleBroker):
storage_policy_index=int(self.policy))
-class ContainerBrokerMigrationMixin(object):
+class ContainerBrokerMigrationMixin(test_db.TestDbBase):
"""
Mixin for running ContainerBroker against databases created with
older schemas.
@@ -5368,6 +5455,7 @@ class ContainerBrokerMigrationMixin(object):
return self.func.__get__(obj, obj_type)
def setUp(self):
+ super(ContainerBrokerMigrationMixin, self).setUp()
self._imported_create_object_table = \
ContainerBroker.create_object_table
ContainerBroker.create_object_table = \
@@ -5408,6 +5496,7 @@ class ContainerBrokerMigrationMixin(object):
self._imported_create_shard_range_table
ContainerBroker.create_policy_stat_table = \
self._imported_create_policy_stat_table
+ # We need to manually teardown and clean the self.tempdir
def premetadata_create_container_info_table(self, conn, put_timestamp,
@@ -5464,7 +5553,8 @@ class TestContainerBrokerBeforeMetadata(ContainerBrokerMigrationMixin,
def setUp(self):
super(TestContainerBrokerBeforeMetadata, self).setUp()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
exc = None
with broker.get() as conn:
@@ -5476,10 +5566,12 @@ class TestContainerBrokerBeforeMetadata(ContainerBrokerMigrationMixin,
def tearDown(self):
super(TestContainerBrokerBeforeMetadata, self).tearDown()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('SELECT metadata FROM container_stat')
+ test_db.TestDbBase.tearDown(self)
def prexsync_create_container_info_table(self, conn, put_timestamp,
@@ -5540,7 +5632,8 @@ class TestContainerBrokerBeforeXSync(ContainerBrokerMigrationMixin,
super(TestContainerBrokerBeforeXSync, self).setUp()
ContainerBroker.create_container_info_table = \
prexsync_create_container_info_table
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
exc = None
with broker.get() as conn:
@@ -5553,10 +5646,12 @@ class TestContainerBrokerBeforeXSync(ContainerBrokerMigrationMixin,
def tearDown(self):
super(TestContainerBrokerBeforeXSync, self).tearDown()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('SELECT x_container_sync_point1 FROM container_stat')
+ test_db.TestDbBase.tearDown(self)
def prespi_create_object_table(self, conn, *args, **kwargs):
@@ -5657,7 +5752,8 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin,
ContainerBroker.create_container_info_table = \
prespi_create_container_info_table
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn:
@@ -5668,10 +5764,12 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin,
def tearDown(self):
super(TestContainerBrokerBeforeSPI, self).tearDown()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('SELECT storage_policy_index FROM container_stat')
+ test_db.TestDbBase.tearDown(self)
@patch_policies
@with_tempdir
@@ -5865,7 +5963,8 @@ class TestContainerBrokerBeforeShardRanges(ContainerBrokerMigrationMixin,
def setUp(self):
super(TestContainerBrokerBeforeShardRanges, self).setUp()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn:
@@ -5875,11 +5974,13 @@ class TestContainerBrokerBeforeShardRanges(ContainerBrokerMigrationMixin,
def tearDown(self):
super(TestContainerBrokerBeforeShardRanges, self).tearDown()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('''SELECT *
FROM shard_range''')
+ test_db.TestDbBase.tearDown(self)
def pre_reported_create_shard_range_table(self, conn):
@@ -5933,7 +6034,8 @@ class TestContainerBrokerBeforeShardRangeReportedColumn(
ContainerBroker.create_shard_range_table = \
pre_reported_create_shard_range_table
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn:
@@ -5944,11 +6046,13 @@ class TestContainerBrokerBeforeShardRangeReportedColumn(
def tearDown(self):
super(TestContainerBrokerBeforeShardRangeReportedColumn,
self).tearDown()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('''SELECT reported
FROM shard_range''')
+ test_db.TestDbBase.tearDown(self)
@with_tempdir
def test_get_shard_ranges_attempts(self, tempdir):
@@ -6161,7 +6265,8 @@ class TestContainerBrokerBeforeShardRangeTombstonesColumn(
ContainerBroker.create_shard_range_table = \
pre_tombstones_create_shard_range_table
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with self.assertRaises(sqlite3.DatabaseError) as raised, \
broker.get() as conn:
@@ -6172,11 +6277,13 @@ class TestContainerBrokerBeforeShardRangeTombstonesColumn(
def tearDown(self):
super(TestContainerBrokerBeforeShardRangeTombstonesColumn,
self).tearDown()
- broker = ContainerBroker(':memory:', account='a', container='c')
+ broker = ContainerBroker(self.get_db_path(), account='a',
+ container='c')
broker.initialize(Timestamp('1').internal, 0)
with broker.get() as conn:
conn.execute('''SELECT tombstones
FROM shard_range''')
+ test_db.TestDbBase.tearDown(self)
class TestUpdateNewItemFromExisting(unittest.TestCase):
View Lorry Controller Status