Merge remote-tracking branch 'gerrit/master' into feature/losffeature/losf

Change-Id: If9d7c63f3c4c15fbccff31e2b77a6911bb95972a

1 files changed, 44 insertions, 9 deletions

diff --git a/swift/common/db.py b/swift/common/db.py
index c6df12aa3..e06baf5c6 100644
--- a/swift/common/db.py
+++ b/swift/common/db.py
@@ -53,6 +53,9 @@ PICKLE_PROTOCOL = 2
 # records will be merged.
 PENDING_CAP = 131072
 
+SQLITE_ARG_LIMIT = 999
+RECLAIM_PAGE_SIZE = 10000
+
 
 def utf8encode(*args):
     return [(s.encode('utf8') if isinstance(s, six.text_type) else s)
@@ -981,16 +984,48 @@ class DatabaseBroker(object):
             with lock_parent_directory(self.pending_file,
                                        self.pending_timeout):
                 self._commit_puts()
-        with self.get() as conn:
-            self._reclaim(conn, age_timestamp, sync_timestamp)
-            self._reclaim_metadata(conn, age_timestamp)
-            conn.commit()
+        marker = ''
+        finished = False
+        while not finished:
+            with self.get() as conn:
+                marker = self._reclaim(conn, age_timestamp, marker)
+                if not marker:
+                    finished = True
+                    self._reclaim_other_stuff(
+                        conn, age_timestamp, sync_timestamp)
+                conn.commit()
+
+    def _reclaim_other_stuff(self, conn, age_timestamp, sync_timestamp):
+        """
+        This is only called once at the end of reclaim after _reclaim has been
+        called for each page.
+        """
+        self._reclaim_sync(conn, sync_timestamp)
+        self._reclaim_metadata(conn, age_timestamp)
+
+    def _reclaim(self, conn, age_timestamp, marker):
+        clean_batch_qry = '''
+            DELETE FROM %s WHERE deleted = 1
+            AND name > ? AND %s < ?
+        ''' % (self.db_contains_type, self.db_reclaim_timestamp)
+        curs = conn.execute('''
+            SELECT name FROM %s WHERE deleted = 1
+            AND name > ?
+            ORDER BY NAME LIMIT 1 OFFSET ?
+        ''' % (self.db_contains_type,), (marker, RECLAIM_PAGE_SIZE))
+        row = curs.fetchone()
+        if row:
+            # do a single book-ended DELETE and bounce out
+            end_marker = row[0]
+            conn.execute(clean_batch_qry + ' AND name <= ?', (
+                marker, age_timestamp, end_marker))
+        else:
+            # delete off the end and reset marker to indicate we're done
+            end_marker = ''
+            conn.execute(clean_batch_qry, (marker, age_timestamp))
+        return end_marker
 
-    def _reclaim(self, conn, age_timestamp, sync_timestamp):
-        conn.execute('''
-            DELETE FROM %s WHERE deleted = 1 AND %s < ?
-        ''' % (self.db_contains_type, self.db_reclaim_timestamp),
-            (age_timestamp,))
+    def _reclaim_sync(self, conn, sync_timestamp):
         try:
             conn.execute('''
                 DELETE FROM outgoing_sync WHERE updated_at < ?