Add support for pulling quarantine stats.

2 files changed, 61 insertions, 8 deletions

diff --git a/bin/swift-recon b/bin/swift-recon
index 4aa38bc7e..4750ea064 100755
--- a/bin/swift-recon
+++ b/bin/swift-recon
@@ -11,6 +11,7 @@ from hashlib import md5
 import datetime
 import eventlet
 import optparse
+import sys
 import os
 
 VERBOSE = False
@@ -83,6 +84,12 @@ def scout_umount(host):
     return url, content, status
 
 
+def scout_quarantine(host):
+    base_url = "http://%s:%s/recon/" % (host[0], host[1])
+    url, content, status = scout(base_url, "quarantined")
+    return url, content, status
+
+
 def get_ringmd5(ringfile):
     stats = {}
     matches = 0
@@ -137,12 +144,11 @@ def async_check():
         print "Async stats: low: %d, high: %d, avg: %d, total: %d" % (low,
             high, average, total)
     else:
-        print "Error: No hosts where available or returned valid information."
+        print "Error: No hosts available or returned valid information."
     print "=" * 79
 
 
 def umount_check():
-    ASYNC_COUNTER = 0
     stats = {}
     hosts = getdevices()
     pool = eventlet.GreenPool(20)
@@ -174,7 +180,7 @@ def replication_check():
         print "[Replication Times] shortest: %s, longest: %s, avg: %s" % \
             (low, high, average)
     else:
-        print "Error: No hosts where available or returned valid information."
+        print "Error: No hosts available or returned valid information."
     print "=" * 79
 
 
@@ -201,7 +207,34 @@ def load_check():
             print "[%s load average] lowest: %s, highest: %s, avg: %s" % \
                 (item, low, high, average)
         else:
-            print "Error: Hosts unavailable or returned valid information."
+            print "Error: No hosts available or returned valid information."
+    print "=" * 79
+
+
+def quarantine_check():
+    objq = {}
+    conq = {}
+    acctq = {}
+    hosts = getdevices()
+    pool = eventlet.GreenPool(20)
+    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    print "[%s] Checking quarantine dirs on %s hosts..." % (now, len(hosts))
+    for url, response, status in pool.imap(scout_quarantine, hosts):
+        if status == 200:
+            objq[url] = response['objects']
+            conq[url] = response['containers']
+            acctq[url] = response['accounts']
+    stats = {"objects": objq, "containers": conq, "accounts": acctq}
+    for item in stats:
+        if len(stats[item]) > 0:
+            low = min(stats[item].values())
+            high = max(stats[item].values())
+            total = sum(stats[item].values())
+            average = total / len(stats[item])
+            print "[Quarantined %s] low: %d, high: %d, avg: %d, total: %d" % \
+                (item, low, high, average, total)
+        else:
+            print "Error: No hosts available or returned valid information."
     print "=" * 79
 
 
@@ -253,7 +286,7 @@ def disk_usage():
         print "Disk usage: lowest: %s%%, highest: %s%%, avg: %s%%" % \
             (low, high, average)
     else:
-        print "Error: No hosts where available or returned valid information."
+        print "Error: No hosts available or returned valid information."
     print "=" * 79
 
 
@@ -261,7 +294,7 @@ def main():
     global VERBOSE, SUPPRESS_ERRORS, swift_dir, pool
     print "=" * 79
     usage = '''
-    usage: %prog [-v] [--suppress] [-a] [-r] [-u] [-d] [-l] [-c] [--objmd5]
+    usage: %prog [-v] [--suppress] [-a] [-r] [-u] [-d] [-l] [--objmd5]
     '''
     args = optparse.OptionParser(usage)
     args.add_option('--verbose', '-v', action="store_true",
@@ -278,14 +311,17 @@ def main():
         help="Get disk usage stats")
     args.add_option('--loadstats', '-l', action="store_true",
         help="Get cluster load average stats")
-    args.add_option('--connstats', '-c', action="store_true",
-        help="Get connection stats")
+    args.add_option('--quarantined', '-q', action="store_true",
+        help="Get cluster quarantine stats")
     args.add_option('--objmd5', action="store_true",
         help="Get md5sums of object.ring.gz and compare to local copy")
     args.add_option('--swiftdir', default="/etc/swift",
         help="Default = /etc/swift")
     options, arguments = args.parse_args()
 
+    if len(sys.argv) <= 1:
+        args.print_help()
+
     swift_dir = options.swiftdir
 
     VERBOSE = options.verbose
@@ -303,6 +339,8 @@ def main():
         disk_usage()
     if options.objmd5:
         get_ringmd5(os.path.join(swift_dir, 'object.ring.gz'))
+    if options.quarantined:
+        quarantine_check()
 
 
 if __name__ == '__main__':
diff --git a/swift/common/middleware/recon.py b/swift/common/middleware/recon.py
index 3b4877908..f2b9e777d 100644
--- a/swift/common/middleware/recon.py
+++ b/swift/common/middleware/recon.py
@@ -154,6 +154,19 @@ class ReconMiddleware(object):
             sums[ringfile] = md5sum.hexdigest()
         return sums
 
+    def get_quarantine_count(self):
+        """get obj/container/account quarantine counts"""
+        qcounts = {"objects": 0, "containers": 0, "accounts": 0}
+        qdir = "quarantined"
+        for device in os.listdir(self.devices):
+            for qtype in qcounts:
+                qtgt = os.path.join(self.devices, device, qdir, qtype)
+                if os.path.exists(qtgt):
+                    linkcount = os.lstat(qtgt).st_nlink
+                    if linkcount > 2:
+                        qcounts[qtype] += linkcount - 2
+        return qcounts
+
     def GET(self, req):
         error = False
         root, type = split_path(req.path, 1, 2, False)
@@ -186,6 +199,8 @@ class ReconMiddleware(object):
                 content = json.dumps(self.get_diskusage())
             elif type == "ringmd5":
                 content = json.dumps(self.get_ring_md5())
+            elif type == "quarantined":
+                content = json.dumps(self.get_quarantine_count())
             else:
                 content = "Invalid path: %s" % req.path
                 return Response(request=req, status="400 Bad Request", \