ReplicatedPG/PG: check snap collections during _scan_list

During _scan_list check the snapcollections corresponding to the
object_info attr on the object.  Report inconsistencies during
scrub_finalize.

Signed-off-by: Samuel Just <sam.just@inktank.com>
Reviewed-by: Sage Weil <sage@inktank.com>

4 files changed, 119 insertions, 2 deletions

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 2f38dac426e..bfcdf7e3d90 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -2682,8 +2682,13 @@ void PG::_scan_list(ScrubMap &map, vector<hobject_t> &ls)
     if (r == 0) {
       ScrubMap::object &o = map.objects[poid];
       o.size = st.st_size;
+      o.nlinks = st.st_nlink;
       assert(!o.negative);
       osd->store->getattrs(coll, poid, o.attrs);
+      if (poid.snap != CEPH_SNAPDIR && poid.snap != CEPH_NOSNAP) {
+	// Check snap collections
+	check_snap_collections(poid, o.attrs, &o.snapcolls);
+      }
       dout(25) << "_scan_list  " << poid << dendl;
     } else {
       dout(25) << "_scan_list  " << poid << " got " << r << ", skipping" << dendl;
@@ -3215,6 +3220,7 @@ void PG::_compare_scrubmaps(const map<int,ScrubMap*> &maps,
 			    map<hobject_t, set<int> > &missing,
 			    map<hobject_t, set<int> > &inconsistent,
 			    map<hobject_t, int> &authoritative,
+			    map<hobject_t, set<int> > &invalid_snapcolls,
 			    ostream &errorstream)
 {
   map<hobject_t,ScrubMap::object>::const_iterator i;
@@ -3241,6 +3247,18 @@ void PG::_compare_scrubmaps(const map<int,ScrubMap*> &maps,
 	  // Take first osd to have it as authoritative
 	  auth = j;
 	} else {
+	  // Check snapcolls
+	  if (k->snap < CEPH_MAXSNAP) {
+	    if (_report_snap_collection_errors(
+		*k,
+		j->first,
+		j->second->objects[*k].attrs,
+		j->second->objects[*k].snapcolls,
+		j->second->objects[*k].nlinks,
+		errorstream)) {
+	      invalid_snapcolls[*k].insert(j->first);
+	    }
+	  }
 	  // Compare 
 	  stringstream ss;
 	  if (!_compare_scrub_objects(auth->second->objects[*k],
@@ -3300,6 +3318,7 @@ void PG::scrub_finalize() {
     // Maps from objects with erros to missing/inconsistent peers
     map<hobject_t, set<int> > missing;
     map<hobject_t, set<int> > inconsistent;
+    map<hobject_t, set<int> > inconsistent_snapcolls;
 
     // Map from object with errors to good peer
     map<hobject_t, int> authoritative;
@@ -3314,9 +3333,22 @@ void PG::scrub_finalize() {
       maps[i] = &scrub_received_maps[acting[i]];
     }
 
-    _compare_scrubmaps(maps, missing, inconsistent, authoritative, ss);
+    _compare_scrubmaps(
+      maps, missing, inconsistent, authoritative,
+      inconsistent_snapcolls,
+      ss);
+
+    for (map<hobject_t, set<int> >::iterator obj = inconsistent_snapcolls.begin();
+	 obj != inconsistent_snapcolls.end();
+	 ++obj) {
+      for (set<int>::iterator j = obj->second.begin(); j != obj->second.end(); ++j) {
+	++errors;
+	ss << info.pgid << " " << mode << " " << " object " << obj->first
+	   << " has inconsistent snapcolls on " << *j << std::endl;
+      }
+    }
 
-    if (authoritative.size()) {
+    if (authoritative.size() || inconsistent_snapcolls.size()) {
       ss << info.pgid << " " << mode << " " << missing.size() << " missing, "
 	 << inconsistent.size() << " inconsistent objects\n";
       dout(2) << ss.str() << dendl;
diff --git a/src/osd/PG.h b/src/osd/PG.h
index 4a292f6b3d0..0354de800c7 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -756,6 +756,7 @@ public:
 			  map<hobject_t, set<int> > &missing,
 			  map<hobject_t, set<int> > &inconsistent,
 			  map<hobject_t, int> &authoritative,
+			  map<hobject_t, set<int> > &inconsistent_snapcolls,
 			  ostream &errorstream);
   void scrub();
   void scrub_finalize();
@@ -766,6 +767,17 @@ public:
   void build_scrub_map(ScrubMap &map);
   void build_inc_scrub_map(ScrubMap &map, eversion_t v);
   virtual int _scrub(ScrubMap &map, int& errors, int& fixed) { return 0; }
+  virtual bool _report_snap_collection_errors(
+    const hobject_t &hoid,
+    int osd,
+    const map<string, bufferptr> &attrs,
+    const set<snapid_t> &snapcolls,
+    uint32_t nlinks,
+    ostream &out) { return false; };
+  virtual void check_snap_collections(
+    const hobject_t &hoid,
+    const map<string, bufferptr> &attrs,
+    set<snapid_t> *snapcolls) {};
   void clear_scrub_reserved();
   void scrub_reserve_replicas();
   void scrub_unreserve_replicas();
diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index 3caa8cd4dce..f0fd24e4640 100644
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -6633,6 +6633,68 @@ int ReplicatedPG::_scrub(ScrubMap& scrubmap, int& errors, int& fixed)
   return errors;
 }
 
+static set<snapid_t> get_expected_snap_colls(
+  const map<string, bufferptr> &attrs,
+  object_info_t *oi = 0)
+{
+  object_info_t _oi;
+  if (!oi)
+    oi = &_oi;
+
+  set<snapid_t> to_check;
+  map<string, bufferptr>::const_iterator oiiter = attrs.find(OI_ATTR);
+  if (oiiter == attrs.end())
+    return to_check;
+
+  bufferlist oiattr;
+  oiattr.push_back(oiiter->second);
+  *oi = object_info_t(oiattr);
+  if (oi->snaps.size() > 0)
+    to_check.insert(*(oi->snaps.begin()));
+  if (oi->snaps.size() > 1)
+    to_check.insert(*(oi->snaps.rbegin()));
+  return to_check;
+}
+
+bool ReplicatedPG::_report_snap_collection_errors(
+  const hobject_t &hoid,
+  int osd,
+  const map<string, bufferptr> &attrs,
+  const set<snapid_t> &snapcolls,
+  uint32_t nlinks,
+  ostream &out)
+{
+  bool errors = false;
+  set<snapid_t> to_check = get_expected_snap_colls(attrs);
+  if (to_check != snapcolls) {
+    out << info.pgid << " osd." << osd << " inconsistent snapcolls on "
+	<< hoid << " found " << snapcolls << " expected " << to_check
+	<< std::endl;
+    errors = true;
+  }
+  return errors;
+}
+
+void ReplicatedPG::check_snap_collections(
+  const hobject_t &hoid,
+  const map<string, bufferptr> &attrs,
+  set<snapid_t> *snapcolls)
+{
+  object_info_t oi;
+  set<snapid_t> to_check = get_expected_snap_colls(attrs, &oi);
+
+  for (set<snapid_t>::iterator i = to_check.begin(); i != to_check.end(); ++i) {
+    struct stat st;
+    int r = osd->store->stat(coll_t(info.pgid, *i), hoid, &st);
+    if (r == -ENOENT) {
+    } else if (r == 0) {
+      snapcolls->insert(*i);
+    } else {
+      assert(0);
+    }
+  }
+}
+
 /*---SnapTrimmer Logging---*/
 #undef dout_prefix
 #define dout_prefix *_dout << pg->gen_prefix() 
diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h
index 6387cf889cb..60aab4b022b 100644
--- a/src/osd/ReplicatedPG.h
+++ b/src/osd/ReplicatedPG.h
@@ -788,6 +788,17 @@ protected:
 
   // -- scrub --
   virtual int _scrub(ScrubMap& map, int& errors, int& fixed);
+  virtual bool _report_snap_collection_errors(
+    const hobject_t &hoid,
+    int osd,
+    const map<string, bufferptr> &attrs,
+    const set<snapid_t> &snapcolls,
+    uint32_t nlinks,
+    ostream &out);
+  virtual void check_snap_collections(
+    const hobject_t &hoid,
+    const map<string, bufferptr> &attrs,
+    set<snapid_t> *snapcolls);
 
   void apply_and_flush_repops(bool requeue);