From 4affecee236b84fb1340b3cccfe82c627320200f Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Wed, 9 Jan 2013 11:56:23 -0800 Subject: ReplicatedPG/PG: check snap collections during _scan_list During _scan_list check the snapcollections corresponding to the object_info attr on the object. Report inconsistencies during scrub_finalize. Signed-off-by: Samuel Just Reviewed-by: Sage Weil --- src/osd/PG.cc | 36 ++++++++++++++++++++++++++-- src/osd/PG.h | 12 ++++++++++ src/osd/ReplicatedPG.cc | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ src/osd/ReplicatedPG.h | 11 +++++++++ 4 files changed, 119 insertions(+), 2 deletions(-) diff --git a/src/osd/PG.cc b/src/osd/PG.cc index 2f38dac426e..bfcdf7e3d90 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -2682,8 +2682,13 @@ void PG::_scan_list(ScrubMap &map, vector &ls) if (r == 0) { ScrubMap::object &o = map.objects[poid]; o.size = st.st_size; + o.nlinks = st.st_nlink; assert(!o.negative); osd->store->getattrs(coll, poid, o.attrs); + if (poid.snap != CEPH_SNAPDIR && poid.snap != CEPH_NOSNAP) { + // Check snap collections + check_snap_collections(poid, o.attrs, &o.snapcolls); + } dout(25) << "_scan_list " << poid << dendl; } else { dout(25) << "_scan_list " << poid << " got " << r << ", skipping" << dendl; @@ -3215,6 +3220,7 @@ void PG::_compare_scrubmaps(const map &maps, map > &missing, map > &inconsistent, map &authoritative, + map > &invalid_snapcolls, ostream &errorstream) { map::const_iterator i; @@ -3241,6 +3247,18 @@ void PG::_compare_scrubmaps(const map &maps, // Take first osd to have it as authoritative auth = j; } else { + // Check snapcolls + if (k->snap < CEPH_MAXSNAP) { + if (_report_snap_collection_errors( + *k, + j->first, + j->second->objects[*k].attrs, + j->second->objects[*k].snapcolls, + j->second->objects[*k].nlinks, + errorstream)) { + invalid_snapcolls[*k].insert(j->first); + } + } // Compare stringstream ss; if (!_compare_scrub_objects(auth->second->objects[*k], @@ -3300,6 +3318,7 @@ void PG::scrub_finalize() { // Maps from objects with erros to missing/inconsistent peers map > missing; map > inconsistent; + map > inconsistent_snapcolls; // Map from object with errors to good peer map authoritative; @@ -3314,9 +3333,22 @@ void PG::scrub_finalize() { maps[i] = &scrub_received_maps[acting[i]]; } - _compare_scrubmaps(maps, missing, inconsistent, authoritative, ss); + _compare_scrubmaps( + maps, missing, inconsistent, authoritative, + inconsistent_snapcolls, + ss); + + for (map >::iterator obj = inconsistent_snapcolls.begin(); + obj != inconsistent_snapcolls.end(); + ++obj) { + for (set::iterator j = obj->second.begin(); j != obj->second.end(); ++j) { + ++errors; + ss << info.pgid << " " << mode << " " << " object " << obj->first + << " has inconsistent snapcolls on " << *j << std::endl; + } + } - if (authoritative.size()) { + if (authoritative.size() || inconsistent_snapcolls.size()) { ss << info.pgid << " " << mode << " " << missing.size() << " missing, " << inconsistent.size() << " inconsistent objects\n"; dout(2) << ss.str() << dendl; diff --git a/src/osd/PG.h b/src/osd/PG.h index 4a292f6b3d0..0354de800c7 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -756,6 +756,7 @@ public: map > &missing, map > &inconsistent, map &authoritative, + map > &inconsistent_snapcolls, ostream &errorstream); void scrub(); void scrub_finalize(); @@ -766,6 +767,17 @@ public: void build_scrub_map(ScrubMap &map); void build_inc_scrub_map(ScrubMap &map, eversion_t v); virtual int _scrub(ScrubMap &map, int& errors, int& fixed) { return 0; } + virtual bool _report_snap_collection_errors( + const hobject_t &hoid, + int osd, + const map &attrs, + const set &snapcolls, + uint32_t nlinks, + ostream &out) { return false; }; + virtual void check_snap_collections( + const hobject_t &hoid, + const map &attrs, + set *snapcolls) {}; void clear_scrub_reserved(); void scrub_reserve_replicas(); void scrub_unreserve_replicas(); diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index 3caa8cd4dce..f0fd24e4640 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -6633,6 +6633,68 @@ int ReplicatedPG::_scrub(ScrubMap& scrubmap, int& errors, int& fixed) return errors; } +static set get_expected_snap_colls( + const map &attrs, + object_info_t *oi = 0) +{ + object_info_t _oi; + if (!oi) + oi = &_oi; + + set to_check; + map::const_iterator oiiter = attrs.find(OI_ATTR); + if (oiiter == attrs.end()) + return to_check; + + bufferlist oiattr; + oiattr.push_back(oiiter->second); + *oi = object_info_t(oiattr); + if (oi->snaps.size() > 0) + to_check.insert(*(oi->snaps.begin())); + if (oi->snaps.size() > 1) + to_check.insert(*(oi->snaps.rbegin())); + return to_check; +} + +bool ReplicatedPG::_report_snap_collection_errors( + const hobject_t &hoid, + int osd, + const map &attrs, + const set &snapcolls, + uint32_t nlinks, + ostream &out) +{ + bool errors = false; + set to_check = get_expected_snap_colls(attrs); + if (to_check != snapcolls) { + out << info.pgid << " osd." << osd << " inconsistent snapcolls on " + << hoid << " found " << snapcolls << " expected " << to_check + << std::endl; + errors = true; + } + return errors; +} + +void ReplicatedPG::check_snap_collections( + const hobject_t &hoid, + const map &attrs, + set *snapcolls) +{ + object_info_t oi; + set to_check = get_expected_snap_colls(attrs, &oi); + + for (set::iterator i = to_check.begin(); i != to_check.end(); ++i) { + struct stat st; + int r = osd->store->stat(coll_t(info.pgid, *i), hoid, &st); + if (r == -ENOENT) { + } else if (r == 0) { + snapcolls->insert(*i); + } else { + assert(0); + } + } +} + /*---SnapTrimmer Logging---*/ #undef dout_prefix #define dout_prefix *_dout << pg->gen_prefix() diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h index 6387cf889cb..60aab4b022b 100644 --- a/src/osd/ReplicatedPG.h +++ b/src/osd/ReplicatedPG.h @@ -788,6 +788,17 @@ protected: // -- scrub -- virtual int _scrub(ScrubMap& map, int& errors, int& fixed); + virtual bool _report_snap_collection_errors( + const hobject_t &hoid, + int osd, + const map &attrs, + const set &snapcolls, + uint32_t nlinks, + ostream &out); + virtual void check_snap_collections( + const hobject_t &hoid, + const map &attrs, + set *snapcolls); void apply_and_flush_repops(bool requeue); -- cgit v1.2.1