From 921c4586f165ce39c17ef8b579c548dc8f6f4500 Mon Sep 17 00:00:00 2001 From: Xiaoxi Chen Date: Fri, 24 Apr 2015 15:32:36 +0800 Subject: [PATCH] os/Filestore:Refactor collection_list_range and collection_list_partial Add collection_list_impl which abstract the common process of coleection list behavior. Todo: Refactor Index::collection_list_partial as well. Signed-off-by: Xiaoxi Chen --- src/os/CollectionIndex.h | 2 +- src/os/FileStore.cc | 95 ++++++++++++++-------------------------- src/os/FileStore.h | 2 + src/os/FlatIndex.cc | 2 +- src/os/FlatIndex.h | 2 +- src/os/HashIndex.cc | 22 +++++----- src/os/HashIndex.h | 4 +- src/os/LFNIndex.cc | 4 +- src/os/LFNIndex.h | 4 +- 9 files changed, 54 insertions(+), 83 deletions(-) diff --git a/src/os/CollectionIndex.h b/src/os/CollectionIndex.h index 5ed41cf1fbef0..eeab2cb0a8e6c 100644 --- a/src/os/CollectionIndex.h +++ b/src/os/CollectionIndex.h @@ -166,7 +166,7 @@ protected: /// List contents of collection by hash virtual int collection_list_partial( const ghobject_t &start, ///< [in] object at which to start - int min_count, ///< [in] get at least min_count objects + const ghobject_t end, ///< [in] list only objects < end int max_count, ///< [in] return at most max_count objects snapid_t seq, ///< [in] list only objects with snap >= seq vector *ls, ///< [out] Listed objects diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc index fc912099630aa..11983205c5d6f 100644 --- a/src/os/FileStore.cc +++ b/src/os/FileStore.cc @@ -4631,7 +4631,7 @@ bool FileStore::collection_empty(coll_t c) vector ls; collection_list_handle_t handle; - r = index->collection_list_partial(ghobject_t(), 1, 1, 0, &ls, NULL); + r = index->collection_list_partial(ghobject_t(), ghobject_t::get_max(), 1, 0, &ls, NULL); if (r < 0) { assert(!m_filestore_fail_eio || r != -EIO); return false; @@ -4640,59 +4640,9 @@ bool FileStore::collection_empty(coll_t c) tracepoint(objectstore, collection_empty_exit, ret); return ret; } - -int FileStore::collection_list_range(coll_t c, ghobject_t start, ghobject_t end, - snapid_t seq, vector *ls) -{ - tracepoint(objectstore, collection_list_range_enter, c.c_str()); - bool done = false; - ghobject_t next = start; - - if (!c.is_temp() && !c.is_meta() && next.hobj.pool < -1) { - coll_t temp = c.get_temp(); - int r = collection_list_range(temp, start, end, seq, ls); - if (r < 0) - return r; - // ... always continue on to non-temp ... - } - - while (!done) { - vector next_objects; - int r = collection_list_partial(c, next, - get_ideal_list_min(), get_ideal_list_max(), - seq, &next_objects, &next); - if (r < 0) - return r; - - ls->insert(ls->end(), next_objects.begin(), next_objects.end()); - - // special case for empty collection - if (ls->empty()) { - break; - } - - while (!ls->empty() && ls->back() >= end) { - ls->pop_back(); - done = true; - } - - if (next >= end) { - done = true; - } - } - - tracepoint(objectstore, collection_list_range_exit, 0); - return 0; -} - -int FileStore::collection_list_partial(coll_t c, ghobject_t start, - int min, int max, snapid_t seq, - vector *ls, ghobject_t *next) +int FileStore::collection_list_impl(coll_t c, ghobject_t start, ghobject_t end, int max, + snapid_t seq, vector *ls, ghobject_t *next) { - tracepoint(objectstore, collection_list_partial_enter, c.c_str()); - dout(10) << "collection_list_partial: " << c << " start " << start << dendl; - assert(next); - if (start.is_max()) return 0; @@ -4720,7 +4670,6 @@ int FileStore::collection_list_partial(coll_t c, ghobject_t start, dout(20) << __func__ << " pool is " << pool << " shard is " << shard << " pgid " << pgid << dendl; } - ghobject_t sep; sep.hobj.pool = -1; sep.set_shard(shard); @@ -4728,7 +4677,7 @@ int FileStore::collection_list_partial(coll_t c, ghobject_t start, if (start < sep) { dout(10) << __func__ << " first checking temp pool" << dendl; coll_t temp = c.get_temp(); - int r = collection_list_partial(temp, start, min, max, seq, ls, next); + int r = collection_list_impl(temp, start, end, max, seq, ls, next); if (r < 0) return r; if (*next != ghobject_t::get_max()) @@ -4749,26 +4698,46 @@ int FileStore::collection_list_partial(coll_t c, ghobject_t start, assert(NULL != index.index); RWLock::RLocker l((index.index)->access_lock); - r = index->collection_list_partial(start, - min, max, seq, - ls, next); + r = index->collection_list_partial(start, end, max, seq, ls, next); + if (r < 0) { assert(!m_filestore_fail_eio || r != -EIO); return r; } - if (ls) - dout(20) << "objects: " << *ls << dendl; + dout(20) << "objects: " << ls << dendl; // HashIndex doesn't know the pool when constructing a 'next' value - if (!next->is_max()) { + if (next && !next->is_max()) { next->hobj.pool = pool; next->set_shard(shard); + dout(20) << " next " << *next << dendl; } - dout(20) << " next " << *next << dendl; - tracepoint(objectstore, collection_list_partial_exit, 0); return 0; } +int FileStore::collection_list_range(coll_t c, ghobject_t start, ghobject_t end, + snapid_t seq, vector *ls) +{ + tracepoint(objectstore, collection_list_range_enter, c.c_str()); + + ghobject_t next; + int r = collection_list_impl(c, start, end, -1, seq, ls, &next); + tracepoint(objectstore, collection_list_range_exit, 0); + return r; +} + +int FileStore::collection_list_partial(coll_t c, ghobject_t start, + int min, int max, snapid_t seq, + vector *ls, ghobject_t *next) +{ + tracepoint(objectstore, collection_list_partial_enter, c.c_str()); + dout(10) << "collection_list_partial: " << c << " start " << start << dendl; + + assert(next); + int r = collection_list_impl(c, start, ghobject_t::get_max(), max, seq, ls, next); + tracepoint(objectstore, collection_list_partial_exit, 0); + return r; +} int FileStore::collection_list(coll_t c, vector& ls) { diff --git a/src/os/FileStore.h b/src/os/FileStore.h index f712b7647d2a2..63c801d1ccaf3 100644 --- a/src/os/FileStore.h +++ b/src/os/FileStore.h @@ -617,6 +617,8 @@ public: const SequencerPosition &spos); // collections + int collection_list_impl(coll_t c, ghobject_t start, ghobject_t end, int max, + snapid_t seq, vector *ls, ghobject_t *next); int list_collections(vector& ls); int list_collections(vector& ls, bool include_temp); int collection_version_current(coll_t c, uint32_t *version); diff --git a/src/os/FlatIndex.cc b/src/os/FlatIndex.cc index 34c42c145b190..c6045228be868 100644 --- a/src/os/FlatIndex.cc +++ b/src/os/FlatIndex.cc @@ -373,7 +373,7 @@ static int get_hobject_from_oinfo(const char *dir, const char *file, } int FlatIndex::collection_list_partial(const ghobject_t &start, - int min_count, + const ghobject_t end, int max_count, snapid_t seq, vector *ls, diff --git a/src/os/FlatIndex.h b/src/os/FlatIndex.h index 0509df46470f6..087799e6d937f 100644 --- a/src/os/FlatIndex.h +++ b/src/os/FlatIndex.h @@ -74,7 +74,7 @@ public: /// @see CollectionIndex int collection_list_partial( const ghobject_t &start, - int min_count, + const ghobject_t end, int max_count, snapid_t seq, vector *ls, diff --git a/src/os/HashIndex.cc b/src/os/HashIndex.cc index 96a0a5af35d98..1e0e1e41fbdae 100644 --- a/src/os/HashIndex.cc +++ b/src/os/HashIndex.cc @@ -322,11 +322,11 @@ int HashIndex::_lookup(const ghobject_t &oid, int HashIndex::_collection_list(vector *ls) { vector path; - return list_by_hash(path, 0, 0, 0, 0, ls); + return list_by_hash(path, ghobject_t::get_max(), 0, 0, 0, ls); } int HashIndex::_collection_list_partial(const ghobject_t &start, - int min_count, + const ghobject_t end, int max_count, snapid_t seq, vector *ls, @@ -336,8 +336,8 @@ int HashIndex::_collection_list_partial(const ghobject_t &start, if (!next) next = &_next; *next = start; - dout(20) << "_collection_list_partial " << start << " " << min_count << "-" << max_count << " ls.size " << ls->size() << dendl; - return list_by_hash(path, min_count, max_count, seq, next, ls); + dout(20) << "_collection_list_partial start:" << start << " end:" << end << "-" << max_count << " ls.size " << ls->size() << dendl; + return list_by_hash(path, end, max_count, seq, next, ls); } int HashIndex::prep_delete() { @@ -816,7 +816,7 @@ int HashIndex::get_path_contents_by_hash(const vector &path, } int HashIndex::list_by_hash(const vector &path, - int min_count, + ghobject_t end, int max_count, snapid_t seq, ghobject_t *next, @@ -841,17 +841,12 @@ int HashIndex::list_by_hash(const vector &path, set >::iterator j = objects.lower_bound( make_pair(*i, ghobject_t())); if (j == objects.end() || j->first != *i) { - if (min_count > 0 && out->size() > (unsigned)min_count) { - if (next) - *next = ghobject_t(hobject_t("", "", CEPH_NOSNAP, hash_prefix_to_hash(*i), -1, "")); - return 0; - } *(next_path.rbegin()) = *(i->rbegin()); ghobject_t next_recurse; if (next) next_recurse = *next; r = list_by_hash(next_path, - min_count, + end, max_count, seq, &next_recurse, @@ -871,6 +866,11 @@ int HashIndex::list_by_hash(const vector &path, *next = j->second; return 0; } + if (j->second >= end) { + if (next) + *next = ghobject_t::get_max(); + return 0; + } if (!next || j->second >= *next) { out->push_back(j->second); } diff --git a/src/os/HashIndex.h b/src/os/HashIndex.h index dad8ce31b8700..afe2364748b18 100644 --- a/src/os/HashIndex.h +++ b/src/os/HashIndex.h @@ -193,7 +193,7 @@ protected: int _collection_list_partial( const ghobject_t &start, - int min_count, + ghobject_t end, int max_count, snapid_t seq, vector *ls, @@ -366,7 +366,7 @@ private: /// List objects in collection in ghobject_t order int list_by_hash( const vector &path, /// [in] Path to list - int min_count, /// [in] List at least min_count + ghobject_t end, /// [in] List only objects < end int max_count, /// [in] List at most max_count snapid_t seq, /// [in] list only objects where snap >= seq ghobject_t *next, /// [in,out] List objects >= *next diff --git a/src/os/LFNIndex.cc b/src/os/LFNIndex.cc index 5b34f9eebd264..7687a66af3274 100644 --- a/src/os/LFNIndex.cc +++ b/src/os/LFNIndex.cc @@ -155,13 +155,13 @@ int LFNIndex::pre_hash_collection(uint32_t pg_num, uint64_t expected_num_objs) int LFNIndex::collection_list_partial(const ghobject_t &start, - int min_count, + const ghobject_t end, int max_count, snapid_t seq, vector *ls, ghobject_t *next) { - return _collection_list_partial(start, min_count, max_count, seq, ls, next); + return _collection_list_partial(start, end, max_count, seq, ls, next); } /* Derived class utility methods */ diff --git a/src/os/LFNIndex.h b/src/os/LFNIndex.h index 5cd35238165cf..000e0ae342347 100644 --- a/src/os/LFNIndex.h +++ b/src/os/LFNIndex.h @@ -192,7 +192,7 @@ public: /// @see CollectionIndex int collection_list_partial( const ghobject_t &start, - int min_count, + const ghobject_t end, int max_count, snapid_t seq, vector *ls, @@ -268,7 +268,7 @@ protected: /// @see CollectionIndex virtual int _collection_list_partial( const ghobject_t &start, - int min_count, + const ghobject_t end, int max_count, snapid_t seq, vector *ls, -- 2.39.5