os/Filestore:Refactor collection_list_range and collection_list_partial

Add collection_list_impl which abstract the common process
of coleection list behavior.

Todo: Refactor Index::collection_list_partial as
well.

Signed-off-by: Xiaoxi Chen <xiaoxi.chen@intel.com>

diff --git a/src/os/CollectionIndex.h b/src/os/CollectionIndex.h
index 5ed41cf1fbef017733b283339fcda50a869c7540..eeab2cb0a8e6ced902cd5b5bb82df7a25ecf25ca 100644 (file)
--- a/src/os/CollectionIndex.h
+++ b/src/os/CollectionIndex.h
@@ -166,7 +166,7 @@ protected:
   /// List contents of collection by hash
   virtual int collection_list_partial(
     const ghobject_t &start, ///< [in] object at which to start
-    int min_count,          ///< [in] get at least min_count objects
+    const ghobject_t end,    ///< [in] list only objects < end
     int max_count,          ///< [in] return at most max_count objects
     snapid_t seq,           ///< [in] list only objects with snap >= seq
     vector<ghobject_t> *ls,  ///< [out] Listed objects

diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
index fc912099630aa9f0e2afb65f6015c5166d32989d..11983205c5d6f0dc4fbc773fd17f5d3b626f3d2f 100644 (file)
--- a/src/os/FileStore.cc
+++ b/src/os/FileStore.cc
@@ -4631,7 +4631,7 @@ bool FileStore::collection_empty(coll_t c)
 
   vector<ghobject_t> ls;
   collection_list_handle_t handle;
-  r = index->collection_list_partial(ghobject_t(), 1, 1, 0, &ls, NULL);
+  r = index->collection_list_partial(ghobject_t(), ghobject_t::get_max(), 1, 0, &ls, NULL);
   if (r < 0) {
     assert(!m_filestore_fail_eio || r != -EIO);
     return false;
@@ -4640,59 +4640,9 @@ bool FileStore::collection_empty(coll_t c)
   tracepoint(objectstore, collection_empty_exit, ret);
   return ret;
 }
-
-int FileStore::collection_list_range(coll_t c, ghobject_t start, ghobject_t end,
-                                     snapid_t seq, vector<ghobject_t> *ls)
-{
-  tracepoint(objectstore, collection_list_range_enter, c.c_str());
-  bool done = false;
-  ghobject_t next = start;
-
-  if (!c.is_temp() && !c.is_meta() && next.hobj.pool < -1) {
-    coll_t temp = c.get_temp();
-    int r = collection_list_range(temp, start, end, seq, ls);
-    if (r < 0)
-      return r;
-    // ... always continue on to non-temp ...
-  }
-
-  while (!done) {
-    vector<ghobject_t> next_objects;
-    int r = collection_list_partial(c, next,
-                                get_ideal_list_min(), get_ideal_list_max(),
-                                seq, &next_objects, &next);
-    if (r < 0)
-      return r;
-
-    ls->insert(ls->end(), next_objects.begin(), next_objects.end());
-
-    // special case for empty collection
-    if (ls->empty()) {
-      break;
-    }
-
-    while (!ls->empty() && ls->back() >= end) {
-      ls->pop_back();
-      done = true;
-    }
-
-    if (next >= end) {
-      done = true;
-    }
-  }
-
-  tracepoint(objectstore, collection_list_range_exit, 0);
-  return 0;
-}
-
-int FileStore::collection_list_partial(coll_t c, ghobject_t start,
-                                      int min, int max, snapid_t seq,
-                                      vector<ghobject_t> *ls, ghobject_t *next)
+int FileStore::collection_list_impl(coll_t c, ghobject_t start, ghobject_t end, int max,
+                                     snapid_t seq, vector<ghobject_t> *ls, ghobject_t *next)
 {
-  tracepoint(objectstore, collection_list_partial_enter, c.c_str());
-  dout(10) << "collection_list_partial: " << c << " start " << start << dendl;
-  assert(next);
-
   if (start.is_max())
     return 0;
 
@@ -4720,7 +4670,6 @@ int FileStore::collection_list_partial(coll_t c, ghobject_t start,
     dout(20) << __func__ << " pool is " << pool << " shard is " << shard
             << " pgid " << pgid << dendl;
   }
-
   ghobject_t sep;
   sep.hobj.pool = -1;
   sep.set_shard(shard);
@@ -4728,7 +4677,7 @@ int FileStore::collection_list_partial(coll_t c, ghobject_t start,
     if (start < sep) {
       dout(10) << __func__ << " first checking temp pool" << dendl;
       coll_t temp = c.get_temp();
-      int r = collection_list_partial(temp, start, min, max, seq, ls, next);
+      int r = collection_list_impl(temp, start, end, max, seq, ls, next);
       if (r < 0)
        return r;
       if (*next != ghobject_t::get_max())
@@ -4749,26 +4698,46 @@ int FileStore::collection_list_partial(coll_t c, ghobject_t start,
   assert(NULL != index.index);
   RWLock::RLocker l((index.index)->access_lock);
 
-  r = index->collection_list_partial(start,
-                                    min, max, seq,
-                                    ls, next);
+  r = index->collection_list_partial(start, end, max, seq, ls, next);
+
   if (r < 0) {
     assert(!m_filestore_fail_eio || r != -EIO);
     return r;
   }
-  if (ls)
-    dout(20) << "objects: " << *ls << dendl;
+  dout(20) << "objects: " << ls << dendl;
 
   // HashIndex doesn't know the pool when constructing a 'next' value
-  if (!next->is_max()) {
+  if (next && !next->is_max()) {
     next->hobj.pool = pool;
     next->set_shard(shard);
+    dout(20) << "  next " << *next << dendl;
   }
-  dout(20) << "  next " << *next << dendl;
 
-  tracepoint(objectstore, collection_list_partial_exit, 0);
   return 0;
 }
+int FileStore::collection_list_range(coll_t c, ghobject_t start, ghobject_t end,
+                                     snapid_t seq, vector<ghobject_t> *ls)
+{
+  tracepoint(objectstore, collection_list_range_enter, c.c_str());
+
+  ghobject_t next;
+  int r = collection_list_impl(c, start, end, -1, seq, ls, &next);
+  tracepoint(objectstore, collection_list_range_exit, 0);
+  return r;
+}
+
+int FileStore::collection_list_partial(coll_t c, ghobject_t start,
+                                      int min, int max, snapid_t seq,
+                                      vector<ghobject_t> *ls, ghobject_t *next)
+{
+  tracepoint(objectstore, collection_list_partial_enter, c.c_str());
+  dout(10) << "collection_list_partial: " << c << " start " << start << dendl;
+
+  assert(next);
+  int r = collection_list_impl(c, start, ghobject_t::get_max(), max, seq, ls, next);
+  tracepoint(objectstore, collection_list_partial_exit, 0);
+  return r;
+}
 
 int FileStore::collection_list(coll_t c, vector<ghobject_t>& ls)
 {  

diff --git a/src/os/FileStore.h b/src/os/FileStore.h
index f712b7647d2a2b92dbb5d8b466ac6e9517ec0bd3..63c801d1ccaf33d42c7d632f5017fa1346291f2a 100644 (file)
--- a/src/os/FileStore.h
+++ b/src/os/FileStore.h
@@ -617,6 +617,8 @@ public:
                                   const SequencerPosition &spos);
 
   // collections
+  int collection_list_impl(coll_t c, ghobject_t start, ghobject_t end, int max,
+                                     snapid_t seq, vector<ghobject_t> *ls, ghobject_t *next);
   int list_collections(vector<coll_t>& ls);
   int list_collections(vector<coll_t>& ls, bool include_temp);
   int collection_version_current(coll_t c, uint32_t *version);

diff --git a/src/os/FlatIndex.cc b/src/os/FlatIndex.cc
index 34c42c145b1902b34e2ea66f03fc3fa0e5f80da3..c6045228be86898fc1658b9b0a6e998fdba73272 100644 (file)
--- a/src/os/FlatIndex.cc
+++ b/src/os/FlatIndex.cc
@@ -373,7 +373,7 @@ static int get_hobject_from_oinfo(const char *dir, const char *file,
 }
 
 int FlatIndex::collection_list_partial(const ghobject_t &start,
-                                      int min_count,
+                                      const ghobject_t end,
                                       int max_count,
                                       snapid_t seq,
                                       vector<ghobject_t> *ls,

diff --git a/src/os/FlatIndex.h b/src/os/FlatIndex.h
index 0509df46470f684c3d80495dfd1ec2811198524d..087799e6d937f28c1e99c6827c9bad834d6499c3 100644 (file)
--- a/src/os/FlatIndex.h
+++ b/src/os/FlatIndex.h
@@ -74,7 +74,7 @@ public:
   /// @see CollectionIndex
   int collection_list_partial(
     const ghobject_t &start,
-    int min_count,
+    const ghobject_t end,
     int max_count,
     snapid_t seq,
     vector<ghobject_t> *ls,

diff --git a/src/os/HashIndex.cc b/src/os/HashIndex.cc
index 96a0a5af35d98cb178dd55a47e8ff39cc5d62584..1e0e1e41fbdae890a764966c3e850c98d854dcfa 100644 (file)
--- a/src/os/HashIndex.cc
+++ b/src/os/HashIndex.cc
@@ -322,11 +322,11 @@ int HashIndex::_lookup(const ghobject_t &oid,
 
 int HashIndex::_collection_list(vector<ghobject_t> *ls) {
   vector<string> path;
-  return list_by_hash(path, 0, 0, 0, 0, ls);
+  return list_by_hash(path, ghobject_t::get_max(), 0, 0, 0, ls);
 }
 
 int HashIndex::_collection_list_partial(const ghobject_t &start,
-                                       int min_count,
+                                       const ghobject_t end,
                                        int max_count,
                                        snapid_t seq,
                                        vector<ghobject_t> *ls,
@@ -336,8 +336,8 @@ int HashIndex::_collection_list_partial(const ghobject_t &start,
   if (!next)
     next = &_next;
   *next = start;
-  dout(20) << "_collection_list_partial " << start << " " << min_count << "-" << max_count << " ls.size " << ls->size() << dendl;
-  return list_by_hash(path, min_count, max_count, seq, next, ls);
+  dout(20) << "_collection_list_partial start:" << start << " end:" << end << "-" << max_count << " ls.size " << ls->size() << dendl;
+  return list_by_hash(path, end, max_count, seq, next, ls);
 }
 
 int HashIndex::prep_delete() {
@@ -816,7 +816,7 @@ int HashIndex::get_path_contents_by_hash(const vector<string> &path,
 }
 
 int HashIndex::list_by_hash(const vector<string> &path,
-                           int min_count,
+                           ghobject_t end,
                            int max_count,
                            snapid_t seq,
                            ghobject_t *next,
@@ -841,17 +841,12 @@ int HashIndex::list_by_hash(const vector<string> &path,
     set<pair<string, ghobject_t> >::iterator j = objects.lower_bound(
       make_pair(*i, ghobject_t()));
     if (j == objects.end() || j->first != *i) {
-      if (min_count > 0 && out->size() > (unsigned)min_count) {
-       if (next)
-         *next = ghobject_t(hobject_t("", "", CEPH_NOSNAP, hash_prefix_to_hash(*i), -1, ""));
-       return 0;
-      }
       *(next_path.rbegin()) = *(i->rbegin());
       ghobject_t next_recurse;
       if (next)
        next_recurse = *next;
       r = list_by_hash(next_path,
-                      min_count,
+                      end,
                       max_count,
                       seq,
                       &next_recurse,
@@ -871,6 +866,11 @@ int HashIndex::list_by_hash(const vector<string> &path,
            *next = j->second;
          return 0;
        }
+       if (j->second >= end) {
+         if (next)
+           *next = ghobject_t::get_max();
+         return 0;
+       }
        if (!next || j->second >= *next) {
          out->push_back(j->second);
        }

diff --git a/src/os/HashIndex.h b/src/os/HashIndex.h
index dad8ce31b8700e8a0156517e576ae5df88151dc7..afe2364748b188fb4e3346f07c739612a1f48450 100644 (file)
--- a/src/os/HashIndex.h
+++ b/src/os/HashIndex.h
@@ -193,7 +193,7 @@ protected:
 
   int _collection_list_partial(
     const ghobject_t &start,
-    int min_count,
+    ghobject_t end,
     int max_count,
     snapid_t seq,
     vector<ghobject_t> *ls,
@@ -366,7 +366,7 @@ private:
   /// List objects in collection in ghobject_t order
   int list_by_hash(
     const vector<string> &path, /// [in] Path to list
-    int min_count,              /// [in] List at least min_count
+    ghobject_t end,             /// [in] List only objects < end
     int max_count,              /// [in] List at most max_count
     snapid_t seq,               /// [in] list only objects where snap >= seq
     ghobject_t *next,            /// [in,out] List objects >= *next

diff --git a/src/os/LFNIndex.cc b/src/os/LFNIndex.cc
index 5b34f9eebd26412a214f244beee9e7e53a305f27..7687a66af3274c12f49423d9719d37f919a94051 100644 (file)
--- a/src/os/LFNIndex.cc
+++ b/src/os/LFNIndex.cc
@@ -155,13 +155,13 @@ int LFNIndex::pre_hash_collection(uint32_t pg_num, uint64_t expected_num_objs)
 
 
 int LFNIndex::collection_list_partial(const ghobject_t &start,
-                                     int min_count,
+                                     const ghobject_t end,
                                      int max_count,
                                      snapid_t seq,
                                      vector<ghobject_t> *ls,
                                      ghobject_t *next)
 {
-  return _collection_list_partial(start, min_count, max_count, seq, ls, next);
+  return _collection_list_partial(start, end, max_count, seq, ls, next);
 }
 
 /* Derived class utility methods */

diff --git a/src/os/LFNIndex.h b/src/os/LFNIndex.h
index 5cd35238165cf70ff33fbc78803b1fb7e94e2879..000e0ae34234713949be0d897a73511c02bd7ea2 100644 (file)
--- a/src/os/LFNIndex.h
+++ b/src/os/LFNIndex.h
@@ -192,7 +192,7 @@ public:
   /// @see CollectionIndex
   int collection_list_partial(
     const ghobject_t &start,
-    int min_count,
+    const ghobject_t end,
     int max_count,
     snapid_t seq,
     vector<ghobject_t> *ls,
@@ -268,7 +268,7 @@ protected:
   /// @see CollectionIndex
   virtual int _collection_list_partial(
     const ghobject_t &start,
-    int min_count,
+    const ghobject_t end,
     int max_count,
     snapid_t seq,
     vector<ghobject_t> *ls,