Add tests for the collection hint OP: 1) Store Test 2) Idempotent Test.

Signed-off-by: Guang Yang (yguang@yahoo-inc.com)

diff --git a/src/test/objectstore/DeterministicOpSequence.cc b/src/test/objectstore/DeterministicOpSequence.cc
index 09c9f5e7ff1122c4c7cb523b2ab32ea207a33211..2d12d34a5b2a9bbefed4752e81b005558cc67f96 100644 (file)
--- a/src/test/objectstore/DeterministicOpSequence.cc
+++ b/src/test/objectstore/DeterministicOpSequence.cc
@@ -26,7 +26,6 @@
 #include <boost/lexical_cast.hpp>
 
 #include "DeterministicOpSequence.h"
-
 #include "common/config.h"
 #include "include/assert.h"
 
@@ -80,6 +79,9 @@ bool DeterministicOpSequence::run_one_op(int op, rngen_t& gen)
   case DSOP_SET_ATTRS:
     ok = do_set_attrs(gen);
     break;
+  case DSOP_COLL_CREATE:
+    ok = do_coll_create(gen);
+    break;
   default:
     assert(0 == "bad op");
   }
@@ -438,6 +440,47 @@ bool DeterministicOpSequence::do_coll_add(rngen_t& gen)
   return true;
 }
 
+bool DeterministicOpSequence::do_coll_create(rngen_t& gen)
+{
+  boost::uniform_int<> pg_num_range(0, 512);
+  int pg_num = pg_num_range(gen);
+
+  // Assume there is 7 OSDs in total, the PGs are evenly distributed across those OSDs
+  int pgs = pg_num / 7;
+
+  boost::uniform_int<> num_objs_range(1, 1024);
+  int num_objs = num_objs_range(gen);
+
+  int pool_id = get_next_pool_id();
+  std::set<int> pg_created;
+  for (int i = 0; i < pgs; i++) {
+    boost::uniform_int<> pg_range(0, pg_num - 1);
+    int pg_id = pg_range(gen);
+    if (pg_created.count(pg_id) > 0)
+      continue;
+    char buf[100];
+    snprintf(buf, 100, "%d.%x_head", pool_id, pg_id);
+    _do_coll_create(coll_t(buf), (uint32_t) pg_num, (uint64_t) num_objs);
+    pg_created.insert(pg_id);
+  }
+  return true;
+}
+
+void DeterministicOpSequence::_do_coll_create(coll_t cid, uint32_t pg_num, uint64_t num_objs)
+{
+  ObjectStore::Transaction t;
+  note_txn(&t);
+  t.create_collection(cid);
+  bufferlist hint;
+  ::encode(pg_num, hint);
+  ::encode(num_objs, hint);
+  t.collection_hint(cid, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS, hint);
+  dout(0) << "Give collection: " << cid << " a hint, pg_num is: " << pg_num << ", num_objs is: "
+    << num_objs << dendl;
+
+  m_store->apply_transaction(t);
+}
+
 void DeterministicOpSequence::_do_touch(coll_t coll, hobject_t& obj)
 {
   ObjectStore::Transaction t;

diff --git a/src/test/objectstore/DeterministicOpSequence.h b/src/test/objectstore/DeterministicOpSequence.h
index 1980c98c3c59adff8d7cf59d6caffb2978db0803..6d4d9ba7bbedb078dfffca869a635e721fa405e2 100644 (file)
--- a/src/test/objectstore/DeterministicOpSequence.h
+++ b/src/test/objectstore/DeterministicOpSequence.h
@@ -42,9 +42,10 @@ class DeterministicOpSequence : public TestObjectStoreState {
     DSOP_COLL_RENAME = 5,
     DSOP_COLL_ADD = 6,
     DSOP_SET_ATTRS = 7,
+    DSOP_COLL_CREATE = 8,
 
     DSOP_FIRST = DSOP_TOUCH,
-    DSOP_LAST = DSOP_SET_ATTRS,
+    DSOP_LAST = DSOP_COLL_CREATE,
   };
 
   int32_t txn;
@@ -66,6 +67,7 @@ class DeterministicOpSequence : public TestObjectStoreState {
   bool do_coll_rename(rngen_t& gen);
   bool do_coll_add(rngen_t& gen);
   bool do_set_attrs(rngen_t& gen);
+  bool do_coll_create(rngen_t& gen);
 
   virtual void _do_touch(coll_t coll, hobject_t& obj);
   virtual void _do_remove(coll_t coll, hobject_t& obj);
@@ -82,6 +84,7 @@ class DeterministicOpSequence : public TestObjectStoreState {
       uint64_t dstoff, bufferlist& bl);
   virtual void _do_coll_add(coll_t orig_coll, coll_t new_coll, hobject_t& obj);
   virtual void _do_coll_rename(coll_t orig_coll, coll_t new_coll);
+  virtual void _do_coll_create(coll_t cid, uint32_t pg_num, uint64_t num_objs);
 
   int _gen_coll_id(rngen_t& gen);
   int _gen_obj_id(rngen_t& gen);

diff --git a/src/test/objectstore/TestObjectStoreState.cc b/src/test/objectstore/TestObjectStoreState.cc
index 37f99c7d310db57f6170531c77d8edce2c9bb2de..e21e8d9e18e899b6288fc7a41084e3982e325ec5 100644 (file)
--- a/src/test/objectstore/TestObjectStoreState.cc
+++ b/src/test/objectstore/TestObjectStoreState.cc
@@ -54,6 +54,13 @@ void TestObjectStoreState::init(int colls, int objs)
 
     t = new ObjectStore::Transaction;
     t->create_collection(entry->m_coll);
+    bufferlist hint;
+    uint32_t pg_num = colls;
+    uint64_t num_objs = uint64_t(objs / colls);
+    ::encode(pg_num, hint);
+    ::encode(num_objs, hint);
+    t->collection_hint(entry->m_coll, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS, hint);
+    dout(5) << "give collection hint, number of objects per collection: " << num_objs << dendl;
     t->touch(META_COLL, entry->m_meta_obj);
 
     for (int i = 0; i < objs; i++) {

diff --git a/src/test/objectstore/TestObjectStoreState.h b/src/test/objectstore/TestObjectStoreState.h
index dad2aab73791c633e770b7ee0f29e62b083b315e..fffa2432408d4801de5889f3fdc0e764ddd2204e 100644 (file)
--- a/src/test/objectstore/TestObjectStoreState.h
+++ b/src/test/objectstore/TestObjectStoreState.h
@@ -91,14 +91,17 @@ public:
 
   coll_entry_t *get_coll(int key, bool erase = false);
   coll_entry_t *get_coll_at(int pos, bool erase = false);
+  int get_next_pool_id() { return m_next_pool++; }
 
  private:
   static const int m_default_num_colls = 30;
+  // The pool ID used for collection creation, ID 0 is preserve for other tests
+  int m_next_pool;
 
  public:
   TestObjectStoreState(ObjectStore *store) :
     m_next_coll_nr(0), m_num_objs_per_coll(10), m_num_objects(0),
-    m_max_in_flight(0), m_finished_lock("Finished Lock") {
+    m_max_in_flight(0), m_finished_lock("Finished Lock"), m_next_pool(1) {
     m_in_flight.set(0);
     m_store.reset(store);
   }

diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc
index 58019e318bb278067124ed850bda6ea7f9b53134..b11641de9edd67d3be6676d764e9ae8c1a93fc88 100644 (file)
--- a/src/test/objectstore/store_test.cc
+++ b/src/test/objectstore/store_test.cc
@@ -117,6 +117,60 @@ TEST_P(StoreTest, SimpleColTest) {
   }
 }
 
+TEST_P(StoreTest, SimpleColPreHashTest) {
+  // Firstly we will need to revert the value making sure
+  // collection hint actually works
+  int merge_threshold = g_ceph_context->_conf->filestore_merge_threshold;
+  std::ostringstream oss;
+  if (merge_threshold > 0) {
+    oss << "-" << merge_threshold;
+    g_ceph_context->_conf->set_val("filestore_merge_threshold", oss.str().c_str());
+  }
+
+  uint32_t pg_num = 128;
+
+  boost::uniform_int<> pg_id_range(0, pg_num);
+  gen_type rng(time(NULL));
+  int pg_id = pg_id_range(rng);
+
+  int objs_per_folder = abs(merge_threshold) * 16 * g_ceph_context->_conf->filestore_split_multiple;
+  boost::uniform_int<> folders_range(5, 256);
+  uint64_t expected_num_objs = (uint64_t)(objs_per_folder * folders_range(rng));
+
+  char buf[100];
+  snprintf(buf, 100, "1.%x_head", pg_id);
+
+  coll_t cid(buf);
+  int r;
+  {
+    // Create a collection along with a hint
+    ObjectStore::Transaction t;
+    t.create_collection(cid);
+    cerr << "create collection" << std::endl;
+    bufferlist hint;
+    ::encode(pg_num, hint);
+    ::encode(expected_num_objs, hint);
+    t.collection_hint(cid, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS, hint);
+    cerr << "collection hint" << std::endl;
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  {
+    // Remove the collection
+    ObjectStore::Transaction t;
+    t.remove_collection(cid);
+    cerr << "remove collection" << std::endl;
+    r = store->apply_transaction(t);
+    ASSERT_EQ(r, 0);
+  }
+  // Revert the config change so that it does not affect the split/merge tests
+  if (merge_threshold > 0) {
+    oss.str("");
+    oss << merge_threshold;
+    g_ceph_context->_conf->set_val("filestore_merge_threshold", oss.str().c_str());
+  }
+}
+
 TEST_P(StoreTest, SimpleObjectTest) {
   int r;
   coll_t cid = coll_t("coll");

diff --git a/src/test/os/TestLFNIndex.cc b/src/test/os/TestLFNIndex.cc
index 3e9f970ceb66dbd262bccee415805fe5c93b201c..3bc5ba01666f67b203dc4a6214f29557967d2f2e 100644 (file)
--- a/src/test/os/TestLFNIndex.cc
+++ b/src/test/os/TestLFNIndex.cc
@@ -87,6 +87,11 @@ protected:
                                       vector<ghobject_t> *ls,
                                       ghobject_t *next
                                       ) { return 0; }
+  virtual int _pre_hash_collection(
+                                   uint32_t pg_num,
+                                   uint64_t expected_num_objs
+                                  ) { return 0; }
+
 };
 
 class TestHASH_INDEX_TAG : public TestWrapLFNIndex, public ::testing::Test {