From: Guang Yang Date: Wed, 9 Jul 2014 07:45:58 +0000 (+0000) Subject: Add tests for the collection hint OP: 1) Store Test 2) Idempotent Test. X-Git-Tag: v0.86~240^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=dbf624e1a236a97d4794c58a19d5de50195d6cfe;p=ceph.git Add tests for the collection hint OP: 1) Store Test 2) Idempotent Test. Signed-off-by: Guang Yang (yguang@yahoo-inc.com) --- diff --git a/src/test/objectstore/DeterministicOpSequence.cc b/src/test/objectstore/DeterministicOpSequence.cc index 09c9f5e7ff11..2d12d34a5b2a 100644 --- a/src/test/objectstore/DeterministicOpSequence.cc +++ b/src/test/objectstore/DeterministicOpSequence.cc @@ -26,7 +26,6 @@ #include #include "DeterministicOpSequence.h" - #include "common/config.h" #include "include/assert.h" @@ -80,6 +79,9 @@ bool DeterministicOpSequence::run_one_op(int op, rngen_t& gen) case DSOP_SET_ATTRS: ok = do_set_attrs(gen); break; + case DSOP_COLL_CREATE: + ok = do_coll_create(gen); + break; default: assert(0 == "bad op"); } @@ -438,6 +440,47 @@ bool DeterministicOpSequence::do_coll_add(rngen_t& gen) return true; } +bool DeterministicOpSequence::do_coll_create(rngen_t& gen) +{ + boost::uniform_int<> pg_num_range(0, 512); + int pg_num = pg_num_range(gen); + + // Assume there is 7 OSDs in total, the PGs are evenly distributed across those OSDs + int pgs = pg_num / 7; + + boost::uniform_int<> num_objs_range(1, 1024); + int num_objs = num_objs_range(gen); + + int pool_id = get_next_pool_id(); + std::set pg_created; + for (int i = 0; i < pgs; i++) { + boost::uniform_int<> pg_range(0, pg_num - 1); + int pg_id = pg_range(gen); + if (pg_created.count(pg_id) > 0) + continue; + char buf[100]; + snprintf(buf, 100, "%d.%x_head", pool_id, pg_id); + _do_coll_create(coll_t(buf), (uint32_t) pg_num, (uint64_t) num_objs); + pg_created.insert(pg_id); + } + return true; +} + +void DeterministicOpSequence::_do_coll_create(coll_t cid, uint32_t pg_num, uint64_t num_objs) +{ + ObjectStore::Transaction t; + note_txn(&t); + t.create_collection(cid); + bufferlist hint; + ::encode(pg_num, hint); + ::encode(num_objs, hint); + t.collection_hint(cid, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS, hint); + dout(0) << "Give collection: " << cid << " a hint, pg_num is: " << pg_num << ", num_objs is: " + << num_objs << dendl; + + m_store->apply_transaction(t); +} + void DeterministicOpSequence::_do_touch(coll_t coll, hobject_t& obj) { ObjectStore::Transaction t; diff --git a/src/test/objectstore/DeterministicOpSequence.h b/src/test/objectstore/DeterministicOpSequence.h index 1980c98c3c59..6d4d9ba7bbed 100644 --- a/src/test/objectstore/DeterministicOpSequence.h +++ b/src/test/objectstore/DeterministicOpSequence.h @@ -42,9 +42,10 @@ class DeterministicOpSequence : public TestObjectStoreState { DSOP_COLL_RENAME = 5, DSOP_COLL_ADD = 6, DSOP_SET_ATTRS = 7, + DSOP_COLL_CREATE = 8, DSOP_FIRST = DSOP_TOUCH, - DSOP_LAST = DSOP_SET_ATTRS, + DSOP_LAST = DSOP_COLL_CREATE, }; int32_t txn; @@ -66,6 +67,7 @@ class DeterministicOpSequence : public TestObjectStoreState { bool do_coll_rename(rngen_t& gen); bool do_coll_add(rngen_t& gen); bool do_set_attrs(rngen_t& gen); + bool do_coll_create(rngen_t& gen); virtual void _do_touch(coll_t coll, hobject_t& obj); virtual void _do_remove(coll_t coll, hobject_t& obj); @@ -82,6 +84,7 @@ class DeterministicOpSequence : public TestObjectStoreState { uint64_t dstoff, bufferlist& bl); virtual void _do_coll_add(coll_t orig_coll, coll_t new_coll, hobject_t& obj); virtual void _do_coll_rename(coll_t orig_coll, coll_t new_coll); + virtual void _do_coll_create(coll_t cid, uint32_t pg_num, uint64_t num_objs); int _gen_coll_id(rngen_t& gen); int _gen_obj_id(rngen_t& gen); diff --git a/src/test/objectstore/TestObjectStoreState.cc b/src/test/objectstore/TestObjectStoreState.cc index 37f99c7d310d..e21e8d9e18e8 100644 --- a/src/test/objectstore/TestObjectStoreState.cc +++ b/src/test/objectstore/TestObjectStoreState.cc @@ -54,6 +54,13 @@ void TestObjectStoreState::init(int colls, int objs) t = new ObjectStore::Transaction; t->create_collection(entry->m_coll); + bufferlist hint; + uint32_t pg_num = colls; + uint64_t num_objs = uint64_t(objs / colls); + ::encode(pg_num, hint); + ::encode(num_objs, hint); + t->collection_hint(entry->m_coll, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS, hint); + dout(5) << "give collection hint, number of objects per collection: " << num_objs << dendl; t->touch(META_COLL, entry->m_meta_obj); for (int i = 0; i < objs; i++) { diff --git a/src/test/objectstore/TestObjectStoreState.h b/src/test/objectstore/TestObjectStoreState.h index dad2aab73791..fffa2432408d 100644 --- a/src/test/objectstore/TestObjectStoreState.h +++ b/src/test/objectstore/TestObjectStoreState.h @@ -91,14 +91,17 @@ public: coll_entry_t *get_coll(int key, bool erase = false); coll_entry_t *get_coll_at(int pos, bool erase = false); + int get_next_pool_id() { return m_next_pool++; } private: static const int m_default_num_colls = 30; + // The pool ID used for collection creation, ID 0 is preserve for other tests + int m_next_pool; public: TestObjectStoreState(ObjectStore *store) : m_next_coll_nr(0), m_num_objs_per_coll(10), m_num_objects(0), - m_max_in_flight(0), m_finished_lock("Finished Lock") { + m_max_in_flight(0), m_finished_lock("Finished Lock"), m_next_pool(1) { m_in_flight.set(0); m_store.reset(store); } diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index 58019e318bb2..b11641de9edd 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -117,6 +117,60 @@ TEST_P(StoreTest, SimpleColTest) { } } +TEST_P(StoreTest, SimpleColPreHashTest) { + // Firstly we will need to revert the value making sure + // collection hint actually works + int merge_threshold = g_ceph_context->_conf->filestore_merge_threshold; + std::ostringstream oss; + if (merge_threshold > 0) { + oss << "-" << merge_threshold; + g_ceph_context->_conf->set_val("filestore_merge_threshold", oss.str().c_str()); + } + + uint32_t pg_num = 128; + + boost::uniform_int<> pg_id_range(0, pg_num); + gen_type rng(time(NULL)); + int pg_id = pg_id_range(rng); + + int objs_per_folder = abs(merge_threshold) * 16 * g_ceph_context->_conf->filestore_split_multiple; + boost::uniform_int<> folders_range(5, 256); + uint64_t expected_num_objs = (uint64_t)(objs_per_folder * folders_range(rng)); + + char buf[100]; + snprintf(buf, 100, "1.%x_head", pg_id); + + coll_t cid(buf); + int r; + { + // Create a collection along with a hint + ObjectStore::Transaction t; + t.create_collection(cid); + cerr << "create collection" << std::endl; + bufferlist hint; + ::encode(pg_num, hint); + ::encode(expected_num_objs, hint); + t.collection_hint(cid, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS, hint); + cerr << "collection hint" << std::endl; + r = store->apply_transaction(t); + ASSERT_EQ(r, 0); + } + { + // Remove the collection + ObjectStore::Transaction t; + t.remove_collection(cid); + cerr << "remove collection" << std::endl; + r = store->apply_transaction(t); + ASSERT_EQ(r, 0); + } + // Revert the config change so that it does not affect the split/merge tests + if (merge_threshold > 0) { + oss.str(""); + oss << merge_threshold; + g_ceph_context->_conf->set_val("filestore_merge_threshold", oss.str().c_str()); + } +} + TEST_P(StoreTest, SimpleObjectTest) { int r; coll_t cid = coll_t("coll"); diff --git a/src/test/os/TestLFNIndex.cc b/src/test/os/TestLFNIndex.cc index 3e9f970ceb66..3bc5ba01666f 100644 --- a/src/test/os/TestLFNIndex.cc +++ b/src/test/os/TestLFNIndex.cc @@ -87,6 +87,11 @@ protected: vector *ls, ghobject_t *next ) { return 0; } + virtual int _pre_hash_collection( + uint32_t pg_num, + uint64_t expected_num_objs + ) { return 0; } + }; class TestHASH_INDEX_TAG : public TestWrapLFNIndex, public ::testing::Test {