From: Sage Weil Date: Sun, 4 Nov 2012 11:52:57 +0000 (-0800) Subject: cls_rgw: reorg test_cls_rgw X-Git-Tag: v0.55~130^2~15^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=0eedc78e5693a53bba1cc6106b62f962c0fe285a;p=ceph.git cls_rgw: reorg test_cls_rgw Signed-off-by: Sage Weil --- diff --git a/src/Makefile.am b/src/Makefile.am index 904822cd4b7..a275ed2683d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -841,7 +841,7 @@ bin_DEBUGPROGRAMS += test_cls_lock if WITH_RADOSGW -test_cls_rgw_SOURCES = test/rgw/test_cls_rgw.cc \ +test_cls_rgw_SOURCES = test/cls_rgw/test_cls_rgw.cc \ test/rados-api/test.cc test_cls_rgw_LDADD = librados.la libcls_rgw_client.a ${UNITTEST_STATIC_LDADD} test_cls_rgw_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} diff --git a/src/test/cls_rgw/test_cls_rgw.cc b/src/test/cls_rgw/test_cls_rgw.cc new file mode 100644 index 00000000000..6ec20aeb674 --- /dev/null +++ b/src/test/cls_rgw/test_cls_rgw.cc @@ -0,0 +1,536 @@ +// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/types.h" +#include "cls/rgw/cls_rgw_client.h" + +#include "gtest/gtest.h" +#include "test/rados-api/test.h" + +#include +#include +#include + +using namespace librados; + +librados::Rados rados; +librados::IoCtx ioctx; +string pool_name; + + +/* must be the first test! */ +TEST(cls_rgw, init) +{ + pool_name = get_temp_pool_name(); + /* create pool */ + ASSERT_EQ("", create_one_pool_pp(pool_name, rados)); + ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx)); +} + + +string str_int(string s, int i) +{ + char buf[32]; + snprintf(buf, sizeof(buf), "-%d", i); + s.append(buf); + + return s; +} + + +class OpMgr { + vector ops; + +public: + OpMgr() {} + ~OpMgr() { + vector::iterator iter; + for (iter = ops.begin(); iter != ops.end(); ++iter) { + ObjectOperation *op = *iter; + delete op; + } + } + + ObjectReadOperation *read_op() { + ObjectReadOperation *op = new ObjectReadOperation; + ops.push_back(op); + return op; + } + + ObjectWriteOperation *write_op() { + ObjectWriteOperation *op = new ObjectWriteOperation; + ops.push_back(op); + return op; + } +}; + +void test_stats(librados::IoCtx& ioctx, string& oid, int category, uint64_t num_entries, uint64_t total_size) +{ + rgw_bucket_dir_header header; + ASSERT_EQ(0, cls_rgw_get_dir_header(ioctx, oid, &header)); + + rgw_bucket_category_stats& stats = header.stats[category]; + ASSERT_EQ(total_size, stats.total_size); + ASSERT_EQ(num_entries, stats.num_entries); +} + +void index_prepare(OpMgr& mgr, librados::IoCtx& ioctx, string& oid, int index_op, string& tag, string& obj, string& loc) +{ + ObjectWriteOperation *op = mgr.write_op(); + cls_rgw_bucket_prepare_op(*op, index_op, tag, obj, loc); + ASSERT_EQ(0, ioctx.operate(oid, op)); +} + +void index_complete(OpMgr& mgr, librados::IoCtx& ioctx, string& oid, int index_op, string& tag, int epoch, string& obj, rgw_bucket_dir_entry_meta& meta) +{ + ObjectWriteOperation *op = mgr.write_op(); + cls_rgw_bucket_complete_op(*op, index_op, tag, epoch, obj, meta); + ASSERT_EQ(0, ioctx.operate(oid, op)); +} + +TEST(cls_rgw, index_basic) +{ + string bucket_oid = str_int("bucket", 0); + + OpMgr mgr; + + ObjectWriteOperation *op = mgr.write_op(); + cls_rgw_bucket_init(*op); + ASSERT_EQ(0, ioctx.operate(bucket_oid, op)); + + uint64_t epoch = 1; + + uint64_t obj_size = 1024; + +#define NUM_OBJS 10 + for (int i = 0; i < NUM_OBJS; i++) { + string obj = str_int("obj", i); + string tag = str_int("tag", i); + string loc = str_int("loc", i); + + index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc); + + test_stats(ioctx, bucket_oid, 0, i, obj_size * i); + + op = mgr.write_op(); + rgw_bucket_dir_entry_meta meta; + meta.category = 0; + meta.size = obj_size; + index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta); + } + + test_stats(ioctx, bucket_oid, 0, NUM_OBJS, obj_size * NUM_OBJS); +} + +TEST(cls_rgw, index_multiple_obj_writers) +{ + string bucket_oid = str_int("bucket", 1); + + OpMgr mgr; + + ObjectWriteOperation *op = mgr.write_op(); + cls_rgw_bucket_init(*op); + ASSERT_EQ(0, ioctx.operate(bucket_oid, op)); + + uint64_t obj_size = 1024; + + string obj = str_int("obj", 0); + string loc = str_int("loc", 0); + /* multi prepare on a single object */ + for (int i = 0; i < NUM_OBJS; i++) { + string tag = str_int("tag", i); + + index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc); + + test_stats(ioctx, bucket_oid, 0, 0, 0); + } + + for (int i = NUM_OBJS; i > 0; i--) { + string tag = str_int("tag", i - 1); + + rgw_bucket_dir_entry_meta meta; + meta.category = 0; + meta.size = obj_size * i; + + index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, i, obj, meta); + + /* verify that object size doesn't change, as we went back with epoch */ + test_stats(ioctx, bucket_oid, 0, 1, obj_size * NUM_OBJS); + } +} + +TEST(cls_rgw, index_remove_object) +{ + string bucket_oid = str_int("bucket", 2); + + OpMgr mgr; + + ObjectWriteOperation *op = mgr.write_op(); + cls_rgw_bucket_init(*op); + ASSERT_EQ(0, ioctx.operate(bucket_oid, op)); + + uint64_t obj_size = 1024; + uint64_t total_size = 0; + + int epoch = 0; + + /* prepare multiple objects */ + for (int i = 0; i < NUM_OBJS; i++) { + string obj = str_int("obj", i); + string tag = str_int("tag", i); + string loc = str_int("loc", i); + + index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc); + + test_stats(ioctx, bucket_oid, 0, i, total_size); + + rgw_bucket_dir_entry_meta meta; + meta.category = 0; + meta.size = i * obj_size; + total_size += i * obj_size; + + index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, ++epoch, obj, meta); + + test_stats(ioctx, bucket_oid, 0, i + 1, total_size); + } + + int i = NUM_OBJS / 2; + string tag_remove = "tag-rm"; + string tag_modify = "tag-mod"; + string obj = str_int("obj", i); + string loc = str_int("loc", i); + + /* prepare both removal and modification on the same object */ + index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, obj, loc); + index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, obj, loc); + + test_stats(ioctx, bucket_oid, 0, NUM_OBJS, total_size); + + rgw_bucket_dir_entry_meta meta; + + /* complete object removal */ + index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, ++epoch, obj, meta); + + /* verify stats correct */ + total_size -= i * obj_size; + test_stats(ioctx, bucket_oid, 0, NUM_OBJS - 1, total_size); + + meta.size = 512; + meta.category = 0; + + /* complete object modification */ + index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, ++epoch, obj, meta); + + /* verify stats correct */ + total_size += meta.size; + test_stats(ioctx, bucket_oid, 0, NUM_OBJS, total_size); + + + /* prepare both removal and modification on the same object, this time we'll + * first complete modification then remove*/ + index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, obj, loc); + index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_modify, obj, loc); + + /* complete modification */ + total_size -= meta.size; + meta.size = i * obj_size * 2; + meta.category = 0; + + /* complete object modification */ + index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, ++epoch, obj, meta); + + /* verify stats correct */ + total_size += meta.size; + test_stats(ioctx, bucket_oid, 0, NUM_OBJS, total_size); + + /* complete object removal */ + index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, ++epoch, obj, meta); + + /* verify stats correct */ + total_size -= meta.size; + test_stats(ioctx, bucket_oid, 0, NUM_OBJS - 1, total_size); +} + +TEST(cls_rgw, index_suggest) +{ + string bucket_oid = str_int("bucket", 3); + + OpMgr mgr; + + ObjectWriteOperation *op = mgr.write_op(); + cls_rgw_bucket_init(*op); + ASSERT_EQ(0, ioctx.operate(bucket_oid, op)); + + uint64_t total_size = 0; + + int epoch = 0; + + int num_objs = 100; + + uint64_t obj_size = 1024; + + /* create multiple objects */ + for (int i = 0; i < num_objs; i++) { + string obj = str_int("obj", i); + string tag = str_int("tag", i); + string loc = str_int("loc", i); + + index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc); + + test_stats(ioctx, bucket_oid, 0, i, total_size); + + rgw_bucket_dir_entry_meta meta; + meta.category = 0; + meta.size = obj_size; + total_size += meta.size; + + index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, ++epoch, obj, meta); + + test_stats(ioctx, bucket_oid, 0, i + 1, total_size); + } + + /* prepare (without completion) some of the objects */ + for (int i = 0; i < num_objs; i += 2) { + string obj = str_int("obj", i); + string tag = str_int("tag-prepare", i); + string loc = str_int("loc", i); + + index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc); + + test_stats(ioctx, bucket_oid, 0, num_objs, total_size); + } + + int actual_num_objs = num_objs; + /* remove half of the objects */ + for (int i = num_objs / 2; i < num_objs; i++) { + string obj = str_int("obj", i); + string tag = str_int("tag-rm", i); + string loc = str_int("loc", i); + + index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc); + + test_stats(ioctx, bucket_oid, 0, actual_num_objs, total_size); + + rgw_bucket_dir_entry_meta meta; + index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag, ++epoch, obj, meta); + + total_size -= obj_size; + actual_num_objs--; + test_stats(ioctx, bucket_oid, 0, actual_num_objs, total_size); + } + + bufferlist updates; + + for (int i = 0; i < num_objs; i += 2) { + string obj = str_int("obj", i); + string tag = str_int("tag-rm", i); + string loc = str_int("loc", i); + + rgw_bucket_dir_entry dirent; + dirent.name = obj; + dirent.locator = loc; + dirent.exists = (i < num_objs / 2); // we removed half the objects + dirent.meta.size = 1024; + + char suggest_op = (i < num_objs / 2 ? CEPH_RGW_UPDATE : CEPH_RGW_REMOVE); + cls_rgw_encode_suggestion(suggest_op, dirent, updates); + } + + op = mgr.write_op(); + cls_rgw_bucket_set_tag_timeout(*op, 1); // short tag timeout + ASSERT_EQ(0, ioctx.operate(bucket_oid, op)); + + sleep(1); + + /* suggest changes! */ + op = mgr.write_op(); + cls_rgw_suggest_changes(*op, updates); + ASSERT_EQ(0, ioctx.operate(bucket_oid, op)); + + /* suggest changes twice! */ + op = mgr.write_op(); + cls_rgw_suggest_changes(*op, updates); + ASSERT_EQ(0, ioctx.operate(bucket_oid, op)); + + test_stats(ioctx, bucket_oid, 0, num_objs / 2, total_size); +} + +/* test garbage collection */ +static void create_obj(cls_rgw_obj& obj, int i, int j) +{ + char buf[32]; + snprintf(buf, sizeof(buf), "-%d.%d", i, j); + obj.pool = "pool"; + obj.pool.append(buf); + obj.oid = "oid"; + obj.oid.append(buf); + obj.key = "key"; + obj.key.append(buf); +} + +static bool cmp_objs(cls_rgw_obj& obj1, cls_rgw_obj& obj2) +{ + return (obj1.pool == obj2.pool) && + (obj1.oid == obj2.oid) && + (obj1.key == obj2.key); +} + + +TEST(cls_rgw, gc_set) +{ + /* add chains */ + string oid = "obj"; + for (int i = 0; i < 10; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "chain-%d", i); + string tag = buf; + librados::ObjectWriteOperation op; + cls_rgw_gc_obj_info info; + + cls_rgw_obj obj1, obj2; + create_obj(obj1, i, 1); + create_obj(obj2, i, 2); + info.chain.objs.push_back(obj1); + info.chain.objs.push_back(obj2); + + op.create(false); // create object + + info.tag = tag; + cls_rgw_gc_set_entry(op, 0, info); + + ASSERT_EQ(0, ioctx.operate(oid, &op)); + } + + bool truncated; + list entries; + string marker; + + /* list chains, verify truncated */ + ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, entries, &truncated)); + ASSERT_EQ(8, (int)entries.size()); + ASSERT_EQ(1, truncated); + + entries.clear(); + + /* list all chains, verify not truncated */ + ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 10, entries, &truncated)); + ASSERT_EQ(10, (int)entries.size()); + ASSERT_EQ(0, truncated); + + /* verify all chains are valid */ + list::iterator iter = entries.begin(); + for (int i = 0; i < 10; i++, ++iter) { + cls_rgw_gc_obj_info& entry = *iter; + + /* create expected chain name */ + char buf[32]; + snprintf(buf, sizeof(buf), "chain-%d", i); + string tag = buf; + + /* verify chain name as expected */ + ASSERT_EQ(entry.tag, tag); + + /* verify expected num of objects in chain */ + ASSERT_EQ(2, (int)entry.chain.objs.size()); + + list::iterator oiter = entry.chain.objs.begin(); + cls_rgw_obj obj1, obj2; + + /* create expected objects */ + create_obj(obj1, i, 1); + create_obj(obj2, i, 2); + + /* assign returned object names */ + cls_rgw_obj& ret_obj1 = *oiter++; + cls_rgw_obj& ret_obj2 = *oiter; + + /* verify objects are as expected */ + ASSERT_EQ(1, (int)cmp_objs(obj1, ret_obj1)); + ASSERT_EQ(1, (int)cmp_objs(obj2, ret_obj2)); + } +} + +TEST(cls_rgw, gc_defer) +{ + librados::IoCtx ioctx; + librados::Rados rados; + + string gc_pool_name = get_temp_pool_name(); + /* create pool */ + ASSERT_EQ("", create_one_pool_pp(gc_pool_name, rados)); + ASSERT_EQ(0, rados.ioctx_create(gc_pool_name.c_str(), ioctx)); + + string oid = "obj"; + string tag = "mychain"; + + librados::ObjectWriteOperation op; + cls_rgw_gc_obj_info info; + + op.create(false); + + info.tag = tag; + + /* create chain */ + cls_rgw_gc_set_entry(op, 0, info); + + ASSERT_EQ(0, ioctx.operate(oid, &op)); + + bool truncated; + list entries; + string marker; + + /* list chains, verify num entries as expected */ + ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated)); + ASSERT_EQ(1, (int)entries.size()); + ASSERT_EQ(0, truncated); + + librados::ObjectWriteOperation op2; + + /* defer chain */ + cls_rgw_gc_defer_entry(op2, 5, tag); + ASSERT_EQ(0, ioctx.operate(oid, &op2)); + + entries.clear(); + + /* verify list doesn't show deferred entry (this may fail if cluster is thrashing) */ + ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated)); + ASSERT_EQ(0, (int)entries.size()); + ASSERT_EQ(0, truncated); + + /* wait enough */ + sleep(5); + + /* verify list shows deferred entry */ + ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated)); + ASSERT_EQ(1, (int)entries.size()); + ASSERT_EQ(0, truncated); + + librados::ObjectWriteOperation op3; + list tags; + tags.push_back(tag); + + /* remove chain */ + cls_rgw_gc_remove(op3, tags); + ASSERT_EQ(0, ioctx.operate(oid, &op3)); + + entries.clear(); + + /* verify entry was removed */ + ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated)); + ASSERT_EQ(0, (int)entries.size()); + ASSERT_EQ(0, truncated); + + /* remove pool */ + ioctx.close(); + ASSERT_EQ(0, destroy_one_pool_pp(gc_pool_name, rados)); +} + + +/* must be last test! */ + +TEST(cls_rgw, finalize) +{ + /* remove pool */ + ioctx.close(); + ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados)); +} diff --git a/src/test/rgw/test_cls_rgw.cc b/src/test/rgw/test_cls_rgw.cc deleted file mode 100644 index 6ec20aeb674..00000000000 --- a/src/test/rgw/test_cls_rgw.cc +++ /dev/null @@ -1,536 +0,0 @@ -// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab - -#include "include/types.h" -#include "cls/rgw/cls_rgw_client.h" - -#include "gtest/gtest.h" -#include "test/rados-api/test.h" - -#include -#include -#include - -using namespace librados; - -librados::Rados rados; -librados::IoCtx ioctx; -string pool_name; - - -/* must be the first test! */ -TEST(cls_rgw, init) -{ - pool_name = get_temp_pool_name(); - /* create pool */ - ASSERT_EQ("", create_one_pool_pp(pool_name, rados)); - ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx)); -} - - -string str_int(string s, int i) -{ - char buf[32]; - snprintf(buf, sizeof(buf), "-%d", i); - s.append(buf); - - return s; -} - - -class OpMgr { - vector ops; - -public: - OpMgr() {} - ~OpMgr() { - vector::iterator iter; - for (iter = ops.begin(); iter != ops.end(); ++iter) { - ObjectOperation *op = *iter; - delete op; - } - } - - ObjectReadOperation *read_op() { - ObjectReadOperation *op = new ObjectReadOperation; - ops.push_back(op); - return op; - } - - ObjectWriteOperation *write_op() { - ObjectWriteOperation *op = new ObjectWriteOperation; - ops.push_back(op); - return op; - } -}; - -void test_stats(librados::IoCtx& ioctx, string& oid, int category, uint64_t num_entries, uint64_t total_size) -{ - rgw_bucket_dir_header header; - ASSERT_EQ(0, cls_rgw_get_dir_header(ioctx, oid, &header)); - - rgw_bucket_category_stats& stats = header.stats[category]; - ASSERT_EQ(total_size, stats.total_size); - ASSERT_EQ(num_entries, stats.num_entries); -} - -void index_prepare(OpMgr& mgr, librados::IoCtx& ioctx, string& oid, int index_op, string& tag, string& obj, string& loc) -{ - ObjectWriteOperation *op = mgr.write_op(); - cls_rgw_bucket_prepare_op(*op, index_op, tag, obj, loc); - ASSERT_EQ(0, ioctx.operate(oid, op)); -} - -void index_complete(OpMgr& mgr, librados::IoCtx& ioctx, string& oid, int index_op, string& tag, int epoch, string& obj, rgw_bucket_dir_entry_meta& meta) -{ - ObjectWriteOperation *op = mgr.write_op(); - cls_rgw_bucket_complete_op(*op, index_op, tag, epoch, obj, meta); - ASSERT_EQ(0, ioctx.operate(oid, op)); -} - -TEST(cls_rgw, index_basic) -{ - string bucket_oid = str_int("bucket", 0); - - OpMgr mgr; - - ObjectWriteOperation *op = mgr.write_op(); - cls_rgw_bucket_init(*op); - ASSERT_EQ(0, ioctx.operate(bucket_oid, op)); - - uint64_t epoch = 1; - - uint64_t obj_size = 1024; - -#define NUM_OBJS 10 - for (int i = 0; i < NUM_OBJS; i++) { - string obj = str_int("obj", i); - string tag = str_int("tag", i); - string loc = str_int("loc", i); - - index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc); - - test_stats(ioctx, bucket_oid, 0, i, obj_size * i); - - op = mgr.write_op(); - rgw_bucket_dir_entry_meta meta; - meta.category = 0; - meta.size = obj_size; - index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta); - } - - test_stats(ioctx, bucket_oid, 0, NUM_OBJS, obj_size * NUM_OBJS); -} - -TEST(cls_rgw, index_multiple_obj_writers) -{ - string bucket_oid = str_int("bucket", 1); - - OpMgr mgr; - - ObjectWriteOperation *op = mgr.write_op(); - cls_rgw_bucket_init(*op); - ASSERT_EQ(0, ioctx.operate(bucket_oid, op)); - - uint64_t obj_size = 1024; - - string obj = str_int("obj", 0); - string loc = str_int("loc", 0); - /* multi prepare on a single object */ - for (int i = 0; i < NUM_OBJS; i++) { - string tag = str_int("tag", i); - - index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc); - - test_stats(ioctx, bucket_oid, 0, 0, 0); - } - - for (int i = NUM_OBJS; i > 0; i--) { - string tag = str_int("tag", i - 1); - - rgw_bucket_dir_entry_meta meta; - meta.category = 0; - meta.size = obj_size * i; - - index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, i, obj, meta); - - /* verify that object size doesn't change, as we went back with epoch */ - test_stats(ioctx, bucket_oid, 0, 1, obj_size * NUM_OBJS); - } -} - -TEST(cls_rgw, index_remove_object) -{ - string bucket_oid = str_int("bucket", 2); - - OpMgr mgr; - - ObjectWriteOperation *op = mgr.write_op(); - cls_rgw_bucket_init(*op); - ASSERT_EQ(0, ioctx.operate(bucket_oid, op)); - - uint64_t obj_size = 1024; - uint64_t total_size = 0; - - int epoch = 0; - - /* prepare multiple objects */ - for (int i = 0; i < NUM_OBJS; i++) { - string obj = str_int("obj", i); - string tag = str_int("tag", i); - string loc = str_int("loc", i); - - index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc); - - test_stats(ioctx, bucket_oid, 0, i, total_size); - - rgw_bucket_dir_entry_meta meta; - meta.category = 0; - meta.size = i * obj_size; - total_size += i * obj_size; - - index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, ++epoch, obj, meta); - - test_stats(ioctx, bucket_oid, 0, i + 1, total_size); - } - - int i = NUM_OBJS / 2; - string tag_remove = "tag-rm"; - string tag_modify = "tag-mod"; - string obj = str_int("obj", i); - string loc = str_int("loc", i); - - /* prepare both removal and modification on the same object */ - index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, obj, loc); - index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, obj, loc); - - test_stats(ioctx, bucket_oid, 0, NUM_OBJS, total_size); - - rgw_bucket_dir_entry_meta meta; - - /* complete object removal */ - index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, ++epoch, obj, meta); - - /* verify stats correct */ - total_size -= i * obj_size; - test_stats(ioctx, bucket_oid, 0, NUM_OBJS - 1, total_size); - - meta.size = 512; - meta.category = 0; - - /* complete object modification */ - index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, ++epoch, obj, meta); - - /* verify stats correct */ - total_size += meta.size; - test_stats(ioctx, bucket_oid, 0, NUM_OBJS, total_size); - - - /* prepare both removal and modification on the same object, this time we'll - * first complete modification then remove*/ - index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, obj, loc); - index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_modify, obj, loc); - - /* complete modification */ - total_size -= meta.size; - meta.size = i * obj_size * 2; - meta.category = 0; - - /* complete object modification */ - index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, ++epoch, obj, meta); - - /* verify stats correct */ - total_size += meta.size; - test_stats(ioctx, bucket_oid, 0, NUM_OBJS, total_size); - - /* complete object removal */ - index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, ++epoch, obj, meta); - - /* verify stats correct */ - total_size -= meta.size; - test_stats(ioctx, bucket_oid, 0, NUM_OBJS - 1, total_size); -} - -TEST(cls_rgw, index_suggest) -{ - string bucket_oid = str_int("bucket", 3); - - OpMgr mgr; - - ObjectWriteOperation *op = mgr.write_op(); - cls_rgw_bucket_init(*op); - ASSERT_EQ(0, ioctx.operate(bucket_oid, op)); - - uint64_t total_size = 0; - - int epoch = 0; - - int num_objs = 100; - - uint64_t obj_size = 1024; - - /* create multiple objects */ - for (int i = 0; i < num_objs; i++) { - string obj = str_int("obj", i); - string tag = str_int("tag", i); - string loc = str_int("loc", i); - - index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc); - - test_stats(ioctx, bucket_oid, 0, i, total_size); - - rgw_bucket_dir_entry_meta meta; - meta.category = 0; - meta.size = obj_size; - total_size += meta.size; - - index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, ++epoch, obj, meta); - - test_stats(ioctx, bucket_oid, 0, i + 1, total_size); - } - - /* prepare (without completion) some of the objects */ - for (int i = 0; i < num_objs; i += 2) { - string obj = str_int("obj", i); - string tag = str_int("tag-prepare", i); - string loc = str_int("loc", i); - - index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc); - - test_stats(ioctx, bucket_oid, 0, num_objs, total_size); - } - - int actual_num_objs = num_objs; - /* remove half of the objects */ - for (int i = num_objs / 2; i < num_objs; i++) { - string obj = str_int("obj", i); - string tag = str_int("tag-rm", i); - string loc = str_int("loc", i); - - index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc); - - test_stats(ioctx, bucket_oid, 0, actual_num_objs, total_size); - - rgw_bucket_dir_entry_meta meta; - index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag, ++epoch, obj, meta); - - total_size -= obj_size; - actual_num_objs--; - test_stats(ioctx, bucket_oid, 0, actual_num_objs, total_size); - } - - bufferlist updates; - - for (int i = 0; i < num_objs; i += 2) { - string obj = str_int("obj", i); - string tag = str_int("tag-rm", i); - string loc = str_int("loc", i); - - rgw_bucket_dir_entry dirent; - dirent.name = obj; - dirent.locator = loc; - dirent.exists = (i < num_objs / 2); // we removed half the objects - dirent.meta.size = 1024; - - char suggest_op = (i < num_objs / 2 ? CEPH_RGW_UPDATE : CEPH_RGW_REMOVE); - cls_rgw_encode_suggestion(suggest_op, dirent, updates); - } - - op = mgr.write_op(); - cls_rgw_bucket_set_tag_timeout(*op, 1); // short tag timeout - ASSERT_EQ(0, ioctx.operate(bucket_oid, op)); - - sleep(1); - - /* suggest changes! */ - op = mgr.write_op(); - cls_rgw_suggest_changes(*op, updates); - ASSERT_EQ(0, ioctx.operate(bucket_oid, op)); - - /* suggest changes twice! */ - op = mgr.write_op(); - cls_rgw_suggest_changes(*op, updates); - ASSERT_EQ(0, ioctx.operate(bucket_oid, op)); - - test_stats(ioctx, bucket_oid, 0, num_objs / 2, total_size); -} - -/* test garbage collection */ -static void create_obj(cls_rgw_obj& obj, int i, int j) -{ - char buf[32]; - snprintf(buf, sizeof(buf), "-%d.%d", i, j); - obj.pool = "pool"; - obj.pool.append(buf); - obj.oid = "oid"; - obj.oid.append(buf); - obj.key = "key"; - obj.key.append(buf); -} - -static bool cmp_objs(cls_rgw_obj& obj1, cls_rgw_obj& obj2) -{ - return (obj1.pool == obj2.pool) && - (obj1.oid == obj2.oid) && - (obj1.key == obj2.key); -} - - -TEST(cls_rgw, gc_set) -{ - /* add chains */ - string oid = "obj"; - for (int i = 0; i < 10; i++) { - char buf[32]; - snprintf(buf, sizeof(buf), "chain-%d", i); - string tag = buf; - librados::ObjectWriteOperation op; - cls_rgw_gc_obj_info info; - - cls_rgw_obj obj1, obj2; - create_obj(obj1, i, 1); - create_obj(obj2, i, 2); - info.chain.objs.push_back(obj1); - info.chain.objs.push_back(obj2); - - op.create(false); // create object - - info.tag = tag; - cls_rgw_gc_set_entry(op, 0, info); - - ASSERT_EQ(0, ioctx.operate(oid, &op)); - } - - bool truncated; - list entries; - string marker; - - /* list chains, verify truncated */ - ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, entries, &truncated)); - ASSERT_EQ(8, (int)entries.size()); - ASSERT_EQ(1, truncated); - - entries.clear(); - - /* list all chains, verify not truncated */ - ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 10, entries, &truncated)); - ASSERT_EQ(10, (int)entries.size()); - ASSERT_EQ(0, truncated); - - /* verify all chains are valid */ - list::iterator iter = entries.begin(); - for (int i = 0; i < 10; i++, ++iter) { - cls_rgw_gc_obj_info& entry = *iter; - - /* create expected chain name */ - char buf[32]; - snprintf(buf, sizeof(buf), "chain-%d", i); - string tag = buf; - - /* verify chain name as expected */ - ASSERT_EQ(entry.tag, tag); - - /* verify expected num of objects in chain */ - ASSERT_EQ(2, (int)entry.chain.objs.size()); - - list::iterator oiter = entry.chain.objs.begin(); - cls_rgw_obj obj1, obj2; - - /* create expected objects */ - create_obj(obj1, i, 1); - create_obj(obj2, i, 2); - - /* assign returned object names */ - cls_rgw_obj& ret_obj1 = *oiter++; - cls_rgw_obj& ret_obj2 = *oiter; - - /* verify objects are as expected */ - ASSERT_EQ(1, (int)cmp_objs(obj1, ret_obj1)); - ASSERT_EQ(1, (int)cmp_objs(obj2, ret_obj2)); - } -} - -TEST(cls_rgw, gc_defer) -{ - librados::IoCtx ioctx; - librados::Rados rados; - - string gc_pool_name = get_temp_pool_name(); - /* create pool */ - ASSERT_EQ("", create_one_pool_pp(gc_pool_name, rados)); - ASSERT_EQ(0, rados.ioctx_create(gc_pool_name.c_str(), ioctx)); - - string oid = "obj"; - string tag = "mychain"; - - librados::ObjectWriteOperation op; - cls_rgw_gc_obj_info info; - - op.create(false); - - info.tag = tag; - - /* create chain */ - cls_rgw_gc_set_entry(op, 0, info); - - ASSERT_EQ(0, ioctx.operate(oid, &op)); - - bool truncated; - list entries; - string marker; - - /* list chains, verify num entries as expected */ - ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated)); - ASSERT_EQ(1, (int)entries.size()); - ASSERT_EQ(0, truncated); - - librados::ObjectWriteOperation op2; - - /* defer chain */ - cls_rgw_gc_defer_entry(op2, 5, tag); - ASSERT_EQ(0, ioctx.operate(oid, &op2)); - - entries.clear(); - - /* verify list doesn't show deferred entry (this may fail if cluster is thrashing) */ - ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated)); - ASSERT_EQ(0, (int)entries.size()); - ASSERT_EQ(0, truncated); - - /* wait enough */ - sleep(5); - - /* verify list shows deferred entry */ - ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated)); - ASSERT_EQ(1, (int)entries.size()); - ASSERT_EQ(0, truncated); - - librados::ObjectWriteOperation op3; - list tags; - tags.push_back(tag); - - /* remove chain */ - cls_rgw_gc_remove(op3, tags); - ASSERT_EQ(0, ioctx.operate(oid, &op3)); - - entries.clear(); - - /* verify entry was removed */ - ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated)); - ASSERT_EQ(0, (int)entries.size()); - ASSERT_EQ(0, truncated); - - /* remove pool */ - ioctx.close(); - ASSERT_EQ(0, destroy_one_pool_pp(gc_pool_name, rados)); -} - - -/* must be last test! */ - -TEST(cls_rgw, finalize) -{ - /* remove pool */ - ioctx.close(); - ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados)); -}