From: Yehuda Sadeh Date: Mon, 31 Oct 2016 22:55:20 +0000 (-0700) Subject: test/rgw: extend obj, manifest unitests X-Git-Tag: v12.0.1~111^2~20 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=fc6fa2f40216e19554d3ab224304cee02c976896;p=ceph.git test/rgw: extend obj, manifest unitests Test rgw_raw_obj and upgrade of old rgw_obj, rgw_bucket and old manifest. Signed-off-by: Yehuda Sadeh --- diff --git a/src/test/rgw/CMakeLists.txt b/src/test/rgw/CMakeLists.txt index c8f47ff016d..e5af903005e 100644 --- a/src/test/rgw/CMakeLists.txt +++ b/src/test/rgw/CMakeLists.txt @@ -20,6 +20,10 @@ add_executable(unittest_http_manager test_http_manager.cc) add_ceph_unittest(unittest_http_manager ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/unittest_http_manager) target_link_libraries(unittest_http_manager rgw_a) +set(test_rgw_a_src + test_rgw_common.cc) +add_library(test_rgw_a STATIC ${test_rgw_a_src}) + # ceph_test_rgw_manifest set(test_rgw_manifest_srcs test_rgw_manifest.cc) add_executable(ceph_test_rgw_manifest @@ -27,6 +31,7 @@ add_executable(ceph_test_rgw_manifest ) target_link_libraries(ceph_test_rgw_manifest rgw_a + test_rgw_a cls_rgw_client cls_lock_client cls_refcount_client @@ -55,6 +60,7 @@ add_executable(ceph_test_rgw_obj ) target_link_libraries(ceph_test_rgw_obj rgw_a + test_rgw_a cls_rgw_client cls_lock_client cls_refcount_client diff --git a/src/test/rgw/test_rgw_common.cc b/src/test/rgw/test_rgw_common.cc new file mode 100644 index 00000000000..d4bba3584ba --- /dev/null +++ b/src/test/rgw/test_rgw_common.cc @@ -0,0 +1,89 @@ +#include "test_rgw_common.h" + +void test_rgw_add_placement(RGWZoneGroup *zonegroup, RGWZoneParams *zone_params, const std::string& name, bool is_default) +{ + zonegroup->placement_targets[name] = { name }; + + RGWZonePlacementInfo& pinfo = zone_params->placement_pools[name]; + pinfo.index_pool = rgw_pool(name + ".index").to_str(); + pinfo.data_pool = rgw_pool(name + ".data").to_str(); + pinfo.data_extra_pool = rgw_pool(name + ".extra").to_str(); + + if (is_default) { + zonegroup->default_placement = name; + } +} + +void test_rgw_init_env(RGWZoneGroup *zonegroup, RGWZoneParams *zone_params) +{ + test_rgw_add_placement(zonegroup, zone_params, "default-placement", true); + +} + +void test_rgw_populate_explicit_placement_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id) +{ + b->tenant = t; + b->name = n; + b->marker = m; + b->bucket_id = id; + b->explicit_placement.data_pool = rgw_pool(dp); + b->explicit_placement.index_pool = rgw_pool(ip); +} + +void test_rgw_populate_old_bucket(old_rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id) +{ + b->tenant = t; + b->name = n; + b->marker = m; + b->bucket_id = id; + b->data_pool = dp; + b->index_pool = ip; +} + +std::string test_rgw_get_obj_oid(const rgw_obj& obj) +{ + std::string oid; + std::string loc; + + get_obj_bucket_and_oid_loc(obj, oid, loc); + return oid; +} + +void test_rgw_init_explicit_placement_bucket(rgw_bucket *bucket, const char *name) +{ + test_rgw_populate_explicit_placement_bucket(bucket, "", name, ".data-pool", ".index-pool", "marker", "bucket-id"); +} + +void test_rgw_init_old_bucket(old_rgw_bucket *bucket, const char *name) +{ + test_rgw_populate_old_bucket(bucket, "", name, ".data-pool", ".index-pool", "marker", "bucket-id"); +} + +void test_rgw_populate_bucket(rgw_bucket *b, const char *t, const char *n, const char *m, const char *id) +{ + b->tenant = t; + b->name = n; + b->marker = m; + b->bucket_id = id; +} + +void test_rgw_init_bucket(rgw_bucket *bucket, const char *name) +{ + test_rgw_populate_bucket(bucket, "", name, "marker", "bucket-id"); +} + +rgw_obj test_rgw_create_obj(const rgw_bucket& bucket, const std::string& name, const std::string& instance, const std::string& ns) +{ + rgw_obj obj(bucket, name); + if (!instance.empty()) { + obj.key.set_instance(instance); + } + if (!ns.empty()) { + obj.key.ns = ns; + } + obj.bucket = bucket; + + return obj; +} + + diff --git a/src/test/rgw/test_rgw_common.h b/src/test/rgw/test_rgw_common.h new file mode 100644 index 00000000000..5530f8b9b36 --- /dev/null +++ b/src/test/rgw/test_rgw_common.h @@ -0,0 +1,505 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 eNovance SAS + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ +#include +#include "common/ceph_json.h" +#include "common/Formatter.h" +#include "rgw/rgw_common.h" +#include "rgw/rgw_rados.h" + +#ifndef CEPH_TEST_RGW_COMMON_H +#define CEPH_TEST_RGW_COMMON_H + +struct old_rgw_bucket { + std::string tenant; + std::string name; + std::string data_pool; + std::string data_extra_pool; /* if not set, then we should use data_pool instead */ + std::string index_pool; + std::string marker; + std::string bucket_id; + + std::string oid; /* + * runtime in-memory only info. If not empty, points to the bucket instance object + */ + + old_rgw_bucket() { } + // cppcheck-suppress noExplicitConstructor + old_rgw_bucket(const std::string& s) : name(s) { + data_pool = index_pool = s; + marker = ""; + } + old_rgw_bucket(const char *n) : name(n) { + data_pool = index_pool = n; + marker = ""; + } + old_rgw_bucket(const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id, const char *h) : + tenant(t), name(n), data_pool(dp), index_pool(ip), marker(m), bucket_id(id) {} + + void encode(bufferlist& bl) const { + ENCODE_START(8, 3, bl); + ::encode(name, bl); + ::encode(data_pool, bl); + ::encode(marker, bl); + ::encode(bucket_id, bl); + ::encode(index_pool, bl); + ::encode(data_extra_pool, bl); + ::encode(tenant, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START_LEGACY_COMPAT_LEN(8, 3, 3, bl); + ::decode(name, bl); + ::decode(data_pool, bl); + if (struct_v >= 2) { + ::decode(marker, bl); + if (struct_v <= 3) { + uint64_t id; + ::decode(id, bl); + char buf[16]; + snprintf(buf, sizeof(buf), "%llu", (long long)id); + bucket_id = buf; + } else { + ::decode(bucket_id, bl); + } + } + if (struct_v >= 5) { + ::decode(index_pool, bl); + } else { + index_pool = data_pool; + } + if (struct_v >= 7) { + ::decode(data_extra_pool, bl); + } + if (struct_v >= 8) { + ::decode(tenant, bl); + } + DECODE_FINISH(bl); + } + + // format a key for the bucket/instance. pass delim=0 to skip a field + std::string get_key(char tenant_delim = '/', + char id_delim = ':') const; + + const std::string& get_data_extra_pool() { + if (data_extra_pool.empty()) { + return data_pool; + } + return data_extra_pool; + } + + void dump(Formatter *f) const; + void decode_json(JSONObj *obj); + static void generate_test_instances(list& o); + + bool operator<(const old_rgw_bucket& b) const { + return name.compare(b.name) < 0; + } +}; +WRITE_CLASS_ENCODER(old_rgw_bucket) + +class old_rgw_obj { + std::string orig_obj; + std::string loc; + std::string object; + std::string instance; +public: + const std::string& get_object() const { return object; } + const std::string& get_orig_obj() const { return orig_obj; } + const std::string& get_loc() const { return loc; } + const std::string& get_instance() const { return instance; } + old_rgw_bucket bucket; + std::string ns; + + bool in_extra_data; /* in-memory only member, does not serialize */ + + // Represents the hash index source for this object once it is set (non-empty) + std::string index_hash_source; + + old_rgw_obj() : in_extra_data(false) {} + old_rgw_obj(old_rgw_bucket& b, const std::string& o) : in_extra_data(false) { + init(b, o); + } + old_rgw_obj(old_rgw_bucket& b, const rgw_obj_key& k) : in_extra_data(false) { + from_index_key(b, k); + } + void init(old_rgw_bucket& b, const std::string& o) { + bucket = b; + set_obj(o); + reset_loc(); + } + void init_ns(old_rgw_bucket& b, const std::string& o, const std::string& n) { + bucket = b; + set_ns(n); + set_obj(o); + reset_loc(); + } + int set_ns(const char *n) { + if (!n) + return -EINVAL; + std::string ns_str(n); + return set_ns(ns_str); + } + int set_ns(const std::string& n) { + if (n[0] == '_') + return -EINVAL; + ns = n; + set_obj(orig_obj); + return 0; + } + int set_instance(const std::string& i) { + if (i[0] == '_') + return -EINVAL; + instance = i; + set_obj(orig_obj); + return 0; + } + + int clear_instance() { + return set_instance(string()); + } + + void set_loc(const std::string& k) { + loc = k; + } + + void reset_loc() { + loc.clear(); + /* + * For backward compatibility. Older versions used to have object locator on all objects, + * however, the orig_obj was the effective object locator. This had the same effect as not + * having object locator at all for most objects but the ones that started with underscore as + * these were escaped. + */ + if (orig_obj[0] == '_' && ns.empty()) { + loc = orig_obj; + } + } + + bool have_null_instance() { + return instance == "null"; + } + + bool have_instance() { + return !instance.empty(); + } + + bool need_to_encode_instance() { + return have_instance() && !have_null_instance(); + } + + void set_obj(const std::string& o) { + object.reserve(128); + + orig_obj = o; + if (ns.empty() && !need_to_encode_instance()) { + if (o.empty()) { + return; + } + if (o.size() < 1 || o[0] != '_') { + object = o; + return; + } + object = "_"; + object.append(o); + } else { + object = "_"; + object.append(ns); + if (need_to_encode_instance()) { + object.append(string(":") + instance); + } + object.append("_"); + object.append(o); + } + reset_loc(); + } + + /* + * get the object's key name as being referred to by the bucket index. + */ + std::string get_index_key_name() const { + if (ns.empty()) { + if (orig_obj.size() < 1 || orig_obj[0] != '_') { + return orig_obj; + } + return std::string("_") + orig_obj; + }; + + char buf[ns.size() + 16]; + snprintf(buf, sizeof(buf), "_%s_", ns.c_str()); + return std::string(buf) + orig_obj; + }; + + void from_index_key(old_rgw_bucket& b, const rgw_obj_key& key) { + if (key.name[0] != '_') { + init(b, key.name); + set_instance(key.instance); + return; + } + if (key.name[1] == '_') { + init(b, key.name.substr(1)); + set_instance(key.instance); + return; + } + ssize_t pos = key.name.find('_', 1); + if (pos < 0) { + /* shouldn't happen, just use key */ + init(b, key.name); + set_instance(key.instance); + return; + } + + init_ns(b, key.name.substr(pos + 1), key.name.substr(1, pos -1)); + set_instance(key.instance); + } + + void get_index_key(rgw_obj_key *key) const { + key->name = get_index_key_name(); + key->instance = instance; + } + + static void parse_ns_field(string& ns, std::string& instance) { + int pos = ns.find(':'); + if (pos >= 0) { + instance = ns.substr(pos + 1); + ns = ns.substr(0, pos); + } else { + instance.clear(); + } + } + + std::string& get_hash_object() { + return index_hash_source.empty() ? orig_obj : index_hash_source; + } + /** + * Translate a namespace-mangled object name to the user-facing name + * existing in the given namespace. + * + * If the object is part of the given namespace, it returns true + * and cuts down the name to the unmangled version. If it is not + * part of the given namespace, it returns false. + */ + static bool translate_raw_obj_to_obj_in_ns(string& obj, std::string& instance, std::string& ns) { + if (obj[0] != '_') { + if (ns.empty()) { + return true; + } + return false; + } + + std::string obj_ns; + bool ret = parse_raw_oid(obj, &obj, &instance, &obj_ns); + if (!ret) { + return ret; + } + + return (ns == obj_ns); + } + + static bool parse_raw_oid(const std::string& oid, std::string *obj_name, std::string *obj_instance, std::string *obj_ns) { + obj_instance->clear(); + obj_ns->clear(); + if (oid[0] != '_') { + *obj_name = oid; + return true; + } + + if (oid.size() >= 2 && oid[1] == '_') { + *obj_name = oid.substr(1); + return true; + } + + if (oid[0] != '_' || oid.size() < 3) // for namespace, min size would be 3: _x_ + return false; + + int pos = oid.find('_', 1); + if (pos <= 1) // if it starts with __, it's not in our namespace + return false; + + *obj_ns = oid.substr(1, pos - 1); + parse_ns_field(*obj_ns, *obj_instance); + + *obj_name = oid.substr(pos + 1); + return true; + } + + /** + * Given a mangled object name and an empty namespace string, this + * function extracts the namespace into the string and sets the object + * name to be the unmangled version. + * + * It returns true after successfully doing so, or + * false if it fails. + */ + static bool strip_namespace_from_object(string& obj, std::string& ns, std::string& instance) { + ns.clear(); + instance.clear(); + if (obj[0] != '_') { + return true; + } + + size_t pos = obj.find('_', 1); + if (pos == std::string::npos) { + return false; + } + + if (obj[1] == '_') { + obj = obj.substr(1); + return true; + } + + size_t period_pos = obj.find('.'); + if (period_pos < pos) { + return false; + } + + ns = obj.substr(1, pos-1); + obj = obj.substr(pos+1, std::string::npos); + + parse_ns_field(ns, instance); + return true; + } + + void set_in_extra_data(bool val) { + in_extra_data = val; + } + + bool is_in_extra_data() const { + return in_extra_data; + } + + void encode(bufferlist& bl) const { + ENCODE_START(5, 3, bl); + ::encode(bucket.name, bl); + ::encode(loc, bl); + ::encode(ns, bl); + ::encode(object, bl); + ::encode(bucket, bl); + ::encode(instance, bl); + if (!ns.empty() || !instance.empty()) { + ::encode(orig_obj, bl); + } + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, bl); + ::decode(bucket.name, bl); + ::decode(loc, bl); + ::decode(ns, bl); + ::decode(object, bl); + if (struct_v >= 2) + ::decode(bucket, bl); + if (struct_v >= 4) + ::decode(instance, bl); + if (ns.empty() && instance.empty()) { + if (object[0] != '_') { + orig_obj = object; + } else { + orig_obj = object.substr(1); + } + } else { + if (struct_v >= 5) { + ::decode(orig_obj, bl); + } else { + ssize_t pos = object.find('_', 1); + if (pos < 0) { + throw buffer::error(); + } + orig_obj = object.substr(pos); + } + } + DECODE_FINISH(bl); + } + + bool operator==(const old_rgw_obj& o) const { + return (object.compare(o.object) == 0) && + (bucket.name.compare(o.bucket.name) == 0) && + (ns.compare(o.ns) == 0) && + (instance.compare(o.instance) == 0); + } + bool operator<(const old_rgw_obj& o) const { + int r = bucket.name.compare(o.bucket.name); + if (r == 0) { + r = bucket.bucket_id.compare(o.bucket.bucket_id); + if (r == 0) { + r = object.compare(o.object); + if (r == 0) { + r = ns.compare(o.ns); + if (r == 0) { + r = instance.compare(o.instance); + } + } + } + } + + return (r < 0); + } +}; +WRITE_CLASS_ENCODER(old_rgw_obj) + +static inline void prepend_old_bucket_marker(const old_rgw_bucket& bucket, const string& orig_oid, string& oid) +{ + if (bucket.marker.empty() || orig_oid.empty()) { + oid = orig_oid; + } else { + oid = bucket.marker; + oid.append("_"); + oid.append(orig_oid); + } +} + +void test_rgw_init_env(RGWZoneGroup *zonegroup, RGWZoneParams *zone_params); + +struct test_rgw_env { + RGWZoneGroup zonegroup; + RGWZoneParams zone_params; + rgw_data_placement_target default_placement; + + test_rgw_env() { + test_rgw_init_env(&zonegroup, &zone_params); + default_placement.data_pool = rgw_pool(zone_params.placement_pools[zonegroup.default_placement].data_pool); + default_placement.data_extra_pool = rgw_pool(zone_params.placement_pools[zonegroup.default_placement].data_extra_pool); + } + + rgw_data_placement_target get_placement(const std::string& placement_id) { + const RGWZonePlacementInfo& pi = zone_params.placement_pools[placement_id]; + rgw_data_placement_target pt; + pt.index_pool = pi.index_pool; + pt.data_pool = pi.data_pool; + pt.data_extra_pool = pi.data_extra_pool; + return pt; + } + + rgw_raw_obj get_raw(const rgw_obj& obj) { + rgw_obj_select s(obj); + return s.get_raw_obj(zonegroup, zone_params); + } + + rgw_raw_obj get_raw(const rgw_obj_select& os) { + return os.get_raw_obj(zonegroup, zone_params); + } +}; + +void test_rgw_add_placement(RGWZoneGroup *zonegroup, RGWZoneParams *zone_params, const std::string& name, bool is_default); +void test_rgw_populate_explicit_placement_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id); +void test_rgw_populate_old_bucket(old_rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id); + +std::string test_rgw_get_obj_oid(const rgw_obj& obj); +void test_rgw_init_explicit_placement_bucket(rgw_bucket *bucket, const char *name); +void test_rgw_init_old_bucket(old_rgw_bucket *bucket, const char *name); +void test_rgw_populate_bucket(rgw_bucket *b, const char *t, const char *n, const char *m, const char *id); +void test_rgw_init_bucket(rgw_bucket *bucket, const char *name); +rgw_obj test_rgw_create_obj(const rgw_bucket& bucket, const std::string& name, const std::string& instance, const std::string& ns); + +#endif + diff --git a/src/test/rgw/test_rgw_manifest.cc b/src/test/rgw/test_rgw_manifest.cc index 2a121854df5..3d4e40c619f 100644 --- a/src/test/rgw/test_rgw_manifest.cc +++ b/src/test/rgw/test_rgw_manifest.cc @@ -13,8 +13,10 @@ */ #include #include "global/global_init.h" +#include "common/ceph_argparse.h" #include "rgw/rgw_common.h" #include "rgw/rgw_rados.h" +#include "test_rgw_common.h" #define GTEST #ifdef GTEST #include @@ -24,25 +26,84 @@ else cout << "(" << #v << "==" << #s << ") PASSED\n"; #define EXPECT_EQ(v, s) ASSERT_EQ(v, s) #define ASSERT_TRUE(c) if(c)cout << "Error at " << __LINE__ << "(" << #c << ")" << "\n"; \ - else cout << "(" << #c << ") PASSED\n"; + else cout << "(" << #c << ") PASSED\n";#define EXPECT_TRUE(c) ASSERT_TRUE(c) #define EXPECT_TRUE(c) ASSERT_TRUE(c) #endif using namespace std; -static void populate_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id) -{ - b->tenant = t; - b->name = n; - b->marker = m; - b->bucket_id = id; - b->placement.data_pool = rgw_pool(dp); - b->placement.index_pool = rgw_pool(ip); -} +struct OldObjManifestPart { + old_rgw_obj loc; /* the object where the data is located */ + uint64_t loc_ofs; /* the offset at that object where the data is located */ + uint64_t size; /* the part size */ -static void init_bucket(rgw_bucket *bucket, const char *ten, const char *name) -{ - populate_bucket(bucket, ten, name, ".data-pool", ".index-pool", "marker.", "bucket-id"); -} + OldObjManifestPart() : loc_ofs(0), size(0) {} + + void encode(bufferlist& bl) const { + ENCODE_START(2, 2, bl); + ::encode(loc, bl); + ::encode(loc_ofs, bl); + ::encode(size, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl); + ::decode(loc, bl); + ::decode(loc_ofs, bl); + ::decode(size, bl); + DECODE_FINISH(bl); + } + + void dump(Formatter *f) const; + static void generate_test_instances(list& o); +}; +WRITE_CLASS_ENCODER(OldObjManifestPart) + +class OldObjManifest { +protected: + map objs; + + uint64_t obj_size; +public: + + OldObjManifest() : obj_size(0) {} + OldObjManifest(const OldObjManifest& rhs) { + *this = rhs; + } + OldObjManifest& operator=(const OldObjManifest& rhs) { + objs = rhs.objs; + obj_size = rhs.obj_size; + return *this; + } + + const map& get_objs() { + return objs; + } + + void append(uint64_t ofs, const OldObjManifestPart& part) { + objs[ofs] = part; + obj_size = max(obj_size, ofs + part.size); + } + + void encode(bufferlist& bl) const { + ENCODE_START(2, 2, bl); + ::encode(obj_size, bl); + ::encode(objs, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START_LEGACY_COMPAT_LEN_32(6, 2, 2, bl); + ::decode(obj_size, bl); + ::decode(objs, bl); + DECODE_FINISH(bl); + } + + bool empty() { + return objs.empty(); + } +}; +WRITE_CLASS_ENCODER(OldObjManifest) void append_head(list *objs, rgw_obj& head) { @@ -52,7 +113,7 @@ void append_head(list *objs, rgw_obj& head) void append_stripes(list *objs, RGWObjManifest& manifest, uint64_t obj_size, uint64_t stripe_size) { string prefix = manifest.get_prefix(); - rgw_bucket bucket = manifest.get_head().bucket; + rgw_bucket bucket = manifest.get_obj().bucket; int i = 0; for (uint64_t ofs = manifest.get_max_head_size(); ofs < obj_size; ofs += stripe_size) { @@ -66,13 +127,13 @@ void append_stripes(list *objs, RGWObjManifest& manifest, uint64_t obj_ } } -static void gen_obj(uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size, +static void gen_obj(test_rgw_env& env, uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size, RGWObjManifest *manifest, rgw_bucket *bucket, rgw_obj *head, RGWObjManifest::generator *gen, list *test_objs) { manifest->set_trivial_rule(head_max_size, stripe_size); - init_bucket(bucket, "", "buck"); + test_rgw_init_bucket(bucket, "buck"); *head = rgw_obj(*bucket, "oid"); gen->create_begin(g_ceph_context, manifest, *bucket, *head); @@ -91,9 +152,11 @@ static void gen_obj(uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_s list::iterator iter = test_objs->begin(); while (ofs < obj_size) { - rgw_obj obj = gen->get_cur_obj(); + rgw_raw_obj obj = gen->get_cur_obj(env.zonegroup, env.zone_params); cout << "obj=" << obj << std::endl; - ASSERT_TRUE(obj == *iter); + rgw_raw_obj test_raw; + rgw_obj_to_raw(env.zonegroup, env.zone_params, *iter, &test_raw); + ASSERT_TRUE(obj == test_raw); ofs = MIN(ofs + gen->cur_stripe_max_size(), obj_size); gen->create_next(ofs); @@ -105,8 +168,10 @@ cout << "obj=" << obj << std::endl; } if (manifest->has_tail()) { - rgw_obj obj = gen->get_cur_obj(); - ASSERT_TRUE(obj == *iter); + rgw_raw_obj obj = gen->get_cur_obj(env.zonegroup, env.zone_params); + rgw_raw_obj test_raw; + rgw_obj_to_raw(env.zonegroup, env.zone_params, *iter, &test_raw); + ASSERT_TRUE(obj == test_raw); ++iter; } ASSERT_TRUE(iter == test_objs->end()); @@ -115,7 +180,44 @@ cout << "obj=" << obj << std::endl; ASSERT_EQ(manifest->has_tail(), (obj_size > head_max_size)); } +static void gen_old_obj(test_rgw_env& env, uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size, + OldObjManifest *manifest, old_rgw_bucket *bucket, old_rgw_obj *head, + list *test_objs) +{ + test_rgw_init_old_bucket(bucket, "buck"); + + *head = old_rgw_obj(*bucket, "obj"); + + OldObjManifestPart part; + part.loc = *head; + part.size = head_max_size; + part.loc_ofs = 0; + + manifest->append(0, part); + test_objs->push_back(part.loc); + + string prefix; + append_rand_alpha(g_ceph_context, prefix, prefix, 16); + + int i = 0; + for (uint64_t ofs = head_max_size; ofs < obj_size; ofs += stripe_size, i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "%s.%d", prefix.c_str(), i); + old_rgw_obj loc(*bucket, buf); + loc.set_ns("shadow"); + OldObjManifestPart part; + part.loc = loc; + part.size = min(stripe_size, obj_size - ofs); + part.loc_ofs = 0; + + manifest->append(ofs, part); + + test_objs->push_back(loc); + } +} + TEST(TestRGWManifest, head_only_obj) { + test_rgw_env env; RGWObjManifest manifest; rgw_bucket bucket; rgw_obj head; @@ -125,7 +227,7 @@ TEST(TestRGWManifest, head_only_obj) { list objs; - gen_obj(obj_size, 512 * 1024, 4 * 1024 * 1024, &manifest, &bucket, &head, &gen, &objs); + gen_obj(env, obj_size, 512 * 1024, 4 * 1024 * 1024, &manifest, &bucket, &head, &gen, &objs); cout << " manifest.get_obj_size()=" << manifest.get_obj_size() << std::endl; cout << " manifest.get_head_size()=" << manifest.get_head_size() << std::endl; @@ -135,18 +237,21 @@ TEST(TestRGWManifest, head_only_obj) { for (iter = manifest.obj_begin(), liter = objs.begin(); iter != manifest.obj_end() && liter != objs.end(); ++iter, ++liter) { - ASSERT_TRUE(*liter == iter.get_location()); + ASSERT_TRUE(env.get_raw(*liter) == env.get_raw(iter.get_location())); } ASSERT_TRUE(iter == manifest.obj_end()); ASSERT_TRUE(liter == objs.end()); + rgw_raw_obj raw_head; + iter = manifest.obj_find(100 * 1024); - ASSERT_TRUE(iter.get_location() == head); + ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(head)); ASSERT_EQ((int)iter.get_stripe_size(), obj_size); } TEST(TestRGWManifest, obj_with_head_and_tail) { + test_rgw_env env; RGWObjManifest manifest; rgw_bucket bucket; rgw_obj head; @@ -158,18 +263,18 @@ TEST(TestRGWManifest, obj_with_head_and_tail) { int stripe_size = 4 * 1024 * 1024; int head_size = 512 * 1024; - gen_obj(obj_size, head_size, stripe_size, &manifest, &bucket, &head, &gen, &objs); + gen_obj(env, obj_size, head_size, stripe_size, &manifest, &bucket, &head, &gen, &objs); list::iterator liter; - rgw_obj last_obj; + rgw_obj_select last_obj; RGWObjManifest::obj_iterator iter; for (iter = manifest.obj_begin(), liter = objs.begin(); iter != manifest.obj_end() && liter != objs.end(); ++iter, ++liter) { - cout << "*liter=" << *liter << " iter.get_location()=" << iter.get_location() << std::endl; - ASSERT_TRUE(*liter == iter.get_location()); + cout << "*liter=" << *liter << " iter.get_location()=" << env.get_raw(iter.get_location()) << std::endl; + ASSERT_TRUE(env.get_raw(*liter) == env.get_raw(iter.get_location())); last_obj = iter.get_location(); } @@ -178,18 +283,19 @@ TEST(TestRGWManifest, obj_with_head_and_tail) { ASSERT_TRUE(liter == objs.end()); iter = manifest.obj_find(100 * 1024); - ASSERT_TRUE(iter.get_location() == head); + ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(head)); ASSERT_EQ((int)iter.get_stripe_size(), head_size); uint64_t ofs = 20 * 1024 * 1024 + head_size; iter = manifest.obj_find(ofs + 100); - ASSERT_TRUE(iter.get_location() == last_obj); + ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(last_obj)); ASSERT_EQ(iter.get_stripe_ofs(), ofs); ASSERT_EQ(iter.get_stripe_size(), obj_size - ofs); } TEST(TestRGWManifest, multipart) { + test_rgw_env env; int num_parts = 16; vector pm(num_parts); rgw_bucket bucket; @@ -224,14 +330,78 @@ TEST(TestRGWManifest, multipart) { RGWObjManifest m; for (int i = 0; i < num_parts; i++) { - m.append(pm[i]); + m.append(pm[i], env.zonegroup, env.zone_params); } RGWObjManifest::obj_iterator iter; for (iter = m.obj_begin(); iter != m.obj_end(); ++iter) { RGWObjManifest::obj_iterator fiter = m.obj_find(iter.get_ofs()); - ASSERT_TRUE(fiter.get_location() == iter.get_location()); + ASSERT_TRUE(env.get_raw(fiter.get_location()) == env.get_raw(iter.get_location())); } ASSERT_EQ(m.get_obj_size(), num_parts * part_size); } +TEST(TestRGWManifest, old_obj_manifest) { + test_rgw_env env; + OldObjManifest old_manifest; + old_rgw_bucket old_bucket; + old_rgw_obj old_head; + + int obj_size = 40 * 1024 * 1024; + uint64_t stripe_size = 4 * 1024 * 1024; + uint64_t head_size = 512 * 1024; + + list old_objs; + + gen_old_obj(env, obj_size, head_size, stripe_size, &old_manifest, &old_bucket, &old_head, &old_objs); + + ASSERT_EQ(old_objs.size(), 11); + + + bufferlist bl; + ::encode(old_manifest , bl); + + RGWObjManifest manifest; + + try { + auto iter = bl.begin(); + ::decode(manifest, iter); + } catch (buffer::error& err) { + ASSERT_TRUE(false); + } + + rgw_raw_obj last_obj; + + RGWObjManifest::obj_iterator iter; + auto liter = old_objs.begin(); + for (iter = manifest.obj_begin(); + iter != manifest.obj_end() && liter != old_objs.end(); + ++iter, ++liter) { + rgw_pool old_pool(liter->bucket.data_pool); + string old_oid; + prepend_old_bucket_marker(old_bucket, liter->get_object(), old_oid); + rgw_raw_obj raw_old(old_pool, old_oid); + cout << "*liter=" << raw_old << " iter.get_location()=" << env.get_raw(iter.get_location()) << std::endl; + ASSERT_EQ(raw_old, env.get_raw(iter.get_location())); + + last_obj = env.get_raw(iter.get_location()); + } + + ASSERT_TRUE(liter == old_objs.end()); + ASSERT_TRUE(iter == manifest.obj_end()); + +} + + +int main(int argc, char **argv) { + vector args; + argv_to_vec(argc, (const char **)argv, args); + env_to_vec(args); + + vector def_args; + global_init(&def_args, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/src/test/rgw/test_rgw_obj.cc b/src/test/rgw/test_rgw_obj.cc index 3b5a7b6d84a..115b0ce6bbb 100644 --- a/src/test/rgw/test_rgw_obj.cc +++ b/src/test/rgw/test_rgw_obj.cc @@ -16,6 +16,8 @@ #include "common/ceph_json.h" #include "common/Formatter.h" #include "rgw/rgw_common.h" +#include "rgw/rgw_rados.h" +#include "test_rgw_common.h" #define GTEST #ifdef GTEST #include @@ -30,21 +32,6 @@ #endif using namespace std; -static void populate_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id) -{ - b->tenant = t; - b->name = n; - b->marker = m; - b->bucket_id = id; - b->placement.data_pool = rgw_pool(dp); - b->placement.index_pool = rgw_pool(ip); -} - -static void init_bucket(rgw_bucket *bucket, const char *name) -{ - populate_bucket(bucket, "", name, ".data-pool", ".index-pool", "marker", "bucket-id"); -} - void check_parsed_correctly(rgw_obj& obj, const string& name, const string& ns, const string& instance) { /* parse_raw_oid() */ @@ -91,7 +78,7 @@ void check_parsed_correctly(rgw_obj& obj, const string& name, const string& ns, void test_obj(const string& name, const string& ns, const string& instance) { rgw_bucket b; - init_bucket(&b, "test"); + test_rgw_init_bucket(&b, "test"); JSONFormatter *formatter = new JSONFormatter(true); @@ -167,3 +154,123 @@ TEST(TestRGWObj, no_underscore) { test_obj("obj", "ns", "v1"); } +template +void dump(JSONFormatter& f, const string& name, const T& entity) +{ + f.open_object_section(name.c_str()); + ::encode_json(name.c_str(), entity, &f); + f.close_section(); + f.flush(cout); +} + +static void test_obj_to_raw(test_rgw_env& env, const rgw_bucket& b, + const string& name, const string& instance, const string& ns, + const string& placement_id) +{ + JSONFormatter f(true); + dump(f, "bucket", b); + rgw_obj obj = test_rgw_create_obj(b, name, instance, ns, placement_id); + dump(f, "obj", obj); + + rgw_raw_obj raw_obj; + + rgw_obj_to_raw(env.zonegroup, env.zone_params, obj, &raw_obj); + dump(f, "raw_obj", raw_obj); + + if (!placement_id.empty()) { + ASSERT_EQ(raw_obj.pool, env.get_placement(placement_id).data_pool); + } else { + ASSERT_EQ(raw_obj.pool, b.explicit_placement.data_pool); + } + ASSERT_EQ(raw_obj.oid, test_rgw_get_obj_oid(obj)); + + rgw_obj new_obj; + rgw_raw_obj_to_obj(b, raw_obj, &new_obj); + + dump(f, "new_obj", new_obj); + + ASSERT_EQ(obj, new_obj); + +} + +TEST(TestRGWObj, obj_to_raw) { + test_rgw_env env; + + rgw_bucket b; + test_rgw_init_bucket(&b, "test"); + + rgw_bucket eb; + test_rgw_init_explicit_placement_bucket(&eb, "ebtest"); + + for (auto name : { "myobj", "_myobj", "_myobj_"}) { + for (auto inst : { "", "inst"}) { + for (auto ns : { "", "ns"}) { + test_obj_to_raw(env, b, name, inst, ns, env.zonegroup.default_placement); + test_obj_to_raw(env, eb, name, inst, ns, string()); + } + } + } +} + +TEST(TestRGWObj, old_to_raw) { + JSONFormatter f(true); + test_rgw_env env; + + old_rgw_bucket eb; + test_rgw_init_old_bucket(&eb, "ebtest"); + + for (auto name : { "myobj", "_myobj", "_myobj_"}) { + for (string inst : { "", "inst"}) { + for (string ns : { "", "ns"}) { + old_rgw_obj old(eb, name); + if (!inst.empty()) { + old.set_instance(inst); + } + if (!ns.empty()) { + old.set_ns(ns); + } + + bufferlist bl; + + ::encode(old, bl); + + rgw_obj new_obj; + rgw_raw_obj raw_obj; + + try { + bufferlist::iterator iter = bl.begin(); + ::decode(new_obj, iter); + + iter = bl.begin(); + ::decode(raw_obj, iter); + } catch (buffer::error& err) { + ASSERT_TRUE(false); + } + + bl.clear(); + + rgw_obj new_obj2; + rgw_raw_obj raw_obj2; + + ::encode(new_obj, bl); + + dump(f, "raw_obj", raw_obj); + dump(f, "new_obj", new_obj); + cout << "raw=" << raw_obj << std::endl; + + try { + bufferlist::iterator iter = bl.begin(); + ::decode(new_obj2, iter); + + iter = bl.begin(); + ::decode(raw_obj2, iter); + } catch (buffer::error& err) { + ASSERT_TRUE(false); + } + + ASSERT_EQ(new_obj, new_obj2); + ASSERT_EQ(raw_obj, raw_obj2); + } + } + } +}