add_ceph_unittest(unittest_http_manager ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/unittest_http_manager)
target_link_libraries(unittest_http_manager rgw_a)
+set(test_rgw_a_src
+ test_rgw_common.cc)
+add_library(test_rgw_a STATIC ${test_rgw_a_src})
+
# ceph_test_rgw_manifest
set(test_rgw_manifest_srcs test_rgw_manifest.cc)
add_executable(ceph_test_rgw_manifest
)
target_link_libraries(ceph_test_rgw_manifest
rgw_a
+ test_rgw_a
cls_rgw_client
cls_lock_client
cls_refcount_client
)
target_link_libraries(ceph_test_rgw_obj
rgw_a
+ test_rgw_a
cls_rgw_client
cls_lock_client
cls_refcount_client
--- /dev/null
+#include "test_rgw_common.h"
+
+void test_rgw_add_placement(RGWZoneGroup *zonegroup, RGWZoneParams *zone_params, const std::string& name, bool is_default)
+{
+ zonegroup->placement_targets[name] = { name };
+
+ RGWZonePlacementInfo& pinfo = zone_params->placement_pools[name];
+ pinfo.index_pool = rgw_pool(name + ".index").to_str();
+ pinfo.data_pool = rgw_pool(name + ".data").to_str();
+ pinfo.data_extra_pool = rgw_pool(name + ".extra").to_str();
+
+ if (is_default) {
+ zonegroup->default_placement = name;
+ }
+}
+
+void test_rgw_init_env(RGWZoneGroup *zonegroup, RGWZoneParams *zone_params)
+{
+ test_rgw_add_placement(zonegroup, zone_params, "default-placement", true);
+
+}
+
+void test_rgw_populate_explicit_placement_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id)
+{
+ b->tenant = t;
+ b->name = n;
+ b->marker = m;
+ b->bucket_id = id;
+ b->explicit_placement.data_pool = rgw_pool(dp);
+ b->explicit_placement.index_pool = rgw_pool(ip);
+}
+
+void test_rgw_populate_old_bucket(old_rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id)
+{
+ b->tenant = t;
+ b->name = n;
+ b->marker = m;
+ b->bucket_id = id;
+ b->data_pool = dp;
+ b->index_pool = ip;
+}
+
+std::string test_rgw_get_obj_oid(const rgw_obj& obj)
+{
+ std::string oid;
+ std::string loc;
+
+ get_obj_bucket_and_oid_loc(obj, oid, loc);
+ return oid;
+}
+
+void test_rgw_init_explicit_placement_bucket(rgw_bucket *bucket, const char *name)
+{
+ test_rgw_populate_explicit_placement_bucket(bucket, "", name, ".data-pool", ".index-pool", "marker", "bucket-id");
+}
+
+void test_rgw_init_old_bucket(old_rgw_bucket *bucket, const char *name)
+{
+ test_rgw_populate_old_bucket(bucket, "", name, ".data-pool", ".index-pool", "marker", "bucket-id");
+}
+
+void test_rgw_populate_bucket(rgw_bucket *b, const char *t, const char *n, const char *m, const char *id)
+{
+ b->tenant = t;
+ b->name = n;
+ b->marker = m;
+ b->bucket_id = id;
+}
+
+void test_rgw_init_bucket(rgw_bucket *bucket, const char *name)
+{
+ test_rgw_populate_bucket(bucket, "", name, "marker", "bucket-id");
+}
+
+rgw_obj test_rgw_create_obj(const rgw_bucket& bucket, const std::string& name, const std::string& instance, const std::string& ns)
+{
+ rgw_obj obj(bucket, name);
+ if (!instance.empty()) {
+ obj.key.set_instance(instance);
+ }
+ if (!ns.empty()) {
+ obj.key.ns = ns;
+ }
+ obj.bucket = bucket;
+
+ return obj;
+}
+
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include <iostream>
+#include "common/ceph_json.h"
+#include "common/Formatter.h"
+#include "rgw/rgw_common.h"
+#include "rgw/rgw_rados.h"
+
+#ifndef CEPH_TEST_RGW_COMMON_H
+#define CEPH_TEST_RGW_COMMON_H
+
+struct old_rgw_bucket {
+ std::string tenant;
+ std::string name;
+ std::string data_pool;
+ std::string data_extra_pool; /* if not set, then we should use data_pool instead */
+ std::string index_pool;
+ std::string marker;
+ std::string bucket_id;
+
+ std::string oid; /*
+ * runtime in-memory only info. If not empty, points to the bucket instance object
+ */
+
+ old_rgw_bucket() { }
+ // cppcheck-suppress noExplicitConstructor
+ old_rgw_bucket(const std::string& s) : name(s) {
+ data_pool = index_pool = s;
+ marker = "";
+ }
+ old_rgw_bucket(const char *n) : name(n) {
+ data_pool = index_pool = n;
+ marker = "";
+ }
+ old_rgw_bucket(const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id, const char *h) :
+ tenant(t), name(n), data_pool(dp), index_pool(ip), marker(m), bucket_id(id) {}
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(8, 3, bl);
+ ::encode(name, bl);
+ ::encode(data_pool, bl);
+ ::encode(marker, bl);
+ ::encode(bucket_id, bl);
+ ::encode(index_pool, bl);
+ ::encode(data_extra_pool, bl);
+ ::encode(tenant, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START_LEGACY_COMPAT_LEN(8, 3, 3, bl);
+ ::decode(name, bl);
+ ::decode(data_pool, bl);
+ if (struct_v >= 2) {
+ ::decode(marker, bl);
+ if (struct_v <= 3) {
+ uint64_t id;
+ ::decode(id, bl);
+ char buf[16];
+ snprintf(buf, sizeof(buf), "%llu", (long long)id);
+ bucket_id = buf;
+ } else {
+ ::decode(bucket_id, bl);
+ }
+ }
+ if (struct_v >= 5) {
+ ::decode(index_pool, bl);
+ } else {
+ index_pool = data_pool;
+ }
+ if (struct_v >= 7) {
+ ::decode(data_extra_pool, bl);
+ }
+ if (struct_v >= 8) {
+ ::decode(tenant, bl);
+ }
+ DECODE_FINISH(bl);
+ }
+
+ // format a key for the bucket/instance. pass delim=0 to skip a field
+ std::string get_key(char tenant_delim = '/',
+ char id_delim = ':') const;
+
+ const std::string& get_data_extra_pool() {
+ if (data_extra_pool.empty()) {
+ return data_pool;
+ }
+ return data_extra_pool;
+ }
+
+ void dump(Formatter *f) const;
+ void decode_json(JSONObj *obj);
+ static void generate_test_instances(list<old_rgw_bucket*>& o);
+
+ bool operator<(const old_rgw_bucket& b) const {
+ return name.compare(b.name) < 0;
+ }
+};
+WRITE_CLASS_ENCODER(old_rgw_bucket)
+
+class old_rgw_obj {
+ std::string orig_obj;
+ std::string loc;
+ std::string object;
+ std::string instance;
+public:
+ const std::string& get_object() const { return object; }
+ const std::string& get_orig_obj() const { return orig_obj; }
+ const std::string& get_loc() const { return loc; }
+ const std::string& get_instance() const { return instance; }
+ old_rgw_bucket bucket;
+ std::string ns;
+
+ bool in_extra_data; /* in-memory only member, does not serialize */
+
+ // Represents the hash index source for this object once it is set (non-empty)
+ std::string index_hash_source;
+
+ old_rgw_obj() : in_extra_data(false) {}
+ old_rgw_obj(old_rgw_bucket& b, const std::string& o) : in_extra_data(false) {
+ init(b, o);
+ }
+ old_rgw_obj(old_rgw_bucket& b, const rgw_obj_key& k) : in_extra_data(false) {
+ from_index_key(b, k);
+ }
+ void init(old_rgw_bucket& b, const std::string& o) {
+ bucket = b;
+ set_obj(o);
+ reset_loc();
+ }
+ void init_ns(old_rgw_bucket& b, const std::string& o, const std::string& n) {
+ bucket = b;
+ set_ns(n);
+ set_obj(o);
+ reset_loc();
+ }
+ int set_ns(const char *n) {
+ if (!n)
+ return -EINVAL;
+ std::string ns_str(n);
+ return set_ns(ns_str);
+ }
+ int set_ns(const std::string& n) {
+ if (n[0] == '_')
+ return -EINVAL;
+ ns = n;
+ set_obj(orig_obj);
+ return 0;
+ }
+ int set_instance(const std::string& i) {
+ if (i[0] == '_')
+ return -EINVAL;
+ instance = i;
+ set_obj(orig_obj);
+ return 0;
+ }
+
+ int clear_instance() {
+ return set_instance(string());
+ }
+
+ void set_loc(const std::string& k) {
+ loc = k;
+ }
+
+ void reset_loc() {
+ loc.clear();
+ /*
+ * For backward compatibility. Older versions used to have object locator on all objects,
+ * however, the orig_obj was the effective object locator. This had the same effect as not
+ * having object locator at all for most objects but the ones that started with underscore as
+ * these were escaped.
+ */
+ if (orig_obj[0] == '_' && ns.empty()) {
+ loc = orig_obj;
+ }
+ }
+
+ bool have_null_instance() {
+ return instance == "null";
+ }
+
+ bool have_instance() {
+ return !instance.empty();
+ }
+
+ bool need_to_encode_instance() {
+ return have_instance() && !have_null_instance();
+ }
+
+ void set_obj(const std::string& o) {
+ object.reserve(128);
+
+ orig_obj = o;
+ if (ns.empty() && !need_to_encode_instance()) {
+ if (o.empty()) {
+ return;
+ }
+ if (o.size() < 1 || o[0] != '_') {
+ object = o;
+ return;
+ }
+ object = "_";
+ object.append(o);
+ } else {
+ object = "_";
+ object.append(ns);
+ if (need_to_encode_instance()) {
+ object.append(string(":") + instance);
+ }
+ object.append("_");
+ object.append(o);
+ }
+ reset_loc();
+ }
+
+ /*
+ * get the object's key name as being referred to by the bucket index.
+ */
+ std::string get_index_key_name() const {
+ if (ns.empty()) {
+ if (orig_obj.size() < 1 || orig_obj[0] != '_') {
+ return orig_obj;
+ }
+ return std::string("_") + orig_obj;
+ };
+
+ char buf[ns.size() + 16];
+ snprintf(buf, sizeof(buf), "_%s_", ns.c_str());
+ return std::string(buf) + orig_obj;
+ };
+
+ void from_index_key(old_rgw_bucket& b, const rgw_obj_key& key) {
+ if (key.name[0] != '_') {
+ init(b, key.name);
+ set_instance(key.instance);
+ return;
+ }
+ if (key.name[1] == '_') {
+ init(b, key.name.substr(1));
+ set_instance(key.instance);
+ return;
+ }
+ ssize_t pos = key.name.find('_', 1);
+ if (pos < 0) {
+ /* shouldn't happen, just use key */
+ init(b, key.name);
+ set_instance(key.instance);
+ return;
+ }
+
+ init_ns(b, key.name.substr(pos + 1), key.name.substr(1, pos -1));
+ set_instance(key.instance);
+ }
+
+ void get_index_key(rgw_obj_key *key) const {
+ key->name = get_index_key_name();
+ key->instance = instance;
+ }
+
+ static void parse_ns_field(string& ns, std::string& instance) {
+ int pos = ns.find(':');
+ if (pos >= 0) {
+ instance = ns.substr(pos + 1);
+ ns = ns.substr(0, pos);
+ } else {
+ instance.clear();
+ }
+ }
+
+ std::string& get_hash_object() {
+ return index_hash_source.empty() ? orig_obj : index_hash_source;
+ }
+ /**
+ * Translate a namespace-mangled object name to the user-facing name
+ * existing in the given namespace.
+ *
+ * If the object is part of the given namespace, it returns true
+ * and cuts down the name to the unmangled version. If it is not
+ * part of the given namespace, it returns false.
+ */
+ static bool translate_raw_obj_to_obj_in_ns(string& obj, std::string& instance, std::string& ns) {
+ if (obj[0] != '_') {
+ if (ns.empty()) {
+ return true;
+ }
+ return false;
+ }
+
+ std::string obj_ns;
+ bool ret = parse_raw_oid(obj, &obj, &instance, &obj_ns);
+ if (!ret) {
+ return ret;
+ }
+
+ return (ns == obj_ns);
+ }
+
+ static bool parse_raw_oid(const std::string& oid, std::string *obj_name, std::string *obj_instance, std::string *obj_ns) {
+ obj_instance->clear();
+ obj_ns->clear();
+ if (oid[0] != '_') {
+ *obj_name = oid;
+ return true;
+ }
+
+ if (oid.size() >= 2 && oid[1] == '_') {
+ *obj_name = oid.substr(1);
+ return true;
+ }
+
+ if (oid[0] != '_' || oid.size() < 3) // for namespace, min size would be 3: _x_
+ return false;
+
+ int pos = oid.find('_', 1);
+ if (pos <= 1) // if it starts with __, it's not in our namespace
+ return false;
+
+ *obj_ns = oid.substr(1, pos - 1);
+ parse_ns_field(*obj_ns, *obj_instance);
+
+ *obj_name = oid.substr(pos + 1);
+ return true;
+ }
+
+ /**
+ * Given a mangled object name and an empty namespace string, this
+ * function extracts the namespace into the string and sets the object
+ * name to be the unmangled version.
+ *
+ * It returns true after successfully doing so, or
+ * false if it fails.
+ */
+ static bool strip_namespace_from_object(string& obj, std::string& ns, std::string& instance) {
+ ns.clear();
+ instance.clear();
+ if (obj[0] != '_') {
+ return true;
+ }
+
+ size_t pos = obj.find('_', 1);
+ if (pos == std::string::npos) {
+ return false;
+ }
+
+ if (obj[1] == '_') {
+ obj = obj.substr(1);
+ return true;
+ }
+
+ size_t period_pos = obj.find('.');
+ if (period_pos < pos) {
+ return false;
+ }
+
+ ns = obj.substr(1, pos-1);
+ obj = obj.substr(pos+1, std::string::npos);
+
+ parse_ns_field(ns, instance);
+ return true;
+ }
+
+ void set_in_extra_data(bool val) {
+ in_extra_data = val;
+ }
+
+ bool is_in_extra_data() const {
+ return in_extra_data;
+ }
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(5, 3, bl);
+ ::encode(bucket.name, bl);
+ ::encode(loc, bl);
+ ::encode(ns, bl);
+ ::encode(object, bl);
+ ::encode(bucket, bl);
+ ::encode(instance, bl);
+ if (!ns.empty() || !instance.empty()) {
+ ::encode(orig_obj, bl);
+ }
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, bl);
+ ::decode(bucket.name, bl);
+ ::decode(loc, bl);
+ ::decode(ns, bl);
+ ::decode(object, bl);
+ if (struct_v >= 2)
+ ::decode(bucket, bl);
+ if (struct_v >= 4)
+ ::decode(instance, bl);
+ if (ns.empty() && instance.empty()) {
+ if (object[0] != '_') {
+ orig_obj = object;
+ } else {
+ orig_obj = object.substr(1);
+ }
+ } else {
+ if (struct_v >= 5) {
+ ::decode(orig_obj, bl);
+ } else {
+ ssize_t pos = object.find('_', 1);
+ if (pos < 0) {
+ throw buffer::error();
+ }
+ orig_obj = object.substr(pos);
+ }
+ }
+ DECODE_FINISH(bl);
+ }
+
+ bool operator==(const old_rgw_obj& o) const {
+ return (object.compare(o.object) == 0) &&
+ (bucket.name.compare(o.bucket.name) == 0) &&
+ (ns.compare(o.ns) == 0) &&
+ (instance.compare(o.instance) == 0);
+ }
+ bool operator<(const old_rgw_obj& o) const {
+ int r = bucket.name.compare(o.bucket.name);
+ if (r == 0) {
+ r = bucket.bucket_id.compare(o.bucket.bucket_id);
+ if (r == 0) {
+ r = object.compare(o.object);
+ if (r == 0) {
+ r = ns.compare(o.ns);
+ if (r == 0) {
+ r = instance.compare(o.instance);
+ }
+ }
+ }
+ }
+
+ return (r < 0);
+ }
+};
+WRITE_CLASS_ENCODER(old_rgw_obj)
+
+static inline void prepend_old_bucket_marker(const old_rgw_bucket& bucket, const string& orig_oid, string& oid)
+{
+ if (bucket.marker.empty() || orig_oid.empty()) {
+ oid = orig_oid;
+ } else {
+ oid = bucket.marker;
+ oid.append("_");
+ oid.append(orig_oid);
+ }
+}
+
+void test_rgw_init_env(RGWZoneGroup *zonegroup, RGWZoneParams *zone_params);
+
+struct test_rgw_env {
+ RGWZoneGroup zonegroup;
+ RGWZoneParams zone_params;
+ rgw_data_placement_target default_placement;
+
+ test_rgw_env() {
+ test_rgw_init_env(&zonegroup, &zone_params);
+ default_placement.data_pool = rgw_pool(zone_params.placement_pools[zonegroup.default_placement].data_pool);
+ default_placement.data_extra_pool = rgw_pool(zone_params.placement_pools[zonegroup.default_placement].data_extra_pool);
+ }
+
+ rgw_data_placement_target get_placement(const std::string& placement_id) {
+ const RGWZonePlacementInfo& pi = zone_params.placement_pools[placement_id];
+ rgw_data_placement_target pt;
+ pt.index_pool = pi.index_pool;
+ pt.data_pool = pi.data_pool;
+ pt.data_extra_pool = pi.data_extra_pool;
+ return pt;
+ }
+
+ rgw_raw_obj get_raw(const rgw_obj& obj) {
+ rgw_obj_select s(obj);
+ return s.get_raw_obj(zonegroup, zone_params);
+ }
+
+ rgw_raw_obj get_raw(const rgw_obj_select& os) {
+ return os.get_raw_obj(zonegroup, zone_params);
+ }
+};
+
+void test_rgw_add_placement(RGWZoneGroup *zonegroup, RGWZoneParams *zone_params, const std::string& name, bool is_default);
+void test_rgw_populate_explicit_placement_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id);
+void test_rgw_populate_old_bucket(old_rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id);
+
+std::string test_rgw_get_obj_oid(const rgw_obj& obj);
+void test_rgw_init_explicit_placement_bucket(rgw_bucket *bucket, const char *name);
+void test_rgw_init_old_bucket(old_rgw_bucket *bucket, const char *name);
+void test_rgw_populate_bucket(rgw_bucket *b, const char *t, const char *n, const char *m, const char *id);
+void test_rgw_init_bucket(rgw_bucket *bucket, const char *name);
+rgw_obj test_rgw_create_obj(const rgw_bucket& bucket, const std::string& name, const std::string& instance, const std::string& ns);
+
+#endif
+
*/
#include <iostream>
#include "global/global_init.h"
+#include "common/ceph_argparse.h"
#include "rgw/rgw_common.h"
#include "rgw/rgw_rados.h"
+#include "test_rgw_common.h"
#define GTEST
#ifdef GTEST
#include <gtest/gtest.h>
else cout << "(" << #v << "==" << #s << ") PASSED\n";
#define EXPECT_EQ(v, s) ASSERT_EQ(v, s)
#define ASSERT_TRUE(c) if(c)cout << "Error at " << __LINE__ << "(" << #c << ")" << "\n"; \
- else cout << "(" << #c << ") PASSED\n";
+ else cout << "(" << #c << ") PASSED\n";#define EXPECT_TRUE(c) ASSERT_TRUE(c)
#define EXPECT_TRUE(c) ASSERT_TRUE(c)
#endif
using namespace std;
-static void populate_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id)
-{
- b->tenant = t;
- b->name = n;
- b->marker = m;
- b->bucket_id = id;
- b->placement.data_pool = rgw_pool(dp);
- b->placement.index_pool = rgw_pool(ip);
-}
+struct OldObjManifestPart {
+ old_rgw_obj loc; /* the object where the data is located */
+ uint64_t loc_ofs; /* the offset at that object where the data is located */
+ uint64_t size; /* the part size */
-static void init_bucket(rgw_bucket *bucket, const char *ten, const char *name)
-{
- populate_bucket(bucket, ten, name, ".data-pool", ".index-pool", "marker.", "bucket-id");
-}
+ OldObjManifestPart() : loc_ofs(0), size(0) {}
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(2, 2, bl);
+ ::encode(loc, bl);
+ ::encode(loc_ofs, bl);
+ ::encode(size, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
+ ::decode(loc, bl);
+ ::decode(loc_ofs, bl);
+ ::decode(size, bl);
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<OldObjManifestPart*>& o);
+};
+WRITE_CLASS_ENCODER(OldObjManifestPart)
+
+class OldObjManifest {
+protected:
+ map<uint64_t, OldObjManifestPart> objs;
+
+ uint64_t obj_size;
+public:
+
+ OldObjManifest() : obj_size(0) {}
+ OldObjManifest(const OldObjManifest& rhs) {
+ *this = rhs;
+ }
+ OldObjManifest& operator=(const OldObjManifest& rhs) {
+ objs = rhs.objs;
+ obj_size = rhs.obj_size;
+ return *this;
+ }
+
+ const map<uint64_t, OldObjManifestPart>& get_objs() {
+ return objs;
+ }
+
+ void append(uint64_t ofs, const OldObjManifestPart& part) {
+ objs[ofs] = part;
+ obj_size = max(obj_size, ofs + part.size);
+ }
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(2, 2, bl);
+ ::encode(obj_size, bl);
+ ::encode(objs, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START_LEGACY_COMPAT_LEN_32(6, 2, 2, bl);
+ ::decode(obj_size, bl);
+ ::decode(objs, bl);
+ DECODE_FINISH(bl);
+ }
+
+ bool empty() {
+ return objs.empty();
+ }
+};
+WRITE_CLASS_ENCODER(OldObjManifest)
void append_head(list<rgw_obj> *objs, rgw_obj& head)
{
void append_stripes(list<rgw_obj> *objs, RGWObjManifest& manifest, uint64_t obj_size, uint64_t stripe_size)
{
string prefix = manifest.get_prefix();
- rgw_bucket bucket = manifest.get_head().bucket;
+ rgw_bucket bucket = manifest.get_obj().bucket;
int i = 0;
for (uint64_t ofs = manifest.get_max_head_size(); ofs < obj_size; ofs += stripe_size) {
}
}
-static void gen_obj(uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size,
+static void gen_obj(test_rgw_env& env, uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size,
RGWObjManifest *manifest, rgw_bucket *bucket, rgw_obj *head, RGWObjManifest::generator *gen,
list<rgw_obj> *test_objs)
{
manifest->set_trivial_rule(head_max_size, stripe_size);
- init_bucket(bucket, "", "buck");
+ test_rgw_init_bucket(bucket, "buck");
*head = rgw_obj(*bucket, "oid");
gen->create_begin(g_ceph_context, manifest, *bucket, *head);
list<rgw_obj>::iterator iter = test_objs->begin();
while (ofs < obj_size) {
- rgw_obj obj = gen->get_cur_obj();
+ rgw_raw_obj obj = gen->get_cur_obj(env.zonegroup, env.zone_params);
cout << "obj=" << obj << std::endl;
- ASSERT_TRUE(obj == *iter);
+ rgw_raw_obj test_raw;
+ rgw_obj_to_raw(env.zonegroup, env.zone_params, *iter, &test_raw);
+ ASSERT_TRUE(obj == test_raw);
ofs = MIN(ofs + gen->cur_stripe_max_size(), obj_size);
gen->create_next(ofs);
}
if (manifest->has_tail()) {
- rgw_obj obj = gen->get_cur_obj();
- ASSERT_TRUE(obj == *iter);
+ rgw_raw_obj obj = gen->get_cur_obj(env.zonegroup, env.zone_params);
+ rgw_raw_obj test_raw;
+ rgw_obj_to_raw(env.zonegroup, env.zone_params, *iter, &test_raw);
+ ASSERT_TRUE(obj == test_raw);
++iter;
}
ASSERT_TRUE(iter == test_objs->end());
ASSERT_EQ(manifest->has_tail(), (obj_size > head_max_size));
}
+static void gen_old_obj(test_rgw_env& env, uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size,
+ OldObjManifest *manifest, old_rgw_bucket *bucket, old_rgw_obj *head,
+ list<old_rgw_obj> *test_objs)
+{
+ test_rgw_init_old_bucket(bucket, "buck");
+
+ *head = old_rgw_obj(*bucket, "obj");
+
+ OldObjManifestPart part;
+ part.loc = *head;
+ part.size = head_max_size;
+ part.loc_ofs = 0;
+
+ manifest->append(0, part);
+ test_objs->push_back(part.loc);
+
+ string prefix;
+ append_rand_alpha(g_ceph_context, prefix, prefix, 16);
+
+ int i = 0;
+ for (uint64_t ofs = head_max_size; ofs < obj_size; ofs += stripe_size, i++) {
+ char buf[32];
+ snprintf(buf, sizeof(buf), "%s.%d", prefix.c_str(), i);
+ old_rgw_obj loc(*bucket, buf);
+ loc.set_ns("shadow");
+ OldObjManifestPart part;
+ part.loc = loc;
+ part.size = min(stripe_size, obj_size - ofs);
+ part.loc_ofs = 0;
+
+ manifest->append(ofs, part);
+
+ test_objs->push_back(loc);
+ }
+}
+
TEST(TestRGWManifest, head_only_obj) {
+ test_rgw_env env;
RGWObjManifest manifest;
rgw_bucket bucket;
rgw_obj head;
list<rgw_obj> objs;
- gen_obj(obj_size, 512 * 1024, 4 * 1024 * 1024, &manifest, &bucket, &head, &gen, &objs);
+ gen_obj(env, obj_size, 512 * 1024, 4 * 1024 * 1024, &manifest, &bucket, &head, &gen, &objs);
cout << " manifest.get_obj_size()=" << manifest.get_obj_size() << std::endl;
cout << " manifest.get_head_size()=" << manifest.get_head_size() << std::endl;
for (iter = manifest.obj_begin(), liter = objs.begin();
iter != manifest.obj_end() && liter != objs.end();
++iter, ++liter) {
- ASSERT_TRUE(*liter == iter.get_location());
+ ASSERT_TRUE(env.get_raw(*liter) == env.get_raw(iter.get_location()));
}
ASSERT_TRUE(iter == manifest.obj_end());
ASSERT_TRUE(liter == objs.end());
+ rgw_raw_obj raw_head;
+
iter = manifest.obj_find(100 * 1024);
- ASSERT_TRUE(iter.get_location() == head);
+ ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(head));
ASSERT_EQ((int)iter.get_stripe_size(), obj_size);
}
TEST(TestRGWManifest, obj_with_head_and_tail) {
+ test_rgw_env env;
RGWObjManifest manifest;
rgw_bucket bucket;
rgw_obj head;
int stripe_size = 4 * 1024 * 1024;
int head_size = 512 * 1024;
- gen_obj(obj_size, head_size, stripe_size, &manifest, &bucket, &head, &gen, &objs);
+ gen_obj(env, obj_size, head_size, stripe_size, &manifest, &bucket, &head, &gen, &objs);
list<rgw_obj>::iterator liter;
- rgw_obj last_obj;
+ rgw_obj_select last_obj;
RGWObjManifest::obj_iterator iter;
for (iter = manifest.obj_begin(), liter = objs.begin();
iter != manifest.obj_end() && liter != objs.end();
++iter, ++liter) {
- cout << "*liter=" << *liter << " iter.get_location()=" << iter.get_location() << std::endl;
- ASSERT_TRUE(*liter == iter.get_location());
+ cout << "*liter=" << *liter << " iter.get_location()=" << env.get_raw(iter.get_location()) << std::endl;
+ ASSERT_TRUE(env.get_raw(*liter) == env.get_raw(iter.get_location()));
last_obj = iter.get_location();
}
ASSERT_TRUE(liter == objs.end());
iter = manifest.obj_find(100 * 1024);
- ASSERT_TRUE(iter.get_location() == head);
+ ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(head));
ASSERT_EQ((int)iter.get_stripe_size(), head_size);
uint64_t ofs = 20 * 1024 * 1024 + head_size;
iter = manifest.obj_find(ofs + 100);
- ASSERT_TRUE(iter.get_location() == last_obj);
+ ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(last_obj));
ASSERT_EQ(iter.get_stripe_ofs(), ofs);
ASSERT_EQ(iter.get_stripe_size(), obj_size - ofs);
}
TEST(TestRGWManifest, multipart) {
+ test_rgw_env env;
int num_parts = 16;
vector <RGWObjManifest> pm(num_parts);
rgw_bucket bucket;
RGWObjManifest m;
for (int i = 0; i < num_parts; i++) {
- m.append(pm[i]);
+ m.append(pm[i], env.zonegroup, env.zone_params);
}
RGWObjManifest::obj_iterator iter;
for (iter = m.obj_begin(); iter != m.obj_end(); ++iter) {
RGWObjManifest::obj_iterator fiter = m.obj_find(iter.get_ofs());
- ASSERT_TRUE(fiter.get_location() == iter.get_location());
+ ASSERT_TRUE(env.get_raw(fiter.get_location()) == env.get_raw(iter.get_location()));
}
ASSERT_EQ(m.get_obj_size(), num_parts * part_size);
}
+TEST(TestRGWManifest, old_obj_manifest) {
+ test_rgw_env env;
+ OldObjManifest old_manifest;
+ old_rgw_bucket old_bucket;
+ old_rgw_obj old_head;
+
+ int obj_size = 40 * 1024 * 1024;
+ uint64_t stripe_size = 4 * 1024 * 1024;
+ uint64_t head_size = 512 * 1024;
+
+ list<old_rgw_obj> old_objs;
+
+ gen_old_obj(env, obj_size, head_size, stripe_size, &old_manifest, &old_bucket, &old_head, &old_objs);
+
+ ASSERT_EQ(old_objs.size(), 11);
+
+
+ bufferlist bl;
+ ::encode(old_manifest , bl);
+
+ RGWObjManifest manifest;
+
+ try {
+ auto iter = bl.begin();
+ ::decode(manifest, iter);
+ } catch (buffer::error& err) {
+ ASSERT_TRUE(false);
+ }
+
+ rgw_raw_obj last_obj;
+
+ RGWObjManifest::obj_iterator iter;
+ auto liter = old_objs.begin();
+ for (iter = manifest.obj_begin();
+ iter != manifest.obj_end() && liter != old_objs.end();
+ ++iter, ++liter) {
+ rgw_pool old_pool(liter->bucket.data_pool);
+ string old_oid;
+ prepend_old_bucket_marker(old_bucket, liter->get_object(), old_oid);
+ rgw_raw_obj raw_old(old_pool, old_oid);
+ cout << "*liter=" << raw_old << " iter.get_location()=" << env.get_raw(iter.get_location()) << std::endl;
+ ASSERT_EQ(raw_old, env.get_raw(iter.get_location()));
+
+ last_obj = env.get_raw(iter.get_location());
+ }
+
+ ASSERT_TRUE(liter == old_objs.end());
+ ASSERT_TRUE(iter == manifest.obj_end());
+
+}
+
+
+int main(int argc, char **argv) {
+ vector<const char*> args;
+ argv_to_vec(argc, (const char **)argv, args);
+ env_to_vec(args);
+
+ vector<const char*> def_args;
+ global_init(&def_args, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
+
#include "common/ceph_json.h"
#include "common/Formatter.h"
#include "rgw/rgw_common.h"
+#include "rgw/rgw_rados.h"
+#include "test_rgw_common.h"
#define GTEST
#ifdef GTEST
#include <gtest/gtest.h>
#endif
using namespace std;
-static void populate_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id)
-{
- b->tenant = t;
- b->name = n;
- b->marker = m;
- b->bucket_id = id;
- b->placement.data_pool = rgw_pool(dp);
- b->placement.index_pool = rgw_pool(ip);
-}
-
-static void init_bucket(rgw_bucket *bucket, const char *name)
-{
- populate_bucket(bucket, "", name, ".data-pool", ".index-pool", "marker", "bucket-id");
-}
-
void check_parsed_correctly(rgw_obj& obj, const string& name, const string& ns, const string& instance)
{
/* parse_raw_oid() */
void test_obj(const string& name, const string& ns, const string& instance)
{
rgw_bucket b;
- init_bucket(&b, "test");
+ test_rgw_init_bucket(&b, "test");
JSONFormatter *formatter = new JSONFormatter(true);
test_obj("obj", "ns", "v1");
}
+template <class T>
+void dump(JSONFormatter& f, const string& name, const T& entity)
+{
+ f.open_object_section(name.c_str());
+ ::encode_json(name.c_str(), entity, &f);
+ f.close_section();
+ f.flush(cout);
+}
+
+static void test_obj_to_raw(test_rgw_env& env, const rgw_bucket& b,
+ const string& name, const string& instance, const string& ns,
+ const string& placement_id)
+{
+ JSONFormatter f(true);
+ dump(f, "bucket", b);
+ rgw_obj obj = test_rgw_create_obj(b, name, instance, ns, placement_id);
+ dump(f, "obj", obj);
+
+ rgw_raw_obj raw_obj;
+
+ rgw_obj_to_raw(env.zonegroup, env.zone_params, obj, &raw_obj);
+ dump(f, "raw_obj", raw_obj);
+
+ if (!placement_id.empty()) {
+ ASSERT_EQ(raw_obj.pool, env.get_placement(placement_id).data_pool);
+ } else {
+ ASSERT_EQ(raw_obj.pool, b.explicit_placement.data_pool);
+ }
+ ASSERT_EQ(raw_obj.oid, test_rgw_get_obj_oid(obj));
+
+ rgw_obj new_obj;
+ rgw_raw_obj_to_obj(b, raw_obj, &new_obj);
+
+ dump(f, "new_obj", new_obj);
+
+ ASSERT_EQ(obj, new_obj);
+
+}
+
+TEST(TestRGWObj, obj_to_raw) {
+ test_rgw_env env;
+
+ rgw_bucket b;
+ test_rgw_init_bucket(&b, "test");
+
+ rgw_bucket eb;
+ test_rgw_init_explicit_placement_bucket(&eb, "ebtest");
+
+ for (auto name : { "myobj", "_myobj", "_myobj_"}) {
+ for (auto inst : { "", "inst"}) {
+ for (auto ns : { "", "ns"}) {
+ test_obj_to_raw(env, b, name, inst, ns, env.zonegroup.default_placement);
+ test_obj_to_raw(env, eb, name, inst, ns, string());
+ }
+ }
+ }
+}
+
+TEST(TestRGWObj, old_to_raw) {
+ JSONFormatter f(true);
+ test_rgw_env env;
+
+ old_rgw_bucket eb;
+ test_rgw_init_old_bucket(&eb, "ebtest");
+
+ for (auto name : { "myobj", "_myobj", "_myobj_"}) {
+ for (string inst : { "", "inst"}) {
+ for (string ns : { "", "ns"}) {
+ old_rgw_obj old(eb, name);
+ if (!inst.empty()) {
+ old.set_instance(inst);
+ }
+ if (!ns.empty()) {
+ old.set_ns(ns);
+ }
+
+ bufferlist bl;
+
+ ::encode(old, bl);
+
+ rgw_obj new_obj;
+ rgw_raw_obj raw_obj;
+
+ try {
+ bufferlist::iterator iter = bl.begin();
+ ::decode(new_obj, iter);
+
+ iter = bl.begin();
+ ::decode(raw_obj, iter);
+ } catch (buffer::error& err) {
+ ASSERT_TRUE(false);
+ }
+
+ bl.clear();
+
+ rgw_obj new_obj2;
+ rgw_raw_obj raw_obj2;
+
+ ::encode(new_obj, bl);
+
+ dump(f, "raw_obj", raw_obj);
+ dump(f, "new_obj", new_obj);
+ cout << "raw=" << raw_obj << std::endl;
+
+ try {
+ bufferlist::iterator iter = bl.begin();
+ ::decode(new_obj2, iter);
+
+ iter = bl.begin();
+ ::decode(raw_obj2, iter);
+ } catch (buffer::error& err) {
+ ASSERT_TRUE(false);
+ }
+
+ ASSERT_EQ(new_obj, new_obj2);
+ ASSERT_EQ(raw_obj, raw_obj2);
+ }
+ }
+ }
+}
projects / ceph.git / commitdiff