From: Orit Wasserman Date: Tue, 28 Feb 2017 14:09:51 +0000 (+0200) Subject: rgw: add cls_rgw_reshard ops X-Git-Tag: ses5-milestone6~8^2~7^2~96 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=450ea1c4cfbc8851607edc69958b847708d7eb68;p=ceph.git rgw: add cls_rgw_reshard ops Signed-off-by: Orit Wasserman --- diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc index 9fe389953540..0f2fcd1da5a0 100644 --- a/src/cls/rgw/cls_rgw.cc +++ b/src/cls/rgw/cls_rgw.cc @@ -3480,6 +3480,141 @@ static int rgw_cls_lc_get_head(cls_method_context_t hctx, bufferlist *in, buffe return 0; } +static int rgw_reshard_add(cls_method_context_t hctx, bufferlist *in, bufferlist *out) +{ + bufferlist::iterator in_iter = in->begin(); + + cls_rgw_reshard_add_op op; + try { + ::decode(op, in_iter); + } catch (buffer::error& err) { + CLS_LOG(1, "ERROR: rgw_reshard_add: failed to decode entry\n"); + return -EINVAL; + } + + bufferlist bl; + ::encode(op.entry, bl); + + string key = op.entry.bucket_name + "." + op.entry.bucket_id; + int ret = cls_cxx_map_set_val(hctx, key, &bl); + if (ret < 0) { + CLS_ERR("error adding reshard job for bucket %s with key %s",op.entry.bucket_name.c_str(), key.c_str()); + return ret; + } + + return ret; +} + +static int rgw_reshard_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) +{ + cls_rgw_reshard_list_op op; + bufferlist::iterator in_iter = in->begin(); + try { + ::decode(op, in_iter); + } catch (buffer::error& err) { + CLS_LOG(1, "ERROR: rgw_cls_rehard_list(): failed to decode entry\n"); + return -EINVAL; + } + cls_rgw_reshard_list_ret op_ret; + bufferlist::iterator iter; + map vals; + string filter_prefix; + int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, op.max, &vals); + if (ret < 0) + return ret; + map::iterator it; + cls_rgw_reshard_entry entry; + for (it = vals.begin(); it != vals.end(); ++it) { + iter = it->second.begin(); + try { + ::decode(entry, iter); + } catch (buffer::error& err) { + CLS_LOG(1, "ERROR: rgw_cls_rehard_list(): failed to decode entry\n"); + return -EIO; + } + op_ret.entries.push_back(entry); + } + op_ret.is_truncated = op.max && (op_ret.entries.size() >= op.max); + ::encode(op_ret, *out); + return 0; +} + + +static int rgw_reshard_get_head(cls_method_context_t hctx, bufferlist *in, bufferlist *out) +{ + bufferlist bl; + int ret = cls_cxx_map_read_header(hctx, &bl); + if (ret < 0) + return ret; + cls_rgw_reshard_entry entry; + if (bl.length() != 0) { + bufferlist::iterator iter = bl.begin(); + try { + ::decode(entry, iter); + } catch (buffer::error& err) { + CLS_LOG(0, "ERROR: rgw_reshard_get_head(): failed to decode entry %s\n",err.what()); + return -EINVAL; + } + } + + cls_rgw_reshard_get_head_ret op_ret; + op_ret.entry = entry; + ::encode(op_ret, *out); + return 0; +} + +static int rgw_reshard_get(cls_method_context_t hctx, bufferlist *in, bufferlist *out) +{ + bufferlist::iterator in_iter = in->begin(); + + cls_rgw_reshard_get_op op; + try { + ::decode(op, in_iter); + } catch (buffer::error& err) { + CLS_LOG(1, "ERROR: rgw_reshard_get: failed to decode entry\n"); + return -EINVAL; + } + + bufferlist bl; + string key = op.entry.bucket_name + "." + op.entry.bucket_id; + int ret = cls_cxx_map_get_val(hctx, key, &bl); + if (ret < 0) + return ret; + cls_rgw_reshard_entry entry; + if (bl.length() != 0) { + bufferlist::iterator iter = bl.begin(); + try { + ::decode(entry, iter); + } catch (buffer::error& err) { + CLS_LOG(0, "ERROR: rgw_cls_reshard_get_head(): failed to decode entry %s\n",err.what()); + return -EINVAL; + } + } + + cls_rgw_reshard_get_ret op_ret; + op_ret.entry = entry; + ::encode(op_ret, *out); + return 0; +} + +static int rgw_reshard_remove(cls_method_context_t hctx, bufferlist *in, bufferlist *out) +{ + bufferlist::iterator in_iter = in->begin(); + + cls_rgw_reshard_remove_op op; + try { + ::decode(op, in_iter); + } catch (buffer::error& err) { + CLS_LOG(1, "ERROR: rgw_cls_rehard_remove: failed to decode entry\n"); + return -EINVAL; + } + + string key = op.bucket_name + "." + op.bucket_id; + int ret = cls_cxx_map_remove_key(hctx, key); + return ret; +} + + CLS_INIT(rgw) { CLS_LOG(1, "Loaded rgw class!"); @@ -3519,6 +3654,11 @@ CLS_INIT(rgw) cls_method_handle_t h_rgw_lc_put_head; cls_method_handle_t h_rgw_lc_get_head; cls_method_handle_t h_rgw_lc_list_entries; + cls_method_handle_t h_rgw_reshard_add; + cls_method_handle_t h_rgw_reshard_list; + cls_method_handle_t h_rgw_reshard_get; + cls_method_handle_t h_rgw_reshard_get_head; + cls_method_handle_t h_rgw_reshard_remove; cls_register(RGW_CLASS, &h_class); @@ -3569,7 +3709,11 @@ CLS_INIT(rgw) cls_register_cxx_method(h_class, RGW_LC_PUT_HEAD, CLS_METHOD_RD| CLS_METHOD_WR, rgw_cls_lc_put_head, &h_rgw_lc_put_head); cls_register_cxx_method(h_class, RGW_LC_GET_HEAD, CLS_METHOD_RD, rgw_cls_lc_get_head, &h_rgw_lc_get_head); cls_register_cxx_method(h_class, RGW_LC_LIST_ENTRIES, CLS_METHOD_RD, rgw_cls_lc_list_entries, &h_rgw_lc_list_entries); - + cls_register_cxx_method(h_class, "reshard_add", CLS_METHOD_RD | CLS_METHOD_WR, rgw_reshard_add, &h_rgw_reshard_add); + cls_register_cxx_method(h_class, "reshard_list", CLS_METHOD_RD, rgw_reshard_list, &h_rgw_reshard_list); + cls_register_cxx_method(h_class, "reshard_get", CLS_METHOD_RD,rgw_reshard_get, &h_rgw_reshard_get); + cls_register_cxx_method(h_class, "reshard_get_head", CLS_METHOD_RD, rgw_reshard_get_head, &h_rgw_reshard_get_head); + cls_register_cxx_method(h_class, "reshard_remove", CLS_METHOD_RD | CLS_METHOD_WR, rgw_reshard_remove, &h_rgw_reshard_remove); return; } diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc index d3dbe89be602..3173cacdd7c3 100644 --- a/src/cls/rgw/cls_rgw_client.cc +++ b/src/cls/rgw/cls_rgw_client.cc @@ -778,3 +778,91 @@ int cls_rgw_lc_list(IoCtx& io_ctx, string& oid, return r; } + +void cls_rgw_reshard_add(librados::ObjectWriteOperation& op, const cls_rgw_reshard_entry& entry) +{ + bufferlist in; + struct cls_rgw_reshard_add_op call; + call.entry = entry; + ::encode(call, in); + op.exec("rgw", "reshard_add", in); +} + +int cls_rgw_reshard_list(librados::IoCtx& io_ctx, const string& oid, string& marker, uint32_t max, + list& entries, bool& is_truncated) +{ + bufferlist in, out; + struct cls_rgw_reshard_list_op call; + call.marker = marker; + call.max = max; + ::encode(call, in); + int r = io_ctx.exec(oid, "rgw", "reshard_list", in, out); + if (r < 0) + return r; + + struct cls_rgw_reshard_list_ret op_ret; + bufferlist::iterator iter = out.begin(); + try { + ::decode(op_ret, iter); + } catch (buffer::error& err) { + return -EIO; + } + + entries.swap(op_ret.entries); + is_truncated = op_ret.is_truncated; + + return 0; +} + +int cls_rgw_reshard_get(librados::IoCtx& io_ctx, const string& oid, cls_rgw_reshard_entry& entry) +{ + bufferlist in, out; + struct cls_rgw_reshard_get_op call; + call.entry = entry; + ::encode(call, in); + int r = io_ctx.exec(oid, "rgw", "reshard_get", in, out); + if (r < 0) + return r; + + struct cls_rgw_reshard_get_ret op_ret; + bufferlist::iterator iter = out.begin(); + try { + ::decode(op_ret, iter); + } catch (buffer::error& err) { + return -EIO; + } + + entry = op_ret.entry; + + return 0; +} + +int cls_rgw_reshard_get_head(librados::IoCtx& io_ctx, const string& oid, cls_rgw_reshard_entry& entry) +{ + bufferlist in, out; + struct cls_rgw_reshard_get_head_op call; + ::encode(call, in); + int r = io_ctx.exec(oid, "rgw", "reshard_get_head", in, out); + if (r < 0) + return r; + + struct cls_rgw_reshard_get_head_ret op_ret; + bufferlist::iterator iter = out.begin(); + try { + ::decode(op_ret, iter); + } catch (buffer::error& err) { + return -EIO; + } + + entry = op_ret.entry; + + return 0; +} + +void cls_rgw_reshard_remove(librados::ObjectWriteOperation& op, const cls_rgw_reshard_entry& entry) +{ + bufferlist in; + struct cls_rgw_reshard_remove_op call; + ::encode(call, in); + op.exec("rgw", "reshard_remove", in); +} diff --git a/src/cls/rgw/cls_rgw_client.h b/src/cls/rgw/cls_rgw_client.h index 2e5bf304697d..497f5df91951 100644 --- a/src/cls/rgw/cls_rgw_client.h +++ b/src/cls/rgw/cls_rgw_client.h @@ -493,9 +493,12 @@ int cls_rgw_lc_list(librados::IoCtx& io_ctx, string& oid, uint32_t max_entries, map& entries); - - - - +/* resharding */ +void cls_rgw_reshard_add(librados::ObjectWriteOperation& op, const cls_rgw_reshard_entry& entry); +int cls_rgw_reshard_list(librados::IoCtx& io_ctx, const string& oid, string& marker, uint32_t max, + list& entries, bool& is_truncated); +int cls_rgw_reshard_get(librados::IoCtx& io_ctx, const string& oid, cls_rgw_reshard_entry& entry); +int cls_rgw_reshard_get_head(librados::IoCtx& io_ctx, const string& oid, cls_rgw_reshard_entry& entry); +void cls_rgw_reshard_remove(librados::ObjectWriteOperation& op, const cls_rgw_reshard_entry& entry); #endif diff --git a/src/cls/rgw/cls_rgw_ops.cc b/src/cls/rgw/cls_rgw_ops.cc index ad2bcdfe45c4..1c547e04a503 100644 --- a/src/cls/rgw/cls_rgw_ops.cc +++ b/src/cls/rgw/cls_rgw_ops.cc @@ -412,3 +412,107 @@ void cls_rgw_bi_log_list_ret::generate_test_instances(listentries.push_back(rgw_bi_log_entry()); ls.back()->truncated = true; } + +void cls_rgw_reshard_add_op::generate_test_instances(list& ls) +{ + ls.push_back(new cls_rgw_reshard_add_op); + ls.push_back(new cls_rgw_reshard_add_op); + list l; + cls_rgw_reshard_entry::generate_test_instances(l); + list::iterator iter = l.begin(); + ls.back()->entry = *(*iter); +} + +void cls_rgw_reshard_add_op::dump(Formatter *f) const +{ + ::encode_json("entry", entry, f); +} + +void cls_rgw_reshard_list_op::generate_test_instances(list& ls) +{ + ls.push_back(new cls_rgw_reshard_list_op); + ls.push_back(new cls_rgw_reshard_list_op); + ls.back()->max = 1000; + ls.back()->marker = "foo"; +} + +void cls_rgw_reshard_list_op::dump(Formatter *f) const +{ + ::encode_json("max", max, f); + ::encode_json("marker", marker, f); +} + +void cls_rgw_reshard_list_ret::generate_test_instances(list& ls) +{ + ls.push_back(new cls_rgw_reshard_list_ret); + ls.push_back(new cls_rgw_reshard_list_ret); + ls.back()->entries.push_back(cls_rgw_reshard_entry()); + ls.back()->is_truncated = true; +} + +void cls_rgw_reshard_list_ret::dump(Formatter *f) const +{ + ::encode_json("entries", entries, f); + ::encode_json("is_truncated", is_truncated, f); +} + +void cls_rgw_reshard_get_op::generate_test_instances(list& ls) +{ + ls.push_back(new cls_rgw_reshard_get_op); + ls.push_back(new cls_rgw_reshard_get_op); +} + +void cls_rgw_reshard_get_op::dump(Formatter *f) const +{ + ::encode_json("entry", entry, f); +} + +void cls_rgw_reshard_get_ret::generate_test_instances(list& ls) +{ + ls.push_back(new cls_rgw_reshard_get_ret); + ls.push_back(new cls_rgw_reshard_get_ret); +} + +void cls_rgw_reshard_get_ret::dump(Formatter *f) const +{ + ::encode_json("entry", entry, f); +} + +void cls_rgw_reshard_get_head_op::generate_test_instances(list& ls) +{ + ls.push_back(new cls_rgw_reshard_get_head_op); + ls.push_back(new cls_rgw_reshard_get_head_op); +} + +void cls_rgw_reshard_get_head_op::dump(Formatter *f) const +{ +} + +void cls_rgw_reshard_get_head_ret::generate_test_instances(list& ls) +{ + ls.push_back(new cls_rgw_reshard_get_head_ret); + ls.push_back(new cls_rgw_reshard_get_head_ret); +} + +void cls_rgw_reshard_get_head_ret::dump(Formatter *f) const +{ + ::encode_json("entry", entry, f); +} + +void cls_rgw_reshard_remove_op::generate_test_instances(list& ls) +{ + ls.push_back(new cls_rgw_reshard_remove_op); + ls.push_back(new cls_rgw_reshard_remove_op); + ls.back()->bucket_name = "foo"; + ls.back()->bucket_id = "bucket_id"; +} + +void cls_rgw_reshard_remove_op::dump(Formatter *f) const +{ + ::encode_json("bucket_name", bucket_name, f); + ::encode_json("bucket_id", bucket_name, f); +} + + + + diff --git a/src/cls/rgw/cls_rgw_ops.h b/src/cls/rgw/cls_rgw_ops.h index 54517454d967..7a034a1ae3e3 100644 --- a/src/cls/rgw/cls_rgw_ops.h +++ b/src/cls/rgw/cls_rgw_ops.h @@ -1145,4 +1145,179 @@ struct cls_rgw_lc_list_entries_ret { }; WRITE_CLASS_ENCODER(cls_rgw_lc_list_entries_ret) +struct cls_rgw_reshard_add_op { + cls_rgw_reshard_entry entry; + + cls_rgw_reshard_add_op() {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(entry, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(entry, bl); + DECODE_FINISH(bl); + } + static void generate_test_instances(list& o); + void dump(Formatter *f) const; +}; +WRITE_CLASS_ENCODER(cls_rgw_reshard_add_op) + +struct cls_rgw_reshard_list_op { + uint32_t max; + string marker; + + cls_rgw_reshard_list_op() {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(max, bl); + ::encode(marker, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(max, bl); + ::decode(marker, bl); + DECODE_FINISH(bl); + } + static void generate_test_instances(list& o); + void dump(Formatter *f) const; +}; +WRITE_CLASS_ENCODER(cls_rgw_reshard_list_op) + + +struct cls_rgw_reshard_list_ret { + list entries; + bool is_truncated; + + cls_rgw_reshard_list_ret() {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(entries, bl); + ::encode(is_truncated, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(entries, bl); + ::decode(is_truncated, bl); + DECODE_FINISH(bl); + } + static void generate_test_instances(list& o); + void dump(Formatter *f) const; +}; +WRITE_CLASS_ENCODER(cls_rgw_reshard_list_ret) + +struct cls_rgw_reshard_get_op { + cls_rgw_reshard_entry entry; + + cls_rgw_reshard_get_op() {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(entry, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(entry, bl); + DECODE_FINISH(bl); + } + static void generate_test_instances(list& o); + void dump(Formatter *f) const; +}; +WRITE_CLASS_ENCODER(cls_rgw_reshard_get_op) + +struct cls_rgw_reshard_get_ret { + cls_rgw_reshard_entry entry; + + cls_rgw_reshard_get_ret() {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(entry, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(entry, bl); + DECODE_FINISH(bl); + } + static void generate_test_instances(list& o); + void dump(Formatter *f) const; +}; +WRITE_CLASS_ENCODER(cls_rgw_reshard_get_ret) + +struct cls_rgw_reshard_get_head_op { + + cls_rgw_reshard_get_head_op() {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + DECODE_FINISH(bl); + } + static void generate_test_instances(list& o); + void dump(Formatter *f) const; +}; +WRITE_CLASS_ENCODER(cls_rgw_reshard_get_head_op) + +struct cls_rgw_reshard_get_head_ret { + cls_rgw_reshard_entry entry; + + cls_rgw_reshard_get_head_ret() {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(entry, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(entry, bl); + DECODE_FINISH(bl); + } + static void generate_test_instances(list& o); + void dump(Formatter *f) const; +}; +WRITE_CLASS_ENCODER(cls_rgw_reshard_get_head_ret) + +struct cls_rgw_reshard_remove_op { + string bucket_name; + string bucket_id; + + cls_rgw_reshard_remove_op() {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(bucket_name, bl); + ::encode(bucket_id, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(bucket_name, bl); + ::decode(bucket_id, bl); + DECODE_FINISH(bl); + } + static void generate_test_instances(list& o); + void dump(Formatter *f) const; +}; +WRITE_CLASS_ENCODER(cls_rgw_reshard_remove_op) + #endif /* CEPH_CLS_RGW_OPS_H */ diff --git a/src/cls/rgw/cls_rgw_types.cc b/src/cls/rgw/cls_rgw_types.cc index 2247f36847bf..5be8b1e326e1 100644 --- a/src/cls/rgw/cls_rgw_types.cc +++ b/src/cls/rgw/cls_rgw_types.cc @@ -565,3 +565,30 @@ void rgw_bucket_dir::dump(Formatter *f) const f->close_section(); } +void cls_rgw_reshard_entry::dump(Formatter *f) const +{ + utime_t ut(time); + encode_json("time",ut, f); + encode_json("tenant", tenant, f); + encode_json("bucket_name", bucket_name, f); + encode_json("bucket_id", bucket_id, f); + encode_json("old_instance_id", old_instance_id, f); + encode_json("new_instance_id", new_instance_id, f); + encode_json("old_num_shards", old_num_shards, f); + encode_json("new_num_shards", new_num_shards, f); + +} + +void cls_rgw_reshard_entry::generate_test_instances(list& ls) +{ + ls.push_back(new cls_rgw_reshard_entry); + ls.push_back(new cls_rgw_reshard_entry); + ls.back()->time = ceph::real_clock::from_ceph_timespec({2, 3}); + ls.back()->tenant = "tenant"; + ls.back()->bucket_name = "bucket1"""; + ls.back()->bucket_id = "bucket_id"; + ls.back()->old_instance_id = "old_instance_id"; + ls.back()->new_instance_id = "new_instance_id"; + ls.back()->old_num_shards = 8; + ls.back()->new_num_shards = 64; +} diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h index f4e5ac83a001..3df336ec99d4 100644 --- a/src/cls/rgw/cls_rgw_types.h +++ b/src/cls/rgw/cls_rgw_types.h @@ -1000,4 +1000,48 @@ struct cls_rgw_lc_obj_head }; WRITE_CLASS_ENCODER(cls_rgw_lc_obj_head) +struct cls_rgw_reshard_entry +{ + ceph::real_time time; + string tenant; + string bucket_name; + string bucket_id; + string old_instance_id; + string new_instance_id; + uint32_t old_num_shards; + uint32_t new_num_shards; + + cls_rgw_reshard_entry() {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(time, bl); + ::encode(tenant, bl); + ::encode(bucket_name, bl); + ::encode(bucket_id, bl); + ::encode(old_instance_id, bl); + ::encode(new_instance_id, bl); + ::encode(old_num_shards, bl); + ::encode(new_num_shards, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(time, bl); + ::decode(tenant, bl); + ::decode(bucket_name, bl); + ::decode(bucket_id, bl); + ::decode(old_instance_id, bl); + ::decode(new_instance_id, bl); + ::decode(old_num_shards, bl); + ::decode(new_num_shards, bl); + DECODE_FINISH(bl); + } + + void dump(Formatter *f) const; + static void generate_test_instances(list& o); +}; +WRITE_CLASS_ENCODER(cls_rgw_reshard_entry) + #endif diff --git a/src/test/encoding/types.h b/src/test/encoding/types.h index 77df9f0d7e64..c39deff889d4 100644 --- a/src/test/encoding/types.h +++ b/src/test/encoding/types.h @@ -336,9 +336,18 @@ TYPE(rgw_cls_read_olh_log_ret) TYPE(rgw_cls_trim_olh_log_op) TYPE(rgw_cls_bucket_clear_olh_op) TYPE(rgw_cls_check_index_ret) +TYPE(cls_rgw_reshard_add_op) +TYPE(cls_rgw_reshard_list_op) +TYPE(cls_rgw_reshard_list_ret) +TYPE(cls_rgw_reshard_get_op) +TYPE(cls_rgw_reshard_get_ret) +TYPE(cls_rgw_reshard_get_head_op) +TYPE(cls_rgw_reshard_get_head_ret) +TYPE(cls_rgw_reshard_remove_op) #include "cls/rgw/cls_rgw_client.h" TYPE(rgw_bi_log_entry) +TYPE(cls_rgw_reshard_entry) #include "cls/user/cls_user_types.h" TYPE(cls_user_bucket)