From 9c347b920346f0799c0ea71f46515ce9b6b75247 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Mon, 20 Oct 2014 15:11:12 -0700 Subject: [PATCH] rgw-admin, cls_rgw: add bi_get objclass operation Signed-off-by: Yehuda Sadeh --- src/cls/rgw/cls_rgw.cc | 48 +++++++++++++++++++++ src/cls/rgw/cls_rgw_client.cc | 28 ++++++++++++ src/cls/rgw/cls_rgw_client.h | 4 ++ src/cls/rgw/cls_rgw_ops.h | 52 ++++++++++++++++++++++ src/cls/rgw/cls_rgw_types.cc | 8 ++++ src/cls/rgw/cls_rgw_types.h | 6 +++ src/rgw/rgw_admin.cc | 81 ++++++++++++++++++++++++++++++++++- src/rgw/rgw_rados.cc | 17 ++++++++ src/rgw/rgw_rados.h | 2 + 9 files changed, 245 insertions(+), 1 deletion(-) diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc index 901ecc5f9f4c1..7c042f409fc18 100644 --- a/src/cls/rgw/cls_rgw.cc +++ b/src/cls/rgw/cls_rgw.cc @@ -33,6 +33,7 @@ cls_method_handle_t h_rgw_bucket_unlink_instance_op; cls_method_handle_t h_rgw_bucket_read_olh_log; cls_method_handle_t h_rgw_bucket_trim_olh_log; cls_method_handle_t h_rgw_obj_remove; +cls_method_handle_t h_rgw_bi_get_op; cls_method_handle_t h_rgw_bi_log_list_op; cls_method_handle_t h_rgw_dir_suggest_changes; cls_method_handle_t h_rgw_user_usage_log_add; @@ -1677,6 +1678,51 @@ static int rgw_obj_remove(cls_method_context_t hctx, bufferlist *in, bufferlist return 0; } +static int rgw_bi_get_op(cls_method_context_t hctx, bufferlist *in, bufferlist *out) +{ + // decode request + rgw_cls_bi_get_op op; + bufferlist::iterator iter = in->begin(); + try { + ::decode(op, iter); + } catch (buffer::error& err) { + CLS_LOG(0, "ERROR: %s(): failed to decode request", __func__); + return -EINVAL; + } + + string idx; + + switch (op.type) { + case PlainIdx: + idx = op.key.name; + break; + case InstanceIdx: + encode_obj_index_key(op.key, &idx); + break; + case OLHIdx: + encode_olh_data_key(op.key, &idx); + break; + default: + CLS_LOG(10, "%s(): invalid key type encoding: %d", __func__, op.type); + return -EINVAL; + } + + rgw_cls_bi_get_ret op_ret; + + op_ret.type = op.type; + op_ret.idx = idx; + + int r = cls_cxx_map_get_val(hctx, idx, &op_ret.data); + if (r < 0) { + CLS_LOG(10, "%s(): cls_cxx_map_get_val() returned %d", __func__, r); + return r; + } + + ::encode(op_ret, *out); + + return 0; +} + int bi_log_record_decode(bufferlist& bl, rgw_bi_log_entry& e) { bufferlist::iterator iter = bl.begin(); @@ -2499,6 +2545,8 @@ void __cls_init() cls_register_cxx_method(h_class, "obj_remove", CLS_METHOD_RD | CLS_METHOD_WR, rgw_obj_remove, &h_rgw_obj_remove); + cls_register_cxx_method(h_class, "bi_get", CLS_METHOD_RD, rgw_bi_get_op, &h_rgw_bi_get_op); + cls_register_cxx_method(h_class, "bi_log_list", CLS_METHOD_RD, rgw_bi_log_list, &h_rgw_bi_log_list_op); cls_register_cxx_method(h_class, "bi_log_trim", CLS_METHOD_RD | CLS_METHOD_WR, rgw_bi_log_trim, &h_rgw_bi_log_list_op); cls_register_cxx_method(h_class, "dir_suggest_changes", CLS_METHOD_RD | CLS_METHOD_WR, rgw_dir_suggest_changes, &h_rgw_dir_suggest_changes); diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc index 9d337d1bbb13f..d9eed8c19b60c 100644 --- a/src/cls/rgw/cls_rgw_client.cc +++ b/src/cls/rgw/cls_rgw_client.cc @@ -103,6 +103,34 @@ void cls_rgw_remove_obj(librados::ObjectWriteOperation& o, list& keep_at o.exec("rgw", "obj_remove", in); } +int cls_rgw_bi_get(librados::IoCtx& io_ctx, const string oid, + BIIndexType index_type, cls_rgw_obj_key& key, + string *idx, + bufferlist *data) +{ + bufferlist in, out; + struct rgw_cls_bi_get_op call; + call.key = key; + call.type = index_type; + ::encode(call, in); + int r = io_ctx.exec(oid, "rgw", "bi_get", in, out); + if (r < 0) + return r; + + struct rgw_cls_bi_get_ret op_ret; + bufferlist::iterator iter = out.begin(); + try { + ::decode(op_ret, iter); + } catch (buffer::error& err) { + return -EIO; + } + + *idx = op_ret.idx; + data->swap(op_ret.data); + + return 0; +} + int cls_rgw_bucket_link_olh(librados::IoCtx& io_ctx, const string& oid, const cls_rgw_obj_key& key, bool delete_marker, const string& op_tag) { diff --git a/src/cls/rgw/cls_rgw_client.h b/src/cls/rgw/cls_rgw_client.h index 6f80e6280d842..771e5e5d3f194 100644 --- a/src/cls/rgw/cls_rgw_client.h +++ b/src/cls/rgw/cls_rgw_client.h @@ -33,6 +33,10 @@ int cls_rgw_list_op(librados::IoCtx& io_ctx, const string& oid, void cls_rgw_remove_obj(librados::ObjectWriteOperation& o, list& keep_attr_prefixes); +int cls_rgw_bi_get(librados::IoCtx& io_ctx, const string oid, + BIIndexType index_type, cls_rgw_obj_key& key, + string *idx, bufferlist *data); + int cls_rgw_bucket_link_olh(librados::IoCtx& io_ctx, const string& oid, const cls_rgw_obj_key& key, bool delete_marker, const string& op_tag); int cls_rgw_bucket_unlink_instance(librados::IoCtx& io_ctx, const string& oid, const cls_rgw_obj_key& key, const string& op_tag); diff --git a/src/cls/rgw/cls_rgw_ops.h b/src/cls/rgw/cls_rgw_ops.h index a80b499db6d1c..308ab6cc5fceb 100644 --- a/src/cls/rgw/cls_rgw_ops.h +++ b/src/cls/rgw/cls_rgw_ops.h @@ -389,6 +389,58 @@ struct rgw_cls_usage_log_add_op { }; WRITE_CLASS_ENCODER(rgw_cls_usage_log_add_op) +struct rgw_cls_bi_get_op { + cls_rgw_obj_key key; + BIIndexType type; /* namespace: plain, instance, olh */ + + rgw_cls_bi_get_op() : type(PlainIdx) {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(key, bl); + ::encode((uint8_t)type, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(key, bl); + uint8_t c; + ::decode(c, bl); + type = (BIIndexType)c; + DECODE_FINISH(bl); + } +}; +WRITE_CLASS_ENCODER(rgw_cls_bi_get_op) + +struct rgw_cls_bi_get_ret { + BIIndexType type; + string idx; + bufferlist data; + + rgw_cls_bi_get_ret() : type(PlainIdx) {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode((uint8_t)type, bl); + ::encode(idx, bl); + ::encode(data, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + uint8_t c; + ::decode(c, bl); + type = (BIIndexType)c; + ::decode(idx, bl); + ::decode(data, bl); + DECODE_FINISH(bl); + } +}; +WRITE_CLASS_ENCODER(rgw_cls_bi_get_ret) + + struct rgw_cls_usage_log_read_op { uint64_t start_epoch; uint64_t end_epoch; diff --git a/src/cls/rgw/cls_rgw_types.cc b/src/cls/rgw/cls_rgw_types.cc index 10d1dfdb5ec17..df1296337a825 100644 --- a/src/cls/rgw/cls_rgw_types.cc +++ b/src/cls/rgw/cls_rgw_types.cc @@ -109,6 +109,14 @@ void rgw_bucket_dir_entry::dump(Formatter *f) const f->close_section(); } +void rgw_bucket_olh_entry::dump(Formatter *f) const +{ + encode_json("key", key, f); + encode_json("delete_marker", delete_marker, f); + encode_json("epoch", epoch, f); + encode_json("pending_log", pending_log, f); +} + void rgw_bucket_olh_log_entry::dump(Formatter *f) const { encode_json("epoch", epoch, f); diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h index 17f214f8788e4..00676d894bfaf 100644 --- a/src/cls/rgw/cls_rgw_types.h +++ b/src/cls/rgw/cls_rgw_types.h @@ -322,6 +322,12 @@ struct rgw_bucket_dir_entry { }; WRITE_CLASS_ENCODER(rgw_bucket_dir_entry) +enum BIIndexType { + InvalidIdx = 0, + PlainIdx = 1, + InstanceIdx = 2, + OLHIdx = 3, +}; enum OLHLogOp { CLS_RGW_OLH_OP_LINK_OLH = 1, diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index aeff06f826378..0062ea232d432 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -221,6 +221,7 @@ enum { OPT_OBJECT_UNLINK, OPT_OBJECT_STAT, OPT_OBJECT_REWRITE, + OPT_BI_GET, OPT_OLH_GET, OPT_OLH_READLOG, OPT_QUOTA_SET, @@ -262,7 +263,8 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more) { *need_more = false; // NOTE: please keep the checks in alphabetical order !!! - if (strcmp(cmd, "bilog") == 0 || + if (strcmp(cmd, "bi") == 0 || + strcmp(cmd, "bilog") == 0 || strcmp(cmd, "bucket") == 0 || strcmp(cmd, "buckets") == 0 || strcmp(cmd, "caps") == 0 || @@ -386,6 +388,9 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more) return OPT_OLH_GET; if (strcmp(cmd, "readlog") == 0) return OPT_OLH_READLOG; + } else if (strcmp(prev_cmd, "bi") == 0) { + if (strcmp(cmd, "get") == 0) + return OPT_BI_GET; } else if (strcmp(prev_cmd, "region") == 0) { if (strcmp(cmd, "get") == 0) return OPT_REGION_GET; @@ -489,6 +494,42 @@ ReplicaLogType get_replicalog_type(const string& name) { return ReplicaLog_Invalid; } +BIIndexType get_bi_index_type(const string& type_str) { + if (type_str == "plain") + return PlainIdx; + if (type_str == "instance") + return InstanceIdx; + if (type_str == "olh") + return OLHIdx; + + return InvalidIdx; +} + +void dump_bi_entry(bufferlist& bl, BIIndexType index_type, Formatter *formatter) +{ + bufferlist::iterator iter = bl.begin(); + switch (index_type) { + case PlainIdx: + case InstanceIdx: + { + rgw_bucket_dir_entry entry; + ::decode(entry, iter); + encode_json("entry", entry, formatter); + } + break; + case OLHIdx: + { + rgw_bucket_olh_entry entry; + ::decode(entry, iter); + encode_json("entry", entry, formatter); + } + break; + default: + assert(0); + break; + } +} + static void show_user_info(RGWUserInfo& info, Formatter *formatter) { encode_json("user_info", info, formatter); @@ -901,10 +942,13 @@ int main(int argc, char **argv) uint64_t max_rewrite_size = ULLONG_MAX; uint64_t min_rewrite_stripe_size = 0; + BIIndexType bi_index_type = PlainIdx; + std::string val; std::ostringstream errs; string err; long long tmp = 0; + for (std::vector::iterator i = args.begin(); i != args.end(); ) { if (ceph_argparse_double_dash(args, i)) { break; @@ -1069,6 +1113,13 @@ int main(int argc, char **argv) cerr << "ERROR: invalid replica log type" << std::endl; return EINVAL; } + } else if (ceph_argparse_witharg(args, i, &val, "--index-type", (char*)NULL)) { + string index_type_str = val; + bi_index_type = get_bi_index_type(index_type_str); + if (bi_index_type == InvalidIdx) { + cerr << "ERROR: invalid bucket index entry type" << std::endl; + return EINVAL; + } } else if (strncmp(*i, "-", 1) == 0) { cerr << "ERROR: invalid flag " << *i << std::endl; return EINVAL; @@ -1931,6 +1982,34 @@ next: formatter->flush(cout); } + if (opt_cmd == OPT_BI_GET) { + RGWBucketInfo bucket_info; + int ret = init_bucket(bucket_name, bucket_info, bucket); + if (ret < 0) { + cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl; + return -ret; + } + rgw_obj obj(bucket, object); + if (!object_version.empty()) { + obj.set_instance(object_version); + } + + string idx; + bufferlist bl; + + ret = store->bi_get(bucket, obj, bi_index_type, &idx, &bl); + if (ret < 0) { + cerr << "ERROR: bi_get(): " << cpp_strerror(-ret) << std::endl; + return -ret; + } + + formatter->open_object_section("result"); + encode_json("idx", idx, formatter); + dump_bi_entry(bl, bi_index_type, formatter); + formatter->close_section(); + formatter->flush(cout); + } + if (opt_cmd == OPT_OBJECT_RM) { RGWBucketInfo bucket_info; int ret = init_bucket(bucket_name, bucket_info, bucket); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 77fc3a76cf0c1..d223452f309bc 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -6573,6 +6573,23 @@ int RGWRados::trim_bi_log_entries(rgw_bucket& bucket, string& start_marker, stri return 0; } +int RGWRados::bi_get(rgw_bucket& bucket, rgw_obj& obj, BIIndexType index_type, string *idx, bufferlist *data) +{ + librados::IoCtx index_ctx; + string oid; + int r = open_bucket_index(bucket, index_ctx, oid); + if (r < 0) + return r; + + cls_rgw_obj_key key(obj.get_index_key_name(), obj.get_instance()); + + int ret = cls_rgw_bi_get(index_ctx, oid, index_type, key, idx, data); + if (ret < 0) + return ret; + + return 0; +} + int RGWRados::gc_operate(string& oid, librados::ObjectWriteOperation *op) { return gc_pool_ctx.operate(oid, op); diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 8850a0563b073..ab3d2478c16b3 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1768,6 +1768,8 @@ public: int list_bi_log_entries(rgw_bucket& bucket, string& marker, uint32_t max, std::list& result, bool *truncated); int trim_bi_log_entries(rgw_bucket& bucket, string& marker, string& end_marker); + int bi_get(rgw_bucket& bucket, rgw_obj& obj, BIIndexType index_type, string *idx, bufferlist *data); + int cls_obj_usage_log_add(const string& oid, rgw_usage_log_info& info); int cls_obj_usage_log_read(string& oid, string& user, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries, string& read_iter, map& usage, bool *is_truncated); -- 2.39.5