From b665b04672e4be1cd0d789b8fbbfdfea5e1ab11d Mon Sep 17 00:00:00 2001 From: Radoslaw Zarzynski Date: Thu, 22 Oct 2015 19:00:21 +0200 Subject: [PATCH] rgw: add support for Bulk Delete API of Swift. Signed-off-by: Radoslaw Zarzynski --- src/rgw/rgw_op.cc | 145 ++++++++++++++++++++++++++++++++++++++ src/rgw/rgw_op.h | 80 +++++++++++++++++++++ src/rgw/rgw_rest.cc | 58 ++++++++++----- src/rgw/rgw_rest.h | 11 ++- src/rgw/rgw_rest_swift.cc | 76 ++++++++++++++++++++ src/rgw/rgw_rest_swift.h | 11 +++ 6 files changed, 364 insertions(+), 17 deletions(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index f5f49848ec87..397f839f2f09 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3772,6 +3772,151 @@ error: } +bool RGWBulkDelete::Deleter::verify_permission(RGWBucketInfo& binfo, + map& battrs, + rgw_obj& obj, + ACLOwner& bucket_owner /* out */) +{ + int ret = 0; + + RGWAccessControlPolicy bacl(store->ctx()); + rgw_obj_key no_obj; + ret = read_policy(store, s, binfo, battrs, &bacl, binfo.bucket, no_obj); + if (ret < 0) { + return false; + } + + RGWAccessControlPolicy oacl(s->cct); + ret = read_policy(store, s, binfo, battrs, &oacl, binfo.bucket, s->object); + if (ret < 0) { + return false; + } + + bucket_owner = bacl.get_owner(); + + return verify_object_permission(s, &bacl, &oacl, RGW_PERM_WRITE); +} + +bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path) +{ + int ret = 0; + + auto& obj_ctx = *static_cast(s->obj_ctx); + RGWBucketInfo binfo; + map battrs; + ret = store->get_bucket_info(obj_ctx, path.bucket_name, binfo, NULL, &battrs); + if (ret < 0) { + goto binfo_fail; + } + + /* We do need a new scope due to goto. */ + { + rgw_obj obj(binfo.bucket, path.obj_key); + obj_ctx.set_atomic(obj); + + RGWRados::Object del_target(store, binfo, obj_ctx, obj); + RGWRados::Object::Delete del_op(&del_target); + + ACLOwner owner; + if (!verify_permission(binfo, battrs, obj, owner)) { + ret = -EACCES; + goto auth_fail; + } + + del_op.params.bucket_owner = binfo.owner; + del_op.params.versioning_status = binfo.versioning_status(); + del_op.params.obj_owner = owner; + + ret = del_op.delete_obj(); + if (ret < 0) { + goto delop_fail; + } + } + + num_deleted++; + return true; + + +binfo_fail: + if (-ENOENT == ret) { + ldout(store->ctx(), 20) << "cannot find bucket = " << path.bucket_name << dendl; + num_unfound++; + } else { + ldout(store->ctx(), 20) << "cannot get bucket info, ret = " << ret << dendl; + + fail_desc_t failed_item = { + .err = ret, + .path = path + }; + failures.push_back(failed_item); + } + return false; + +auth_fail: + ldout(store->ctx(), 20) << "wrong auth for " << path << dendl; + { + fail_desc_t failed_item = { + .err = ret, + .path = path + }; + failures.push_back(failed_item); + } + return false; + +delop_fail: + if (-ENOENT == ret) { + ldout(store->ctx(), 20) << "cannot find the object" << dendl; + num_unfound++; + } else { + fail_desc_t failed_item = { + .err = ret, + .path = path + }; + failures.push_back(failed_item); + } + return false; +} + +bool RGWBulkDelete::Deleter::delete_chunk(const std::list& paths) +{ + ldout(store->ctx(), 20) << "in delete_chunk" << dendl; + for (auto path : paths) { + ldout(store->ctx(), 20) << "bulk deleting path: " << path << dendl; + delete_single(path); + } + + return true; +} + +int RGWBulkDelete::verify_permission() +{ + return 0; +} + +void RGWBulkDelete::pre_exec() +{ + rgw_bucket_object_pre_exec(s); +} + +void RGWBulkDelete::execute() +{ + deleter = std::unique_ptr(new Deleter(store, s)); + + bool is_truncated = false; + do { + list items; + + int ret = get_data(items, &is_truncated); + if (ret < 0) { + return; + } + + ret = deleter->delete_chunk(items); + } while (!ret && is_truncated); + + return; +} + RGWHandler::~RGWHandler() { } diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 0bd49a874449..cc3967500ab3 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -13,6 +13,7 @@ #include +#include #include #include #include @@ -64,6 +65,7 @@ enum RGWOpType { RGW_OP_LIST_MULTIPART, RGW_OP_LIST_BUCKET_MULTIPARTS, RGW_OP_DELETE_MULTI_OBJ, + RGW_OP_BULK_DELETE }; /** @@ -1090,6 +1092,84 @@ public: virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; } }; + +class RGWBulkDelete : public RGWOp { +public: + struct acct_path_t { + std::string bucket_name; + rgw_obj_key obj_key; + }; + + struct fail_desc_t { + int err; + acct_path_t path; + }; + + class Deleter { + protected: + unsigned int num_deleted; + unsigned int num_unfound; + std::list failures; + + RGWRados * const store; + req_state * const s; + + public: + Deleter(RGWRados * const str, req_state * const s) + : num_deleted(0), + num_unfound(0), + store(str), + s(s) { + } + + unsigned int get_num_deleted() const { + return num_deleted; + } + + unsigned int get_num_unfound() const { + return num_unfound; + } + + const std::list get_failures() const { + return failures; + } + + bool verify_permission(RGWBucketInfo& binfo, + map& battrs, + rgw_obj& obj, + ACLOwner& bucket_owner /* out */); + bool delete_single(const acct_path_t& path); + bool delete_chunk(const std::list& paths); + }; + /* End of Deleter subclass */ + +protected: + int ret; + std::unique_ptr deleter; + +public: + RGWBulkDelete() + : ret(0), + deleter(nullptr) { + } + + int verify_permission(); + void pre_exec(); + void execute(); + + virtual int get_data(std::list& items, + bool * is_truncated) = 0; + virtual void send_response() = 0; + + virtual const string name() { return "bulk_delete"; } + virtual RGWOpType get_type() { return RGW_OP_BULK_DELETE; } + virtual uint32_t op_mask() { return RGW_OP_TYPE_DELETE; } +}; + +inline ostream& operator<<(ostream& out, const RGWBulkDelete::acct_path_t &o) { + return out << o.bucket_name << "/" << o.obj_key; +} + class RGWDeleteMultiObj : public RGWOp { protected: int ret; diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 767ad1ecfc3d..2a761f720f61 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -277,31 +277,51 @@ void rgw_flush_formatter(struct req_state *s, Formatter *formatter) } } -void set_req_state_err(struct req_state *s, int err_no) +void set_req_state_err(struct rgw_err& err, /* out */ + int err_no, /* in */ + const int prot_flags) /* in */ { const struct rgw_http_errors *r; if (err_no < 0) err_no = -err_no; - s->err.ret = -err_no; - if (s->prot_flags & RGW_REST_SWIFT) { + err.ret = -err_no; + if (prot_flags & RGW_REST_SWIFT) { r = search_err(err_no, RGW_HTTP_SWIFT_ERRORS, ARRAY_LEN(RGW_HTTP_SWIFT_ERRORS)); if (r) { - s->err.http_ret = r->http_ret; - s->err.s3_code = r->s3_code; + err.http_ret = r->http_ret; + err.s3_code = r->s3_code; return; } } r = search_err(err_no, RGW_HTTP_ERRORS, ARRAY_LEN(RGW_HTTP_ERRORS)); if (r) { - s->err.http_ret = r->http_ret; - s->err.s3_code = r->s3_code; + err.http_ret = r->http_ret; + err.s3_code = r->s3_code; return; } dout(0) << "WARNING: set_req_state_err err_no=" << err_no << " resorting to 500" << dendl; - s->err.http_ret = 500; - s->err.s3_code = "UnknownError"; + err.http_ret = 500; + err.s3_code = "UnknownError"; +} + +void set_req_state_err(struct req_state * const s, const int err_no) +{ + if (s) { + set_req_state_err(s->err, err_no, s->prot_flags); + } +} + +void dump_errno(int http_ret, string& out) { + stringstream ss; + + ss << http_ret << " " << http_status_names[http_ret]; + out = ss.str(); +} + +void dump_errno(const struct rgw_err &err, string& out) { + dump_errno(err.http_ret, out); } void dump_errno(struct req_state *s) @@ -311,11 +331,11 @@ void dump_errno(struct req_state *s) dump_status(s, buf, http_status_names[s->err.http_ret]); } -void dump_errno(struct req_state *s, int err) +void dump_errno(struct req_state *s, int http_ret) { char buf[32]; - snprintf(buf, sizeof(buf), "%d", err); - dump_status(s, buf, http_status_names[s->err.http_ret]); + snprintf(buf, sizeof(buf), "%d", http_ret); + dump_status(s, buf, http_status_names[http_ret]); } void dump_string_header(struct req_state *s, const char *name, const char *val) @@ -1159,11 +1179,17 @@ int RGWHandler_ObjStore::allocate_formatter(struct req_state *s, int default_typ switch (s->format) { case RGW_FORMAT_PLAIN: - s->formatter = new RGWFormatter_Plain; - break; + { + const bool use_kv_syntax = s->info.args.exists("bulk-delete"); + s->formatter = new RGWFormatter_Plain(use_kv_syntax); + break; + } case RGW_FORMAT_XML: - s->formatter = new XMLFormatter(false); - break; + { + const bool lowercase_underscore = s->info.args.exists("bulk-delete"); + s->formatter = new XMLFormatter(false, lowercase_underscore); + break; + } case RGW_FORMAT_JSON: s->formatter = new JSONFormatter(false); break; diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index b9355e6457b0..e17627ca9a83 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -270,6 +270,12 @@ public: int get_params(); }; +class RGWBulkDelete_ObjStore : public RGWBulkDelete { +public: + RGWBulkDelete_ObjStore() {} + ~RGWBulkDelete_ObjStore() {} +}; + class RGWDeleteMultiObj_ObjStore : public RGWDeleteMultiObj { public: RGWDeleteMultiObj_ObjStore() {} @@ -368,9 +374,12 @@ public: static const int64_t NO_CONTENT_LENGTH = -1; +extern void set_req_state_err(struct rgw_err &err, int err_no, int prot_flags); extern void set_req_state_err(struct req_state *s, int err_no); +extern void dump_errno(int http_ret, string& out); +extern void dump_errno(const struct rgw_err &err, string& out); extern void dump_errno(struct req_state *s); -extern void dump_errno(struct req_state *s, int ret); +extern void dump_errno(struct req_state *s, int http_ret); extern void end_header(struct req_state *s, RGWOp *op = NULL, const char *content_type = NULL, diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index 500794f84672..70658dee431f 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -938,6 +938,71 @@ void RGWOptionsCORS_ObjStore_SWIFT::send_response() end_header(s, NULL); } +int RGWBulkDelete_ObjStore_SWIFT::get_data(list& items, + bool * const is_truncated) +{ + const size_t MAX_LINE_SIZE = 2048; + + RGWClientIOStreamBuf ciosb(*s->cio, (size_t)s->cct->_conf->rgw_max_chunk_size); + istream cioin(&ciosb); + + char buf[MAX_LINE_SIZE]; + while (cioin.getline(buf, sizeof(buf))) { + string path_str(buf); + + ldout(s->cct, 20) << "extracted Bulk Delete entry: " << path_str << dendl; + + const size_t sep_pos = path_str.find('/'); + if (string::npos == sep_pos) { + ldout(s->cct, 20) << "wrongly formatted item: " << path_str << dendl; + continue; + } + + RGWBulkDelete::acct_path_t path; + path.bucket_name = path_str.substr(0, sep_pos); + path.obj_key = path_str.substr(sep_pos + 1); + + items.push_back(path); + } + + *is_truncated = false; + return 0; +} + +void RGWBulkDelete_ObjStore_SWIFT::send_response() +{ + set_req_state_err(s, ret); + dump_errno(s); + end_header(s, NULL); + + s->formatter->open_object_section("delete"); + + s->formatter->dump_int("Number Deleted", deleter->get_num_deleted()); + s->formatter->dump_int("Number Not Found", deleter->get_num_unfound()); + s->formatter->dump_string("Response Body", ""); + s->formatter->dump_string("Response Status", "200 OK"); + s->formatter->open_array_section("Errors"); + for (const auto fail_desc : deleter->get_failures()) { + rgw_err err; + set_req_state_err(err, fail_desc.err, s->prot_flags); + string status; + dump_errno(err, status); + + stringstream ss_name; + ss_name << fail_desc.path; + + s->formatter->open_array_section("object"); + s->formatter->dump_string("Name", ss_name.str()); + s->formatter->dump_string("Status", status); + s->formatter->close_section(); + } + s->formatter->close_section(); + + s->formatter->close_section(); + + rgw_flush_formatter_and_reset(s, s->formatter); +} + RGWOp *RGWHandler_ObjStore_Service_SWIFT::op_get() { return new RGWListBuckets_ObjStore_SWIFT; @@ -950,9 +1015,20 @@ RGWOp *RGWHandler_ObjStore_Service_SWIFT::op_head() RGWOp *RGWHandler_ObjStore_Service_SWIFT::op_post() { + if (s->info.args.exists("bulk-delete")) { + return new RGWBulkDelete_ObjStore_SWIFT; + } return new RGWPutMetadataAccount_ObjStore_SWIFT; } +RGWOp *RGWHandler_ObjStore_Service_SWIFT::op_delete() +{ + if (s->info.args.exists("bulk-delete")) { + return new RGWBulkDelete_ObjStore_SWIFT; + } + return NULL; +} + RGWOp *RGWHandler_ObjStore_Bucket_SWIFT::get_obj_op(bool get_data) { if (is_acl_op()) { diff --git a/src/rgw/rgw_rest_swift.h b/src/rgw/rgw_rest_swift.h index 66d8c81bcfec..7647cb125556 100644 --- a/src/rgw/rgw_rest_swift.h +++ b/src/rgw/rgw_rest_swift.h @@ -167,6 +167,16 @@ public: void send_response(); }; +class RGWBulkDelete_ObjStore_SWIFT : public RGWBulkDelete_ObjStore { +public: + RGWBulkDelete_ObjStore_SWIFT() {} + ~RGWBulkDelete_ObjStore_SWIFT() {} + + int get_data(std::list& items, + bool * is_truncated); + void send_response(); +}; + class RGWHandler_ObjStore_SWIFT : public RGWHandler_ObjStore { friend class RGWRESTMgr_SWIFT; protected: @@ -193,6 +203,7 @@ protected: RGWOp *op_get(); RGWOp *op_head(); RGWOp *op_post(); + RGWOp *op_delete(); public: RGWHandler_ObjStore_Service_SWIFT() {} virtual ~RGWHandler_ObjStore_Service_SWIFT() {} -- 2.47.3