From 06b9a84851463524c0c13992d72c58b082b66a5c Mon Sep 17 00:00:00 2001 From: Radoslaw Zarzynski Date: Wed, 30 Nov 2016 21:57:26 +0100 Subject: [PATCH] rgw: implement the full response generation in BulkUpload of Swift API. Signed-off-by: Radoslaw Zarzynski --- src/rgw/rgw_op.cc | 20 +++++++++++++++ src/rgw/rgw_op.h | 24 ++++++++++++++++++ src/rgw/rgw_rest.cc | 10 +++++--- src/rgw/rgw_rest_swift.cc | 51 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 3 deletions(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 7d7b019e0ab..72bfefb4fe8 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -5498,6 +5498,8 @@ void RGWBulkDelete::execute() } +constexpr std::initializer_list RGWBulkUploadOp::terminal_errors; + int RGWBulkUploadOp::verify_permission() { if (s->auth.identity->is_anonymous()) { @@ -5955,12 +5957,21 @@ void RGWBulkUploadOp::execute() op_ret = handle_file(header->get_filename(), header->get_filesize(), body); + if (! op_ret) { + /* Only regular files counts. */ + num_created++; + } else { + failures.emplace_back(op_ret, header->get_filename().to_string()); + } break; } case rgw::tar::FileType::DIRECTORY: { ldout(s->cct, 2) << "bulk upload: handling regular directory" << dendl; op_ret = handle_dir(header->get_filename()); + if (op_ret < 0 && op_ret != -ERR_BUCKET_EXISTS) { + failures.emplace_back(op_ret, header->get_filename().to_string()); + } break; } default: { @@ -5969,6 +5980,15 @@ void RGWBulkUploadOp::execute() break; } } + + /* In case of any problems with sub-request authorization Swift simply + * terminates whole upload immediately. */ + if (boost::algorithm::contains(std::initializer_list{ op_ret }, + terminal_errors)) { + ldout(s->cct, 2) << "bulk upload: terminating due to ret=" << op_ret + << dendl; + break; + } } else { ldout(s->cct, 2) << "bulk upload: an empty block" << dendl; op_ret = 0; diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 2ab94c363df..158cc84794d 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -390,6 +391,25 @@ class RGWBulkUploadOp : public RGWOp { boost::optional dir_ctx; protected: + class fail_desc_t { + public: + fail_desc_t(const int err, std::string path) + : err(err), + path(std::move(path)) { + } + + const int err; + const std::string path; + }; + + static constexpr std::initializer_list terminal_errors = { + -EACCES, -EPERM + }; + + /* FIXME: boost::container::small_vector failures; */ + std::vector failures; + size_t num_created; + class StreamGetter; class DecoratedStreamGetter; class AlignedStreamGetter; @@ -411,6 +431,10 @@ protected: int handle_dir(boost::string_ref path); public: + RGWBulkUploadOp() + : num_created(0) { + } + void init(RGWRados* const store, struct req_state* const s, RGWHandler* const h) override { diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 0a5fac47aa7..c7d516afd61 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -1577,17 +1577,21 @@ int RGWHandler_REST::allocate_formatter(struct req_state *s, const string& mm = s->info.args.get("multipart-manifest"); const bool multipart_delete = (mm.compare("delete") == 0); - + const bool swift_bulkupload = s->prot_flags & RGW_REST_SWIFT && + s->info.args.exists("extract-archive"); switch (s->format) { case RGW_FORMAT_PLAIN: { - const bool use_kv_syntax = s->info.args.exists("bulk-delete") || multipart_delete; + const bool use_kv_syntax = s->info.args.exists("bulk-delete") || + multipart_delete || swift_bulkupload; s->formatter = new RGWFormatter_Plain(use_kv_syntax); break; } case RGW_FORMAT_XML: { - const bool lowercase_underscore = s->info.args.exists("bulk-delete") || multipart_delete; + const bool lowercase_underscore = s->info.args.exists("bulk-delete") || + multipart_delete || swift_bulkupload; + s->formatter = new XMLFormatter(false, lowercase_underscore); break; } diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index b0fa219b3b7..d02d4afc292 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -1490,6 +1490,57 @@ void RGWBulkUploadOp_ObjStore_SWIFT::send_response() end_header(s, this /* RGWOp */, nullptr /* contype */, CHUNKED_TRANSFER_ENCODING); rgw_flush_formatter_and_reset(s, s->formatter); + + s->formatter->open_object_section("delete"); + + std::string resp_status; + std::string resp_body; + + if (! failures.empty()) { + rgw_err err; + + const auto last_err = { failures.back().err }; + if (boost::algorithm::contains(last_err, terminal_errors)) { + /* The terminal errors are affecting the status of the whole upload. */ + set_req_state_err(err, failures.back().err, s->prot_flags); + } else { + set_req_state_err(err, ERR_INVALID_REQUEST, s->prot_flags); + } + + dump_errno(err, resp_status); + } else if (0 == num_created && failures.empty()) { + /* Nothing created, nothing failed. This means the archive contained no + * entity we could understand (regular file or directory). We need to + * send 400 Bad Request to an HTTP client in the internal status field. */ + dump_errno(400, resp_status); + resp_body = "Invalid Tar File: No Valid Files"; + } else { + /* 200 OK */ + dump_errno(201, resp_status); + } + + encode_json("Number Files Created", num_created, s->formatter); + encode_json("Response Body", resp_body, s->formatter); + encode_json("Response Status", resp_status, s->formatter); + + s->formatter->open_array_section("Errors"); + for (const auto& fail_desc : failures) { + s->formatter->open_array_section("object"); + + encode_json("Name", fail_desc.path, s->formatter); + + rgw_err err; + set_req_state_err(err, fail_desc.err, s->prot_flags); + std::string status; + dump_errno(err, status); + encode_json("Status", status, s->formatter); + + s->formatter->close_section(); + } + s->formatter->close_section(); + + s->formatter->close_section(); + rgw_flush_formatter_and_reset(s, s->formatter); } -- 2.39.5