From 1aee7f98a2e7e89a2ed00d5e6b7f32a1e5be2649 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Fri, 3 Jun 2011 14:02:51 -0700 Subject: [PATCH] rgw: use clone_range for multi upload completion --- src/rgw/rgw_access.h | 4 +++- src/rgw/rgw_common.h | 3 +++ src/rgw/rgw_fs.h | 2 ++ src/rgw/rgw_op.cc | 33 +++++++++++++++++++++++++++++---- src/rgw/rgw_rados.cc | 15 +++++++++++++++ src/rgw/rgw_rados.h | 2 ++ src/rgw/rgw_rest.cc | 3 +++ 7 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/rgw/rgw_access.h b/src/rgw/rgw_access.h index 7338884428ee2..c7a1798fbf565 100644 --- a/src/rgw/rgw_access.h +++ b/src/rgw/rgw_access.h @@ -138,7 +138,9 @@ public: virtual void finish_get_obj(void **handle) = 0; - /** + virtual int clone_range(std::string& bucket, std::string& dst_oid, off_t dst_ofs, + std::string& src_oid, off_t src_ofs, size_t size, std::string& loc) = 0; + /** * a simple object read without keeping state */ virtual int read(std::string& bucket, std::string& oid, off_t ofs, size_t size, bufferlist& bl) = 0; diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 27868e5c83b97..757128f9b4d89 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -81,6 +81,9 @@ extern string rgw_root_bucket; #define ERR_INVALID_DIGEST 2004 #define ERR_BAD_DIGEST 2005 #define ERR_UNRESOLVABLE_EMAIL 2006 +#define ERR_INVALID_PART 2007 +#define ERR_INVALID_PART_ORDER 2008 +#define ERR_NO_SUCH_UPLOAD 2009 typedef void *RGWAccessHandle; diff --git a/src/rgw/rgw_fs.h b/src/rgw/rgw_fs.h index f4b37c57ac75e..8058c3309c6bf 100644 --- a/src/rgw/rgw_fs.h +++ b/src/rgw/rgw_fs.h @@ -23,6 +23,8 @@ public: map& attrs, bool exclusive); int put_obj_data(std::string& id, std::string& bucket, std::string& obj, std::string& loc, const char *data, off_t ofs, size_t size, time_t *mtime); + int clone_range(std::string& bucket, std::string& dst_oid, off_t dst_ofs, + std::string& src_oid, off_t src_ofs, size_t size, std::string& loc) { return -ENOTSUP; } int copy_obj(std::string& id, std::string& dest_bucket, std::string& dest_obj, std::string& src_bucket, std::string& src_obj, time_t *mtime, diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 99e2813e861ce..804d0ee0c4ad6 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -914,7 +914,10 @@ void RGWCompleteMultipart::execute() RGWMultiXMLParser parser; string obj = s->object_str; map obj_parts; + map::iterator obj_iter; RGWAccessControlPolicy policy; + off_t ofs = 0; + string prefix; ret = get_params(); if (ret < 0) @@ -941,17 +944,39 @@ void RGWCompleteMultipart::execute() goto done; } - for (iter = parts->parts.begin(); iter != parts->parts.end(); ++iter) { - RGW_LOG(0) << "part: " << iter->first << " etag: " << iter->second << dendl; - } - obj.append("."); + prefix = obj; obj.append(upload_id); ret = get_multiparts_info(s, obj, obj_parts, policy); + if (ret == -ENOENT) + ret = -ERR_NO_SUCH_UPLOAD; if (ret < 0) goto done; + for (iter = parts->parts.begin(), obj_iter = obj_parts.begin(); + iter != parts->parts.end(); + ++iter, ++obj_iter) { + if (iter->first != (int)obj_iter->first) { + RGW_LOG(0) << "parts num mismatch: next requested: " << iter->first << " next uploaded: " << obj_iter->first << dendl; + ret = -ERR_INVALID_PART; + goto done; + } + if (iter->second.compare(obj_iter->second.etag) != 0) { + RGW_LOG(0) << "part: " << iter->first << " etag: " << iter->second << dendl; + ret = -ERR_INVALID_PART; + goto done; + } + } + + for (obj_iter = obj_parts.begin(); obj_iter != obj_parts.end(); ++obj_iter) { + obj = prefix; + char buf[16]; + snprintf(buf, 16, "%d", obj_iter->second.num); + obj.append(buf); + rgwstore->clone_range(s->bucket_str, s->object_str, ofs, obj, 0, obj_iter->second.size, s->object_str); + } + done: send_response(); } diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 3315dd065c0d5..8b1697406cb5b 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -669,6 +669,21 @@ done_err: return r; } +int RGWRados::clone_range(std::string& bucket, std::string& dst_oid, off_t dst_ofs, + std::string& src_oid, off_t src_ofs, size_t size, std::string& loc) +{ + librados::IoCtx io_ctx; + + int r = open_bucket_ctx(bucket, io_ctx); + if (r < 0) + return r; + + io_ctx.locator_set_key(loc); + + return io_ctx.clone_range(dst_oid, dst_ofs, src_oid, src_ofs, size); +} + + int RGWRados::get_obj(void **handle, std::string& bucket, std::string& oid, string& loc, char **data, off_t ofs, off_t end) diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index f82d347c420e4..30c47b2e74038 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -46,6 +46,8 @@ public: map& attrs, bool exclusive); virtual int put_obj_data(std::string& id, std::string& bucket, std::string& obj, std::string& loc, const char *data, off_t ofs, size_t len, time_t *mtime); + virtual int clone_range(std::string& bucket, std::string& dst_oid, off_t dst_ofs, + std::string& src_oid, off_t src_ofs, size_t size, std::string& loc); /** Copy an object, with many extra options */ virtual int copy_obj(std::string& id, std::string& dest_bucket, std::string& dest_obj, std::string& src_bucket, std::string& src_obj, diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 23a2c50dca90d..f9218e7d9ee4b 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -44,10 +44,13 @@ const static struct rgw_html_errors RGW_HTML_ERRORS[] = { { ERR_INVALID_BUCKET_NAME, 400, "InvalidBucketName" }, { ERR_INVALID_OBJECT_NAME, 400, "InvalidObjectName" }, { ERR_UNRESOLVABLE_EMAIL, 400, "UnresolvableGrantByEmailAddress" }, + { ERR_INVALID_PART, 400, "InvalidPart" }, + { ERR_INVALID_PART_ORDER, 400, "InvalidPartOrder" }, { EACCES, 403, "AccessDenied" }, { EPERM, 403, "AccessDenied" }, { ENOENT, 404, "NoSuchKey" }, { ERR_NO_SUCH_BUCKET, 404, "NoSuchBucket" }, + { ERR_NO_SUCH_UPLOAD, 404, "NoSuchUpload" }, { ERR_METHOD_NOT_ALLOWED, 405, "MethodNotAllowed" }, { ETIMEDOUT, 408, "RequestTimeout" }, { EEXIST, 409, "BucketAlreadyExists" }, -- 2.39.5