rgw: implement atomic multipart upload, atomic copy

diff --git a/src/rgw/rgw_access.h b/src/rgw/rgw_access.h
index 751c324684cbbdb47443749755b877aad911f99c..c7e87a7dc727317b0fe8d1f5ee2d30f8554b6a58 100644 (file)
--- a/src/rgw/rgw_access.h
+++ b/src/rgw/rgw_access.h
@@ -17,6 +17,13 @@ public:
   virtual bool filter(string& name, string& key) = 0;
 };
 
+struct RGWCloneRangeInfo {
+  rgw_obj src;
+  off_t src_ofs;
+  off_t dst_ofs;
+  size_t len;
+};
+
 /**
  * Abstract class defining the interface for storage devices used by RGW.
  */
@@ -162,8 +169,19 @@ public:
                           size_t size) = 0;
 
   virtual int clone_obj(rgw_obj& dst_obj, off_t dst_ofs,
-                        rgw_obj& src_obj, off_t src_ofs,
-                        size_t size,
+                          rgw_obj& src_obj, off_t src_ofs,
+                          size_t size, map<string, bufferlist> attrs) {
+    RGWCloneRangeInfo info;
+    vector<RGWCloneRangeInfo> v;
+    info.src = src_obj;
+    info.src_ofs = src_ofs;
+    info.len = size;
+    v.push_back(info);
+    return clone_objs(dst_obj, v, attrs);
+  }
+
+  virtual int clone_objs(rgw_obj& dst_obj,
+                        vector<RGWCloneRangeInfo>& ranges,
                         map<string, bufferlist> attrs) { return -ENOTSUP; }
  /**
    * a simple object read without keeping state

diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 59d0f4247f54b1ea638080364748510ed1204ef5..e835c7953efb66de77637ed909ab42ed1506b10e 100644 (file)
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -656,6 +656,15 @@ static inline int rgw_str_to_bool(const char *s, int def_val)
           strcasecmp(s, "1") == 0);
 }
 
+static inline void append_rand_alpha(string& src, string& dest, int len)
+{
+  dest = src;
+  char buf[len + 1];
+  gen_rand_alphanumeric(buf, len);
+  dest.append("_");
+  dest.append(buf);
+}
+
 /** */
 extern int parse_time(const char *time_str, time_t *time);
 /** Check if a user has a permission on that ACL */

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 08a2e49885a85e0ac912a7c9e0975ec9d5669f70..da5aa127da1ae3313dbb5c99c5737000a9f0c908 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -1144,6 +1144,7 @@ void RGWCompleteMultipart::execute()
   rgw_obj meta_obj;
   rgw_obj target_obj;
   RGWMPObj mp;
+  vector<RGWCloneRangeInfo> ranges;
 
 
   ret = get_params();
@@ -1215,15 +1216,23 @@ void RGWCompleteMultipart::execute()
   ret = rgwstore->put_obj_meta(s->user.user_id, target_obj, NULL, attrs, false);
   if (ret < 0)
     goto done;
-
+  
   for (obj_iter = obj_parts.begin(); obj_iter != obj_parts.end(); ++obj_iter) {
     string oid = mp.get_part(obj_iter->second.num);
     rgw_obj src_obj(s->bucket_str, oid, s->object_str, mp_ns);
-    ret = rgwstore->clone_range(target_obj, ofs, src_obj, 0, obj_iter->second.size);
-    if (ret < 0)
-      goto done;
+
+    RGWCloneRangeInfo range;
+    range.src = src_obj;
+    range.src_ofs = 0;
+    range.dst_ofs = ofs;
+    range.len = obj_iter->second.size;
+    ranges.push_back(range);
+
     ofs += obj_iter->second.size;
   }
+  ret = rgwstore->clone_objs(target_obj, ranges, attrs);
+  if (ret < 0)
+    goto done;
 
   // now erase all parts
   for (obj_iter = obj_parts.begin(); obj_iter != obj_parts.end(); ++obj_iter) {

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 7b5f0537306b654903d590b01fe2432116c9ea5f..38c2863eb5df9e9218b13fe5d0a090300cc054fa 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -432,6 +432,13 @@ int RGWRados::copy_obj(std::string& id, rgw_obj& dest_obj,
   size_t total_len;
   time_t lastmod;
   map<string, bufferlist>::iterator iter;
+  rgw_obj tmp_obj = dest_obj;
+  string tmp_oid;
+
+  append_rand_alpha(dest_obj.key, tmp_oid, 32);
+  tmp_obj.set_key(tmp_oid);
+
+  rgw_obj tmp_dest;
 
   RGW_LOG(5) << "Copy object " << src_obj.bucket << ":" << src_obj.object << " => " << dest_obj.bucket << ":" << dest_obj.object << dendl;
 
@@ -452,7 +459,7 @@ int RGWRados::copy_obj(std::string& id, rgw_obj& dest_obj,
 
     // In the first call to put_obj_data, we pass ofs == -1 so that it will do
     // a write_full, wiping out whatever was in the object before this
-    r = put_obj_data(id, dest_obj, data, ((ofs == 0) ? -1 : ofs), ret);
+    r = put_obj_data(id, tmp_obj, data, ((ofs == 0) ? -1 : ofs), ret);
     free(data);
     if (r < 0)
       goto done_err;
@@ -465,12 +472,19 @@ int RGWRados::copy_obj(std::string& id, rgw_obj& dest_obj,
   }
   attrs = attrset;
 
-  ret = put_obj_meta(id, dest_obj, mtime, attrs, false);
+  ret = clone_obj(dest_obj, 0, tmp_obj, 0, end, attrs);
+  if (mtime)
+    obj_stat(tmp_obj, NULL, mtime);
+
+  r = rgwstore->delete_obj(id, tmp_obj);
+  if (r < 0)
+    RGW_LOG(0) << "ERROR: could not remove " << tmp_obj << dendl;
 
   finish_get_obj(&handle);
 
   return ret;
 done_err:
+  rgwstore->delete_obj(id, tmp_obj);
   finish_get_obj(&handle);
   return r;
 }
@@ -865,13 +879,12 @@ int RGWRados::clone_range(rgw_obj& dst_obj, off_t dst_ofs,
   return io_ctx.clone_range(dst_oid, dst_ofs, src_oid, src_ofs, size);
 }
 
-int RGWRados::clone_obj(rgw_obj& dst_obj, off_t dst_ofs,
-                        rgw_obj& src_obj, off_t src_ofs, size_t size,
+int RGWRados::clone_objs(rgw_obj& dst_obj,
+                        vector<RGWCloneRangeInfo>& ranges,
                         map<string, bufferlist> attrs)
 {
   std::string& bucket = dst_obj.bucket;
   std::string& dst_oid = dst_obj.object;
-  std::string& src_oid = src_obj.object;
   librados::IoCtx io_ctx;
 
   int r = open_bucket_ctx(bucket, io_ctx);
@@ -887,7 +900,11 @@ int RGWRados::clone_obj(rgw_obj& dst_obj, off_t dst_ofs,
     bufferlist& bl = iter->second;
     op.setxattr(name.c_str(), bl);
   }
-  op.clone_range(dst_ofs, src_oid, src_ofs, size);
+  vector<RGWCloneRangeInfo>::iterator range_iter;
+  for (range_iter = ranges.begin(); range_iter != ranges.end(); ++range_iter) {
+    RGWCloneRangeInfo& range = *range_iter;
+    op.clone_range(range.dst_ofs, range.src.object, range.src_ofs, range.len);
+  }
 
   bufferlist outbl;
   int ret = io_ctx.operate(dst_oid, &op, &outbl);

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 8948a95cf706913be2964140da066b2afa457ca1..48ff6cf63513e440390fba541260a6cff6e9aecc 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -54,8 +54,8 @@ public:
   virtual bool aio_completed(void *handle);
   virtual int clone_range(rgw_obj& dst_obj, off_t dst_ofs,
                           rgw_obj& src_obj, off_t src_ofs, size_t size);
-  virtual int clone_obj(rgw_obj& dst_obj, off_t dst_ofs,
-                        rgw_obj& src_obj, off_t src_ofs, size_t size,
+  virtual int clone_objs(rgw_obj& dst_obj, 
+                        vector<RGWCloneRangeInfo>& ranges,
                         map<string, bufferlist> attrs);
 
   /** Copy an object, with many extra options */