rados tool: copy object in chunks

Instead of reading the entire object and then writing it,
we read it in chunks.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

diff --git a/src/rados.cc b/src/rados.cc
index 94ca5c1e9249da6fc3839e2c111c8bf802b2f09f..42c0d1d568f0c6bc512d70cd21de492a598a8a44 100644 (file)
--- a/src/rados.cc
+++ b/src/rados.cc
@@ -173,7 +173,8 @@ static int do_copy(IoCtx& io_ctx, const char *objname, IoCtx& target_ctx, const
   librados::ObjectReadOperation read_op;
   string start_after;
 
-  read_op.read(0, 0, &outdata, NULL);
+#define COPY_CHUNK_SIZE (4 * 1024 * 1024)
+  read_op.read(0, COPY_CHUNK_SIZE, &outdata, NULL);
 
   map<std::string, bufferlist> attrset;
   read_op.getxattrs(&attrset, NULL);
@@ -213,6 +214,20 @@ static int do_copy(IoCtx& io_ctx, const char *objname, IoCtx& target_ctx, const
     return ret;
   }
 
+  uint64_t off = 0;
+
+  while (outdata.length() == COPY_CHUNK_SIZE) {
+    off += outdata.length();
+    outdata.clear();
+    ret = io_ctx.read(oid, outdata, COPY_CHUNK_SIZE, off); 
+    if (ret < 0)
+      goto err;
+
+    ret = target_ctx.write(target_oid, outdata, outdata.length(), off);
+    if (ret < 0)
+      goto err;
+  }
+
   /* iterate through source omap and update target. This is not atomic */
   while (omap.size() == OMAP_CHUNK) {
     /* now start_after should point at the last entry */    
@@ -233,7 +248,6 @@ static int do_copy(IoCtx& io_ctx, const char *objname, IoCtx& target_ctx, const
       goto err;
   }
 
-
   return 0;
 
 err: