From: Yehuda Sadeh <yehuda@redhat.com>
Date: Thu, 19 Oct 2017 00:41:00 +0000 (-0700)
Subject: rgw: aws sync, initial work on multipart data parts creation
X-Git-Tag: v13.1.0~270^2~74
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=48da099d9db4159b8155d5367c45ff8e8128e916;p=ceph.git

rgw: aws sync, initial work on multipart data parts creation

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
---

diff --git a/src/rgw/rgw_sync_module_aws.cc b/src/rgw/rgw_sync_module_aws.cc
index 571947d99f55..94ce689f99d2 100644
--- a/src/rgw/rgw_sync_module_aws.cc
+++ b/src/rgw/rgw_sync_module_aws.cc
@@ -401,6 +401,19 @@ class RGWAWSStreamObjToCloudMultipartCR : public RGWCoroutine {
 
   string upload_id;
 
+  uint32_t part_size;
+  int num_parts;
+
+  struct part_info {
+    int part_num;
+    uint64_t ofs;
+    uint64_t size;
+    string etag;
+  };
+
+  int cur_part{0};
+  uint64_t cur_ofs{0};
+
 public:
   RGWAWSStreamObjToCloudMultipartCR(RGWDataSyncEnv *_sync_env,
                                 RGWRESTConn *_source_conn,
@@ -415,27 +428,37 @@ public:
                                                    src_obj(_src_obj),
                                                    dest_obj(_dest_obj),
                                                    obj_size(_obj_size),
-                                                   mtime(_mtime) {}
+                                                   mtime(_mtime) {
+#warning flexible part size needed
+    part_size = 5 * 1024 * 1024;
 
-  int operate() {
-    reenter(this) {
-#if 0
-      /* init input */
-      in_crf.reset(new RGWRESTStreamGetCRF(cct, get_env(), this, sync_env, source_conn, src_obj));
+    num_parts = (obj_size + part_size - 1) / part_size;
+  }
 
-      /* init output */
-      out_crf.reset(new RGWAWSStreamPutCRF(cct, get_env(), this, sync_env, dest_conn,
-                                           dest_obj));
 
-      yield call(new RGWStreamSpliceCR(cct, sync_env->http_manager, in_crf, out_crf));
+  int operate() {
+    reenter(this) {
+      yield call(new RGWAWSInitMultipartCR(sync_env, dest_conn, dest_obj, obj_size, mtime, &upload_id));
       if (retcode < 0) {
         return set_cr_error(retcode);
       }
-#endif
 
-      yield call(new RGWAWSInitMultipartCR(sync_env, dest_conn, dest_obj, obj_size, mtime, &upload_id));
-      if (retcode < 0) {
-        return set_cr_error(retcode);
+      for (cur_part = 1; cur_part <= num_parts; ++cur_part) {
+        yield {
+          part_info cur_part_info;
+          cur_part_info.part_num = cur_part + 1;
+          cur_part_info.ofs = cur_ofs;
+          cur_part_info.size = std::min((uint64_t)part_size, obj_size - cur_ofs);
+
+          cur_ofs += part_size;
+
+          call(new RGWAWSStreamObjToCloudMultipartPartCR(sync_env,
+                                                             source_conn, src_obj,
+                                                             dest_conn, dest_obj,
+                                                             upload_id,
+                                                             cur_part_info.ofs, cur_part_info.size,
+                                                             cur_part_info.part_num));
+        }
       }
 
       return set_cr_done();