rgw: get/put read and write by chunks

diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 90601f8f62714beabf112c60de2bea3b89d076d0..1f03602c87c2da1fae1871eccb6a3a4902b19bbe 100644 (file)
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -36,6 +36,8 @@ using namespace std;
 
 #define USER_INFO_VER 2
 
+#define RGW_MAX_CHUNK_SIZE     (4*1024*1024)
+
 typedef void *RGWAccessHandle;
 
 /** Store error returns for output at a different point in the program */

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index fae8c62d0140d76c9dbf06584dd5a316d0606c5b..4b6696af74b8ab15ef1b8befbe84fd449f340941 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -340,7 +340,18 @@ void RGWPutObj::execute()
     }
 
     MD5_Init(&c);
-    MD5_Update(&c, data, (unsigned long)len);
+    do {
+      get_data();
+      if (len > 0) {
+        MD5_Update(&c, data, (unsigned long)len);
+        ret = rgwstore->put_obj_data(s->user.user_id, s->bucket_str, s->object_str, data, ofs, len, NULL);
+        free(data);
+        if (ret < 0)
+          goto done;
+        ofs += len;
+      }
+    } while ( len > 0);
+
     MD5_Final(m, &c);
 
     buf_to_hex(m, MD5_DIGEST_LENGTH, calc_md5);
@@ -368,10 +379,9 @@ void RGWPutObj::execute()
 
     get_request_metadata(s, attrs);
 
-    ret = rgwstore->put_obj(s->user.user_id, s->bucket_str, s->object_str, data, len, NULL, attrs);
+    ret = rgwstore->put_obj_meta(s->user.user_id, s->bucket_str, s->object_str, NULL, attrs);
   }
 done:
-  free(data);
   send_response();
 }
 

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 8bf432044c745951c717fcf61f1fb409cc3942d9..eb47b2e4274d7ac7683dc10dafc26d2873f9f42b 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -169,6 +169,7 @@ class RGWPutObj : public RGWOp {
 protected:
   int ret;
   size_t len;
+  off_t ofs;
   char *data;
   struct rgw_err err;
   char *supplied_md5_b64;
@@ -181,12 +182,14 @@ public:
     RGWOp::init(s);
     ret = 0;
     len = 0;
+    ofs = 0;
     data = NULL;
     supplied_md5_b64 = NULL;
   }
   void execute();
 
   virtual int get_params() = 0;
+  virtual int get_data() = 0;
   virtual void send_response() = 0;
 };
 

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index bbcc619774857f5ef19d4389d4f1b617d1e5f856..24e54f23e90d6f7c4e2d6e5f3d7b279b74684cac 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -614,6 +614,9 @@ int RGWRados::get_obj(void *handle,
   else
     len = end - ofs + 1;
 
+  if (len > RGW_MAX_CHUNK_SIZE)
+    len = RGW_MAX_CHUNK_SIZE;
+
   cout << "rados->read ofs=" << ofs << " len=" << len << std::endl;
   int r = rados->read(state->pool, oid, ofs, bl, len);
   cout << "rados->read r=" << r << std::endl;
@@ -623,7 +626,7 @@ int RGWRados::get_obj(void *handle,
     memcpy(*data, bl.c_str(), bl.length());
   }
 
-  if (r < 0 || !len || (ofs + len - 1 == end)) {
+  if (r < 0 || !len || ((off_t)(ofs + len - 1) == end)) {
     rados->close_pool(state->pool);
     delete state;
   }

diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index f34d1ff6ab68da800728eb0aa8536b502235cd18..f79ca08dc1a2ea5dc30e5490f06f13039b32e8b9 100644 (file)
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -317,7 +317,17 @@ void RGWDeleteBucket_REST::send_response()
 
 int RGWPutObj_REST::get_params()
 {
-  size_t cl = atoll(s->length);
+  supplied_md5_b64 = FCGX_GetParam("HTTP_CONTENT_MD5", s->fcgx->envp);
+
+  return 0;
+}
+
+int RGWPutObj_REST::get_data()
+{
+  size_t cl = atoll(s->length) - ofs;
+  if (cl > RGW_MAX_CHUNK_SIZE)
+    cl = RGW_MAX_CHUNK_SIZE;
+  len = 0;
   if (cl) {
     data = (char *)malloc(cl);
     if (!data)

diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h
index 20e5f8e9f021bb213290f2bf21595b617d254465..dda33d4db0996524c7a7e22b92ad96da66404e96 100644 (file)
--- a/src/rgw/rgw_rest.h
+++ b/src/rgw/rgw_rest.h
@@ -58,6 +58,7 @@ public:
   ~RGWPutObj_REST() {}
 
   int get_params();
+  int get_data();
   void send_response();
 };