From: Yehuda Sadeh <yehuda@hq.newdream.net>
Date: Mon, 19 Jul 2010 23:50:43 +0000 (-0700)
Subject: rgw: get/put read and write by chunks
X-Git-Tag: v0.21~87
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=925e2092486bbc78f011065172524d6c550ae7c6;p=ceph.git

rgw: get/put read and write by chunks
---

diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 90601f8f627..1f03602c87c 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -36,6 +36,8 @@ using namespace std;
 
 #define USER_INFO_VER 2
 
+#define RGW_MAX_CHUNK_SIZE	(4*1024*1024)
+
 typedef void *RGWAccessHandle;
 
 /** Store error returns for output at a different point in the program */
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index fae8c62d014..4b6696af74b 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -340,7 +340,18 @@ void RGWPutObj::execute()
     }
 
     MD5_Init(&c);
-    MD5_Update(&c, data, (unsigned long)len);
+    do {
+      get_data();
+      if (len > 0) {
+        MD5_Update(&c, data, (unsigned long)len);
+        ret = rgwstore->put_obj_data(s->user.user_id, s->bucket_str, s->object_str, data, ofs, len, NULL);
+        free(data);
+        if (ret < 0)
+          goto done;
+        ofs += len;
+      }
+    } while ( len > 0);
+
     MD5_Final(m, &c);
 
     buf_to_hex(m, MD5_DIGEST_LENGTH, calc_md5);
@@ -368,10 +379,9 @@ void RGWPutObj::execute()
 
     get_request_metadata(s, attrs);
 
-    ret = rgwstore->put_obj(s->user.user_id, s->bucket_str, s->object_str, data, len, NULL, attrs);
+    ret = rgwstore->put_obj_meta(s->user.user_id, s->bucket_str, s->object_str, NULL, attrs);
   }
 done:
-  free(data);
   send_response();
 }
 
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 8bf432044c7..eb47b2e4274 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -169,6 +169,7 @@ class RGWPutObj : public RGWOp {
 protected:
   int ret;
   size_t len;
+  off_t ofs;
   char *data;
   struct rgw_err err;
   char *supplied_md5_b64;
@@ -181,12 +182,14 @@ public:
     RGWOp::init(s);
     ret = 0;
     len = 0;
+    ofs = 0;
     data = NULL;
     supplied_md5_b64 = NULL;
   }
   void execute();
 
   virtual int get_params() = 0;
+  virtual int get_data() = 0;
   virtual void send_response() = 0;
 };
 
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index bbcc6197748..24e54f23e90 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -614,6 +614,9 @@ int RGWRados::get_obj(void *handle,
   else
     len = end - ofs + 1;
 
+  if (len > RGW_MAX_CHUNK_SIZE)
+    len = RGW_MAX_CHUNK_SIZE;
+
   cout << "rados->read ofs=" << ofs << " len=" << len << std::endl;
   int r = rados->read(state->pool, oid, ofs, bl, len);
   cout << "rados->read r=" << r << std::endl;
@@ -623,7 +626,7 @@ int RGWRados::get_obj(void *handle,
     memcpy(*data, bl.c_str(), bl.length());
   }
 
-  if (r < 0 || !len || (ofs + len - 1 == end)) {
+  if (r < 0 || !len || ((off_t)(ofs + len - 1) == end)) {
     rados->close_pool(state->pool);
     delete state;
   }
diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index f34d1ff6ab6..f79ca08dc1a 100644
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -317,7 +317,17 @@ void RGWDeleteBucket_REST::send_response()
 
 int RGWPutObj_REST::get_params()
 {
-  size_t cl = atoll(s->length);
+  supplied_md5_b64 = FCGX_GetParam("HTTP_CONTENT_MD5", s->fcgx->envp);
+
+  return 0;
+}
+
+int RGWPutObj_REST::get_data()
+{
+  size_t cl = atoll(s->length) - ofs;
+  if (cl > RGW_MAX_CHUNK_SIZE)
+    cl = RGW_MAX_CHUNK_SIZE;
+  len = 0;
   if (cl) {
     data = (char *)malloc(cl);
     if (!data)
diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h
index 20e5f8e9f02..dda33d4db09 100644
--- a/src/rgw/rgw_rest.h
+++ b/src/rgw/rgw_rest.h
@@ -58,6 +58,7 @@ public:
   ~RGWPutObj_REST() {}
 
   int get_params();
+  int get_data();
   void send_response();
 };