From: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
Date: Thu, 18 May 2017 09:52:08 +0000 (+0800)
Subject: rgw: optimize generating torrent file. Object data won't stay in memory
X-Git-Tag: v12.1.0~10^2~40^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F15153%2Fhead;p=ceph.git

rgw: optimize generating torrent file. Object data won't stay in memory
now.

Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
---

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 844b1dad5b0..351d2de86b4 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -3044,8 +3044,8 @@ void RGWPutObj::execute()
       hash.Update((const byte *)data.c_str(), data.length());
     }
 
-    /* save data for producing torrent data */
-    torrent.save_data(data_in);
+    /* update torrrent */
+    torrent.update(data_in);
 
     /* do we need this operation to be synchronous? if we're dealing with an object with immutable
      * head, e.g., multipart object we need to make sure we're the first one writing to this object
@@ -3236,7 +3236,7 @@ void RGWPutObj::execute()
   {
     torrent.init(s, store);
     torrent.set_create_date(mtime);
-    op_ret =  torrent.handle_data();
+    op_ret =  torrent.complete();
     if (0 != op_ret)
     {
       ldout(s->cct, 0) << "ERROR: torrent.handle_data() returned " << op_ret << dendl;
diff --git a/src/rgw/rgw_torrent.cc b/src/rgw/rgw_torrent.cc
index 5e440c5d11f..c1f8aaac0bd 100644
--- a/src/rgw/rgw_torrent.cc
+++ b/src/rgw/rgw_torrent.cc
@@ -27,7 +27,6 @@ seed::~seed()
 {
   seed::info.sha1_bl.clear();
   bl.clear();
-  torrent_bl.clear();
   s = NULL;
   store = NULL;
 }
@@ -90,15 +89,35 @@ bool seed::get_flag()
   return is_torrent;
 }
 
-void seed::save_data(bufferlist &bl)
+void seed::update(bufferlist &bl)
 {
   if (!is_torrent)
   {
     return;
   }
-
   info.len += bl.length();
-  torrent_bl.push_back(bl);
+  sha1(&h, bl, bl.length());
+}
+
+int seed::complete()
+{
+  uint64_t remain = info.len%info.piece_length;
+  uint8_t  remain_len = ((remain > 0)? 1 : 0);
+  sha_len = (info.len/info.piece_length + remain_len)*CEPH_CRYPTO_SHA1_DIGESTSIZE;
+
+  int ret = 0;
+   /* produce torrent data */
+  do_encode();
+
+  /* save torrent data into OMAP */
+  ret = save_torrent_file();
+  if (0 != ret)
+  {
+    ldout(s->cct, 0) << "ERROR: failed to save_torrent_file() ret= "<< ret << dendl;
+    return ret;
+  }
+
+  return 0;
 }
 
 off_t seed::get_data_len()
@@ -151,49 +170,6 @@ void seed::sha1(SHA1 *h, bufferlist &bl, off_t bl_len)
   }
 }
 
-int seed::sha1_process()
-{
-  uint64_t remain = info.len%info.piece_length;
-  uint8_t  remain_len = ((remain > 0)? 1 : 0);
-  sha_len = (info.len/info.piece_length + remain_len)*CEPH_CRYPTO_SHA1_DIGESTSIZE;
-
-  SHA1 h;
-  list<bufferlist>::iterator iter = torrent_bl.begin();
-  for (; iter != torrent_bl.end(); ++iter)
-  {
-    bufferlist &bl_info = *iter;
-    sha1(&h, bl_info, (*iter).length());
-  }
-
-  return 0;
-}
-
-int seed::handle_data()
-{
-  int ret = 0;
-
-  /* sha1 process */
-  ret = sha1_process();
-  if (0 != ret)
-  {
-    ldout(s->cct, 0) << "ERROR: failed to sha1_process() ret= "<< ret << dendl;
-    return ret;
-  }
-
-  /* produce torrent data */
-  do_encode();
-
-  /* save torrent data into OMAP */
-  ret = save_torrent_file();
-  if (0 != ret)
-  {
-    ldout(s->cct, 0) << "ERROR: failed to save_torrent_file() ret= "<< ret << dendl;
-    return ret;
-  }
-
-  return 0;
-}
-
 int seed::get_params()
 {
   is_torrent = true;
diff --git a/src/rgw/rgw_torrent.h b/src/rgw/rgw_torrent.h
index 6d2b28cd91a..e6aa5bf6695 100644
--- a/src/rgw/rgw_torrent.h
+++ b/src/rgw/rgw_torrent.h
@@ -104,10 +104,10 @@ private:
   uint64_t sha_len;  // sha1 length
   bool is_torrent;  // flag
   bufferlist bl;  // bufflist ready to send
-  list<bufferlist> torrent_bl;   // meate data
 
   struct req_state *s;
   RGWRados *store;
+  SHA1 h;
 
   TorrentBencode dencode;
 public:
@@ -122,17 +122,16 @@ public:
   off_t get_data_len();
   bool get_flag();
 
-  int handle_data();
-  void save_data(bufferlist &bl);
   void set_create_date(ceph::real_time& value);
-  void set_info_name(const string& value); 
+  void set_info_name(const string& value);
+  void update(bufferlist &bl);
+  int complete();
 
 private:
   void do_encode ();
   void set_announce();
   void set_exist(bool exist);
   void set_info_pieces(char *buff);
-  int sha1_process();
   void sha1(SHA1 *h, bufferlist &bl, off_t bl_len);
   int save_torrent_file();
 };