rgw/torrent: PutObj uses the filter to write RGW_ATTR_TORRENT

Signed-off-by: Casey Bodley <cbodley@redhat.com>

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index 29dc8cdb93b8efce875f47033e0d4f1a2ae9cf86..7c7965e1df9cff7837a1d2ede3afe6c1af5ae53d 100644 (file)
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -3130,6 +3130,16 @@ options:
   services:
   - rgw
   with_legacy: true
+- name: rgw_torrent_max_size
+  type: size
+  level: advanced
+  desc: Objects over this size will not store torrent info.
+  default: 5_G
+  services:
+  - rgw
+  see_also:
+  - rgw_torrent_flag
+  with_legacy: true
 - name: rgw_dynamic_resharding
   type: bool
   level: basic

diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 9695335ba1a10eceda00e8ddbd92d966041d6aef..7cabd8e4ca690ce2c52313cae7f3de6a6e202f5f 100644 (file)
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -133,6 +133,7 @@ using ceph::crypto::MD5;
 #define RGW_ATTR_OLH_PENDING_PREFIX RGW_ATTR_OLH_PREFIX "pending."
 
 #define RGW_ATTR_COMPRESSION    RGW_ATTR_PREFIX "compression"
+#define RGW_ATTR_TORRENT        RGW_ATTR_PREFIX "torrent"
 
 #define RGW_ATTR_APPEND_PART_NUM    RGW_ATTR_PREFIX "append_part_num"
 

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index e113b9731721c688d25ffb9f704b197c79a4f9db..605f50f2436c32b40cd8ca9ef812896185ed48b5 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -3893,6 +3893,24 @@ static CompressorRef get_compressor_plugin(const req_state *s,
   return Compressor::create(s->cct, alg);
 }
 
+auto RGWPutObj::get_torrent_filter(rgw::sal::DataProcessor* cb)
+    -> std::optional<RGWPutObj_Torrent>
+{
+  auto& conf = get_cct()->_conf;
+  if (!conf->rgw_torrent_flag) {
+    return std::nullopt; // torrent generation disabled
+  }
+  const auto max_len = conf->rgw_torrent_max_size;
+  const auto piece_len = conf->rgw_torrent_sha_unit;
+  if (!max_len || !piece_len) {
+    return std::nullopt; // invalid configuration
+  }
+  if (crypt_http_responses.count("x-amz-server-side-encryption-customer-algorithm")) {
+    return std::nullopt; // downloading the torrent would require customer keys
+  }
+  return RGWPutObj_Torrent{cb, max_len, piece_len};
+}
+
 int RGWPutObj::get_lua_filter(std::unique_ptr<rgw::sal::DataProcessor>* filter, rgw::sal::DataProcessor* cb) {
   std::string script;
   const auto rc = rgw::lua::read_script(s, s->penv.lua.manager.get(), s->bucket_tenant, s->yield, rgw::lua::context::putData, script);
@@ -4099,7 +4117,8 @@ void RGWPutObj::execute(optional_yield y)
 
   const auto& compression_type = driver->get_compression_type(*pdest_placement);
   CompressorRef plugin;
-  boost::optional<RGWPutObj_Compress> compressor;
+  std::optional<RGWPutObj_Compress> compressor;
+  std::optional<RGWPutObj_Torrent> torrent;
 
   std::unique_ptr<rgw::sal::DataProcessor> encrypt;
   std::unique_ptr<rgw::sal::DataProcessor> run_lua;
@@ -4124,6 +4143,9 @@ void RGWPutObj::execute(optional_yield y)
         s->object->set_compressed();
       }
     }
+    if (torrent = get_torrent_filter(filter); torrent) {
+      filter = &*torrent;
+    }
     // run lua script before data is compressed and encrypted - last filter runs first
     op_ret = get_lua_filter(&run_lua, filter);
     if (op_ret < 0) {
@@ -4161,9 +4183,6 @@ void RGWPutObj::execute(optional_yield y)
       hash.Update((const unsigned char *)data.c_str(), data.length());
     }
 
-    /* update torrrent */
-    torrent.update(data);
-
     op_ret = filter->process(std::move(data), ofs);
     if (op_ret < 0) {
       ldpp_dout(this, 20) << "processor->process() returned ret="
@@ -4217,6 +4236,14 @@ void RGWPutObj::execute(optional_yield y)
         << ", orig_size=" << cs_info.orig_size
         << ", blocks=" << cs_info.blocks.size() << dendl;
   }
+  if (torrent) {
+    auto bl = torrent->bencode_torrent(s->object->get_name());
+    if (bl.length()) {
+      ldpp_dout(this, 20) << "storing " << bl.length()
+         << " bytes of torrent info in " << RGW_ATTR_TORRENT << dendl;
+      attrs[RGW_ATTR_TORRENT] = std::move(bl);
+    }
+  }
 
   buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5);
 
@@ -4286,19 +4313,6 @@ void RGWPutObj::execute(optional_yield y)
                                s->yield);
   tracepoint(rgw_op, processor_complete_exit, s->req_id.c_str());
 
-  /* produce torrent */
-  if (s->cct->_conf->rgw_torrent_flag && (ofs == torrent.get_data_len()))
-  {
-    torrent.init(s, driver);
-    torrent.set_create_date(mtime);
-    op_ret =  torrent.complete(y);
-    if (0 != op_ret)
-    {
-      ldpp_dout(this, 0) << "ERROR: torrent.handle_data() returned " << op_ret << dendl;
-      return;
-    }
-  }
-
   // send request to notification manager
   int ret = res->publish_commit(this, s->obj_size, mtime, etag, s->object->get_instance());
   if (ret < 0) {

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index d29dc014153839c652962e7aa4e77a18bfda2b3e..2484d7a41ad5e31f20c7da466fa4da693a947925 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -1189,7 +1189,6 @@ WRITE_CLASS_ENCODER(RGWSLOInfo)
 
 class RGWPutObj : public RGWOp {
 protected:
-  seed torrent;
   off_t ofs;
   const char *supplied_md5_b64;
   const char *supplied_etag;
@@ -1285,6 +1284,9 @@ public:
                                  rgw::sal::DataProcessor *cb) {
     return 0;
   }
+  // if configured, construct a filter to generate torrent metadata
+  auto get_torrent_filter(rgw::sal::DataProcessor *cb)
+      -> std::optional<RGWPutObj_Torrent>;
 
   // get lua script to run as a "put object" filter
   int get_lua_filter(std::unique_ptr<rgw::sal::DataProcessor>* filter,

diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index 54a39a9a549a7b7464d4c66534cc809f304b1060..88f1e7c3a6d0fd05cff6a6f107ddb56a6d3375fa 100644 (file)
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -1033,19 +1033,6 @@ int RGWPutObj_ObjStore::verify_params()
 
 int RGWPutObj_ObjStore::get_params(optional_yield y)
 {
-  /* start gettorrent */
-  if (s->cct->_conf->rgw_torrent_flag)
-  {
-    int ret = 0;
-    ret = torrent.get_params();
-    ldpp_dout(s, 5) << "NOTICE:  open produce torrent file " << dendl;
-    if (ret < 0)
-    {
-      return ret;
-    }
-    torrent.set_info_name(s->object->get_name());
-  }
-  /* end gettorrent */
   supplied_md5_b64 = s->info.env->get("HTTP_CONTENT_MD5");
 
   return 0;