From a89bb547523ecb918a8ce2747fc214a291a20edb Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Sun, 19 Feb 2023 00:48:04 -0500 Subject: [PATCH] rgw/torrent: PutObj uses the filter to write RGW_ATTR_TORRENT Signed-off-by: Casey Bodley --- src/common/options/rgw.yaml.in | 10 +++++++ src/rgw/rgw_common.h | 1 + src/rgw/rgw_op.cc | 48 ++++++++++++++++++++++------------ src/rgw/rgw_op.h | 4 ++- src/rgw/rgw_rest.cc | 13 --------- 5 files changed, 45 insertions(+), 31 deletions(-) diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index 29dc8cdb93b8e..7c7965e1df9cf 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -3130,6 +3130,16 @@ options: services: - rgw with_legacy: true +- name: rgw_torrent_max_size + type: size + level: advanced + desc: Objects over this size will not store torrent info. + default: 5_G + services: + - rgw + see_also: + - rgw_torrent_flag + with_legacy: true - name: rgw_dynamic_resharding type: bool level: basic diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 9695335ba1a10..7cabd8e4ca690 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -133,6 +133,7 @@ using ceph::crypto::MD5; #define RGW_ATTR_OLH_PENDING_PREFIX RGW_ATTR_OLH_PREFIX "pending." #define RGW_ATTR_COMPRESSION RGW_ATTR_PREFIX "compression" +#define RGW_ATTR_TORRENT RGW_ATTR_PREFIX "torrent" #define RGW_ATTR_APPEND_PART_NUM RGW_ATTR_PREFIX "append_part_num" diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index e113b9731721c..605f50f2436c3 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3893,6 +3893,24 @@ static CompressorRef get_compressor_plugin(const req_state *s, return Compressor::create(s->cct, alg); } +auto RGWPutObj::get_torrent_filter(rgw::sal::DataProcessor* cb) + -> std::optional +{ + auto& conf = get_cct()->_conf; + if (!conf->rgw_torrent_flag) { + return std::nullopt; // torrent generation disabled + } + const auto max_len = conf->rgw_torrent_max_size; + const auto piece_len = conf->rgw_torrent_sha_unit; + if (!max_len || !piece_len) { + return std::nullopt; // invalid configuration + } + if (crypt_http_responses.count("x-amz-server-side-encryption-customer-algorithm")) { + return std::nullopt; // downloading the torrent would require customer keys + } + return RGWPutObj_Torrent{cb, max_len, piece_len}; +} + int RGWPutObj::get_lua_filter(std::unique_ptr* filter, rgw::sal::DataProcessor* cb) { std::string script; const auto rc = rgw::lua::read_script(s, s->penv.lua.manager.get(), s->bucket_tenant, s->yield, rgw::lua::context::putData, script); @@ -4099,7 +4117,8 @@ void RGWPutObj::execute(optional_yield y) const auto& compression_type = driver->get_compression_type(*pdest_placement); CompressorRef plugin; - boost::optional compressor; + std::optional compressor; + std::optional torrent; std::unique_ptr encrypt; std::unique_ptr run_lua; @@ -4124,6 +4143,9 @@ void RGWPutObj::execute(optional_yield y) s->object->set_compressed(); } } + if (torrent = get_torrent_filter(filter); torrent) { + filter = &*torrent; + } // run lua script before data is compressed and encrypted - last filter runs first op_ret = get_lua_filter(&run_lua, filter); if (op_ret < 0) { @@ -4161,9 +4183,6 @@ void RGWPutObj::execute(optional_yield y) hash.Update((const unsigned char *)data.c_str(), data.length()); } - /* update torrrent */ - torrent.update(data); - op_ret = filter->process(std::move(data), ofs); if (op_ret < 0) { ldpp_dout(this, 20) << "processor->process() returned ret=" @@ -4217,6 +4236,14 @@ void RGWPutObj::execute(optional_yield y) << ", orig_size=" << cs_info.orig_size << ", blocks=" << cs_info.blocks.size() << dendl; } + if (torrent) { + auto bl = torrent->bencode_torrent(s->object->get_name()); + if (bl.length()) { + ldpp_dout(this, 20) << "storing " << bl.length() + << " bytes of torrent info in " << RGW_ATTR_TORRENT << dendl; + attrs[RGW_ATTR_TORRENT] = std::move(bl); + } + } buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5); @@ -4286,19 +4313,6 @@ void RGWPutObj::execute(optional_yield y) s->yield); tracepoint(rgw_op, processor_complete_exit, s->req_id.c_str()); - /* produce torrent */ - if (s->cct->_conf->rgw_torrent_flag && (ofs == torrent.get_data_len())) - { - torrent.init(s, driver); - torrent.set_create_date(mtime); - op_ret = torrent.complete(y); - if (0 != op_ret) - { - ldpp_dout(this, 0) << "ERROR: torrent.handle_data() returned " << op_ret << dendl; - return; - } - } - // send request to notification manager int ret = res->publish_commit(this, s->obj_size, mtime, etag, s->object->get_instance()); if (ret < 0) { diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index d29dc01415383..2484d7a41ad5e 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -1189,7 +1189,6 @@ WRITE_CLASS_ENCODER(RGWSLOInfo) class RGWPutObj : public RGWOp { protected: - seed torrent; off_t ofs; const char *supplied_md5_b64; const char *supplied_etag; @@ -1285,6 +1284,9 @@ public: rgw::sal::DataProcessor *cb) { return 0; } + // if configured, construct a filter to generate torrent metadata + auto get_torrent_filter(rgw::sal::DataProcessor *cb) + -> std::optional; // get lua script to run as a "put object" filter int get_lua_filter(std::unique_ptr* filter, diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 54a39a9a549a7..88f1e7c3a6d0f 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -1033,19 +1033,6 @@ int RGWPutObj_ObjStore::verify_params() int RGWPutObj_ObjStore::get_params(optional_yield y) { - /* start gettorrent */ - if (s->cct->_conf->rgw_torrent_flag) - { - int ret = 0; - ret = torrent.get_params(); - ldpp_dout(s, 5) << "NOTICE: open produce torrent file " << dendl; - if (ret < 0) - { - return ret; - } - torrent.set_info_name(s->object->get_name()); - } - /* end gettorrent */ supplied_md5_b64 = s->info.env->get("HTTP_CONTENT_MD5"); return 0; -- 2.39.5