From: Ved-vampir Date: Wed, 2 Mar 2016 11:55:54 +0000 (+0300) Subject: rgw: replace multiple compression attributes by struct X-Git-Tag: v11.1.0~429^2~37 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=44d397d4c8b57a751219c18d94a9e64fe963ead9;p=ceph.git rgw: replace multiple compression attributes by struct Signed-off-by: Alyona Kiseleva --- diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 33486b51c61b..f81f857b2cbf 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1525,7 +1525,7 @@ OPTION(rgw_list_bucket_min_readahead, OPT_INT, 1000) // minimum number of entrie OPTION(rgw_rest_getusage_op_compat, OPT_BOOL, false) // dump description of total stats for s3 GetUsage API OPTION(rgw_compression_enabled, OPT_BOOL, true) // to use compression on rgw level -OPTION(rgw_compression_type, OPT_STR, "zlib") // type of compressor +OPTION(rgw_compression_type, OPT_STR, "none") // type of compressor, none to not use OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter OPTION(throttler_perf_counter, OPT_BOOL, true) // enable/disable throttler perf counter diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index e9c77e71b618..192344828eb6 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -109,7 +109,6 @@ using ceph::crypto::MD5; #define RGW_ATTR_OLH_PENDING_PREFIX RGW_ATTR_OLH_PREFIX "pending." #define RGW_ATTR_COMPRESSION RGW_ATTR_PREFIX "compression" -#define RGW_ATTR_COMPRESSION_ORIG_SIZE RGW_ATTR_PREFIX "orig_size" /* RGW File Attributes */ #define RGW_ATTR_UNIX_KEY1 RGW_ATTR_PREFIX "unix-key1" diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 982c25a0f5b0..e5f2366dad75 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1266,22 +1266,20 @@ int RGWGetObj::get_data_cb(bufferlist& bl, off_t bl_ofs, off_t bl_len) gc_invalidate_time += (s->cct->_conf->rgw_gc_obj_min_wait / 2); } // compression stuff - bool need_decompress = (attrs.find(RGW_ATTR_COMPRESSION) != attrs.end()); - if (need_decompress) { // or it's the first part and flag is set + if (need_decompress) { ldout(s->cct, 10) << "Compression for rgw is enabled, decompress part" << dendl; - string compression_type = attrs[RGW_ATTR_COMPRESSION].c_str(); - CompressorRef compressor = Compressor::create(s->cct, compression_type); + CompressorRef compressor = Compressor::create(s->cct, cs_info.compression_type); if (!compressor.get()) { // if compressor isn't available - error, because cannot return decompressed data? - lderr(s->cct) << "Cannot load compressor of type " << compression_type + lderr(s->cct) << "Cannot load compressor of type " << cs_info.compression_type << "for rgw, check rgw_compression_type config option" << dendl; - return -1; + return -EIO; } else { bufferlist out_bl; int cr = compressor->decompress(bl, out_bl); - if (cr != 0) { + if (cr < 0) { lderr(s->cct) << "Compression failed with exit code " << cr << dendl; - return -1; + return cr; } return send_response_data(out_bl, bl_ofs, out_bl.length()); } @@ -1437,6 +1435,16 @@ void RGWGetObj::execute() return; } + op_ret = rgw_compression_info_from_attrset(attrs, need_decompress, cs_info); + if (op_ret < 0) { + lderr(s->cct) << "ERROR: failed to decode compression info, cannot decompress" << dendl; + goto done_err; + } + if (need_decompress) { + total_len = cs_info.orig_size; + } + + /* Check whether the object has expired. Swift API documentation * stands that we should return 404 Not Found in such case. */ if (need_object_expiration() && object_is_expired(attrs)) { @@ -3009,13 +3017,11 @@ void RGWPutObj::execute() if (processor->is_compressed()) { bufferlist tmp; - tmp.append(s->cct->_conf->rgw_compression_type.c_str(), s->cct->_conf->rgw_compression_type.length()+1); + RGWCompressionInfo cs_info; + cs_info.compression_type = s->cct->_conf->rgw_compression_type; + cs_info.orig_size = s->obj_size; + ::encode(cs_info, tmp); attrs[RGW_ATTR_COMPRESSION] = tmp; - tmp.clear(); - char sz [20]; - snprintf(sz, sizeof(sz), "%lu", s->obj_size); - tmp.append(sz, strlen(sz)+1); - attrs[RGW_ATTR_COMPRESSION_ORIG_SIZE] = tmp; } diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index d93b0cad921b..9e9eef14873c 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -135,6 +135,10 @@ protected: string lo_etag; bool rgwx_stat; /* extended rgw stat operation */ + // compression attrs + RGWCompressionInfo cs_info; + bool need_decompress; + int init_common(); public: RGWGetObj() { diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index fb277cba58dd..abe23089de15 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -2370,8 +2370,9 @@ int RGWPutObjProcessor_Atomic::handle_data(bufferlist& bl, off_t ofs, MD5 *hash, bufferlist in_bl; // compression stuff + bool compression_enabled = store->ctx()->_conf->rgw_compression_type != "none"; if ((ofs > 0 && compressed) || // if previous part was compressed - (ofs == 0 && store->ctx()->_conf->rgw_compression_enabled)) { // or it's the first part and flag is set + (ofs == 0 && compression_enabled)) { // or it's the first part and flag is set ldout(store->ctx(), 10) << "Compression for rgw is enabled, compress part" << dendl; CompressorRef compressor = Compressor::create(store->ctx(), store->ctx()->_conf->rgw_compression_type); if (!compressor.get()) { @@ -9052,17 +9053,13 @@ int RGWRados::Object::Read::prepare(int64_t *pofs, int64_t *pend) return r; } - int dec_size = 0; if (params.attrs) { *params.attrs = astate->attrset; if (cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20)) { for (iter = params.attrs->begin(); iter != params.attrs->end(); ++iter) { - ldout(cct, 20) << "Read xattr: " << iter->first << "=" << iter->second << dendl; + ldout(cct, 20) << "Read xattr: " << iter->first << dendl; } } - if (params.attrs->find(RGW_ATTR_COMPRESSION) != params.attrs->end()) { - dec_size = atoi(params.attrs->at(RGW_ATTR_COMPRESSION_ORIG_SIZE).c_str()); - } } /* Convert all times go GMT to make them compatible */ @@ -9139,12 +9136,8 @@ int RGWRados::Object::Read::prepare(int64_t *pofs, int64_t *pend) *pofs = ofs; if (pend) *pend = end; - if (params.read_size) { - if (dec_size) - *params.read_size = dec_size; - else - *params.read_size = (ofs <= end ? end + 1 - ofs : 0); - } + if (params.read_size) + *params.read_size = (ofs <= end ? end + 1 - ofs : 0); if (params.obj_size) *params.obj_size = astate->size; if (params.lastmod) @@ -12892,3 +12885,20 @@ int RGWRados::delete_obj_aio(rgw_obj& obj, rgw_bucket& bucket, return ret; } +int rgw_compression_info_from_attrset(map& attrs, bool& need_decompress, RGWCompressionInfo& cs_info) { + if (attrs.find(RGW_ATTR_COMPRESSION) != attrs.end()) { + bufferlist::iterator bliter = attrs[RGW_ATTR_COMPRESSION].begin(); + try { + ::decode(cs_info, bliter); + } catch (buffer::error& err) { + return -EIO; + } + need_decompress = true; + return 0; + } else { + need_decompress = false; + return 0; + } +} + + diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 0c1ace424358..6b42c97e4f42 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -76,6 +76,30 @@ static inline void get_obj_bucket_and_oid_loc(const rgw_obj& obj, rgw_bucket& bu int rgw_policy_from_attrset(CephContext *cct, map& attrset, RGWAccessControlPolicy *policy); +struct RGWCompressionInfo { + string compression_type; + uint64_t orig_size; + + RGWCompressionInfo() : compression_type("none"), orig_size(0) {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(compression_type, bl); + ::encode(orig_size, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(compression_type, bl); + ::decode(orig_size, bl); + DECODE_FINISH(bl); + } +}; +WRITE_CLASS_ENCODER(RGWCompressionInfo) + +int rgw_compression_info_from_attrset(map& attrs, bool& need_decompress, RGWCompressionInfo& cs_info); + struct RGWOLHInfo { rgw_obj target; bool removed; diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index b02fde2b896c..860aed7ded08 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -23,8 +23,6 @@ #include "rgw_client_io.h" #include "rgw_resolve.h" -#include "compressor/Compressor.h" - #include #define dout_subsys ceph_subsys_rgw