From 7f5fdcedee201afcbfc19bf303fd414b6a72758a Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Mon, 21 Sep 2015 22:14:22 -0700 Subject: [PATCH] rgw: store slo info when putting object Signed-off-by: Yehuda Sadeh Signed-off-by: Radoslaw Zarzynski Conflicts: src/rgw/rgw_op.cc src/rgw/rgw_op.h --- src/rgw/rgw_common.h | 1 + src/rgw/rgw_op.cc | 26 ++++++++++++++++++++++++- src/rgw/rgw_op.h | 40 ++++++++++++++++++++++++++++++++++++++- src/rgw/rgw_rest.h | 29 ++++++++++++++++++++++++++++ src/rgw/rgw_rest_swift.cc | 7 +++++-- 5 files changed, 99 insertions(+), 4 deletions(-) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 641e2d57c9297..89115b2dd8f4b 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -71,6 +71,7 @@ using ceph::crypto::MD5; #define RGW_ATTR_SHADOW_OBJ RGW_ATTR_PREFIX "shadow_name" #define RGW_ATTR_MANIFEST RGW_ATTR_PREFIX "manifest" #define RGW_ATTR_USER_MANIFEST RGW_ATTR_PREFIX "user_manifest" +#define RGW_ATTR_SLO_MANIFEST RGW_ATTR_PREFIX "slo_manifest" #define RGW_ATTR_TEMPURL_KEY1 RGW_ATTR_META_PREFIX "temp-url-key" #define RGW_ATTR_TEMPURL_KEY2 RGW_ATTR_META_PREFIX "temp-url-key-2" diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index fdc51db7707d2..5b5776697bf6a 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1873,6 +1873,17 @@ static int encode_dlo_manifest_attr(const char * const dlo_manifest, return 0; } +static void complete_etag(MD5& hash, string *etag) +{ + char etag_buf[CEPH_CRYPTO_MD5_DIGESTSIZE]; + char etag_buf_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; + + hash.Final((byte *)etag_buf); + buf_to_hex((const unsigned char *)etag_buf, CEPH_CRYPTO_MD5_DIGESTSIZE, etag_buf_str); + + *etag = etag_buf_str; +} + void RGWPutObj::execute() { RGWPutObjProcessor *processor = NULL; @@ -1887,7 +1898,7 @@ void RGWPutObj::execute() map::iterator iter; bool multipart; - bool need_calc_md5 = (dlo_manifest == NULL); + bool need_calc_md5 = (dlo_manifest == NULL) && (slo_info == NULL); perfcounter->inc(l_rgw_put); @@ -2011,6 +2022,7 @@ void RGWPutObj::execute() goto done; } s->obj_size = ofs; + perfcounter->inc(l_rgw_put_b, s->obj_size); ret = store->check_quota(s->bucket_owner.get_id(), s->bucket, @@ -2043,6 +2055,18 @@ void RGWPutObj::execute() ldout(s->cct, 0) << "bad user manifest: " << dlo_manifest << dendl; goto done; } + complete_etag(hash, &etag); + ldout(s->cct, 10) << __func__ << ": calculated md5 for user manifest: " << etag << dendl; + } + + if (slo_info) { + bufferlist manifest_bl; + ::encode(*slo_info, manifest_bl); + attrs[RGW_ATTR_SLO_MANIFEST] = manifest_bl; + + hash.Update((byte *)slo_info->raw_data, slo_info->raw_data_len); + complete_etag(hash, &etag); + ldout(s->cct, 10) << __func__ << ": calculated md5 for user manifest: " << etag << dendl; } if (supplied_etag && etag.compare(supplied_etag) != 0) { diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index ecb3ef8ff6d9b..0a7d7cdb01fb4 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -430,12 +430,51 @@ struct rgw_slo_entry { string etag; uint64_t size_bytes; + rgw_slo_entry() : size_bytes(0) {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(path, bl); + ::encode(etag, bl); + ::encode(size_bytes, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(path, bl); + ::decode(etag, bl); + ::decode(size_bytes, bl); + DECODE_FINISH(bl); + } + void decode_json(JSONObj *obj); }; +WRITE_CLASS_ENCODER(rgw_slo_entry) struct RGWSLOInfo { vector entries; + char *raw_data; /* in memory only */ + int raw_data_len; + + RGWSLOInfo() : raw_data(NULL), raw_data_len(0) {} + ~RGWSLOInfo() { + free(raw_data); + } + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(entries, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(entries, bl); + DECODE_FINISH(bl); + } }; +WRITE_CLASS_ENCODER(RGWSLOInfo) class RGWPutObj : public RGWOp { @@ -455,7 +494,6 @@ protected: RGWSLOInfo *slo_info; time_t mtime; - uint64_t olh_epoch; string version_id; diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index e17627ca9a83c..9b49a1ed72919 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -59,6 +59,35 @@ int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out, int max_len, return 0; } +template +int rgw_rest_get_json_input_keep_data(CephContext *cct, req_state *s, T& out, int max_len, char **pdata, int *len) +{ + int rv, data_len; + char *data; + + if ((rv = rgw_rest_read_all_input(s, &data, &data_len, max_len)) < 0) { + return rv; + } + + if (!data_len) { + return -EINVAL; + } + + *len = data_len; + + JSONParser parser; + + if (!parser.parse(data, data_len)) { + free(data); + return -EINVAL; + } + + decode_json_obj(out, &parser); + + *pdata = data; + return 0; +} + class RESTArgs { public: diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index ac9ffc0816398..f8c39f12dc0b1 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -577,16 +577,19 @@ int RGWPutObj_ObjStore_SWIFT::get_params() uint64_t max_len = s->cct->_conf->rgw_max_slo_entries * MAX_SLO_ENTRY_SIZE; slo_info = new RGWSLOInfo; - int r = rgw_rest_get_json_input(s->cct, s, slo_info->entries, max_len, NULL); + + int r = rgw_rest_get_json_input_keep_data(s->cct, s, slo_info->entries, max_len, &slo_info->raw_data, &slo_info->raw_data_len); if (r < 0) { ldout(s->cct, 5) << "failed to read input for slo r=" << r << dendl; return r; } - if (slo_info->entries.size() > s->cct->_conf->rgw_max_slo_entries) { + if ((int64_t)slo_info->entries.size() > s->cct->_conf->rgw_max_slo_entries) { ldout(s->cct, 5) << "too many entries in slo request: " << slo_info->entries.size() << dendl; return -EINVAL; } + + ofs = slo_info->raw_data_len; } return RGWPutObj_ObjStore::get_params(); -- 2.39.5