From b758348447e60af23b114aa47c28f151ffd97792 Mon Sep 17 00:00:00 2001 From: Pavan Rallabhandi Date: Fri, 14 Apr 2017 21:42:45 +0530 Subject: [PATCH] rgw: add a field to store generic user data in the bucket index, that can be populated/fetched via a configurable custom http header Signed-off-by: Pavan Rallabhandi (cherry picked from commit abca7a86c3cfbb58fafb5d057d9d6f5017a53704) Conflicts: src/rgw/rgw_op.cc Signature fixes for RGWPutObjProcessor_Multipart::do_complete(). src/rgw/rgw_op.h A new member field `crypt_http_responses` is added in the class `RGWPutObj` in master version, which is not required in Kraken. src/rgw/rgw_rados.cc In RGWRados::Bucket::UpdateIndex::complete, RGWObjEnt has been removed in master, which has to be retained in Kraken. In RGWRados::cls_obj_complete_op, user_data is added to the rgw_bucket_dir_entry_meta In RGWRados::cls_bucket_list, the user_data field of RGWObjEnt is populated. src/rgw/rgw_rados.h In UpdateIndex::complete(), remove_objs is of type rgw_obj_key in Kraken instead of rgw_obj_index_key RGWPutObjProcessor_Multipart is not part of this file in Kraken. src/rgw/rgw_rest_swift.cc In RGWListBucket_ObjStore_SWIFT::send_response(), there is no meta struct in Kraken. src/rgw/rgw_common.h Add user_data field in RGWObjEnt structure. src/rgw/rgw_json_enc.cc Add user_data field while dumping RGWObjEnt. --- src/cls/rgw/cls_rgw_types.cc | 2 ++ src/cls/rgw/cls_rgw_types.h | 8 ++++++-- src/common/config_opts.h | 2 ++ src/rgw/rgw_common.h | 1 + src/rgw/rgw_json_enc.cc | 1 + src/rgw/rgw_op.cc | 7 ++++--- src/rgw/rgw_op.h | 1 + src/rgw/rgw_rados.cc | 16 +++++++++++----- src/rgw/rgw_rados.h | 11 ++++++----- src/rgw/rgw_rest_swift.cc | 9 +++++++++ 10 files changed, 43 insertions(+), 15 deletions(-) diff --git a/src/cls/rgw/cls_rgw_types.cc b/src/cls/rgw/cls_rgw_types.cc index 1b8edbb667d13..7d9c83bfad8ef 100644 --- a/src/cls/rgw/cls_rgw_types.cc +++ b/src/cls/rgw/cls_rgw_types.cc @@ -61,6 +61,7 @@ void rgw_bucket_dir_entry_meta::dump(Formatter *f) const encode_json("owner_display_name", owner_display_name, f); encode_json("content_type", content_type, f); encode_json("accounted_size", accounted_size, f); + encode_json("user_data", user_data, f); } void rgw_bucket_dir_entry_meta::decode_json(JSONObj *obj) { @@ -75,6 +76,7 @@ void rgw_bucket_dir_entry_meta::decode_json(JSONObj *obj) { JSONDecoder::decode_json("owner_display_name", owner_display_name, obj); JSONDecoder::decode_json("content_type", content_type, obj); JSONDecoder::decode_json("accounted_size", accounted_size, obj); + JSONDecoder::decode_json("user_data", user_data, obj); } void rgw_bucket_dir_entry::generate_test_instances(list& o) diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h index 144c5af203f90..01fd260dccf6a 100644 --- a/src/cls/rgw/cls_rgw_types.h +++ b/src/cls/rgw/cls_rgw_types.h @@ -95,12 +95,13 @@ struct rgw_bucket_dir_entry_meta { string owner_display_name; string content_type; uint64_t accounted_size; + string user_data; rgw_bucket_dir_entry_meta() : category(0), size(0), accounted_size(0) { } void encode(bufferlist &bl) const { - ENCODE_START(4, 3, bl); + ENCODE_START(5, 3, bl); ::encode(category, bl); ::encode(size, bl); ::encode(mtime, bl); @@ -109,10 +110,11 @@ struct rgw_bucket_dir_entry_meta { ::encode(owner_display_name, bl); ::encode(content_type, bl); ::encode(accounted_size, bl); + ::encode(user_data, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator &bl) { - DECODE_START_LEGACY_COMPAT_LEN(4, 3, 3, bl); + DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, bl); ::decode(category, bl); ::decode(size, bl); ::decode(mtime, bl); @@ -125,6 +127,8 @@ struct rgw_bucket_dir_entry_meta { ::decode(accounted_size, bl); else accounted_size = size; + if (struct_v >= 5) + ::decode(user_data, bl); DECODE_FINISH(bl); } void dump(Formatter *f) const; diff --git a/src/common/config_opts.h b/src/common/config_opts.h index aacd183a161d7..b9965192a9a51 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1612,3 +1612,5 @@ OPTION(rgw_torrent_sha_unit, OPT_INT, 512*1024) // torrent field piece length OPTION(internal_safe_to_start_threads, OPT_BOOL, false) OPTION(debug_deliberately_leak_memory, OPT_BOOL, false) + +OPTION(rgw_swift_custom_header, OPT_STR, "") // option to enable swift custom headers diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 468e82462e520..4e740b03f9699 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -1392,6 +1392,7 @@ struct RGWObjEnt { string tag; uint32_t flags; uint64_t versioned_epoch; + string user_data; RGWObjEnt() : flags(0), versioned_epoch(0) {} diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc index ca4b1e372037b..4feb61925fb48 100644 --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@ -750,6 +750,7 @@ void RGWObjEnt::dump(Formatter *f) const encode_json("content_type", content_type, f); encode_json("tag", tag, f); encode_json("flags", flags, f); + encode_json("user_data", user_data, f); } void RGWBucketEnt::dump(Formatter *f) const diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 6f29be48ecbbd..30c25aa0b006b 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -2583,7 +2583,7 @@ protected: int do_complete(size_t accounted_size, const string& etag, real_time *mtime, real_time set_mtime, map& attrs, real_time delete_at, const char *if_match, - const char *if_nomatch) override; + const char *if_nomatch, const string *user_data) override; public: bool immutable_head() { return true; } @@ -2661,7 +2661,7 @@ int RGWPutObjProcessor_Multipart::do_complete(size_t accounted_size, map& attrs, real_time delete_at, const char *if_match, - const char *if_nomatch) + const char *if_nomatch, const string *user_data) { complete_writing_data(); @@ -3175,7 +3175,8 @@ void RGWPutObj::execute() } op_ret = processor->complete(s->obj_size, etag, &mtime, real_time(), attrs, - (delete_at ? *delete_at : real_time()), if_match, if_nomatch); + (delete_at ? *delete_at : real_time()), if_match, if_nomatch, + (user_data.empty() ? nullptr : &user_data)); /* produce torrent */ if (s->cct->_conf->rgw_torrent_flag && (ofs == torrent.get_data_len())) diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index ae51f0bf09bd5..bc85ec99d77cf 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -719,6 +719,7 @@ protected: uint64_t olh_epoch; string version_id; bufferlist bl_aux; + string user_data; boost::optional delete_at; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 48a2344bf8dc2..0c2dbb29a2c76 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -2214,9 +2214,9 @@ void RGWObjVersionTracker::generate_new_write_ver(CephContext *cct) int RGWPutObjProcessor::complete(size_t accounted_size, const string& etag, real_time *mtime, real_time set_mtime, map& attrs, real_time delete_at, - const char *if_match, const char *if_nomatch) + const char *if_match, const char *if_nomatch, const string *user_data) { - int r = do_complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at, if_match, if_nomatch); + int r = do_complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at, if_match, if_nomatch, user_data); if (r < 0) return r; @@ -2549,7 +2549,7 @@ int RGWPutObjProcessor_Atomic::do_complete(size_t accounted_size, const string& map& attrs, real_time delete_at, const char *if_match, - const char *if_nomatch) { + const char *if_nomatch, const string *user_data) { int r = complete_writing_data(); if (r < 0) return r; @@ -2574,6 +2574,7 @@ int RGWPutObjProcessor_Atomic::do_complete(size_t accounted_size, const string& obj_op.meta.flags = PUT_OBJ_CREATE; obj_op.meta.olh_epoch = olh_epoch; obj_op.meta.delete_at = delete_at; + obj_op.meta.user_data = user_data; r = obj_op.write_meta(obj_len, accounted_size, attrs); if (r < 0) { @@ -6479,7 +6480,7 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si r = index_op->complete(poolid, epoch, size, accounted_size, meta.set_mtime, etag, content_type, &acl_bl, - meta.category, meta.remove_objs); + meta.category, meta.remove_objs, meta.user_data); if (r < 0) goto done_cancel; @@ -9403,7 +9404,7 @@ int RGWRados::Bucket::UpdateIndex::complete(int64_t poolid, uint64_t epoch, const string& content_type, bufferlist *acl_bl, RGWObjCategory category, - list *remove_objs) + list *remove_objs, const string *user_data) { if (blind) { return 0; @@ -9422,6 +9423,9 @@ int RGWRados::Bucket::UpdateIndex::complete(int64_t poolid, uint64_t epoch, ent.accounted_size = accounted_size; ent.mtime = ut; ent.etag = etag; + if (user_data) + ent.user_data = *user_data; + ACLOwner owner; if (acl_bl && acl_bl->length()) { int ret = store->decode_policy(*acl_bl, &owner); @@ -11970,6 +11974,7 @@ int RGWRados::cls_obj_complete_op(BucketShard& bs, RGWModifyOp op, string& tag, dir_meta.owner_display_name = ent.owner_display_name; dir_meta.content_type = ent.content_type; dir_meta.category = category; + dir_meta.user_data = ent.user_data; rgw_bucket_entry_ver ver; ver.pool = pool; @@ -12090,6 +12095,7 @@ int RGWRados::cls_bucket_list(rgw_bucket& bucket, int shard_id, rgw_obj_key& sta e.tag = dirent.tag; e.flags = dirent.flags; e.versioned_epoch = dirent.versioned_epoch; + e.user_data = dirent.meta.user_data; bool force_check = force_check_filter && force_check_filter(dirent.key.name); if ((!dirent.exists && !dirent.is_delete_marker()) || !dirent.pending_map.empty() || force_check) { diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index c3475a28d6b12..e2c5936151bea 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -2452,10 +2452,11 @@ public: uint64_t olh_epoch; ceph::real_time delete_at; bool canceled; + const string *user_data; MetaParams() : mtime(NULL), rmattrs(NULL), data(NULL), manifest(NULL), ptag(NULL), remove_objs(NULL), category(RGW_OBJ_CATEGORY_MAIN), flags(0), - if_match(NULL), if_nomatch(NULL), olh_epoch(0), canceled(false) {} + if_match(NULL), if_nomatch(NULL), olh_epoch(0), canceled(false), user_data(nullptr) {} } meta; explicit Write(RGWRados::Object *_target) : target(_target) {} @@ -2588,7 +2589,7 @@ public: uint64_t accounted_size, ceph::real_time& ut, const string& etag, const string& content_type, bufferlist *acl_bl, RGWObjCategory category, - list *remove_objs); + list *remove_objs, const string *user_data = nullptr); int complete_del(int64_t poolid, uint64_t epoch, ceph::real_time& removed_mtime, /* mtime of removed object */ list *remove_objs); @@ -3356,7 +3357,7 @@ protected: virtual int do_complete(size_t accounted_size, const string& etag, ceph::real_time *mtime, ceph::real_time set_mtime, map& attrs, ceph::real_time delete_at, - const char *if_match, const char *if_nomatch) = 0; + const char *if_match, const char *if_nomatch, const string *user_data) = 0; public: RGWPutObjProcessor(RGWObjectCtx& _obj_ctx, RGWBucketInfo& _bi) : store(NULL), @@ -3373,7 +3374,7 @@ public: int complete(size_t accounted_size, const string& etag, ceph::real_time *mtime, ceph::real_time set_mtime, map& attrs, ceph::real_time delete_at, - const char *if_match = NULL, const char *if_nomatch = NULL); + const char *if_match = NULL, const char *if_nomatch = NULL, const string *user_data = nullptr); CephContext *ctx(); @@ -3451,7 +3452,7 @@ protected: int do_complete(size_t accounted_size, const string& etag, ceph::real_time *mtime, ceph::real_time set_mtime, map& attrs, ceph::real_time delete_at, - const char *if_match, const char *if_nomatch) override; + const char *if_match, const char *if_nomatch, const string *user_data) override; int prepare_next_part(off_t ofs); int complete_parts(); diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index 2086b4ca72997..9421f4fe8b818 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -306,6 +306,8 @@ void RGWListBucket_ObjStore_SWIFT::send_response() s->formatter->dump_string("name", key.name); s->formatter->dump_string("hash", iter->etag); s->formatter->dump_int("bytes", iter->accounted_size); + if (!iter->user_data.empty()) + s->formatter->dump_string("user_custom_data", iter->user_data); string single_content_type = iter->content_type; if (iter->content_type.size()) { // content type might hold multiple values, just dump the last one @@ -758,6 +760,13 @@ int RGWPutObj_ObjStore_SWIFT::get_params() return r; } + if (!s->cct->_conf->rgw_swift_custom_header.empty()) { + string custom_header = s->cct->_conf->rgw_swift_custom_header; + if (s->info.env->exists(custom_header.c_str())) { + user_data = s->info.env->get(custom_header.c_str()); + } + } + dlo_manifest = s->info.env->get("HTTP_X_OBJECT_MANIFEST"); bool exists; string multipart_manifest = s->info.args.get("multipart-manifest", &exists); -- 2.39.5