From: Pavan Rallabhandi Date: Fri, 14 Apr 2017 16:12:45 +0000 (+0530) Subject: rgw: add a field to store generic user data in the bucket index, X-Git-Tag: v10.2.10~55^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=178245d543d680bbdc7f2da07c431fc745da1143;p=ceph.git rgw: add a field to store generic user data in the bucket index, that can be populated/fetched via a configurable custom http header Signed-off-by: Pavan Rallabhandi (cherry picked from commit abca7a86c3cfbb58fafb5d057d9d6f5017a53704) Conflicts: src/rgw/rgw_op.cc Number of arguments to RGWPutObjProcessor_Multipart::do_complete has been changed in master with an extra field accounted_size added, which is not required in Jewel. Number of arguments for the processor->complete() has been changed in master, which is not required in Jewel. src/rgw/rgw_op.h A new member field `crypt_http_responses` is added in the class `RGWPutObj` in master, which is not required in Jewel. src/rgw/rgw_rados.cc In the function RGWPutObjProcessor::complete() there is an extra argument to do_complete() in master, which is not required in Jewel. In the function RGWPutObjProcessor_Atomic::do_complete, a call to obj_op.write_meta has accounted_size, which is not required in Jewel. There is an extra argument accounted_size in RGWRados::Bucket::UpdateIndex::complete in master, which is not required in Jewel. In RGWRados::Bucket::UpdateIndex::complete, RGWObjEnt has been removed in master, which has to be retained in Jewel. In RGWRados::cls_obj_complete_op, user_data is added to the rgw_bucket_dir_entry_meta structure. In RGWRados::cls_bucket_list, the user_data field of RGWObjEnt is populated. src/rgw/rgw_rados.h In UpdateIndex::complete(), remove_objs is of type rgw_obj_key in Jewel instead of rgw_obj_index_key Corrected various function declarations as appropriate. RGWPutObjProcessor_Multipart is not part of this file in Jewel. src/rgw/rgw_rest_swift.cc In RGWListBucket_ObjStore_SWIFT::send_response(), accounted_size is not required in Jewel. src/rgw/rgw_common.h Add user_data field in RGWObjEnt structure. src/rgw/rgw_json_enc.cc Add user_data field while dumping RGWObjEnt. --- diff --git a/src/cls/rgw/cls_rgw_types.cc b/src/cls/rgw/cls_rgw_types.cc index 27a413e865b9..481c064dbc34 100644 --- a/src/cls/rgw/cls_rgw_types.cc +++ b/src/cls/rgw/cls_rgw_types.cc @@ -61,6 +61,7 @@ void rgw_bucket_dir_entry_meta::dump(Formatter *f) const encode_json("owner_display_name", owner_display_name, f); encode_json("content_type", content_type, f); encode_json("accounted_size", accounted_size, f); + encode_json("user_data", user_data, f); } void rgw_bucket_dir_entry_meta::decode_json(JSONObj *obj) { @@ -75,6 +76,7 @@ void rgw_bucket_dir_entry_meta::decode_json(JSONObj *obj) { JSONDecoder::decode_json("owner_display_name", owner_display_name, obj); JSONDecoder::decode_json("content_type", content_type, obj); JSONDecoder::decode_json("accounted_size", accounted_size, obj); + JSONDecoder::decode_json("user_data", user_data, obj); } void rgw_bucket_dir_entry::generate_test_instances(list& o) diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h index c7942ac7bdb5..454138974fed 100644 --- a/src/cls/rgw/cls_rgw_types.h +++ b/src/cls/rgw/cls_rgw_types.h @@ -95,12 +95,13 @@ struct rgw_bucket_dir_entry_meta { string owner_display_name; string content_type; uint64_t accounted_size; + string user_data; rgw_bucket_dir_entry_meta() : category(0), size(0), accounted_size(0) { } void encode(bufferlist &bl) const { - ENCODE_START(4, 3, bl); + ENCODE_START(5, 3, bl); ::encode(category, bl); ::encode(size, bl); ::encode(mtime, bl); @@ -109,10 +110,11 @@ struct rgw_bucket_dir_entry_meta { ::encode(owner_display_name, bl); ::encode(content_type, bl); ::encode(accounted_size, bl); + ::encode(user_data, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator &bl) { - DECODE_START_LEGACY_COMPAT_LEN(4, 3, 3, bl); + DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, bl); ::decode(category, bl); ::decode(size, bl); ::decode(mtime, bl); @@ -125,6 +127,8 @@ struct rgw_bucket_dir_entry_meta { ::decode(accounted_size, bl); else accounted_size = size; + if (struct_v >= 5) + ::decode(user_data, bl); DECODE_FINISH(bl); } void dump(Formatter *f) const; diff --git a/src/common/config_opts.h b/src/common/config_opts.h index a941edbf9eb7..e28f673e9b6c 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1474,3 +1474,5 @@ OPTION(throttler_perf_counter, OPT_BOOL, true) // enable/disable throttler perf OPTION(internal_safe_to_start_threads, OPT_BOOL, false) OPTION(debug_deliberately_leak_memory, OPT_BOOL, false) + +OPTION(rgw_swift_custom_header, OPT_STR, "") // option to enable swift custom headers diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index e164850e0370..c5f851ad21a4 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -1332,6 +1332,7 @@ struct RGWObjEnt { string tag; uint32_t flags; uint64_t versioned_epoch; + string user_data; RGWObjEnt() : size(0), flags(0), versioned_epoch(0) {} diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc index 05939630d8ac..0a844b1c771b 100644 --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@ -701,6 +701,7 @@ void RGWObjEnt::dump(Formatter *f) const encode_json("content_type", content_type, f); encode_json("tag", tag, f); encode_json("flags", flags, f); + encode_json("user_data", user_data, f); } void RGWBucketEnt::dump(Formatter *f) const diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index b0e1f276352f..0144810ea30b 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -2299,7 +2299,7 @@ protected: int prepare(RGWRados *store, string *oid_rand); int do_complete(string& etag, real_time *mtime, real_time set_mtime, map& attrs, real_time delete_at, - const char *if_match = NULL, const char *if_nomatch = NULL); + const char *if_match = NULL, const char *if_nomatch = NULL, const string *user_data = nullptr); public: bool immutable_head() { return true; } @@ -2373,7 +2373,7 @@ static bool is_v2_upload_id(const string& upload_id) int RGWPutObjProcessor_Multipart::do_complete(string& etag, real_time *mtime, real_time set_mtime, map& attrs, real_time delete_at, - const char *if_match, const char *if_nomatch) + const char *if_match, const char *if_nomatch, const string *user_data) { complete_writing_data(); @@ -2794,7 +2794,8 @@ void RGWPutObj::execute() } op_ret = processor->complete(etag, &mtime, real_time(), attrs, - (delete_at ? *delete_at : real_time()), if_match, if_nomatch); + (delete_at ? *delete_at : real_time()), if_match, if_nomatch, + (user_data.empty() ? nullptr : &user_data)); done: dispose_processor(processor); diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 4ddebd836a6d..ee01f32e7929 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -669,6 +669,7 @@ protected: uint64_t olh_epoch; string version_id; bufferlist bl_aux; + string user_data; boost::optional delete_at; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index f2f8a4781216..449cc0acc37b 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -2146,9 +2146,9 @@ void RGWObjVersionTracker::generate_new_write_ver(CephContext *cct) int RGWPutObjProcessor::complete(string& etag, real_time *mtime, real_time set_mtime, map& attrs, real_time delete_at, - const char *if_match, const char * if_nomatch) + const char *if_match, const char *if_nomatch, const string *user_data) { - int r = do_complete(etag, mtime, set_mtime, attrs, delete_at, if_match, if_nomatch); + int r = do_complete(etag, mtime, set_mtime, attrs, delete_at, if_match, if_nomatch, user_data); if (r < 0) return r; @@ -2492,7 +2492,7 @@ int RGWPutObjProcessor_Atomic::complete_writing_data() int RGWPutObjProcessor_Atomic::do_complete(string& etag, real_time *mtime, real_time set_mtime, map& attrs, real_time delete_at, const char *if_match, - const char *if_nomatch) { + const char *if_nomatch, const string *user_data) { int r = complete_writing_data(); if (r < 0) return r; @@ -2517,6 +2517,7 @@ int RGWPutObjProcessor_Atomic::do_complete(string& etag, real_time *mtime, real_ obj_op.meta.flags = PUT_OBJ_CREATE; obj_op.meta.olh_epoch = olh_epoch; obj_op.meta.delete_at = delete_at; + obj_op.meta.user_data = user_data; r = obj_op.write_meta(obj_len, attrs); if (r < 0) { @@ -6286,7 +6287,7 @@ int RGWRados::Object::Write::write_meta(uint64_t size, r = index_op.complete(poolid, epoch, size, meta.set_mtime, etag, content_type, &acl_bl, - meta.category, meta.remove_objs); + meta.category, meta.remove_objs, meta.user_data); if (r < 0) goto done_cancel; @@ -8977,7 +8978,7 @@ int RGWRados::Bucket::UpdateIndex::prepare(RGWModifyOp op) int RGWRados::Bucket::UpdateIndex::complete(int64_t poolid, uint64_t epoch, uint64_t size, ceph::real_time& ut, string& etag, string& content_type, bufferlist *acl_bl, RGWObjCategory category, - list *remove_objs) + list *remove_objs, const string *user_data) { if (blind) { return 0; @@ -8995,6 +8996,9 @@ int RGWRados::Bucket::UpdateIndex::complete(int64_t poolid, uint64_t epoch, uint ent.size = size; ent.mtime = ut; ent.etag = etag; + if (user_data) + ent.user_data = *user_data; + ACLOwner owner; if (acl_bl && acl_bl->length()) { int ret = store->decode_policy(*acl_bl, &owner); @@ -11536,6 +11540,7 @@ int RGWRados::cls_obj_complete_op(BucketShard& bs, RGWModifyOp op, string& tag, dir_meta.owner_display_name = ent.owner_display_name; dir_meta.content_type = ent.content_type; dir_meta.category = category; + dir_meta.user_data = ent.user_data; rgw_bucket_entry_ver ver; ver.pool = pool; @@ -11655,6 +11660,7 @@ int RGWRados::cls_bucket_list(rgw_bucket& bucket, int shard_id, rgw_obj_key& sta e.tag = dirent.tag; e.flags = dirent.flags; e.versioned_epoch = dirent.versioned_epoch; + e.user_data = dirent.meta.user_data; bool force_check = force_check_filter && force_check_filter(dirent.key.name); if ((!dirent.exists && !dirent.is_delete_marker()) || !dirent.pending_map.empty() || force_check) { diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 5bf125d88b09..021c7c1d6828 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -2305,10 +2305,11 @@ public: uint64_t olh_epoch; ceph::real_time delete_at; bool canceled; + const string *user_data; MetaParams() : mtime(NULL), rmattrs(NULL), data(NULL), manifest(NULL), ptag(NULL), remove_objs(NULL), category(RGW_OBJ_CATEGORY_MAIN), flags(0), - if_match(NULL), if_nomatch(NULL), olh_epoch(0), canceled(false) {} + if_match(NULL), if_nomatch(NULL), olh_epoch(0), canceled(false), user_data(nullptr) {} } meta; explicit Write(RGWRados::Object *_target) : target(_target) {} @@ -2435,7 +2436,7 @@ public: int complete(int64_t poolid, uint64_t epoch, uint64_t size, ceph::real_time& ut, string& etag, string& content_type, bufferlist *acl_bl, RGWObjCategory category, - list *remove_objs); + list *remove_objs, const string *user_data = nullptr); int complete_del(int64_t poolid, uint64_t epoch, ceph::real_time& removed_mtime, /* mtime of removed object */ list *remove_objs); @@ -3157,7 +3158,7 @@ protected: virtual int do_complete(string& etag, ceph::real_time *mtime, ceph::real_time set_mtime, map& attrs, ceph::real_time delete_at, - const char *if_match = NULL, const char *if_nomatch = NULL) = 0; + const char *if_match = NULL, const char *if_nomatch = NULL, const string *user_data = nullptr) = 0; public: RGWPutObjProcessor(RGWObjectCtx& _obj_ctx, RGWBucketInfo& _bi) : store(NULL), obj_ctx(_obj_ctx), is_complete(false), bucket_info(_bi), canceled(false) {} @@ -3173,7 +3174,7 @@ public: } virtual int complete(string& etag, ceph::real_time *mtime, ceph::real_time set_mtime, map& attrs, ceph::real_time delete_at, - const char *if_match = NULL, const char *if_nomatch = NULL); + const char *if_match = NULL, const char *if_nomatch = NULL, const string *user_data = nullptr); CephContext *ctx(); @@ -3247,7 +3248,7 @@ protected: int write_data(bufferlist& bl, off_t ofs, void **phandle, rgw_obj *pobj, bool exclusive); virtual int do_complete(string& etag, ceph::real_time *mtime, ceph::real_time set_mtime, map& attrs, ceph::real_time delete_at, - const char *if_match = NULL, const char *if_nomatch = NULL); + const char *if_match = NULL, const char *if_nomatch = NULL, const string *user_data = NULL); int prepare_next_part(off_t ofs); int complete_parts(); diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index f53582c1782a..1532d1eeff54 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -275,6 +275,8 @@ void RGWListBucket_ObjStore_SWIFT::send_response() s->formatter->dump_string("name", key.name); s->formatter->dump_string("hash", iter->etag); s->formatter->dump_int("bytes", iter->size); + if (!iter->user_data.empty()) + s->formatter->dump_string("user_custom_data", iter->user_data); string single_content_type = iter->content_type; if (iter->content_type.size()) { // content type might hold multiple values, just dump the last one @@ -670,6 +672,13 @@ int RGWPutObj_ObjStore_SWIFT::get_params() return r; } + if (!s->cct->_conf->rgw_swift_custom_header.empty()) { + string custom_header = s->cct->_conf->rgw_swift_custom_header; + if (s->info.env->exists(custom_header.c_str())) { + user_data = s->info.env->get(custom_header.c_str()); + } + } + dlo_manifest = s->info.env->get("HTTP_X_OBJECT_MANIFEST"); bool exists; string multipart_manifest = s->info.args.get("multipart-manifest", &exists);