rgw: add a field to store generic user data in the bucket index,

that can be populated/fetched via a configurable custom http header

Signed-off-by: Pavan Rallabhandi <prallabhandi@walmartlabs.com>
(cherry picked from commit abca7a86c3cfbb58fafb5d057d9d6f5017a53704)

Conflicts:
	src/rgw/rgw_op.cc
		Number of arguments to RGWPutObjProcessor_Multipart::do_complete has been changed in master with an extra field accounted_size added, which is not required in Jewel.
        	Number of arguments for the processor->complete() has been changed in master, which is not required in Jewel.

	src/rgw/rgw_op.h
		A new member field `crypt_http_responses` is added in the class `RGWPutObj` in master, which is not required in Jewel.

	src/rgw/rgw_rados.cc
        	In the function RGWPutObjProcessor::complete() there is an extra argument to do_complete() in master, which is not required in Jewel.
        	In the function RGWPutObjProcessor_Atomic::do_complete, a call to obj_op.write_meta has accounted_size, which is not required in Jewel.
        	There is an extra argument accounted_size in RGWRados::Bucket::UpdateIndex::complete in master, which is not required in Jewel.
        	In RGWRados::Bucket::UpdateIndex::complete, RGWObjEnt has been removed in master, which has to be retained in Jewel.
        	In RGWRados::cls_obj_complete_op, user_data is added to the rgw_bucket_dir_entry_meta structure.
        	In RGWRados::cls_bucket_list, the user_data field of RGWObjEnt is populated.

	src/rgw/rgw_rados.h
        	In UpdateIndex::complete(), remove_objs is of type rgw_obj_key in Jewel instead of rgw_obj_index_key
        	Corrected various function declarations as appropriate.
        	RGWPutObjProcessor_Multipart is not part of this file in Jewel.

	src/rgw/rgw_rest_swift.cc
		In RGWListBucket_ObjStore_SWIFT::send_response(), accounted_size is not required in Jewel.

	src/rgw/rgw_common.h
		Add user_data field in RGWObjEnt structure.

	src/rgw/rgw_json_enc.cc
		Add user_data field while dumping RGWObjEnt.

diff --git a/src/cls/rgw/cls_rgw_types.cc b/src/cls/rgw/cls_rgw_types.cc
index 27a413e865b948a555f098cf6a4eeba0cab33533..481c064dbc34880781aba7d7a07cd91ec875818d 100644 (file)
--- a/src/cls/rgw/cls_rgw_types.cc
+++ b/src/cls/rgw/cls_rgw_types.cc
@@ -61,6 +61,7 @@ void rgw_bucket_dir_entry_meta::dump(Formatter *f) const
   encode_json("owner_display_name", owner_display_name, f);
   encode_json("content_type", content_type, f);
   encode_json("accounted_size", accounted_size, f);
+  encode_json("user_data", user_data, f);
 }
 
 void rgw_bucket_dir_entry_meta::decode_json(JSONObj *obj) {
@@ -75,6 +76,7 @@ void rgw_bucket_dir_entry_meta::decode_json(JSONObj *obj) {
   JSONDecoder::decode_json("owner_display_name", owner_display_name, obj);
   JSONDecoder::decode_json("content_type", content_type, obj);
   JSONDecoder::decode_json("accounted_size", accounted_size, obj);
+  JSONDecoder::decode_json("user_data", user_data, obj);
 }
 
 void rgw_bucket_dir_entry::generate_test_instances(list<rgw_bucket_dir_entry*>& o)

diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h
index c7942ac7bdb598241d6fed38c597733cbb92ed76..454138974fedb34591ed8cd63754e48beba8d39b 100644 (file)
--- a/src/cls/rgw/cls_rgw_types.h
+++ b/src/cls/rgw/cls_rgw_types.h
@@ -95,12 +95,13 @@ struct rgw_bucket_dir_entry_meta {
   string owner_display_name;
   string content_type;
   uint64_t accounted_size;
+  string user_data;
 
   rgw_bucket_dir_entry_meta() :
   category(0), size(0), accounted_size(0) { }
 
   void encode(bufferlist &bl) const {
-    ENCODE_START(4, 3, bl);
+    ENCODE_START(5, 3, bl);
     ::encode(category, bl);
     ::encode(size, bl);
     ::encode(mtime, bl);
@@ -109,10 +110,11 @@ struct rgw_bucket_dir_entry_meta {
     ::encode(owner_display_name, bl);
     ::encode(content_type, bl);
     ::encode(accounted_size, bl);
+    ::encode(user_data, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::iterator &bl) {
-    DECODE_START_LEGACY_COMPAT_LEN(4, 3, 3, bl);
+    DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, bl);
     ::decode(category, bl);
     ::decode(size, bl);
     ::decode(mtime, bl);
@@ -125,6 +127,8 @@ struct rgw_bucket_dir_entry_meta {
       ::decode(accounted_size, bl);
     else
       accounted_size = size;
+    if (struct_v >= 5)
+      ::decode(user_data, bl);
     DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index a941edbf9eb72454fe49d84cc3ecd8d5f254efd6..e28f673e9b6c911b2fc88100f2b82caf1d31e95c 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -1474,3 +1474,5 @@ OPTION(throttler_perf_counter, OPT_BOOL, true) // enable/disable throttler perf
 OPTION(internal_safe_to_start_threads, OPT_BOOL, false)
 
 OPTION(debug_deliberately_leak_memory, OPT_BOOL, false)
+
+OPTION(rgw_swift_custom_header, OPT_STR, "") // option to enable swift custom headers

diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index e164850e0370dad7789baa8e1b5692f25e2710e5..c5f851ad21a4cd7fe0c659dc57ccdca79b309fb0 100644 (file)
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1332,6 +1332,7 @@ struct RGWObjEnt {
   string tag;
   uint32_t flags;
   uint64_t versioned_epoch;
+  string user_data;
 
   RGWObjEnt() : size(0), flags(0), versioned_epoch(0) {}
 

diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc
index 05939630d8ac0b7e9dfbc50580d809f55172de31..0a844b1c771bb7d1bfccc09c2d98b691145225d7 100644 (file)
--- a/src/rgw/rgw_json_enc.cc
+++ b/src/rgw/rgw_json_enc.cc
@@ -701,6 +701,7 @@ void RGWObjEnt::dump(Formatter *f) const
   encode_json("content_type", content_type, f);
   encode_json("tag", tag, f);
   encode_json("flags", flags, f);
+  encode_json("user_data", user_data, f);
 }
 
 void RGWBucketEnt::dump(Formatter *f) const

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index b0e1f276352f4485718e0248cb6ccf578077ab88..0144810ea30b43e5e9bd58eabf93ee2c47727ccb 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -2299,7 +2299,7 @@ protected:
   int prepare(RGWRados *store, string *oid_rand);
   int do_complete(string& etag, real_time *mtime, real_time set_mtime,
                   map<string, bufferlist>& attrs, real_time delete_at,
-                  const char *if_match = NULL, const char *if_nomatch = NULL);
+                  const char *if_match = NULL, const char *if_nomatch = NULL, const string *user_data = nullptr);
 
 public:
   bool immutable_head() { return true; }
@@ -2373,7 +2373,7 @@ static bool is_v2_upload_id(const string& upload_id)
 
 int RGWPutObjProcessor_Multipart::do_complete(string& etag, real_time *mtime, real_time set_mtime,
                                               map<string, bufferlist>& attrs, real_time delete_at,
-                                              const char *if_match, const char *if_nomatch)
+                                              const char *if_match, const char *if_nomatch, const string *user_data)
 {
   complete_writing_data();
 
@@ -2794,7 +2794,8 @@ void RGWPutObj::execute()
   }
 
   op_ret = processor->complete(etag, &mtime, real_time(), attrs,
-                               (delete_at ? *delete_at : real_time()), if_match, if_nomatch);
+                               (delete_at ? *delete_at : real_time()), if_match, if_nomatch,
+                               (user_data.empty() ? nullptr : &user_data));
 
 done:
   dispose_processor(processor);

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 4ddebd836a6dcc6f9e285cfca27e61bb3e6a704c..ee01f32e79299a4437bf7326c5a33b89f81fca4f 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -669,6 +669,7 @@ protected:
   uint64_t olh_epoch;
   string version_id;
   bufferlist bl_aux;
+  string user_data;
 
   boost::optional<ceph::real_time> delete_at;
 

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index f2f8a4781216f3245123adc0fa738d18a3e16e50..449cc0acc37b32e825389b5ad0d771f9b83dd08f 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -2146,9 +2146,9 @@ void RGWObjVersionTracker::generate_new_write_ver(CephContext *cct)
 
 int RGWPutObjProcessor::complete(string& etag, real_time *mtime, real_time set_mtime,
                                  map<string, bufferlist>& attrs, real_time delete_at,
-                                 const char *if_match, const char * if_nomatch)
+                                 const char *if_match, const char *if_nomatch, const string *user_data)
 {
-  int r = do_complete(etag, mtime, set_mtime, attrs, delete_at, if_match, if_nomatch);
+  int r = do_complete(etag, mtime, set_mtime, attrs, delete_at, if_match, if_nomatch, user_data);
   if (r < 0)
     return r;
 
@@ -2492,7 +2492,7 @@ int RGWPutObjProcessor_Atomic::complete_writing_data()
 int RGWPutObjProcessor_Atomic::do_complete(string& etag, real_time *mtime, real_time set_mtime,
                                            map<string, bufferlist>& attrs, real_time delete_at,
                                            const char *if_match,
-                                           const char *if_nomatch) {
+                                           const char *if_nomatch, const string *user_data) {
   int r = complete_writing_data();
   if (r < 0)
     return r;
@@ -2517,6 +2517,7 @@ int RGWPutObjProcessor_Atomic::do_complete(string& etag, real_time *mtime, real_
   obj_op.meta.flags = PUT_OBJ_CREATE;
   obj_op.meta.olh_epoch = olh_epoch;
   obj_op.meta.delete_at = delete_at;
+  obj_op.meta.user_data = user_data;
 
   r = obj_op.write_meta(obj_len, attrs);
   if (r < 0) {
@@ -6286,7 +6287,7 @@ int RGWRados::Object::Write::write_meta(uint64_t size,
 
   r = index_op.complete(poolid, epoch, size,
                         meta.set_mtime, etag, content_type, &acl_bl,
-                        meta.category, meta.remove_objs);
+                        meta.category, meta.remove_objs, meta.user_data);
   if (r < 0)
     goto done_cancel;
 
@@ -8977,7 +8978,7 @@ int RGWRados::Bucket::UpdateIndex::prepare(RGWModifyOp op)
 
 int RGWRados::Bucket::UpdateIndex::complete(int64_t poolid, uint64_t epoch, uint64_t size,
                                     ceph::real_time& ut, string& etag, string& content_type, bufferlist *acl_bl, RGWObjCategory category,
-                                    list<rgw_obj_key> *remove_objs)
+                                    list<rgw_obj_key> *remove_objs, const string *user_data)
 {
   if (blind) {
     return 0;
@@ -8995,6 +8996,9 @@ int RGWRados::Bucket::UpdateIndex::complete(int64_t poolid, uint64_t epoch, uint
   ent.size = size;
   ent.mtime = ut;
   ent.etag = etag;
+  if (user_data)
+    ent.user_data = *user_data;
+
   ACLOwner owner;
   if (acl_bl && acl_bl->length()) {
     int ret = store->decode_policy(*acl_bl, &owner);
@@ -11536,6 +11540,7 @@ int RGWRados::cls_obj_complete_op(BucketShard& bs, RGWModifyOp op, string& tag,
   dir_meta.owner_display_name = ent.owner_display_name;
   dir_meta.content_type = ent.content_type;
   dir_meta.category = category;
+  dir_meta.user_data = ent.user_data;
 
   rgw_bucket_entry_ver ver;
   ver.pool = pool;
@@ -11655,6 +11660,7 @@ int RGWRados::cls_bucket_list(rgw_bucket& bucket, int shard_id, rgw_obj_key& sta
     e.tag = dirent.tag;
     e.flags = dirent.flags;
     e.versioned_epoch = dirent.versioned_epoch;
+    e.user_data = dirent.meta.user_data;
 
     bool force_check = force_check_filter && force_check_filter(dirent.key.name);
     if ((!dirent.exists && !dirent.is_delete_marker()) || !dirent.pending_map.empty() || force_check) {

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 5bf125d88b0902d1ea0dbbd43091ccc2c0a4475f..021c7c1d68289bfe3db06e84687f41e14e0042e1 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -2305,10 +2305,11 @@ public:
         uint64_t olh_epoch;
         ceph::real_time delete_at;
         bool canceled;
+        const string *user_data;
 
         MetaParams() : mtime(NULL), rmattrs(NULL), data(NULL), manifest(NULL), ptag(NULL),
                  remove_objs(NULL), category(RGW_OBJ_CATEGORY_MAIN), flags(0),
-                 if_match(NULL), if_nomatch(NULL), olh_epoch(0), canceled(false) {}
+                 if_match(NULL), if_nomatch(NULL), olh_epoch(0), canceled(false), user_data(nullptr) {}
       } meta;
 
       explicit Write(RGWRados::Object *_target) : target(_target) {}
@@ -2435,7 +2436,7 @@ public:
       int complete(int64_t poolid, uint64_t epoch, uint64_t size,
                    ceph::real_time& ut, string& etag, string& content_type,
                    bufferlist *acl_bl, RGWObjCategory category,
-                  list<rgw_obj_key> *remove_objs);
+                  list<rgw_obj_key> *remove_objs, const string *user_data = nullptr);
       int complete_del(int64_t poolid, uint64_t epoch,
                        ceph::real_time& removed_mtime, /* mtime of removed object */
                        list<rgw_obj_key> *remove_objs);
@@ -3157,7 +3158,7 @@ protected:
 
   virtual int do_complete(string& etag, ceph::real_time *mtime, ceph::real_time set_mtime,
                           map<string, bufferlist>& attrs, ceph::real_time delete_at,
-                          const char *if_match = NULL, const char *if_nomatch = NULL) = 0;
+                          const char *if_match = NULL, const char *if_nomatch = NULL, const string *user_data = nullptr) = 0;
 
 public:
   RGWPutObjProcessor(RGWObjectCtx& _obj_ctx, RGWBucketInfo& _bi) : store(NULL), obj_ctx(_obj_ctx), is_complete(false), bucket_info(_bi), canceled(false) {}
@@ -3173,7 +3174,7 @@ public:
   }
   virtual int complete(string& etag, ceph::real_time *mtime, ceph::real_time set_mtime,
                        map<string, bufferlist>& attrs, ceph::real_time delete_at,
-                       const char *if_match = NULL, const char *if_nomatch = NULL);
+                       const char *if_match = NULL, const char *if_nomatch = NULL, const string *user_data = nullptr);
 
   CephContext *ctx();
 
@@ -3247,7 +3248,7 @@ protected:
   int write_data(bufferlist& bl, off_t ofs, void **phandle, rgw_obj *pobj, bool exclusive);
   virtual int do_complete(string& etag, ceph::real_time *mtime, ceph::real_time set_mtime,
                           map<string, bufferlist>& attrs, ceph::real_time delete_at,
-                          const char *if_match = NULL, const char *if_nomatch = NULL);
+                          const char *if_match = NULL, const char *if_nomatch = NULL, const string *user_data = NULL);
 
   int prepare_next_part(off_t ofs);
   int complete_parts();

diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index f53582c1782a46fd43b598e6d44393e32000cdcd..1532d1eeff54d662f86d70e1698670c60688e95c 100644 (file)
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -275,6 +275,8 @@ void RGWListBucket_ObjStore_SWIFT::send_response()
       s->formatter->dump_string("name", key.name);
       s->formatter->dump_string("hash", iter->etag);
       s->formatter->dump_int("bytes", iter->size);
+      if (!iter->user_data.empty())
+        s->formatter->dump_string("user_custom_data", iter->user_data);
       string single_content_type = iter->content_type;
       if (iter->content_type.size()) {
         // content type might hold multiple values, just dump the last one
@@ -670,6 +672,13 @@ int RGWPutObj_ObjStore_SWIFT::get_params()
     return r;
   }
 
+  if (!s->cct->_conf->rgw_swift_custom_header.empty()) {
+    string custom_header = s->cct->_conf->rgw_swift_custom_header;
+    if (s->info.env->exists(custom_header.c_str())) {
+      user_data = s->info.env->get(custom_header.c_str());
+    }
+  }
+
   dlo_manifest = s->info.env->get("HTTP_X_OBJECT_MANIFEST");
   bool exists;
   string multipart_manifest = s->info.args.get("multipart-manifest", &exists);