rgw: add a field to store generic user data in the bucket index,

that can be populated/fetched via a configurable custom http header

Signed-off-by: Pavan Rallabhandi <prallabhandi@walmartlabs.com>
(cherry picked from commit abca7a86c3cfbb58fafb5d057d9d6f5017a53704)

Conflicts:
	src/rgw/rgw_op.cc
         	Signature fixes for RGWPutObjProcessor_Multipart::do_complete().

	src/rgw/rgw_op.h
        	A new member field `crypt_http_responses` is added in the class `RGWPutObj` in master version, which is not required in Kraken.

	src/rgw/rgw_rados.cc
        	In RGWRados::Bucket::UpdateIndex::complete, RGWObjEnt has been removed in master, which has to be retained in Kraken.
        	In RGWRados::cls_obj_complete_op, user_data is added to the rgw_bucket_dir_entry_meta
        	In RGWRados::cls_bucket_list, the user_data field of RGWObjEnt is populated.

	src/rgw/rgw_rados.h
        	In UpdateIndex::complete(), remove_objs is of type rgw_obj_key in Kraken instead of rgw_obj_index_key
        	RGWPutObjProcessor_Multipart is not part of this file in Kraken.

	src/rgw/rgw_rest_swift.cc
		In RGWListBucket_ObjStore_SWIFT::send_response(), there is no meta struct in Kraken.

        src/rgw/rgw_common.h
        	Add user_data field in RGWObjEnt structure.

        src/rgw/rgw_json_enc.cc
        	Add user_data field while dumping RGWObjEnt.

diff --git a/src/cls/rgw/cls_rgw_types.cc b/src/cls/rgw/cls_rgw_types.cc
index 1b8edbb667d13e51d9a9737ef70415c67d9b3cdd..7d9c83bfad8ef68fa8fcccf5903d14740c92d39a 100644 (file)
--- a/src/cls/rgw/cls_rgw_types.cc
+++ b/src/cls/rgw/cls_rgw_types.cc
@@ -61,6 +61,7 @@ void rgw_bucket_dir_entry_meta::dump(Formatter *f) const
   encode_json("owner_display_name", owner_display_name, f);
   encode_json("content_type", content_type, f);
   encode_json("accounted_size", accounted_size, f);
+  encode_json("user_data", user_data, f);
 }
 
 void rgw_bucket_dir_entry_meta::decode_json(JSONObj *obj) {
@@ -75,6 +76,7 @@ void rgw_bucket_dir_entry_meta::decode_json(JSONObj *obj) {
   JSONDecoder::decode_json("owner_display_name", owner_display_name, obj);
   JSONDecoder::decode_json("content_type", content_type, obj);
   JSONDecoder::decode_json("accounted_size", accounted_size, obj);
+  JSONDecoder::decode_json("user_data", user_data, obj);
 }
 
 void rgw_bucket_dir_entry::generate_test_instances(list<rgw_bucket_dir_entry*>& o)

diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h
index 144c5af203f909491d2714953613397e1fdb7415..01fd260dccf6aa6b1af9dac5b0408f457da7722b 100644 (file)
--- a/src/cls/rgw/cls_rgw_types.h
+++ b/src/cls/rgw/cls_rgw_types.h
@@ -95,12 +95,13 @@ struct rgw_bucket_dir_entry_meta {
   string owner_display_name;
   string content_type;
   uint64_t accounted_size;
+  string user_data;
 
   rgw_bucket_dir_entry_meta() :
   category(0), size(0), accounted_size(0) { }
 
   void encode(bufferlist &bl) const {
-    ENCODE_START(4, 3, bl);
+    ENCODE_START(5, 3, bl);
     ::encode(category, bl);
     ::encode(size, bl);
     ::encode(mtime, bl);
@@ -109,10 +110,11 @@ struct rgw_bucket_dir_entry_meta {
     ::encode(owner_display_name, bl);
     ::encode(content_type, bl);
     ::encode(accounted_size, bl);
+    ::encode(user_data, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::iterator &bl) {
-    DECODE_START_LEGACY_COMPAT_LEN(4, 3, 3, bl);
+    DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, bl);
     ::decode(category, bl);
     ::decode(size, bl);
     ::decode(mtime, bl);
@@ -125,6 +127,8 @@ struct rgw_bucket_dir_entry_meta {
       ::decode(accounted_size, bl);
     else
       accounted_size = size;
+    if (struct_v >= 5)
+      ::decode(user_data, bl);
     DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index aacd183a161d72ec20d37a2e5b6a112a51fbf468..b9965192a9a5124f931bb961cb6dc011ed5971e0 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -1612,3 +1612,5 @@ OPTION(rgw_torrent_sha_unit, OPT_INT, 512*1024)    // torrent field piece length
 OPTION(internal_safe_to_start_threads, OPT_BOOL, false)
 
 OPTION(debug_deliberately_leak_memory, OPT_BOOL, false)
+
+OPTION(rgw_swift_custom_header, OPT_STR, "") // option to enable swift custom headers

diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 468e82462e52030b4da6ba95e8ec1ea68438529b..4e740b03f96999572cec58d9e28b9075b9332e49 100644 (file)
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1392,6 +1392,7 @@ struct RGWObjEnt {
   string tag;
   uint32_t flags;
   uint64_t versioned_epoch;
+  string user_data;
 
   RGWObjEnt() : flags(0), versioned_epoch(0) {}
 

diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc
index ca4b1e372037ba77a5f5810cbf37648e73c65b60..4feb61925fb4857352af93e9d1f43f90361d8815 100644 (file)
--- a/src/rgw/rgw_json_enc.cc
+++ b/src/rgw/rgw_json_enc.cc
@@ -750,6 +750,7 @@ void RGWObjEnt::dump(Formatter *f) const
   encode_json("content_type", content_type, f);
   encode_json("tag", tag, f);
   encode_json("flags", flags, f);
+  encode_json("user_data", user_data, f);
 }
 
 void RGWBucketEnt::dump(Formatter *f) const

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 6f29be48ecbbdf67065e56997476224f9e2b23c8..30c25aa0b006b219acb3a55316cc93ac2995cf83 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -2583,7 +2583,7 @@ protected:
   int do_complete(size_t accounted_size, const string& etag, real_time *mtime,
                   real_time set_mtime, map<string, bufferlist>& attrs,
                   real_time delete_at, const char *if_match,
-                  const char *if_nomatch) override;
+                  const char *if_nomatch, const string *user_data) override;
 
 public:
   bool immutable_head() { return true; }
@@ -2661,7 +2661,7 @@ int RGWPutObjProcessor_Multipart::do_complete(size_t accounted_size,
                                               map<string, bufferlist>& attrs,
                                               real_time delete_at,
                                               const char *if_match,
-                                              const char *if_nomatch)
+                                              const char *if_nomatch, const string *user_data)
 {
   complete_writing_data();
 
@@ -3175,7 +3175,8 @@ void RGWPutObj::execute()
   }
 
   op_ret = processor->complete(s->obj_size, etag, &mtime, real_time(), attrs,
-                               (delete_at ? *delete_at : real_time()), if_match, if_nomatch);
+                               (delete_at ? *delete_at : real_time()), if_match, if_nomatch,
+                               (user_data.empty() ? nullptr : &user_data));
 
   /* produce torrent */
   if (s->cct->_conf->rgw_torrent_flag && (ofs == torrent.get_data_len()))

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index ae51f0bf09bd51e31fae02858a35e4efd573bab3..bc85ec99d77cf5e220f8e54e99221b9ade2c844e 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -719,6 +719,7 @@ protected:
   uint64_t olh_epoch;
   string version_id;
   bufferlist bl_aux;
+  string user_data;
 
   boost::optional<ceph::real_time> delete_at;
 

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 48a2344bf8dc2b0eca6d763ddd72eea6ddfce8b1..0c2dbb29a2c76a2bd5d2028779ab795fd63e17a5 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -2214,9 +2214,9 @@ void RGWObjVersionTracker::generate_new_write_ver(CephContext *cct)
 int RGWPutObjProcessor::complete(size_t accounted_size, const string& etag,
                                  real_time *mtime, real_time set_mtime,
                                  map<string, bufferlist>& attrs, real_time delete_at,
-                                 const char *if_match, const char *if_nomatch)
+                                 const char *if_match, const char *if_nomatch, const string *user_data)
 {
-  int r = do_complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at, if_match, if_nomatch);
+  int r = do_complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at, if_match, if_nomatch, user_data);
   if (r < 0)
     return r;
 
@@ -2549,7 +2549,7 @@ int RGWPutObjProcessor_Atomic::do_complete(size_t accounted_size, const string&
                                            map<string, bufferlist>& attrs,
                                            real_time delete_at,
                                            const char *if_match,
-                                           const char *if_nomatch) {
+                                           const char *if_nomatch, const string *user_data) {
   int r = complete_writing_data();
   if (r < 0)
     return r;
@@ -2574,6 +2574,7 @@ int RGWPutObjProcessor_Atomic::do_complete(size_t accounted_size, const string&
   obj_op.meta.flags = PUT_OBJ_CREATE;
   obj_op.meta.olh_epoch = olh_epoch;
   obj_op.meta.delete_at = delete_at;
+  obj_op.meta.user_data = user_data;
 
   r = obj_op.write_meta(obj_len, accounted_size, attrs);
   if (r < 0) {
@@ -6479,7 +6480,7 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
 
   r = index_op->complete(poolid, epoch, size, accounted_size,
                         meta.set_mtime, etag, content_type, &acl_bl,
-                        meta.category, meta.remove_objs);
+                        meta.category, meta.remove_objs, meta.user_data);
   if (r < 0)
     goto done_cancel;
 
@@ -9403,7 +9404,7 @@ int RGWRados::Bucket::UpdateIndex::complete(int64_t poolid, uint64_t epoch,
                                             const string& content_type,
                                             bufferlist *acl_bl,
                                             RGWObjCategory category,
-                                            list<rgw_obj_key> *remove_objs)
+                                            list<rgw_obj_key> *remove_objs, const string *user_data)
 {
   if (blind) {
     return 0;
@@ -9422,6 +9423,9 @@ int RGWRados::Bucket::UpdateIndex::complete(int64_t poolid, uint64_t epoch,
   ent.accounted_size = accounted_size;
   ent.mtime = ut;
   ent.etag = etag;
+  if (user_data)
+    ent.user_data = *user_data;
+
   ACLOwner owner;
   if (acl_bl && acl_bl->length()) {
     int ret = store->decode_policy(*acl_bl, &owner);
@@ -11970,6 +11974,7 @@ int RGWRados::cls_obj_complete_op(BucketShard& bs, RGWModifyOp op, string& tag,
   dir_meta.owner_display_name = ent.owner_display_name;
   dir_meta.content_type = ent.content_type;
   dir_meta.category = category;
+  dir_meta.user_data = ent.user_data;
 
   rgw_bucket_entry_ver ver;
   ver.pool = pool;
@@ -12090,6 +12095,7 @@ int RGWRados::cls_bucket_list(rgw_bucket& bucket, int shard_id, rgw_obj_key& sta
     e.tag = dirent.tag;
     e.flags = dirent.flags;
     e.versioned_epoch = dirent.versioned_epoch;
+    e.user_data = dirent.meta.user_data;
 
     bool force_check = force_check_filter && force_check_filter(dirent.key.name);
     if ((!dirent.exists && !dirent.is_delete_marker()) || !dirent.pending_map.empty() || force_check) {

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index c3475a28d6b1295661a240ef8ff7a81050e326bc..e2c5936151bea0bbb3524ad60224a7d255101afd 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -2452,10 +2452,11 @@ public:
         uint64_t olh_epoch;
         ceph::real_time delete_at;
         bool canceled;
+        const string *user_data;
 
         MetaParams() : mtime(NULL), rmattrs(NULL), data(NULL), manifest(NULL), ptag(NULL),
                  remove_objs(NULL), category(RGW_OBJ_CATEGORY_MAIN), flags(0),
-                 if_match(NULL), if_nomatch(NULL), olh_epoch(0), canceled(false) {}
+                 if_match(NULL), if_nomatch(NULL), olh_epoch(0), canceled(false), user_data(nullptr) {}
       } meta;
 
       explicit Write(RGWRados::Object *_target) : target(_target) {}
@@ -2588,7 +2589,7 @@ public:
                    uint64_t accounted_size, ceph::real_time& ut,
                    const string& etag, const string& content_type,
                    bufferlist *acl_bl, RGWObjCategory category,
-                  list<rgw_obj_key> *remove_objs);
+                  list<rgw_obj_key> *remove_objs, const string *user_data = nullptr);
       int complete_del(int64_t poolid, uint64_t epoch,
                        ceph::real_time& removed_mtime, /* mtime of removed object */
                        list<rgw_obj_key> *remove_objs);
@@ -3356,7 +3357,7 @@ protected:
   virtual int do_complete(size_t accounted_size, const string& etag,
                           ceph::real_time *mtime, ceph::real_time set_mtime,
                           map<string, bufferlist>& attrs, ceph::real_time delete_at,
-                          const char *if_match, const char *if_nomatch) = 0;
+                          const char *if_match, const char *if_nomatch, const string *user_data) = 0;
 
 public:
   RGWPutObjProcessor(RGWObjectCtx& _obj_ctx, RGWBucketInfo& _bi) : store(NULL), 
@@ -3373,7 +3374,7 @@ public:
   int complete(size_t accounted_size, const string& etag,
                ceph::real_time *mtime, ceph::real_time set_mtime,
                map<string, bufferlist>& attrs, ceph::real_time delete_at,
-               const char *if_match = NULL, const char *if_nomatch = NULL);
+               const char *if_match = NULL, const char *if_nomatch = NULL, const string *user_data = nullptr);
 
   CephContext *ctx();
 
@@ -3451,7 +3452,7 @@ protected:
   int do_complete(size_t accounted_size, const string& etag,
                   ceph::real_time *mtime, ceph::real_time set_mtime,
                   map<string, bufferlist>& attrs, ceph::real_time delete_at,
-                  const char *if_match, const char *if_nomatch) override;
+                  const char *if_match, const char *if_nomatch, const string *user_data) override;
 
   int prepare_next_part(off_t ofs);
   int complete_parts();

diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 2086b4ca7299762bb04084d8e13efd6a1098e727..9421f4fe8b818cf46d1e40713fa7b8c334becc49 100644 (file)
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -306,6 +306,8 @@ void RGWListBucket_ObjStore_SWIFT::send_response()
       s->formatter->dump_string("name", key.name);
       s->formatter->dump_string("hash", iter->etag);
       s->formatter->dump_int("bytes", iter->accounted_size);
+      if (!iter->user_data.empty())
+        s->formatter->dump_string("user_custom_data", iter->user_data);
       string single_content_type = iter->content_type;
       if (iter->content_type.size()) {
         // content type might hold multiple values, just dump the last one
@@ -758,6 +760,13 @@ int RGWPutObj_ObjStore_SWIFT::get_params()
     return r;
   }
 
+  if (!s->cct->_conf->rgw_swift_custom_header.empty()) {
+    string custom_header = s->cct->_conf->rgw_swift_custom_header;
+    if (s->info.env->exists(custom_header.c_str())) {
+      user_data = s->info.env->get(custom_header.c_str());
+    }
+  }
+
   dlo_manifest = s->info.env->get("HTTP_X_OBJECT_MANIFEST");
   bool exists;
   string multipart_manifest = s->info.args.get("multipart-manifest", &exists);