rgw: new api to configure bucket's custom keys for mdsearch

POST /bucket?mdsearch
x-rgw-meta-search: <x-amz-meta-key>[;<str|int|date>][,...]

note that x-amz-meta-search header can also be used.

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 529dfebe09846ec74166b85207d54056788c8026..c1a6bed0807ffb2ad33ce53dcb30f46fa0dacfd2 100644 (file)
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -466,6 +466,7 @@ enum RGWOpType {
   RGW_OP_GET_OBJ_LAYOUT,
   RGW_OP_BULK_UPLOAD,
   RGW_OP_METADATA_SEARCH,
+  RGW_OP_CONFIG_BUCKET_META_SEARCH,
 };
 
 class RGWAccessControlPolicy;
@@ -1168,9 +1169,11 @@ struct RGWBucketInfo
   bool swift_versioning;
   string swift_ver_location;
 
+  map<string, uint32_t> mdsearch_config;
+
 
   void encode(bufferlist& bl) const {
-     ENCODE_START(17, 4, bl);
+     ENCODE_START(18, 4, bl);
      ::encode(bucket, bl);
      ::encode(owner.id, bl);
      ::encode(flags, bl);
@@ -1194,10 +1197,11 @@ struct RGWBucketInfo
        ::encode(swift_ver_location, bl);
      }
      ::encode(creation_time, bl);
+     ::encode(mdsearch_config, bl);
      ENCODE_FINISH(bl);
   }
   void decode(bufferlist::iterator& bl) {
-    DECODE_START_LEGACY_COMPAT_LEN_32(17, 4, 4, bl);
+    DECODE_START_LEGACY_COMPAT_LEN_32(18, 4, 4, bl);
      ::decode(bucket, bl);
      if (struct_v >= 2) {
        string s;
@@ -1254,6 +1258,9 @@ struct RGWBucketInfo
      if (struct_v >= 17) {
        ::decode(creation_time, bl);
      }
+     if (struct_v >= 18) {
+       ::decode(mdsearch_config, bl);
+     }
      DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;

diff --git a/src/rgw/rgw_es_query.h b/src/rgw/rgw_es_query.h
index 834eb38e687536cc842d45e92c8671a455e56276..52711be8609a10416da27aa9f83de2740fb6e30d 100644 (file)
--- a/src/rgw/rgw_es_query.h
+++ b/src/rgw/rgw_es_query.h
@@ -64,10 +64,10 @@ class ESQueryNode;
 
 struct ESEntityTypeMap {
   enum EntityType {
-    ES_ENTITY_NONE,
-    ES_ENTITY_STR,
-    ES_ENTITY_INT,
-    ES_ENTITY_DATE,
+    ES_ENTITY_NONE = 0,
+    ES_ENTITY_STR  = 1,
+    ES_ENTITY_INT  = 2,
+    ES_ENTITY_DATE = 3,
   };
 
   map<string, EntityType> m;

diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc
index aff97d3c3edf6688bb0a946925f3a15c60537395..f5f3612239eb2d6e7da141aa2920a8880d7cc451 100644 (file)
--- a/src/rgw/rgw_json_enc.cc
+++ b/src/rgw/rgw_json_enc.cc
@@ -737,6 +737,7 @@ void RGWBucketInfo::dump(Formatter *f) const
   encode_json("swift_versioning", swift_versioning, f);
   encode_json("swift_ver_location", swift_ver_location, f);
   encode_json("index_type", (uint32_t)index_type, f);
+  encode_json("mdsearch_config", mdsearch_config, f);
 }
 
 void RGWBucketInfo::decode_json(JSONObj *obj) {
@@ -768,6 +769,7 @@ void RGWBucketInfo::decode_json(JSONObj *obj) {
   uint32_t it;
   JSONDecoder::decode_json("index_type", it, obj);
   index_type = (RGWBucketIndexType)it;
+  JSONDecoder::decode_json("mdsearch_config", mdsearch_config, obj);
 }
 
 void rgw_obj_key::dump(Formatter *f) const

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 6caa57c067433320cc1b63925d99f53dd7286801..7656a0b60b2269510dc4756f1097a1cfb8e580e0 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -6308,6 +6308,38 @@ void RGWGetObjLayout::execute()
 }
 
 
+int RGWConfigBucketMetaSearch::verify_permission()
+{
+  if (!s->auth.identity->is_owner_of(s->bucket_owner.get_id())) {
+    return -EACCES;
+  }
+
+  return 0;
+}
+
+void RGWConfigBucketMetaSearch::pre_exec()
+{
+  rgw_bucket_object_pre_exec(s);
+}
+
+void RGWConfigBucketMetaSearch::execute()
+{
+  op_ret = get_params();
+  if (op_ret < 0) {
+    ldout(s->cct, 20) << "NOTICE: get_params() returned ret=" << op_ret << dendl;
+    return;
+  }
+
+  s->bucket_info.mdsearch_config = mdsearch_config;
+
+  op_ret = store->put_bucket_instance_info(s->bucket_info, false, real_time(), &s->bucket_attrs);
+  if (op_ret < 0) {
+    ldout(s->cct, 0) << "NOTICE: put_bucket_info on bucket=" << s->bucket.name << " returned err=" << op_ret << dendl;
+    return;
+  }
+}
+
+
 RGWHandler::~RGWHandler()
 {
 }

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index ed7812cf80b5679122f746b99e594a3f713b814d..8debd7e2efea802fe1824a6a44ad6e4f64f5d610 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -2015,4 +2015,21 @@ public:
 };
 
 
+class RGWConfigBucketMetaSearch : public RGWOp {
+protected:
+  std::map<std::string, uint32_t> mdsearch_config;
+public:
+  RGWConfigBucketMetaSearch() {}
+
+  int verify_permission();
+  void pre_exec();
+  void execute();
+
+  virtual int get_params() = 0;
+  virtual void send_response() = 0;
+  virtual const string name() { return "config_bucket_meta_search"; }
+  virtual RGWOpType get_type() { return RGW_OP_CONFIG_BUCKET_META_SEARCH; }
+  virtual uint32_t op_mask() { return RGW_OP_TYPE_WRITE; }
+};
+
 #endif /* CEPH_RGW_OP_H */

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index c8a3f8371fa8e31ed95a64fdccc1602d5da94965..d73b4666e3a207c15715596228cefcd6a685e4da 100644 (file)
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -29,6 +29,8 @@
 #include "rgw_auth_keystone.h"
 #include "rgw_auth_registry.h"
 
+#include "rgw_es_query.h"
+
 #include <typeinfo> // for 'typeid'
 
 #include "rgw_ldap.h"
@@ -2713,6 +2715,77 @@ void RGWGetObjLayout_ObjStore_S3::send_response()
   rgw_flush_formatter(s, &f);
 }
 
+int RGWConfigBucketMetaSearch_ObjStore_S3::get_params()
+{
+  auto iter = s->info.x_meta_map.find("x-amz-meta-search");
+  if (iter == s->info.x_meta_map.end()) {
+    s->err.message = "X-Rgw-Meta-Search header not provided";
+    ldout(s->cct, 20) << s->err.message << dendl;
+    return -EINVAL;
+  }
+
+  list<string> expressions;
+  get_str_list(iter->second, ",", expressions);
+
+  for (auto& expression : expressions) {
+    vector<string> args;
+    get_str_vec(expression, ";", args);
+
+    if (args.empty()) {
+      s->err.message = "invalid empty expression";
+      ldout(s->cct, 20) << s->err.message << dendl;
+      return -EINVAL;
+    }
+    if (args.size() > 2) {
+      s->err.message = string("invalid expression: ") + expression;
+      ldout(s->cct, 20) << s->err.message << dendl;
+      return -EINVAL;
+    }
+
+    string key = boost::algorithm::to_lower_copy(rgw_trim_whitespace(args[0]));
+    string val;
+    if (args.size() > 1) {
+      val = boost::algorithm::to_lower_copy(rgw_trim_whitespace(args[1]));
+    }
+
+#define X_AMZ_META_PREFIX "x-amz-meta-"
+    if (!boost::algorithm::starts_with(key, X_AMZ_META_PREFIX)) {
+      s->err.message = string("invalid expression, key must start with '" X_AMZ_META_PREFIX "' : ") + expression;
+      ldout(s->cct, 20) << s->err.message << dendl;
+      return -EINVAL;
+    }
+
+    key = key.substr(sizeof(X_AMZ_META_PREFIX) - 1);
+
+    ESEntityTypeMap::EntityType entity_type;
+
+    if (val.empty() || val == "str" || val == "string") {
+      entity_type = ESEntityTypeMap::ES_ENTITY_STR;
+    } else if (val == "int" || val == "integer") {
+      entity_type = ESEntityTypeMap::ES_ENTITY_INT;
+    } else if (val == "date" || val == "datetime") {
+      entity_type = ESEntityTypeMap::ES_ENTITY_DATE;
+    } else {
+      s->err.message = string("invalid entity type: ") + val;
+      ldout(s->cct, 20) << s->err.message << dendl;
+      return -EINVAL;
+    }
+
+    mdsearch_config[key] = entity_type;
+  }
+
+  return 0;
+}
+
+void RGWConfigBucketMetaSearch_ObjStore_S3::send_response()
+{
+  if (op_ret)
+    set_req_state_err(s, op_ret);
+  dump_errno(s);
+  end_header(s, this);
+}
+
+
 RGWOp *RGWHandler_REST_Service_S3::op_get()
 {
   if (is_usage_op()) {
@@ -2859,6 +2932,10 @@ RGWOp *RGWHandler_REST_Bucket_S3::op_post()
     return new RGWDeleteMultiObj_ObjStore_S3;
   }
 
+  if (s->info.args.exists("mdsearch")) {
+    return new RGWConfigBucketMetaSearch_ObjStore_S3;
+  }
+
   return new RGWPostObj_ObjStore_S3;
 }
 

diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h
index 8dcf242b283be9341fb1efd365c6882b9fd8909a..aad591aec5a415332a6daca9720a6796e5912727 100644 (file)
--- a/src/rgw/rgw_rest_s3.h
+++ b/src/rgw/rgw_rest_s3.h
@@ -418,6 +418,14 @@ public:
   void send_response();
 };
 
+class RGWConfigBucketMetaSearch_ObjStore_S3 : public RGWConfigBucketMetaSearch {
+public:
+  RGWConfigBucketMetaSearch_ObjStore_S3() {}
+  ~RGWConfigBucketMetaSearch_ObjStore_S3() {}
+
+  int get_params() override;
+  void send_response() override;
+};
 
 class RGW_Auth_S3 {
 private: