rgw: support for generic entities types in es

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index ddbd8fae70871bd90d0b692cb1b88de3641511d7..365cd86cb4abf11ecee7e1ddacc6e229e2802f8a 100644 (file)
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -550,7 +550,7 @@ bool parse_iso8601(const char *s, struct tm *t, uint32_t *pns, bool extended_for
   trim_whitespace(p, str);
   int len = str.size();
 
-  if (len == 1 && str[0] == 'Z')
+  if (len == 0 || (len == 1 && str[0] == 'Z'))
     return true;
 
   if (str[0] != '.' ||

diff --git a/src/rgw/rgw_es_main.cc b/src/rgw/rgw_es_main.cc
index e6972b62bbc97f19b6858899932128b9a4e6473e..fb705193624e13cbd34cc7959ac88d4754bc6daf 100644 (file)
--- a/src/rgw/rgw_es_main.cc
+++ b/src/rgw/rgw_es_main.cc
@@ -2,6 +2,10 @@
 #include <string>
 #include <iostream>
 
+#include "global/global_init.h"
+#include "global/global_context.h"
+
+#include "common/ceph_argparse.h"
 #include "common/ceph_json.h"
 #include "rgw_es_query.h"
 
@@ -9,6 +13,15 @@ using namespace std;
 
 int main(int argc, char *argv[])
 {
+  vector<const char*> args;
+  argv_to_vec(argc, (const char **)argv, args);
+  env_to_vec(args);
+
+  auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
+                         CODE_ENVIRONMENT_UTILITY, 0);
+
+  common_init_finish(g_ceph_context);
+
   list<string> infix;
 
   string expr;
@@ -21,6 +34,13 @@ int main(int argc, char *argv[])
 
   ESQueryCompiler es_query(expr, nullptr, "x-amz-meta-");
 
+  map<string, ESEntityTypeMap::EntityType> generic_map = { {"key", ESEntityTypeMap::ES_ENTITY_STR},
+                                                           {"instance", ESEntityTypeMap::ES_ENTITY_STR},
+                                                           {"lastmodified", ESEntityTypeMap::ES_ENTITY_DATE},
+                                                           {"size", ESEntityTypeMap::ES_ENTITY_DATE} };
+  ESEntityTypeMap gm(generic_map);
+  es_query.set_generic_type_map(&gm);
+
   map<string, ESEntityTypeMap::EntityType> custom_map = { {"str", ESEntityTypeMap::ES_ENTITY_STR},
                                                           {"int", ESEntityTypeMap::ES_ENTITY_INT},
                                                           {"date", ESEntityTypeMap::ES_ENTITY_DATE} };

diff --git a/src/rgw/rgw_es_query.cc b/src/rgw/rgw_es_query.cc
index bd6bb2fe2dd45ce38763e262fcb4f77aa6f6de82..58c2b5eb77d5b1404f3077270a2a7a4b3d3d8ab8 100644 (file)
--- a/src/rgw/rgw_es_query.cc
+++ b/src/rgw/rgw_es_query.cc
@@ -113,15 +113,9 @@ public:
   ESQueryNode(ESQueryCompiler *_compiler) : compiler(_compiler) {}
   virtual ~ESQueryNode() {}
 
-  virtual bool init(ESQueryStack *s) = 0;
+  virtual bool init(ESQueryStack *s, ESQueryNode **pnode) = 0;
 
   virtual void dump(Formatter *f) const = 0;
-
-  virtual bool leaf_field_name(string *name) {
-    return false;
-  }
-
-  virtual void leaf_field_rename(const string& new_name) {}
 };
 
 static bool alloc_node(ESQueryCompiler *compiler, ESQueryStack *s, ESQueryNode **pnode);
@@ -133,7 +127,7 @@ class ESQueryNode_Bool : public ESQueryNode {
 public:
   ESQueryNode_Bool(ESQueryCompiler *compiler) : ESQueryNode(compiler) {}
   ESQueryNode_Bool(ESQueryCompiler *compiler, const string& _op, ESQueryNode *_first, ESQueryNode *_second) :ESQueryNode(compiler), op(_op), first(_first), second(_second) {}
-  bool init(ESQueryStack *s) {
+  bool init(ESQueryStack *s, ESQueryNode **pnode) override {
     bool valid = s->pop(&op);
     if (!valid) {
       return false;
@@ -143,6 +137,7 @@ public:
     if (!valid) {
       return false;
     }
+    *pnode = this;
     return true;
   }
   virtual ~ESQueryNode_Bool() {
@@ -162,46 +157,129 @@ public:
 
 };
 
+class ESQueryNodeLeafVal {
+public:
+  ESQueryNodeLeafVal() = default;
+  virtual ~ESQueryNodeLeafVal() {}
+
+  virtual bool init(const string& str_val) = 0;
+  virtual void encode_json(const string& field, Formatter *f) const  = 0;
+};
+
+class ESQueryNodeLeafVal_Str : public ESQueryNodeLeafVal {
+  string val;
+public:
+  ESQueryNodeLeafVal_Str() {}
+  bool init(const string& str_val) override {
+    val = str_val;
+    return true;
+  }
+  void encode_json(const string& field, Formatter *f) const {
+    ::encode_json(field.c_str(), val.c_str(), f);
+  }
+};
+
+class ESQueryNodeLeafVal_Int : public ESQueryNodeLeafVal {
+  int64_t val;
+public:
+  ESQueryNodeLeafVal_Int() {}
+  bool init(const string& str_val) override {
+    string err;
+    val = strict_strtoll(str_val.c_str(), 10, &err);
+    if (!err.empty()) {
+      return false;
+    }
+    return true;
+  }
+  void encode_json(const string& field, Formatter *f) const {
+    ::encode_json(field.c_str(), val, f);
+  }
+};
+
+class ESQueryNodeLeafVal_Date : public ESQueryNodeLeafVal {
+  ceph::real_time val;
+public:
+  ESQueryNodeLeafVal_Date() {}
+  bool init(const string& str_val) override {
+    if (parse_time(str_val.c_str(), &val) < 0) {
+      return false;
+    }
+    return true;
+  }
+  void encode_json(const string& field, Formatter *f) const {
+    utime_t ut(val);
+    ::encode_json(field.c_str(), ut, f);
+  }
+};
+
 class ESQueryNode_Op : public ESQueryNode {
 protected:
   string op;
   string field;
-  string val;
+  ESQueryNodeLeafVal *val{nullptr};
+  ESEntityTypeMap::EntityType entity_type{ESEntityTypeMap::ES_ENTITY_NONE};
+
+  bool val_from_str(const string& str_val) {
+    switch (entity_type) {
+      case ESEntityTypeMap::ES_ENTITY_DATE:
+        val = new ESQueryNodeLeafVal_Date;
+        break;
+      case ESEntityTypeMap::ES_ENTITY_INT:
+        val = new ESQueryNodeLeafVal_Int;
+        break;
+      default:
+        val = new ESQueryNodeLeafVal_Str;
+    }
+    return val->init(str_val);
+  }
 public:
   ESQueryNode_Op(ESQueryCompiler *compiler) : ESQueryNode(compiler) {}
-  bool init(ESQueryStack *s) {
+  ~ESQueryNode_Op() {
+    delete val;
+  }
+  virtual bool init(ESQueryStack *s, ESQueryNode **pnode) override {
+    string str_val;
     bool valid = s->pop(&op) &&
-      s->pop(&val) &&
+      s->pop(&str_val) &&
       s->pop(&field);
     if (!valid) {
       return false;
     }
+    ESQueryNode *effective_node;
+    if (!handle_nested(&effective_node)) {
+      return false;
+    }
+    if (!val_from_str(str_val)) {
+      return false;
+    }
+    *pnode = effective_node;
     return true;
   }
+  bool handle_nested(ESQueryNode **pnode);
 
   virtual void dump(Formatter *f) const = 0;
-  bool leaf_field_name(string *name) override {
-    *name = field;
-    return true;
-  }
-
-  void leaf_field_rename(const string& new_name) override {
-    field = new_name;
-  }
 };
 
 class ESQueryNode_Op_Equal : public ESQueryNode_Op {
+  string str_val;
 public:
   ESQueryNode_Op_Equal(ESQueryCompiler *compiler) : ESQueryNode_Op(compiler) {}
   ESQueryNode_Op_Equal(ESQueryCompiler *compiler, const string& f, const string& v) : ESQueryNode_Op(compiler) {
     op = "==";
     field = f;
-    val = v;
+    str_val = v;
+  }
+
+  bool init(ESQueryStack *s, ESQueryNode **pnode) override {
+    if (op.empty()) {
+      return ESQueryNode_Op::init(s, pnode);
+    }
+    return val_from_str(str_val);
   }
 
   virtual void dump(Formatter *f) const {
     f->open_object_section("term");
-    encode_json(field.c_str(), val.c_str(), f);
+    val->encode_json(field, f);
     f->close_section();
   }
 };
@@ -214,7 +292,7 @@ public:
   virtual void dump(Formatter *f) const {
     f->open_object_section("range");
     f->open_object_section(field.c_str());
-    encode_json(range_str.c_str(), val.c_str(), f);
+    val->encode_json(range_str, f);
     f->close_section();
     f->close_section();
   }
@@ -279,6 +357,44 @@ string ESQueryNode_Op_Nested<ceph::real_time>::type_str() const {
   return "date";
 }
 
+bool ESQueryNode_Op::handle_nested(ESQueryNode **pnode)
+{
+  string field_name = field;
+  const string& custom_prefix = compiler->get_custom_prefix();
+  if (!boost::algorithm::starts_with(field_name, custom_prefix)) {
+    *pnode = this;
+    auto m = compiler->get_generic_type_map();
+    if (m) {
+      return m->find(field_name, &entity_type);
+    }
+    return false;
+  }
+
+  field_name = field_name.substr(custom_prefix.size());
+  auto m = compiler->get_custom_type_map();
+  if (m) {
+    m->find(field_name, &entity_type);
+    /* ignoring returned bool, for now just treat it as string */
+  }
+
+  ESQueryNode_Op_Nested_Parent *new_node;
+  switch (entity_type) {
+    case ESEntityTypeMap::ES_ENTITY_INT:
+      new_node = new ESQueryNode_Op_Nested<int64_t>(compiler, field_name, this);
+      break;
+    case ESEntityTypeMap::ES_ENTITY_DATE:
+      new_node = new ESQueryNode_Op_Nested<ceph::real_time>(compiler, field_name, this);
+      break;
+    default:
+      new_node = new ESQueryNode_Op_Nested<string>(compiler, field_name, this);
+  }
+    
+  field = new_node->get_custom_leaf_field_name();
+  *pnode = new_node;
+
+  return true;
+}
+
 static bool is_bool_op(const string& str)
 {
   return (str == "or" || str == "and");
@@ -314,39 +430,10 @@ static bool alloc_node(ESQueryCompiler *compiler, ESQueryStack *s, ESQueryNode *
     node = new ESQueryNode_Op_Range(compiler, iter->second);
   }
 
-  if (!node->init(s)) {
+  if (!node->init(s, pnode)) {
     delete node;
     return false;
   }
-  string field_name;
-  string custom_prefix = compiler->get_custom_prefix();
-  if (node->leaf_field_name(&field_name) &&
-      boost::algorithm::starts_with(field_name, custom_prefix)) {
-    ESQueryNode *leaf_node = node;
-    field_name = field_name.substr(custom_prefix.size());
-    ESEntityTypeMap::EntityType entity_type = ESEntityTypeMap::ES_ENTITY_NONE;
-    auto m = compiler->get_custom_type_map();
-    if (m) {
-      entity_type = m->find(field_name);
-    }
-
-    ESQueryNode_Op_Nested_Parent *new_node;
-    switch (entity_type) {
-      case ESEntityTypeMap::ES_ENTITY_INT:
-        new_node = new ESQueryNode_Op_Nested<int64_t>(compiler, field_name, node);
-        break;
-      case ESEntityTypeMap::ES_ENTITY_DATE:
-        new_node = new ESQueryNode_Op_Nested<ceph::real_time>(compiler, field_name, node);
-        break;
-      default:
-        new_node = new ESQueryNode_Op_Nested<string>(compiler, field_name, node);
-    }
-    
-    leaf_node->leaf_field_rename(new_node->get_custom_leaf_field_name());
-
-    node = new_node;
-  }
-  *pnode = node;
   return true;
 }
 

diff --git a/src/rgw/rgw_es_query.h b/src/rgw/rgw_es_query.h
index 79b221fd469ec3ca2692a355e2757401689bf99d..913c95e097ca6e8a11b21533897b809b185c3867 100644 (file)
--- a/src/rgw/rgw_es_query.h
+++ b/src/rgw/rgw_es_query.h
@@ -74,13 +74,15 @@ struct ESEntityTypeMap {
 
   ESEntityTypeMap(map<string, EntityType>& _m) : m(_m) {}
 
-  EntityType find(const string& entity) {
+  bool find(const string& entity, EntityType *ptype) {
     auto i = m.find(entity);
     if (i != m.end()) {
-      return i->second;
+      *ptype = i->second;
+      return true;
     }
 
-    return ES_ENTITY_NONE;
+    *ptype = ES_ENTITY_NONE;
+    return false;
   }
 };
 
@@ -95,6 +97,7 @@ class ESQueryCompiler {
 
   list<pair<string, string> > eq_conds;
 
+  ESEntityTypeMap *generic_type_map{nullptr};
   ESEntityTypeMap *custom_type_map{nullptr};
 
 public:
@@ -108,6 +111,13 @@ public:
   bool compile();
   void dump(Formatter *f) const;
   
+  void set_generic_type_map(ESEntityTypeMap *entity_map) {
+    generic_type_map = entity_map;
+  }
+
+  ESEntityTypeMap *get_generic_type_map() {
+    return generic_type_map;
+  }
   const string& get_custom_prefix() { return custom_prefix; }
 
   void set_custom_type_map(ESEntityTypeMap *entity_map) {