rgw: configurable index type

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/ceph-erasure-code-corpus b/ceph-erasure-code-corpus
index c332279082854effc00ea93c226f2f211f73631d..b0d1137d31e4b36b72ccae9c0a9a13de2ec82faa 160000 (submodule)
--- a/ceph-erasure-code-corpus
+++ b/ceph-erasure-code-corpus
@@ -1 +1 @@
-Subproject commit c332279082854effc00ea93c226f2f211f73631d
+Subproject commit b0d1137d31e4b36b72ccae9c0a9a13de2ec82faa

diff --git a/src/civetweb b/src/civetweb
index 8d271315a541218caada366f84a2690fdbd474a2..061cd700231eb8e6e3e6e075e2245debb332b00c 160000 (submodule)
--- a/src/civetweb
+++ b/src/civetweb
@@ -1 +1 @@
-Subproject commit 8d271315a541218caada366f84a2690fdbd474a2
+Subproject commit 061cd700231eb8e6e3e6e075e2245debb332b00c

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 23da4478db725634a7c5d9b6281982cdf261be06..77830d16cad8ae6744496dc4854c3afd658a56f2 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -1843,18 +1843,21 @@ public:
       parse_bucket(entry, tenant_name, bucket_name);
 
       rgw_bucket bucket;
+      RGWZonePlacementInfo rule_info;
       ret = store->set_bucket_location_by_rule(bci.info.placement_rule,
-                                           tenant_name, bucket_name, bucket);
+                                           tenant_name, bucket_name, bucket, &rule_info);
       if (ret < 0) {
         ldout(store->ctx(), 0) << "ERROR: select_bucket_placement() returned " << ret << dendl;
         return ret;
       }
       bci.info.bucket.data_pool = bucket.data_pool;
       bci.info.bucket.index_pool = bucket.index_pool;
+      bci.info.index_type = rule_info.index_type;
     } else {
       /* existing bucket, keep its placement pools */
       bci.info.bucket.data_pool = old_bci.info.bucket.data_pool;
       bci.info.bucket.index_pool = old_bci.info.bucket.index_pool;
+      bci.info.index_type = old_bci.info.index_type;
     }
 
     // are we actually going to perform this put, or is it too old?

diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 2ea635a205e1f30938564713a1fd75218e26de4d..3607817e9a151829b8ce094ead53685551cf0f2f 100644 (file)
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -851,6 +851,11 @@ enum RGWBucketFlags {
   BUCKET_VERSIONS_SUSPENDED = 0x4,
 };
 
+enum RGWBucketIndexType {
+  RGWBIType_Normal = 0,
+  RGWBIType_Indexless = 1,
+};
+
 struct RGWBucketInfo
 {
   enum BIShardsHashType {
@@ -885,8 +890,10 @@ struct RGWBucketInfo
   bool has_website;
   RGWBucketWebsiteConf website_conf;
 
+  RGWBucketIndexType index_type;
+
   void encode(bufferlist& bl) const {
-     ENCODE_START(14, 4, bl);
+     ENCODE_START(15, 4, bl);
      ::encode(bucket, bl);
      ::encode(owner.id, bl);
      ::encode(flags, bl);
@@ -904,6 +911,7 @@ struct RGWBucketInfo
      if (has_website) {
        ::encode(website_conf, bl);
      }
+     ::encode((uint32_t)index_type, bl);
      ENCODE_FINISH(bl);
   }
   void decode(bufferlist::iterator& bl) {
@@ -945,6 +953,13 @@ struct RGWBucketInfo
          website_conf = RGWBucketWebsiteConf();
        }
      }
+     if (struct_v >= 15) {
+       uint32_t it;
+       ::decode(it, bl);
+       index_type = (RGWBucketIndexType)it;
+     } else {
+       index_type = RGWBIType_Normal;
+     }
      DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;

diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc
index dabe282e9b4611f71ed6da50c61ae7fed486bf5a..54b1e8f945db025f884db7a85bb52d5543903d5b 100644 (file)
--- a/src/rgw/rgw_json_enc.cc
+++ b/src/rgw/rgw_json_enc.cc
@@ -808,6 +808,7 @@ void RGWZonePlacementInfo::dump(Formatter *f) const
   encode_json("index_pool", index_pool, f);
   encode_json("data_pool", data_pool, f);
   encode_json("data_extra_pool", data_extra_pool, f);
+  encode_json("index_type", (uint32_t)index_type, f);
 }
 
 void RGWZonePlacementInfo::decode_json(JSONObj *obj)
@@ -815,6 +816,9 @@ void RGWZonePlacementInfo::decode_json(JSONObj *obj)
   JSONDecoder::decode_json("index_pool", index_pool, obj);
   JSONDecoder::decode_json("data_pool", data_pool, obj);
   JSONDecoder::decode_json("data_extra_pool", data_extra_pool, obj);
+  uint32_t it;
+  JSONDecoder::decode_json("index_type", it, obj);
+  index_type = (RGWBucketIndexType)it;
 }
 
 void RGWZoneParams::decode_json(JSONObj *obj)

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 41b5e42deeab229d193d54528740485f214573b0..84fcd6539f5cc3a00cd3465786d169552ead8fcc 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -1829,7 +1829,7 @@ void RGWCreateBucket::execute()
     op_ret = store->select_bucket_placement(*(s->user), zonegroup_id,
                                            placement_rule,
                                            s->bucket_tenant, s->bucket_name,
-                                           bucket, &selected_placement_rule);
+                                           bucket, &selected_placement_rule, nullptr);
     if (selected_placement_rule != s->bucket_info.placement_rule) {
       op_ret = -EEXIST;
       return;

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 74162fb053849007645c34fddf0152edef079a53..9d7f568fd76cfbe095a56693ca3577f3bbab8e6c 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -4951,12 +4951,14 @@ int RGWRados::create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
                            bool exclusive)
 {
 #define MAX_CREATE_RETRIES 20 /* need to bound retries */
-  string selected_placement_rule;
+  string selected_placement_rule_name;
+  RGWZonePlacementInfo rule_info;
+
   for (int i = 0; i < MAX_CREATE_RETRIES; i++) {
     int ret = 0;
     ret = select_bucket_placement(owner, zonegroup_id, placement_rule,
                                   bucket.tenant, bucket.name, bucket,
-                                  &selected_placement_rule);
+                                  &selected_placement_rule_name, &rule_info);
     if (ret < 0)
       return ret;
     bufferlist bl;
@@ -4998,7 +5000,8 @@ int RGWRados::create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
     info.bucket = bucket;
     info.owner = owner.user_id;
     info.zonegroup = zonegroup_id;
-    info.placement_rule = selected_placement_rule;
+    info.placement_rule = selected_placement_rule_name;
+    info.index_type = rule_info.index_type;
     info.num_shards = bucket_index_max_shards;
     info.bucket_index_shard_hash_type = RGWBucketInfo::MOD;
     info.requester_pays = false;
@@ -5054,7 +5057,9 @@ int RGWRados::create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
 }
 
 int RGWRados::select_new_bucket_location(RGWUserInfo& user_info, const string& zonegroup_id, const string& request_rule,
-                                         const string& tenant_name, const string& bucket_name, rgw_bucket& bucket, string *pselected_rule)
+                                         const string& tenant_name, const string& bucket_name, rgw_bucket& bucket, string *pselected_rule_name,
+                                         RGWZonePlacementInfo *rule_info)
+
 {
   /* first check that rule exists within the specific zonegroup */
   RGWZoneGroup zonegroup;
@@ -5078,28 +5083,27 @@ int RGWRados::select_new_bucket_location(RGWUserInfo& user_info, const string& z
     return -EIO;
   }
 
-  if (!rule.empty()) {
-    map<string, RGWZoneGroupPlacementTarget>::iterator titer = zonegroup.placement_targets.find(rule);
-    if (titer == zonegroup.placement_targets.end()) {
-      ldout(cct, 0) << "could not find placement rule " << rule << " within zonegroup " << dendl;
-      return -EINVAL;
-    }
+  map<string, RGWZoneGroupPlacementTarget>::iterator titer = zonegroup.placement_targets.find(rule);
+  if (titer == zonegroup.placement_targets.end()) {
+    ldout(cct, 0) << "could not find placement rule " << rule << " within zonegroup " << dendl;
+    return -EINVAL;
+  }
 
-    /* now check tag for the rule, whether user is permitted to use rule */
-    RGWZoneGroupPlacementTarget& target_rule = titer->second;
-    if (!target_rule.user_permitted(user_info.placement_tags)) {
-      ldout(cct, 0) << "user not permitted to use placement rule" << dendl;
-      return -EPERM;
-    }
+  /* now check tag for the rule, whether user is permitted to use rule */
+  RGWZoneGroupPlacementTarget& target_rule = titer->second;
+  if (!target_rule.user_permitted(user_info.placement_tags)) {
+    ldout(cct, 0) << "user not permitted to use placement rule" << dendl;
+    return -EPERM;
   }
 
-  if (pselected_rule)
-    *pselected_rule = rule;
-  
-  return set_bucket_location_by_rule(rule, tenant_name, bucket_name, bucket);
+  if (pselected_rule_name)
+    *pselected_rule_name = rule;
+
+  return set_bucket_location_by_rule(rule, tenant_name, bucket_name, bucket, rule_info);
 }
 
-int RGWRados::set_bucket_location_by_rule(const string& location_rule, const string& tenant_name, const string& bucket_name, rgw_bucket& bucket)
+int RGWRados::set_bucket_location_by_rule(const string& location_rule, const string& tenant_name, const string& bucket_name, rgw_bucket& bucket,
+                                         RGWZonePlacementInfo *rule_info)
 {
   bucket.tenant = tenant_name;
   bucket.name = bucket_name;
@@ -5108,7 +5112,7 @@ int RGWRados::set_bucket_location_by_rule(const string& location_rule, const str
     /* we can only reach here if we're trying to set a bucket location from a bucket
      * created on a different zone, using a legacy / default pool configuration
      */
-    return select_legacy_bucket_placement(tenant_name, bucket_name, bucket);
+    return select_legacy_bucket_placement(tenant_name, bucket_name, bucket, rule_info);
   }
 
   /*
@@ -5135,25 +5139,31 @@ int RGWRados::set_bucket_location_by_rule(const string& location_rule, const str
   bucket.data_extra_pool = placement_info.data_extra_pool;
   bucket.index_pool = placement_info.index_pool;
 
+  if (rule_info) {
+    *rule_info = placement_info;
+  }
+
   return 0;
 }
 
 int RGWRados::select_bucket_placement(RGWUserInfo& user_info, const string& zonegroup_id, const string& placement_rule,
                                       const string& tenant_name, const string& bucket_name, rgw_bucket& bucket,
-                                      string *pselected_rule)
+                                      string *pselected_rule_name, RGWZonePlacementInfo *rule_info)
 {
   if (!get_zone_params().placement_pools.empty()) {
     return select_new_bucket_location(user_info, zonegroup_id, placement_rule,
-                                      tenant_name, bucket_name, bucket, pselected_rule);
+                                      tenant_name, bucket_name, bucket, pselected_rule_name, rule_info);
   }
 
-  if (pselected_rule)
-    pselected_rule->clear();
+  if (pselected_rule_name) {
+    pselected_rule_name->clear();
+  }
 
-  return select_legacy_bucket_placement(tenant_name, bucket_name, bucket);
+  return select_legacy_bucket_placement(tenant_name, bucket_name, bucket, rule_info);
 }
 
-int RGWRados::select_legacy_bucket_placement(const string& tenant_name, const string& bucket_name, rgw_bucket& bucket)
+int RGWRados::select_legacy_bucket_placement(const string& tenant_name, const string& bucket_name, rgw_bucket& bucket,
+                                             RGWZonePlacementInfo *rule_info)
 {
   bufferlist map_bl;
   map<string, bufferlist> m;
@@ -5228,6 +5238,11 @@ read_omap:
   bucket.data_pool = pool_name;
   bucket.index_pool = pool_name;
 
+  rule_info->data_pool = pool_name;
+  rule_info->data_extra_pool = pool_name;
+  rule_info->index_pool = pool_name;
+  rule_info->index_type = RGWBIType_Normal;
+
   return 0;
 }
 

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 4a9659346c56e38d3869b733723087a4f836b14a..9d45fa75ebe68621fcc2bc75d17ce3b69be13c2a 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -809,22 +809,31 @@ struct RGWZonePlacementInfo {
   string index_pool;
   string data_pool;
   string data_extra_pool; /* if not set we should use data_pool */
+  RGWBucketIndexType index_type;
+
+  RGWZonePlacementInfo() : index_type(RGWBIType_Normal) {}
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(4, 1, bl);
+    ENCODE_START(5, 1, bl);
     ::encode(index_pool, bl);
     ::encode(data_pool, bl);
     ::encode(data_extra_pool, bl);
+    ::encode((uint32_t)index_type, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::iterator& bl) {
-    DECODE_START(4, bl);
+    DECODE_START(5, bl);
     ::decode(index_pool, bl);
     ::decode(data_pool, bl);
     if (struct_v >= 4) {
       ::decode(data_extra_pool, bl);
     }
+    if (struct_v >= 5) {
+      uint32_t it;
+      ::decode(it, bl);
+      index_type = (RGWBucketIndexType)it;
+    }
     DECODE_FINISH(bl);
   }
   const string& get_data_extra_pool() {
@@ -2010,11 +2019,15 @@ public:
    */
   virtual int init_bucket_index(rgw_bucket& bucket, int num_shards);
   int select_bucket_placement(RGWUserInfo& user_info, const string& zonegroup_id, const string& rule,
-                              const string& tenant_name, const string& bucket_name, rgw_bucket& bucket, string *pselected_rule);
-  int select_legacy_bucket_placement(const string& tenant_name, const string& bucket_name, rgw_bucket& bucket);
+                              const string& tenant_name, const string& bucket_name, rgw_bucket& bucket, string *pselected_rule_name,
+                              RGWZonePlacementInfo *rule_info);
+  int select_legacy_bucket_placement(const string& tenant_name, const string& bucket_name, rgw_bucket& bucket,
+                                     RGWZonePlacementInfo *rule_info);
   int select_new_bucket_location(RGWUserInfo& user_info, const string& zonegroup_id, const string& rule,
-                                 const string& tenant_name, const string& bucket_name, rgw_bucket& bucket, string *pselected_rule);
-  int set_bucket_location_by_rule(const string& location_rule, const string& tenant_name, const string& bucket_name, rgw_bucket& bucket);
+                                 const string& tenant_name, const string& bucket_name, rgw_bucket& bucket, string *pselected_rule_name,
+                                 RGWZonePlacementInfo *rule_info);
+  int set_bucket_location_by_rule(const string& location_rule, const string& tenant_name, const string& bucket_name, rgw_bucket& bucket,
+                                  RGWZonePlacementInfo *rule_info);
   virtual int create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
                             const string& zonegroup_id,
                             const string& placement_rule,