rgw: extend manifest to avoid old style manifest

In case we hit issue #8269 we'd like to avoid creating an old style
manifest. Since we need to have parts that use different prefix we add a
new rule param that overrides the manifest prefix.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
(cherry picked from commit 545d8ad1d2dfe53b967ab1bd17dfeb1780abbb5e)

diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc
index 4df7dafb4bfc7125bdcc5122b81df2742041b688..a1986782d11124c362c1fcdc722c5084c1d23006 100644 (file)
--- a/src/rgw/rgw_json_enc.cc
+++ b/src/rgw/rgw_json_enc.cc
@@ -46,6 +46,7 @@ void RGWObjManifestRule::dump(Formatter *f) const
   encode_json("start_ofs", start_ofs, f);
   encode_json("part_size", part_size, f);
   encode_json("stripe_max_size", stripe_max_size, f);
+  encode_json("override_prefix", override_prefix, f);
 }
 
 void RGWObjManifest::dump(Formatter *f) const

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 3be8b78c880b8df18bdb151d6616207a6c8b5ecd..36690ce1c9c0ffc962bdb189b0ac6b9dc61c8a24 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -561,7 +561,10 @@ void RGWObjManifest::obj_iterator::seek(uint64_t o)
     rule_iter = manifest->rules.begin();
     stripe_ofs = 0;
     stripe_size = manifest->get_head_size();
-    cur_part_id = rule_iter->second.start_part_num;
+    if (rule_iter != manifest->rules.end()) {
+      cur_part_id = rule_iter->second.start_part_num;
+      cur_override_prefix = rule_iter->second.override_prefix;
+    }
     update_location();
     return;
   }
@@ -572,6 +575,11 @@ void RGWObjManifest::obj_iterator::seek(uint64_t o)
     --rule_iter;
   }
 
+  if (rule_iter == manifest->rules.end()) {
+    update_location();
+    return;
+  }
+
   RGWObjManifestRule& rule = rule_iter->second;
 
   if (rule.part_size > 0) {
@@ -601,6 +609,8 @@ void RGWObjManifest::obj_iterator::seek(uint64_t o)
     stripe_size = next - stripe_ofs;
   }
 
+  cur_override_prefix = rule.override_prefix;
+
   update_location();
 }
 
@@ -618,7 +628,7 @@ void RGWObjManifest::obj_iterator::update_location()
     return;
   }
 
-  manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, &location);
+  manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, &cur_override_prefix, &location);
 }
 
 void RGWObjManifest::obj_iterator::operator++()
@@ -699,6 +709,8 @@ void RGWObjManifest::obj_iterator::operator++()
     stripe_size = MIN(rule->part_size - (stripe_ofs - part_ofs), rule->stripe_max_size);
   }
 
+  cur_override_prefix = rule->override_prefix;
+
   ofs = stripe_ofs;
   if (ofs > obj_size) {
     ofs = obj_size;
@@ -746,7 +758,7 @@ int RGWObjManifest::generator::create_begin(CephContext *cct, RGWObjManifest *_m
   
   cur_part_id = rule.start_part_num;
 
-  manifest->get_implicit_location(cur_part_id, cur_stripe, 0, &cur_obj);
+  manifest->get_implicit_location(cur_part_id, cur_stripe, 0, NULL, &cur_obj);
 
   manifest->update_iterators();
 
@@ -780,7 +792,7 @@ int RGWObjManifest::generator::create_next(uint64_t ofs)
   manifest->set_obj_size(ofs);
 
 
-  manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, &cur_obj);
+  manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, NULL, &cur_obj);
 
   manifest->update_iterators();
 
@@ -797,9 +809,14 @@ const RGWObjManifest::obj_iterator& RGWObjManifest::obj_end()
   return end_iter;
 }
 
-void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, rgw_obj *location)
+void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_obj *location)
 {
-  string oid = prefix;
+  string oid;
+  if (!override_prefix || override_prefix->empty()) {
+    oid = prefix;
+  } else {
+    oid = *override_prefix;
+  }
   string ns;
 
   if (!cur_part_id) {
@@ -857,10 +874,14 @@ int RGWObjManifest::append(RGWObjManifest& m)
     return 0;
   }
 
+  string override_prefix;
+
   if (prefix.empty()) {
     prefix = m.prefix;
-  } else if (prefix != m.prefix) {
-    return append_explicit(m);
+  }
+
+  if (prefix != m.prefix) {
+    override_prefix = m.prefix;
   }
 
   map<uint64_t, RGWObjManifestRule>::iterator miter = m.rules.begin();
@@ -882,9 +903,15 @@ int RGWObjManifest::append(RGWObjManifest& m)
       next_rule.part_size = m.obj_size - next_rule.start_ofs;
     }
 
+    if (override_prefix != rule.override_prefix) {
+      append_rules(m, miter, &override_prefix);
+      break;
+    }
+
     if (rule.part_size != next_rule.part_size ||
-        rule.stripe_max_size != next_rule.stripe_max_size) {
-      append_rules(m, miter);
+        rule.stripe_max_size != next_rule.stripe_max_size ||
+        rule.override_prefix != next_rule.override_prefix) {
+      append_rules(m, miter, NULL);
       break;
     }
 
@@ -894,7 +921,7 @@ int RGWObjManifest::append(RGWObjManifest& m)
     }
 
     if (expected_part_num != next_rule.start_part_num) {
-      append_rules(m, miter);
+      append_rules(m, miter, NULL);
       break;
     }
   }
@@ -904,11 +931,14 @@ int RGWObjManifest::append(RGWObjManifest& m)
   return 0;
 }
 
-void RGWObjManifest::append_rules(RGWObjManifest& m, map<uint64_t, RGWObjManifestRule>::iterator& miter)
+void RGWObjManifest::append_rules(RGWObjManifest& m, map<uint64_t, RGWObjManifestRule>::iterator& miter,
+                                  string *override_prefix)
 {
   for (; miter != m.rules.end(); ++miter) {
     RGWObjManifestRule rule = miter->second;
     rule.start_ofs += obj_size;
+    if (override_prefix)
+      rule.override_prefix = *override_prefix;
     rules[rule.start_ofs] = rule;
   }
 }

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 7cf47649af9d5e5f946236a866a36c0928540fcd..51ebccc2c5e1b7241cfbf43b707ae2e699aff010 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -139,26 +139,29 @@ struct RGWObjManifestRule {
   uint64_t start_ofs;
   uint64_t part_size; /* each part size, 0 if there's no part size, meaning it's unlimited */
   uint64_t stripe_max_size; /* underlying obj max size */
+  string override_prefix;
 
   RGWObjManifestRule() : start_part_num(0), start_ofs(0), part_size(0), stripe_max_size(0) {}
   RGWObjManifestRule(uint32_t _start_part_num, uint64_t _start_ofs, uint64_t _part_size, uint64_t _stripe_max_size) :
                        start_part_num(_start_part_num), start_ofs(_start_ofs), part_size(_part_size), stripe_max_size(_stripe_max_size) {}
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     ::encode(start_part_num, bl);
     ::encode(start_ofs, bl);
     ::encode(part_size, bl);
     ::encode(stripe_max_size, bl);
+    ::encode(override_prefix, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::iterator& bl) {
-    DECODE_START(1,  bl);
+    DECODE_START(2, bl);
     ::decode(start_part_num, bl);
     ::decode(start_ofs, bl);
     ::decode(part_size, bl);
     ::decode(stripe_max_size, bl);
+    ::decode(override_prefix, bl);
     DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;
@@ -183,7 +186,7 @@ protected:
 
   void convert_to_explicit();
   int append_explicit(RGWObjManifest& m);
-  void append_rules(RGWObjManifest& m, map<uint64_t, RGWObjManifestRule>::iterator& iter);
+  void append_rules(RGWObjManifest& m, map<uint64_t, RGWObjManifestRule>::iterator& iter, string *override_prefix);
 
   void update_iterators() {
     begin_iter.seek(0);
@@ -223,7 +226,7 @@ public:
     objs.swap(_objs);
   }
 
-  void get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, rgw_obj *location);
+  void get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_obj *location);
 
   void set_trivial_rule(uint64_t tail_ofs, uint64_t stripe_max_size) {
     RGWObjManifestRule rule(0, tail_ofs, 0, stripe_max_size);
@@ -358,6 +361,7 @@ public:
 
     int cur_part_id;
     int cur_stripe;
+    string cur_override_prefix;
 
     rgw_obj location;