From: Yehuda Sadeh Date: Tue, 6 May 2014 22:35:20 +0000 (-0700) Subject: rgw: extend manifest to avoid old style manifest X-Git-Tag: v0.81~28^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=545d8ad1d2dfe53b967ab1bd17dfeb1780abbb5e;p=ceph.git rgw: extend manifest to avoid old style manifest In case we hit issue #8269 we'd like to avoid creating an old style manifest. Since we need to have parts that use different prefix we add a new rule param that overrides the manifest prefix. Signed-off-by: Yehuda Sadeh --- diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc index 4df7dafb4bfc..a1986782d111 100644 --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@ -46,6 +46,7 @@ void RGWObjManifestRule::dump(Formatter *f) const encode_json("start_ofs", start_ofs, f); encode_json("part_size", part_size, f); encode_json("stripe_max_size", stripe_max_size, f); + encode_json("override_prefix", override_prefix, f); } void RGWObjManifest::dump(Formatter *f) const diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 4b1c3559af4d..d8e278cf64ba 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -561,7 +561,10 @@ void RGWObjManifest::obj_iterator::seek(uint64_t o) rule_iter = manifest->rules.begin(); stripe_ofs = 0; stripe_size = manifest->get_head_size(); - cur_part_id = rule_iter->second.start_part_num; + if (rule_iter != manifest->rules.end()) { + cur_part_id = rule_iter->second.start_part_num; + cur_override_prefix = rule_iter->second.override_prefix; + } update_location(); return; } @@ -572,6 +575,11 @@ void RGWObjManifest::obj_iterator::seek(uint64_t o) --rule_iter; } + if (rule_iter == manifest->rules.end()) { + update_location(); + return; + } + RGWObjManifestRule& rule = rule_iter->second; if (rule.part_size > 0) { @@ -601,6 +609,8 @@ void RGWObjManifest::obj_iterator::seek(uint64_t o) stripe_size = MIN(stripe_size, rule.stripe_max_size); } + cur_override_prefix = rule.override_prefix; + update_location(); } @@ -618,7 +628,7 @@ void RGWObjManifest::obj_iterator::update_location() return; } - manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, &location); + manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, &cur_override_prefix, &location); } void RGWObjManifest::obj_iterator::operator++() @@ -699,6 +709,8 @@ void RGWObjManifest::obj_iterator::operator++() stripe_size = MIN(rule->part_size - (stripe_ofs - part_ofs), rule->stripe_max_size); } + cur_override_prefix = rule->override_prefix; + ofs = stripe_ofs; if (ofs > obj_size) { ofs = obj_size; @@ -746,7 +758,7 @@ int RGWObjManifest::generator::create_begin(CephContext *cct, RGWObjManifest *_m cur_part_id = rule.start_part_num; - manifest->get_implicit_location(cur_part_id, cur_stripe, 0, &cur_obj); + manifest->get_implicit_location(cur_part_id, cur_stripe, 0, NULL, &cur_obj); manifest->update_iterators(); @@ -780,7 +792,7 @@ int RGWObjManifest::generator::create_next(uint64_t ofs) manifest->set_obj_size(ofs); - manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, &cur_obj); + manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, NULL, &cur_obj); manifest->update_iterators(); @@ -797,9 +809,14 @@ const RGWObjManifest::obj_iterator& RGWObjManifest::obj_end() return end_iter; } -void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, rgw_obj *location) +void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_obj *location) { - string oid = prefix; + string oid; + if (!override_prefix || override_prefix->empty()) { + oid = prefix; + } else { + oid = *override_prefix; + } string ns; if (!cur_part_id) { @@ -857,10 +874,14 @@ int RGWObjManifest::append(RGWObjManifest& m) return 0; } + string override_prefix; + if (prefix.empty()) { prefix = m.prefix; - } else if (prefix != m.prefix) { - return append_explicit(m); + } + + if (prefix != m.prefix) { + override_prefix = m.prefix; } map::iterator miter = m.rules.begin(); @@ -882,9 +903,15 @@ int RGWObjManifest::append(RGWObjManifest& m) next_rule.part_size = m.obj_size - next_rule.start_ofs; } + if (override_prefix != rule.override_prefix) { + append_rules(m, miter, &override_prefix); + break; + } + if (rule.part_size != next_rule.part_size || - rule.stripe_max_size != next_rule.stripe_max_size) { - append_rules(m, miter); + rule.stripe_max_size != next_rule.stripe_max_size || + rule.override_prefix != next_rule.override_prefix) { + append_rules(m, miter, NULL); break; } @@ -894,7 +921,7 @@ int RGWObjManifest::append(RGWObjManifest& m) } if (expected_part_num != next_rule.start_part_num) { - append_rules(m, miter); + append_rules(m, miter, NULL); break; } } @@ -904,11 +931,14 @@ int RGWObjManifest::append(RGWObjManifest& m) return 0; } -void RGWObjManifest::append_rules(RGWObjManifest& m, map::iterator& miter) +void RGWObjManifest::append_rules(RGWObjManifest& m, map::iterator& miter, + string *override_prefix) { for (; miter != m.rules.end(); ++miter) { RGWObjManifestRule rule = miter->second; rule.start_ofs += obj_size; + if (override_prefix) + rule.override_prefix = *override_prefix; rules[rule.start_ofs] = rule; } } diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 3f6276b316c0..9a8ff69f67ba 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -139,26 +139,29 @@ struct RGWObjManifestRule { uint64_t start_ofs; uint64_t part_size; /* each part size, 0 if there's no part size, meaning it's unlimited */ uint64_t stripe_max_size; /* underlying obj max size */ + string override_prefix; RGWObjManifestRule() : start_part_num(0), start_ofs(0), part_size(0), stripe_max_size(0) {} RGWObjManifestRule(uint32_t _start_part_num, uint64_t _start_ofs, uint64_t _part_size, uint64_t _stripe_max_size) : start_part_num(_start_part_num), start_ofs(_start_ofs), part_size(_part_size), stripe_max_size(_stripe_max_size) {} void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); ::encode(start_part_num, bl); ::encode(start_ofs, bl); ::encode(part_size, bl); ::encode(stripe_max_size, bl); + ::encode(override_prefix, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); + DECODE_START(2, bl); ::decode(start_part_num, bl); ::decode(start_ofs, bl); ::decode(part_size, bl); ::decode(stripe_max_size, bl); + ::decode(override_prefix, bl); DECODE_FINISH(bl); } void dump(Formatter *f) const; @@ -183,7 +186,7 @@ protected: void convert_to_explicit(); int append_explicit(RGWObjManifest& m); - void append_rules(RGWObjManifest& m, map::iterator& iter); + void append_rules(RGWObjManifest& m, map::iterator& iter, string *override_prefix); void update_iterators() { begin_iter.seek(0); @@ -223,7 +226,7 @@ public: objs.swap(_objs); } - void get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, rgw_obj *location); + void get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_obj *location); void set_trivial_rule(uint64_t tail_ofs, uint64_t stripe_max_size) { RGWObjManifestRule rule(0, tail_ofs, 0, stripe_max_size); @@ -358,6 +361,7 @@ public: int cur_part_id; int cur_stripe; + string cur_override_prefix; rgw_obj location;