From: Yingxin Cheng Date: Mon, 28 Sep 2020 14:57:54 +0000 (+0800) Subject: crimson/onode-staged-tree: cleanup, introduce explicit INDEX_LAST X-Git-Tag: v16.1.0~359^2~25 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=66f20c8164a26ccc06dfb9deb7e931a1726eeeec;p=ceph.git crimson/onode-staged-tree: cleanup, introduce explicit INDEX_LAST Signed-off-by: Yingxin Cheng --- diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/fwd.h b/src/crimson/os/seastore/onode_manager/staged-fltree/fwd.h index c25d92fb8de5..477c939fcdb1 100644 --- a/src/crimson/os/seastore/onode_manager/staged-fltree/fwd.h +++ b/src/crimson/os/seastore/onode_manager/staged-fltree/fwd.h @@ -44,6 +44,9 @@ using NodeImplURef = std::unique_ptr; using level_t = uint8_t; constexpr auto INDEX_END = std::numeric_limits::max(); +constexpr auto INDEX_LAST = INDEX_END - 0xf; +constexpr auto INDEX_UPPER_BOUND = INDEX_END - 0xff; +inline bool is_valid_index(size_t index) { return index < INDEX_UPPER_BOUND; } // TODO: decide by NODE_BLOCK_SIZE using node_offset_t = uint16_t; diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/node.cc b/src/crimson/os/seastore/onode_manager/staged-fltree/node.cc index 36900606443a..eb28b6119447 100644 --- a/src/crimson/os/seastore/onode_manager/staged-fltree/node.cc +++ b/src/crimson/os/seastore/onode_manager/staged-fltree/node.cc @@ -428,7 +428,7 @@ void InternalNode::track_insert( Ref insert_child, Ref nxt_child) { // update tracks auto pos_upper_bound = insert_pos; - pos_upper_bound.index_by_stage(insert_stage) = INDEX_END; + pos_upper_bound.index_by_stage(insert_stage) = INDEX_UPPER_BOUND; auto first = tracked_child_nodes.lower_bound(insert_pos); auto last = tracked_child_nodes.lower_bound(pos_upper_bound); std::vector nodes; @@ -698,7 +698,7 @@ Ref LeafNode::track_insert( const onode_t* p_onode) { // update cursor position auto pos_upper_bound = insert_pos; - pos_upper_bound.index_by_stage(insert_stage) = INDEX_END; + pos_upper_bound.index_by_stage(insert_stage) = INDEX_UPPER_BOUND; auto first = tracked_cursors.lower_bound(insert_pos); auto last = tracked_cursors.lower_bound(pos_upper_bound); std::vector p_cursors; diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/node.h b/src/crimson/os/seastore/onode_manager/staged-fltree/node.h index 328fe454bc5f..1364c448306e 100644 --- a/src/crimson/os/seastore/onode_manager/staged-fltree/node.h +++ b/src/crimson/os/seastore/onode_manager/staged-fltree/node.h @@ -244,7 +244,7 @@ class InternalNode final : public Node { private: // XXX: leverage intrusive data structure to control memory overhead - // track the current living child nodes by position + // track the current living child nodes by position (can be end) std::map tracked_child_nodes; InternalNodeImpl* impl; }; @@ -324,7 +324,7 @@ class LeafNode final : public Node { private: // XXX: leverage intrusive data structure to control memory overhead - // track the current living cursors by position + // track the current living cursors by position (cannot be end) std::map tracked_cursors; LeafNodeImpl* impl; layout_version_t layout_version = 0; diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/node_layout.h b/src/crimson/os/seastore/onode_manager/staged-fltree/node_layout.h index 32da56011e15..c7ecfe7ee97a 100644 --- a/src/crimson/os/seastore/onode_manager/staged-fltree/node_layout.h +++ b/src/crimson/os/seastore/onode_manager/staged-fltree/node_layout.h @@ -459,7 +459,7 @@ class NodeLayoutT final : public InternalNodeImpl, public LeafNodeImpl { insert_size = STAGE_T::template insert_size(key, packed_value); } else { std::tie(insert_stage, insert_size) = STAGE_T::evaluate_insert( - node_stage, key, packed_value, cast_down(insert_pos), true); + node_stage, key, packed_value, cast_down(insert_pos), false); } return {insert_stage, insert_size}; } else { @@ -485,8 +485,13 @@ class NodeLayoutT final : public InternalNodeImpl, public LeafNodeImpl { const MatchHistory& history, match_stat_t mstat, search_position_t& insert_pos) const override { if constexpr (NODE_TYPE == node_type_t::LEAF) { - return STAGE_T::evaluate_insert( - key, value, history, mstat, cast_down(insert_pos)); + if (unlikely(is_empty())) { + assert(insert_pos.is_end()); + return {STAGE, STAGE_T::template insert_size(key, value)}; + } else { + return STAGE_T::evaluate_insert( + key, value, history, mstat, cast_down(insert_pos)); + } } else { assert(false && "impossible path"); } diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/stages/item_iterator_stage.cc b/src/crimson/os/seastore/onode_manager/staged-fltree/stages/item_iterator_stage.cc index 42160d2dd08e..fa006687408d 100644 --- a/src/crimson/os/seastore/onode_manager/staged-fltree/stages/item_iterator_stage.cc +++ b/src/crimson/os/seastore/onode_manager/staged-fltree/stages/item_iterator_stage.cc @@ -86,42 +86,45 @@ template class ITER_INST(node_type_t::INTERNAL)::Appender; template template -bool APPEND_T::append(const ITER_T& src, size_t& items, index_t type) { +bool APPEND_T::append(const ITER_T& src, size_t& items) { auto p_end = src.p_end(); - if (items != INDEX_END) { + bool append_till_end = false; + if (is_valid_index(items)) { for (auto i = 1u; i <= items; ++i) { if (!src.has_next()) { assert(i == items); - type = index_t::end; + append_till_end = true; break; } ++src; } - } else if (type != index_t::none) { + } else { + if (items == INDEX_END) { + append_till_end = true; + } else { + assert(items == INDEX_LAST); + } items = 0; while (src.has_next()) { ++src; ++items; } - if (type == index_t::end) { + if (append_till_end) { ++items; } - } else { - assert(false); } + const char* p_start; - if (type == index_t::end) { - // include last + if (append_till_end) { p_start = src.p_start(); } else { - // exclude last p_start = src.p_end(); } assert(p_end >= p_start); size_t append_size = p_end - p_start; p_append -= append_size; p_mut->copy_in_absolute(p_append, p_start, append_size); - return type == index_t::end; + return append_till_end; } template diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/stages/item_iterator_stage.h b/src/crimson/os/seastore/onode_manager/staged-fltree/stages/item_iterator_stage.h index 3c1725d5f041..e2783c8eccd5 100644 --- a/src/crimson/os/seastore/onode_manager/staged-fltree/stages/item_iterator_stage.h +++ b/src/crimson/os/seastore/onode_manager/staged-fltree/stages/item_iterator_stage.h @@ -92,7 +92,6 @@ class item_iterator_t { static size_t trim_at( NodeExtentMutable&, const item_iterator_t&, size_t trimmed); - enum class index_t { none, last, end }; template class Appender; @@ -120,7 +119,7 @@ class item_iterator_t::Appender { public: Appender(NodeExtentMutable* p_mut, char* p_append) : p_mut{p_mut}, p_append{p_append} {} - bool append(const item_iterator_t& src, size_t& items, index_t type); + bool append(const item_iterator_t& src, size_t& items); char* wrap() { return p_append; } std::tuple open_nxt(const key_get_type&); std::tuple open_nxt(const full_key_t&); diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/stages/stage.h b/src/crimson/os/seastore/onode_manager/staged-fltree/stages/stage.h index 647c0958ebb4..e9c533e02ea7 100644 --- a/src/crimson/os/seastore/onode_manager/staged-fltree/stages/stage.h +++ b/src/crimson/os/seastore/onode_manager/staged-fltree/stages/stage.h @@ -153,7 +153,7 @@ struct staged { static void _left_or_right(size_t& split_index, size_t insert_index, std::optional& is_insert_left) { assert(!is_insert_left.has_value()); - assert(split_index != INDEX_END); + assert(is_valid_index(split_index)); if constexpr (is_exclusive) { if (split_index <= insert_index) { // ...[s_index-1] |!| (i_index) [s_index]... @@ -317,15 +317,18 @@ struct staged { std::optional& is_insert_left) { assert(!is_end()); assert(index() == 0); - assert(insert_index <= container.keys() || insert_index == INDEX_END); - // replace the unknown INDEX_END value - if (insert_index == INDEX_END) { - if constexpr (!is_exclusive) { + // replace insert_index placeholder + if constexpr (!is_exclusive) { + if (insert_index == INDEX_LAST) { insert_index = container.keys() - 1; - } else { + } + } else { + if (insert_index == INDEX_END) { insert_index = container.keys(); } } + assert(insert_index <= container.keys()); + auto start_size_1 = start_size + extra_size; auto f_get_used_size = [this, start_size, start_size_1, insert_index, insert_size] (size_t index) { @@ -380,29 +383,26 @@ struct staged { return current_size; } - // Note: possible to return an end iterater if - // to_index == INDEX_END && to_stage == STAGE + // Note: possible to return an end iterater if to_index == INDEX_END template - void copy_out_until(typename container_t::template Appender& appender, - size_t& to_index, - match_stage_t to_stage) { - assert(to_stage <= STAGE); + void copy_out_until( + typename container_t::template Appender& appender, size_t& to_index) { auto num_keys = container.keys(); size_t items; if (to_index == INDEX_END) { - if (to_stage == STAGE) { - items = num_keys - _index; - appender.append(container, _index, items); - _index = num_keys; - } else { - assert(!is_end()); - items = num_keys - 1 - _index; - appender.append(container, _index, items); - _index = num_keys - 1; - } + items = num_keys - _index; + appender.append(container, _index, items); + _index = num_keys; + to_index = _index; + } else if (to_index == INDEX_LAST) { + assert(!is_end()); + items = num_keys - 1 - _index; + appender.append(container, _index, items); + _index = num_keys - 1; to_index = _index; } else { assert(_index <= to_index); + assert(to_index <= num_keys); items = to_index - _index; appender.append(container, _index, items); _index = to_index; @@ -589,7 +589,7 @@ struct staged { if constexpr (!is_exclusive) { if (is_last()) { assert(split_index == index()); - if (insert_index == INDEX_END) { + if (insert_index == INDEX_LAST) { insert_index = index(); } assert(insert_index <= index()); @@ -667,35 +667,27 @@ struct staged { return current_size; } - // Note: possible to return an end iterater if - // to_index == INDEX_END && to_stage == STAGE + // Note: possible to return an end iterater if to_index == INDEX_END template - void copy_out_until(typename container_t::template Appender& appender, - size_t& to_index, - match_stage_t to_stage) { - assert(to_stage <= STAGE); + void copy_out_until( + typename container_t::template Appender& appender, size_t& to_index) { if (is_end()) { assert(!container.has_next()); - assert(to_stage == STAGE); - assert(to_index == index() || to_index == INDEX_END); - to_index = index(); + if (to_index == INDEX_END) { + to_index = index(); + } + assert(to_index == index()); return; } - typename container_t::index_t type; size_t items; - if (to_index == INDEX_END) { - if (to_stage == STAGE) { - type = container_t::index_t::end; - } else { - type = container_t::index_t::last; - } - items = INDEX_END; + if (to_index == INDEX_END || to_index == INDEX_LAST) { + items = to_index; } else { + assert(is_valid_index(to_index)); assert(index() <= to_index); - type = container_t::index_t::none; items = to_index - index(); } - if (appender.append(container, items, type)) { + if (appender.append(container, items)) { set_end(); } to_index = index(); @@ -760,7 +752,7 @@ struct staged { * -> insert to left/right (exclusive, can be end) * -> split_size * seek_split(start_size, extra_size, target_size) -> split_size - * copy_out_until(appender, to_index, to_stage) (can be end) + * copy_out_until(appender, to_index) (can be end) * trim_until(mut) -> trim_size * (!IS_BOTTOM) trim_at(mut, trimmed) -> trim_size * static: @@ -1000,17 +992,15 @@ struct staged { template > static std::enable_if_t evaluate_insert( const container_t& container, const full_key_t& key, - const value_t& value, position_t& position, bool is_current) { + const value_t& value, position_t& position, bool evaluate_last) { auto iter = iterator_t(container); auto& index = position.index; - if (!is_current) { - index = INDEX_END; - } - if (index == INDEX_END) { + if (evaluate_last || index == INDEX_END) { iter.seek_last(); index = iter.index(); // evaluate the previous index } else { + assert(is_valid_index(index)); // evaluate the current index iter.seek_at(index); auto match = compare_to(key, iter.get_key()); @@ -1022,10 +1012,10 @@ struct staged { // insert into the current index auto nxt_container = iter.get_nxt_container(); return NXT_STAGE_T::evaluate_insert( - nxt_container, key, value, position.nxt, true); + nxt_container, key, value, position.nxt, false); } } else { - assert(is_current && match == MatchKindCMP::NE); + assert(match == MatchKindCMP::NE); if (index == 0) { // already the first index, so insert at the current index return {STAGE, insert_size(key, value)}; @@ -1052,7 +1042,7 @@ struct staged { // insert into the previous index auto nxt_container = iter.get_nxt_container(); return NXT_STAGE_T::evaluate_insert( - nxt_container, key, value, position.nxt, false); + nxt_container, key, value, position.nxt, true); } } } @@ -1074,10 +1064,10 @@ struct staged { bool compensate = NXT_STAGE_T:: compensate_insert_position_at(stage, position.nxt); if (compensate) { - assert(index != INDEX_END); + assert(is_valid_index(index)); if (index == 0) { // insert into the *last* index of the current stage - index = INDEX_END; + index = INDEX_LAST; return true; } else { --index; @@ -1090,6 +1080,16 @@ struct staged { } } + static void patch_insert_end(position_t& insert_pos, match_stage_t insert_stage) { + assert(insert_stage <= STAGE); + if (insert_stage == STAGE) { + insert_pos.index = INDEX_END; + } else if constexpr (!IS_BOTTOM) { + insert_pos.index = INDEX_LAST; + NXT_STAGE_T::patch_insert_end(insert_pos.nxt, insert_stage); + } + } + template > static std::enable_if_t evaluate_insert( const full_key_t& key, const onode_t& value, @@ -1125,6 +1125,10 @@ struct staged { } } + if (position.is_end()) { + patch_insert_end(position, insert_stage); + } + node_offset_t insert_size = insert_size_at(insert_stage, key, value); return {insert_stage, insert_size}; @@ -1154,19 +1158,26 @@ struct staged { assert(stage <= STAGE); auto iter = iterator_t(container); auto& index = position.index; - if (index == INDEX_END) { - iter.seek_last(); - } else { - iter.seek_till_end(index); - } bool do_insert = false; if (stage == STAGE) { if (index == INDEX_END) { + iter.seek_last(); iter.set_end(); + index = iter.index(); + } else { + assert(is_valid_index(index)); + iter.seek_till_end(index); } do_insert = true; } else { // stage < STAGE + if (index == INDEX_LAST) { + iter.seek_last(); + index = iter.index(); + } else { + assert(is_valid_index(index)); + iter.seek_till_end(index); + } if constexpr (SPLIT) { if (iter.is_end()) { // insert at the higher stage due to split @@ -1178,9 +1189,6 @@ struct staged { assert(!iter.is_end()); } } - if (index == INDEX_END) { - index = iter.index(); - } if (do_insert) { if constexpr (!IS_BOTTOM) { @@ -1216,9 +1224,10 @@ struct staged { position_t& position, match_stage_t& stage, node_offset_t& _insert_size) { auto p_left_bound = container.p_left_bound(); if (unlikely(!container.keys())) { - if (position == position_t::end()) { + if (position.is_end()) { position = position_t::begin(); assert(stage == STAGE); + assert(_insert_size == insert_size(key, value)); } else if (position == position_t::begin()) { // when insert into a trimmed and empty left node stage = STAGE; @@ -1421,6 +1430,7 @@ struct staged { assert(current_size <= target_size); iterator_t& split_iter = split_at.get(); current_size = split_iter.seek_split(current_size, extra_size, target_size); + assert(!split_iter.is_end()); if constexpr (!IS_BOTTOM) { NXT_STAGE_T::recursively_locate_split( current_size, extra_size + split_iter.size_to_nxt(), @@ -1442,14 +1452,25 @@ struct staged { insert_index, insert_size, is_insert_left); assert(is_insert_left.has_value()); if (*is_insert_left == false && split_iter.index() == insert_index) { + // split_iter can be end + // found the lower-bound of target_size // ...[s_index-1] |!| (i_index) [s_index]... return; - } - assert(!split_iter.is_end()); - if (split_iter.index() == 0) { - extra_size += iterator_t::header_size(); } else { - extra_size = 0; + // Already considered insert effect in the current stage. + // Look into the next stage to identify the target_size lower-bound w/o + // insert effect. + assert(!split_iter.is_end()); + if (split_iter.index() == 0) { + extra_size += iterator_t::header_size(); + } else { + extra_size = 0; + } + if constexpr (!IS_BOTTOM) { + NXT_STAGE_T::recursively_locate_split( + current_size, extra_size + split_iter.size_to_nxt(), + target_size, split_at.nxt()); + } } } else { if constexpr (!IS_BOTTOM) { @@ -1464,6 +1485,10 @@ struct staged { extra_size = 0; } if (!is_insert_left.has_value()) { + // Considered insert effect in the current stage, and insert happens + // in the lower stage. + // Look into the next stage to identify the target_size lower-bound w/ + // insert effect. assert(split_iter.index() == insert_index); NXT_STAGE_T::recursively_locate_split_inserted( current_size, extra_size + split_iter.size_to_nxt(), target_size, @@ -1471,17 +1496,21 @@ struct staged { is_insert_left, split_at.nxt()); assert(is_insert_left.has_value()); return; + } else { + // is_insert_left.has_value() == true + // Insert will *not* happen in the lower stage. + // Need to look into the next stage to identify the target_size + // lower-bound w/ insert effect + NXT_STAGE_T::recursively_locate_split( + current_size, extra_size + split_iter.size_to_nxt(), + target_size, split_at.nxt()); + return; } } else { assert(false && "impossible path"); + return; } } - if constexpr (!IS_BOTTOM) { - NXT_STAGE_T::recursively_locate_split( - current_size, extra_size + split_iter.size_to_nxt(), - target_size, split_at.nxt()); - } - return; } /* @@ -1522,14 +1551,11 @@ struct staged { appender = typename container_t::template Appender(p_mut, p_start); _index = 0; } - // possible to make src_iter end if - // to_index == INDEX_END && to_stage == STAGE - void append_until( - StagedIterator& src_iter, size_t& to_index, match_stage_t to_stage) { + // possible to make src_iter end if to_index == INDEX_END + void append_until(StagedIterator& src_iter, size_t& to_index) { assert(!require_wrap_nxt); - assert(to_stage <= STAGE); auto s_index = src_iter.index(); - src_iter.get().template copy_out_until(*appender, to_index, to_stage); + src_iter.get().template copy_out_until(*appender, to_index); assert(src_iter.index() == to_index); assert(to_index >= s_index); auto increment = (to_index - s_index); @@ -1614,37 +1640,40 @@ struct staged { }; template - static void _append_range(StagedIterator& src_iter, StagedAppender& appender, - size_t& to_index, match_stage_t stage) { + static void _append_range( + StagedIterator& src_iter, StagedAppender& appender, size_t& to_index) { if (src_iter.is_end()) { + // append done assert(to_index == INDEX_END); - assert(stage == STAGE); to_index = src_iter.index(); } else if constexpr (!IS_BOTTOM) { if (appender.in_progress()) { - // we are in the progress of appending - auto to_index_nxt = INDEX_END; + // appender has appended something at the current item, + // cannot append the current item as-a-whole + size_t to_index_nxt = INDEX_END; NXT_STAGE_T::template _append_range( - src_iter.nxt(), appender.get_nxt(), - to_index_nxt, STAGE - 1); + src_iter.nxt(), appender.get_nxt(), to_index_nxt); ++src_iter; appender.wrap_nxt(); } else if (src_iter.in_progress()) { + // src_iter is not at the beginning of the current item, // cannot append the current item as-a-whole - auto to_index_nxt = INDEX_END; + size_t to_index_nxt = INDEX_END; NXT_STAGE_T::template _append_range( - src_iter.nxt(), appender.open_nxt(src_iter.get_key()), - to_index_nxt, STAGE - 1); + src_iter.nxt(), appender.open_nxt(src_iter.get_key()), to_index_nxt); ++src_iter; appender.wrap_nxt(); + } else { + // we can safely append the current item as-a-whole } } - appender.append_until(src_iter, to_index, stage); + appender.append_until(src_iter, to_index); } template static void _append_into(StagedIterator& src_iter, StagedAppender& appender, position_t& position, match_stage_t stage) { + assert(position.index == src_iter.index()); // reaches the last item if (stage == STAGE) { // done, end recursion @@ -1653,7 +1682,7 @@ struct staged { } } else { assert(stage < STAGE); - // process append in the next stage + // proceed append in the next stage NXT_STAGE_T::template append_until( src_iter.nxt(), appender.open_nxt(src_iter.get_key()), position.nxt, stage); @@ -1668,13 +1697,18 @@ struct staged { assert(from_index <= to_index); if constexpr (IS_BOTTOM) { assert(stage == STAGE); - appender.append_until(src_iter, to_index, stage); + appender.append_until(src_iter, to_index); } else { assert(stage <= STAGE); if (src_iter.index() == to_index) { _append_into(src_iter, appender, position, stage); } else { - _append_range(src_iter, appender, to_index, stage); + if (to_index == INDEX_END) { + assert(stage == STAGE); + } else if (to_index == INDEX_LAST) { + assert(stage < STAGE); + } + _append_range(src_iter, appender, to_index); _append_into(src_iter, appender, position, stage); } } diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/stages/stage_types.h b/src/crimson/os/seastore/onode_manager/staged-fltree/stages/stage_types.h index 55144e10f0c8..7aaa6e10e21e 100644 --- a/src/crimson/os/seastore/onode_manager/staged-fltree/stages/stage_types.h +++ b/src/crimson/os/seastore/onode_manager/staged-fltree/stages/stage_types.h @@ -130,7 +130,14 @@ struct staged_position_t { static_assert(STAGE > STAGE_BOTTOM && STAGE <= STAGE_TOP); using me_t = staged_position_t; using nxt_t = staged_position_t; - bool is_end() const { return index == INDEX_END; } + bool is_end() const { + if (index == INDEX_END) { + return true; + } else { + assert(is_valid_index(index)); + return false; + } + } size_t& index_by_stage(match_stage_t stage) { assert(stage <= STAGE); if (STAGE == stage) { @@ -157,9 +164,10 @@ struct staged_position_t { bool operator!=(const me_t& o) const { return cmp(o) != 0; } me_t& operator-=(const me_t& o) { - assert(o.index != INDEX_END); + assert(is_valid_index(o.index)); assert(index >= o.index); if (index != INDEX_END) { + assert(is_valid_index(index)); index -= o.index; if (index == 0) { nxt -= o.nxt; @@ -180,8 +188,11 @@ template std::ostream& operator<<(std::ostream& os, const staged_position_t& pos) { if (pos.index == INDEX_END) { os << "END"; + } else if (pos.index == INDEX_LAST) { + os << "LAST"; } else { os << pos.index; + assert(is_valid_index(pos.index)); } return os << ", " << pos.nxt; } @@ -189,7 +200,14 @@ std::ostream& operator<<(std::ostream& os, const staged_position_t& pos) template <> struct staged_position_t { using me_t = staged_position_t; - bool is_end() const { return index == INDEX_END; } + bool is_end() const { + if (index == INDEX_END) { + return true; + } else { + assert(is_valid_index(index)); + return false; + } + } size_t& index_by_stage(match_stage_t stage) { assert(stage == STAGE_BOTTOM); return index; @@ -212,9 +230,10 @@ struct staged_position_t { bool operator!=(const me_t& o) const { return cmp(o) != 0; } me_t& operator-=(const me_t& o) { - assert(o.index != INDEX_END); + assert(is_valid_index(o.index)); assert(index >= o.index); if (index != INDEX_END) { + assert(is_valid_index(index)); index -= o.index; } return *this; @@ -228,10 +247,14 @@ struct staged_position_t { template <> inline std::ostream& operator<<(std::ostream& os, const staged_position_t& pos) { if (pos.index == INDEX_END) { - return os << "END"; + os << "END"; + } else if (pos.index == INDEX_LAST) { + os << "LAST"; } else { - return os << pos.index; + os << pos.index; + assert(is_valid_index(pos.index)); } + return os; } using search_position_t = staged_position_t;