From d0933eef607068bb5a2301940a52a8ab75bb00f7 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Wed, 4 Dec 2013 18:23:17 -0800 Subject: [PATCH] test/osd: add append support to WriteOp Signed-off-by: Samuel Just --- src/test/osd/Object.cc | 23 +++++++++++++-- src/test/osd/Object.h | 48 ++++++++++++++++++++++--------- src/test/osd/RadosModel.h | 60 ++++++++++++++++++++++++++++----------- src/test/osd/TestRados.cc | 31 +++++++++++++++----- 4 files changed, 123 insertions(+), 39 deletions(-) diff --git a/src/test/osd/Object.cc b/src/test/osd/Object.cc index eb8e81d5861e3..e8b41443ea5ca 100644 --- a/src/test/osd/Object.cc +++ b/src/test/osd/Object.cc @@ -38,7 +38,26 @@ ostream &operator<<(ostream &out, const ContDesc &rhs) << ")"; } -void VarLenGenerator::get_ranges(const ContDesc &cont, interval_set &out) { +void AppendGenerator::get_ranges_map( + const ContDesc &cont, map &out) { + RandWrap rand(cont.seqnum); + uint64_t pos = off; + uint64_t limit = off + get_append_size(cont); + while (pos < limit) { + uint64_t segment_length = ( + rand() % (max_append_size - min_append_size)) + min_append_size; + assert(segment_length < max_append_size); + assert(segment_length >= min_append_size); + if (segment_length + pos > limit) { + segment_length = limit - pos; + } + out.insert(make_pair(pos, segment_length)); + pos += segment_length; + } +} + +void VarLenGenerator::get_ranges_map( + const ContDesc &cont, map &out) { RandWrap rand(cont.seqnum); uint64_t pos = 0; uint64_t limit = get_length(cont); @@ -51,7 +70,7 @@ void VarLenGenerator::get_ranges(const ContDesc &cont, interval_set &o segment_length = limit - pos; } if (include) { - out.insert(pos, segment_length); + out.insert(make_pair(pos, segment_length)); include = false; } else { include = true; diff --git a/src/test/osd/Object.h b/src/test/osd/Object.h index 22446b2bdebbc..06547457af2ae 100644 --- a/src/test/osd/Object.h +++ b/src/test/osd/Object.h @@ -97,7 +97,18 @@ public: virtual uint64_t get_length(const ContDesc &in) = 0; - virtual void get_ranges(const ContDesc &in, interval_set &ranges) = 0; + virtual void get_ranges_map( + const ContDesc &cont, map &out) = 0; + void get_ranges(const ContDesc &cont, interval_set &out) { + map ranges; + get_ranges_map(cont, ranges); + for (map::iterator i = ranges.begin(); + i != ranges.end(); + ++i) { + out.insert(i->first, i->second); + } + } + virtual iterator_impl *get_iterator_impl(const ContDesc &in) = 0; @@ -169,8 +180,6 @@ public: } }; - virtual void get_ranges(const ContDesc &cont, interval_set &out) = 0; - ContentsGenerator::iterator_impl *get_iterator_impl(const ContDesc &in) { RandGenerator::iterator_impl *i = new iterator_impl(in, this); return i; @@ -198,7 +207,8 @@ public: max_length(length), min_stride_size(min_stride_size), max_stride_size(max_stride_size) {} - void get_ranges(const ContDesc &cont, interval_set &out); + void get_ranges_map( + const ContDesc &cont, map &out); uint64_t get_length(const ContDesc &in) { RandWrap rand(in.seqnum); return (rand() % max_length); @@ -209,8 +219,9 @@ class AttrGenerator : public RandGenerator { uint64_t max_len; public: AttrGenerator(uint64_t max_len) : max_len(max_len) {} - void get_ranges(const ContDesc &cont, interval_set &out) { - out.insert(0, get_length(cont)); + void get_ranges_map( + const ContDesc &cont, map &out) { + out.insert(make_pair(0, get_length(cont))); } uint64_t get_length(const ContDesc &in) { RandWrap rand(in.seqnum); @@ -228,18 +239,27 @@ public: class AppendGenerator : public RandGenerator { uint64_t off; - uint64_t max_len; - uint64_t max_intervals; + uint64_t min_append_size; + uint64_t max_append_size; + uint64_t max_append_total; public: - AppendGenerator(uint64_t off, uint64_t max_len, uint64_t max_intervals) : - off(off), max_len(max_len), max_intervals(max_intervals) {} - uint64_t get_length(const ContDesc &in) { + AppendGenerator( + uint64_t off, + uint64_t min_append_size, + uint64_t max_append_size, + uint64_t max_append_total) : + off(off), min_append_size(min_append_size), + max_append_size(max_append_size), + max_append_total(max_append_total) {} + uint64_t get_append_size(const ContDesc &in) { RandWrap rand(in.seqnum); - return off + (rand() % max_len); + return rand() % max_append_total; } - void get_ranges(const ContDesc &cont, interval_set &out) { - out.insert(off, get_length(cont)); + uint64_t get_length(const ContDesc &in) { + return off + get_append_size(in); } + void get_ranges_map( + const ContDesc &cont, map &out); }; class ObjectDesc { diff --git a/src/test/osd/RadosModel.h b/src/test/osd/RadosModel.h index 41ee37e7ba274..5d451a2cfc0c7 100644 --- a/src/test/osd/RadosModel.h +++ b/src/test/osd/RadosModel.h @@ -58,7 +58,8 @@ enum TestOpType { TEST_OP_IS_DIRTY, TEST_OP_CACHE_FLUSH, TEST_OP_CACHE_TRY_FLUSH, - TEST_OP_CACHE_EVICT + TEST_OP_CACHE_EVICT, + TEST_OP_APPEND }; class TestWatchContext : public librados::WatchCtx { @@ -698,12 +699,15 @@ public: librados::ObjectWriteOperation write_op; bufferlist rbuffer; + bool do_append; + WriteOp(int n, RadosTestContext *context, const string &oid, + bool do_append, TestOpStat *stat = 0) : TestOp(n, context, stat), - oid(oid), waiting_on(0), last_acked_tid(0) + oid(oid), waiting_on(0), last_acked_tid(0), do_append(do_append) {} void _begin() @@ -716,44 +720,66 @@ public: cont = ContDesc(context->seq_num, context->current_snap, context->seq_num, prefix); - ContentsGenerator *cont_gen = new VarLenGenerator( - context->max_size, context->min_stride_size, context->max_stride_size); + ContentsGenerator *cont_gen; + if (do_append) { + ObjectDesc old_value; + bool found = context->find_object(oid, &old_value); + uint64_t prev_length = found && old_value.has_contents() ? + old_value.most_recent_gen()->get_length(old_value.most_recent()) : + 0; + cont_gen = new AppendGenerator( + prev_length, + context->min_stride_size, + context->max_stride_size, + 3 * context->max_stride_size); + } else { + cont_gen = new VarLenGenerator( + context->max_size, context->min_stride_size, context->max_stride_size); + } context->update_object(cont_gen, oid, cont); context->oid_in_use.insert(oid); context->oid_not_in_use.erase(oid); - interval_set ranges; + map ranges; - cont_gen->get_ranges(cont, ranges); + cont_gen->get_ranges_map(cont, ranges); std::cout << num << ": seq_num " << context->seq_num << " ranges " << ranges << std::endl; context->seq_num++; context->state_lock.Unlock(); - waiting_on = ranges.num_intervals(); + waiting_on = ranges.size(); //cout << " waiting_on = " << waiting_on << std::endl; ContentsGenerator::iterator gen_pos = cont_gen->get_iterator(cont); uint64_t tid = 1; - for (interval_set::iterator i = ranges.begin(); + for (map::iterator i = ranges.begin(); i != ranges.end(); ++i, ++tid) { bufferlist to_write; - gen_pos.seek(i.get_start()); - for (uint64_t k = 0; k != i.get_len(); ++k, ++gen_pos) { + gen_pos.seek(i->first); + for (uint64_t k = 0; k != i->second; ++k, ++gen_pos) { to_write.append(*gen_pos); } - assert(to_write.length() == i.get_len()); + assert(to_write.length() == i->second); assert(to_write.length() > 0); - std::cout << num << ": writing " << context->prefix+oid << " from " << i.get_start() - << " to " << i.get_len() + i.get_start() << " tid " << tid << std::endl; + std::cout << num << ": writing " << context->prefix+oid + << " from " << i->first + << " to " << i->first + i->second << " tid " << tid << std::endl; pair *cb_arg = new pair(this, new TestOp::CallbackInfo(tid)); librados::AioCompletion *completion = context->rados.aio_create_completion((void*) cb_arg, &write_callback, NULL); waiting.insert(completion); - context->io_ctx.aio_write(context->prefix+oid, completion, - to_write, i.get_len(), i.get_start()); + librados::ObjectWriteOperation op; + if (do_append) { + op.append(to_write); + } else { + op.write(i->first, to_write); + } + context->io_ctx.aio_operate( + context->prefix+oid, completion, + &op); } bufferlist contbl; @@ -767,7 +793,9 @@ public: waiting.insert(completion); waiting_on++; write_op.setxattr("_header", contbl); - write_op.truncate(cont_gen->get_length(cont)); + if (!do_append) { + write_op.truncate(cont_gen->get_length(cont)); + } context->io_ctx.aio_operate( context->prefix+oid, completion, &write_op); diff --git a/src/test/osd/TestRados.cc b/src/test/osd/TestRados.cc index d3430b8202b67..1def0622f10f6 100644 --- a/src/test/osd/TestRados.cc +++ b/src/test/osd/TestRados.cc @@ -26,9 +26,12 @@ public: int objects, map op_weights, TestOpStat *stats, - int max_seconds) : - m_nextop(NULL), m_op(0), m_ops(ops), m_seconds(max_seconds), m_objects(objects), m_stats(stats), - m_total_weight(0) + int max_seconds, + bool ec_pool) : + m_nextop(NULL), m_op(0), m_ops(ops), m_seconds(max_seconds), + m_objects(objects), m_stats(stats), + m_total_weight(0), + m_ec_pool(ec_pool) { m_start = time(0); for (map::const_iterator it = op_weights.begin(); @@ -50,7 +53,11 @@ public: oid << m_op; cout << m_op << ": write initial oid " << oid.str() << std::endl; context.oid_not_flushing.insert(oid.str()); - return new WriteOp(m_op, &context, oid.str()); + if (m_ec_pool) { + return new WriteOp(m_op, &context, oid.str(), true); + } else { + return new WriteOp(m_op, &context, oid.str(), false); + } } else if (m_op >= m_ops) { return NULL; } @@ -97,7 +104,7 @@ private: oid = *(rand_choose(context.oid_not_in_use)); cout << m_op << ": " << "write oid " << oid << " current snap is " << context.current_snap << std::endl; - return new WriteOp(m_op, &context, oid, m_stats); + return new WriteOp(m_op, &context, oid, false, m_stats); case TEST_OP_DELETE: oid = *(rand_choose(context.oid_not_in_use)); @@ -194,6 +201,12 @@ private: return new CacheEvictOp(m_op, &context, oid, m_stats); } + case TEST_OP_APPEND: + oid = *(rand_choose(context.oid_not_in_use)); + cout << "append oid " << oid << " current snap is " + << context.current_snap << std::endl; + return new WriteOp(m_op, &context, oid, true, m_stats); + default: cerr << m_op << ": Invalid op type " << type << std::endl; assert(0); @@ -209,6 +222,7 @@ private: TestOpStat *m_stats; map m_weight_sums; unsigned int m_total_weight; + bool m_ec_pool; }; int main(int argc, char **argv) @@ -241,6 +255,7 @@ int main(int argc, char **argv) { TEST_OP_CACHE_FLUSH, "cache_flush", true }, { TEST_OP_CACHE_TRY_FLUSH, "cache_try_flush", true }, { TEST_OP_CACHE_EVICT, "cache_evict", true }, + { TEST_OP_APPEND, "append", true }, { TEST_OP_READ /* grr */, NULL }, }; @@ -351,8 +366,10 @@ int main(int argc, char **argv) id); TestOpStat stats; - WeightedTestGenerator gen = WeightedTestGenerator(ops, objects, - op_weights, &stats, max_seconds); + WeightedTestGenerator gen = WeightedTestGenerator( + ops, objects, + op_weights, &stats, max_seconds, + ec_pool); int r = context.init(); if (r < 0) { cerr << "Error initializing rados test context: " -- 2.39.5