From: Adam Kupczyk Date: Fri, 26 Jul 2024 16:33:08 +0000 (+0000) Subject: test/store_test: Make ceph_test_objectstore run faster X-Git-Tag: testing/wip-pdonnell-testing-20240916.200549-debug~48^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=8452eb889043396588c45b1101a387f6785e582b;p=ceph-ci.git test/store_test: Make ceph_test_objectstore run faster 1) Queue transactions together 2) Make fsck less frequent Signed-off-by: Adam Kupczyk --- diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index a12acc06000..70a2644ef84 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -4282,7 +4282,7 @@ public: gen_type *rng; ObjectStore *store; ObjectStore::CollectionHandle ch; - + std::vector tt; ceph::mutex lock = ceph::make_mutex("State lock"); ceph::condition_variable cond; @@ -4296,6 +4296,24 @@ public: } }; + int queue_transactions() + { + int result = 0; + if (!tt.empty()) { + result = store->queue_transactions(ch, tt); + tt.clear(); + } + return result; + } + int maybe_queue_transactions() + { + int result = 0; + if ((rand() % 7) == 0 || tt.size() > 10) { + result = store->queue_transactions(ch, tt); + tt.clear(); + } + return result; + } class C_SyntheticOnReadable : public Context { public: SyntheticWorkloadState *state; @@ -4413,6 +4431,7 @@ public: } void shutdown() { ghobject_t next; + queue_transactions(); while (1) { vector objects; int r = collection_list(store, ch, next, ghobject_t::get_max(), 10, @@ -4433,6 +4452,7 @@ public: queue_transaction(store, ch, std::move(t)); } void statfs(store_statfs_t& stat) { + queue_transactions(); store->statfs(&stat); } @@ -4468,6 +4488,7 @@ public: void wait_for_done() { std::unique_lock locker{lock}; + queue_transactions(); cond.wait(locker, [this] { return in_flight == 0; }); } @@ -4541,7 +4562,7 @@ public: wait_for_ready(locker); ghobject_t new_obj = object_gen->create_object(rng); available_objects.erase(new_obj); - ObjectStore::Transaction t; + ObjectStore::Transaction& t = tt.emplace_back(); t.touch(cid, new_obj); boost::uniform_int<> u(17, 22); boost::uniform_int<> v(12, 17); @@ -4554,7 +4575,7 @@ public: if (!contents.count(new_obj)) contents[new_obj] = Object(); t.register_on_applied(new C_SyntheticOnReadable(this, new_obj)); - int status = store->queue_transaction(ch, std::move(t)); + int status = maybe_queue_transactions(); return status; } @@ -4577,7 +4598,7 @@ public: new_obj.generation++; available_objects.erase(new_obj); - ObjectStore::Transaction t; + ObjectStore::Transaction& t = tt.emplace_back(); t.collection_move_rename(cid, old_obj, cid, new_obj); ++in_flight; in_flight_objects.insert(old_obj); @@ -4586,7 +4607,7 @@ public: contents[new_obj].data = contents[old_obj].data; contents.erase(old_obj); t.register_on_applied(new C_SyntheticOnStash(this, old_obj, new_obj)); - int status = store->queue_transaction(ch, std::move(t)); + int status = maybe_queue_transactions(); return status; } @@ -4610,7 +4631,7 @@ public: new_obj.hobj.set_hash(old_obj.hobj.get_hash()); available_objects.erase(new_obj); - ObjectStore::Transaction t; + ObjectStore::Transaction& t = tt.emplace_back(); t.clone(cid, old_obj, new_obj); ++in_flight; in_flight_objects.insert(old_obj); @@ -4619,7 +4640,7 @@ public: contents[new_obj].data = contents[old_obj].data; t.register_on_applied(new C_SyntheticOnClone(this, old_obj, new_obj)); - int status = store->queue_transaction(ch, std::move(t)); + int status = maybe_queue_transactions(); return status; } @@ -4669,7 +4690,7 @@ public: << " (size " << srcdata.length() << ") to " << dstoff << "~" << len << std::endl; - ObjectStore::Transaction t; + ObjectStore::Transaction& t = tt.emplace_back(); t.clone_range(cid, old_obj, new_obj, srcoff, len, dstoff); ++in_flight; in_flight_objects.insert(old_obj); @@ -4701,7 +4722,7 @@ public: } t.register_on_applied(new C_SyntheticOnClone(this, old_obj, new_obj)); - int status = store->queue_transaction(ch, std::move(t)); + int status = maybe_queue_transactions(); return status; } @@ -4715,7 +4736,7 @@ public: ghobject_t new_obj = get_uniform_random_object(locker); available_objects.erase(new_obj); - ObjectStore::Transaction t; + ObjectStore::Transaction& t = tt.emplace_back(); boost::uniform_int<> u1(0, max_object_len - max_write_len); boost::uniform_int<> u2(0, max_write_len); @@ -4750,7 +4771,7 @@ public: ++in_flight; in_flight_objects.insert(new_obj); t.register_on_applied(new C_SyntheticOnReadable(this, new_obj)); - int status = store->queue_transaction(ch, std::move(t)); + int status = maybe_queue_transactions(); return status; } @@ -4763,7 +4784,7 @@ public: ghobject_t obj = get_uniform_random_object(locker); available_objects.erase(obj); - ObjectStore::Transaction t; + ObjectStore::Transaction& t = tt.emplace_back(); boost::uniform_int<> choose(0, max_object_len); size_t len = choose(*rng); @@ -4784,7 +4805,7 @@ public: } t.register_on_applied(new C_SyntheticOnReadable(this, obj)); - int status = store->queue_transaction(ch, std::move(t)); + int status = maybe_queue_transactions(); return status; } @@ -4797,7 +4818,7 @@ public: ghobject_t new_obj = get_uniform_random_object(locker); available_objects.erase(new_obj); - ObjectStore::Transaction t; + ObjectStore::Transaction& t = tt.emplace_back(); boost::uniform_int<> u1(0, max_object_len - max_write_len); boost::uniform_int<> u2(0, max_write_len); @@ -4825,12 +4846,13 @@ public: ++in_flight; in_flight_objects.insert(new_obj); t.register_on_applied(new C_SyntheticOnReadable(this, new_obj)); - int status = store->queue_transaction(ch, std::move(t)); + int status = maybe_queue_transactions(); return status; } void read() { EnterExit ee("read"); + queue_transactions(); boost::uniform_int<> u1(0, max_object_len/2); boost::uniform_int<> u2(0, max_object_len); uint64_t offset = u1(*rng); @@ -4880,7 +4902,7 @@ public: ghobject_t obj = get_uniform_random_object(locker); available_objects.erase(obj); - ObjectStore::Transaction t; + ObjectStore::Transaction& t = tt.emplace_back(); boost::uniform_int<> u0(1, max_attr_size); boost::uniform_int<> u1(4, max_attr_name_len); @@ -4915,7 +4937,7 @@ public: ++in_flight; in_flight_objects.insert(obj); t.register_on_applied(new C_SyntheticOnReadable(this, obj)); - int status = store->queue_transaction(ch, std::move(t)); + int status = maybe_queue_transactions(); return status; } @@ -4928,7 +4950,7 @@ public: ghobject_t obj = get_next_object(locker); available_objects.erase(obj); - ObjectStore::Transaction t; + ObjectStore::Transaction& t = tt.emplace_back(); map> attrs; set keys; @@ -4944,12 +4966,13 @@ public: ++in_flight; in_flight_objects.insert(obj); t.register_on_applied(new C_SyntheticOnReadable(this, obj)); - int status = store->queue_transaction(ch, std::move(t)); + int status = maybe_queue_transactions(); return status; } void getattrs() { EnterExit ee("getattrs"); + queue_transactions(); ghobject_t obj; map expected; { @@ -4979,6 +5002,7 @@ public: void getattr() { EnterExit ee("getattr"); + queue_transactions(); ghobject_t obj; int r; int retry; @@ -5036,19 +5060,20 @@ public: } available_objects.erase(obj); - ObjectStore::Transaction t; + ObjectStore::Transaction& t = tt.emplace_back(); t.rmattr(cid, obj, it->first); contents[obj].attrs.erase(it->first); ++in_flight; in_flight_objects.insert(obj); t.register_on_applied(new C_SyntheticOnReadable(this, obj)); - int status = store->queue_transaction(ch, std::move(t)); + int status = maybe_queue_transactions(); return status; } void fsck(bool deep) { std::unique_lock locker{lock}; + queue_transactions(); EnterExit ee("fsck"); cond.wait(locker, [this] { return in_flight == 0; }); ch.reset(); @@ -5062,6 +5087,7 @@ public: void scan() { std::unique_lock locker{lock}; EnterExit ee("scan"); + queue_transactions(); cond.wait(locker, [this] { return in_flight == 0; }); vector objects; set objects_set, objects_set2; @@ -5119,6 +5145,7 @@ public: void stat() { EnterExit ee("stat"); + queue_transactions(); ghobject_t hoid; uint64_t expected; { @@ -5153,14 +5180,14 @@ public: if (!can_unlink()) return -ENOENT; ghobject_t to_remove = get_uniform_random_object(locker); - ObjectStore::Transaction t; + ObjectStore::Transaction& t = tt.emplace_back(); t.remove(cid, to_remove); ++in_flight; available_objects.erase(to_remove); in_flight_objects.insert(to_remove); contents.erase(to_remove); t.register_on_applied(new C_SyntheticOnReadable(this, to_remove)); - int status = store->queue_transaction(ch, std::move(t)); + int status = maybe_queue_transactions(); return status; } @@ -5199,35 +5226,36 @@ void StoreTestBase::doSyntheticTest( cerr << "Op " << i << std::endl; test_obj.print_internal_state(); } - boost::uniform_int<> true_false(0, 999); + boost::uniform_int<> true_false(0, 9999); int val = true_false(rng); - if (val > 998) { + if (val > 9998) { test_obj.fsck(true); - } else if (val > 997) { + } else if (val > 9997) { test_obj.fsck(false); - } else if (val > 970) { + } else if (val > 9900) { test_obj.scan(); - } else if (val > 950) { + } else if (val > 9500) { test_obj.stat(); - } else if (val > 850) { + } else if (val > 8500) { test_obj.zero(); - } else if (val > 800) { + } else if (val > 8000) { test_obj.unlink(); - } else if (val > 550) { + } else if (val > 5500) { test_obj.write(); - } else if (val > 500) { + } else if (val > 5000) { test_obj.clone(); - } else if (val > 450) { + } else if (val > 4500) { test_obj.clone_range(); - } else if (val > 300) { + } else if (val > 3000) { test_obj.stash(); - } else if (val > 100) { + } else if (val > 1000) { test_obj.read(); } else { test_obj.truncate(); } } test_obj.wait_for_done(); + test_obj.fsck(true); test_obj.shutdown(); } @@ -5256,7 +5284,7 @@ void StoreTestBase::doSyntheticLimitedTest( cerr << "Op " << i << std::endl; test_obj.print_internal_state(); } - boost::uniform_int<> true_false(0, 9999 /*999*/); + boost::uniform_int<> true_false(0, 99999 /*999*/); int val = true_false(rng); auto option = [&](int range) -> bool { if (val == -1) { @@ -5270,21 +5298,22 @@ void StoreTestBase::doSyntheticLimitedTest( return false; } }; - if (option(1)) test_obj.fsck(true); - if (option(1)) test_obj.fsck(false); - if (option(1)) test_obj.scan(); - if (option(497)) test_obj.stat(); - if (option(1000)) test_obj.zero(); - if (option(1500)) test_obj.read(); - if (option(1500)) test_obj.write(); - if (option(500)) test_obj.truncate(); - if (option(1000)) test_obj.clone_range(); - if (option(1000)) test_obj.stash(); - if (option(1500)) test_obj.unlink(); - if (option(1500)) test_obj.clone(); + if (option(3)) test_obj.fsck(true); + if (option(3)) test_obj.fsck(false); + if (option(94)) test_obj.scan(); + if (option(4900)) test_obj.stat(); + if (option(10000)) test_obj.zero(); + if (option(15000)) test_obj.read(); + if (option(15000)) test_obj.write(); + if (option(5000)) test_obj.truncate(); + if (option(10000)) test_obj.clone_range(); + if (option(10000)) test_obj.stash(); + if (option(15000)) test_obj.unlink(); + if (option(15000)) test_obj.clone(); ceph_assert(val == -1); } test_obj.wait_for_done(); + test_obj.fsck(true); test_obj.shutdown(); }