From: Greg Farnum Date: Sat, 11 Jun 2016 00:01:09 +0000 (-0700) Subject: test: build a correctness test for the ObjectCacher X-Git-Tag: v10.2.3~37^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=3446fa498266ec2902a1d4d9215de4e4b1d9c455;p=ceph.git test: build a correctness test for the ObjectCacher For now it's very specifically designed to reproduce http://tracker.ceph.com/issues/16002, but it can be extended to other patterns in the future. Signed-off-by: Greg Farnum (cherry picked from commit 0fd55a9886dd8da344c23a5e9898ee5c5061e8f9) --- diff --git a/src/test/osdc/object_cacher_stress.cc b/src/test/osdc/object_cacher_stress.cc index ffe84c242ce..a753295e212 100644 --- a/src/test/osdc/object_cacher_stress.cc +++ b/src/test/osdc/object_cacher_stress.cc @@ -176,20 +176,178 @@ int stress_test(uint64_t num_ops, uint64_t num_objs, int correctness_test(uint64_t delay_ns) { + std::cerr << "starting correctness test" << std::endl; Mutex lock("object_cacher_stress::object_cacher"); MemWriteback writeback(g_ceph_context, &lock, delay_ns); ObjectCacher obc(g_ceph_context, "test", writeback, lock, NULL, NULL, - g_conf->client_oc_size, - g_conf->client_oc_max_objects, - g_conf->client_oc_max_dirty, - g_conf->client_oc_target_dirty, + 1<<21, // max cache size, 2MB + 1, // max objects, just one + 1<<18, // max dirty, 256KB + 1<<17, // target dirty, 128KB g_conf->client_oc_max_dirty_age, true); obc.start(); + std::cerr << "just start()ed ObjectCacher" << std::endl; + + SnapContext snapc; + ceph_tid_t journal_tid = 0; + std::string oid("correctness_test_obj"); + ObjectCacher::ObjectSet object_set(NULL, 0, 0); + ceph::bufferlist zeroes_bl; + zeroes_bl.append_zero(1<<20); + + // set up a 4MB all-zero object + std::cerr << "writing 4x1MB object" << std::endl; + std::map create_finishers; + for (int i = 0; i < 4; ++i) { + ObjectCacher::OSDWrite *wr = obc.prepare_write(snapc, zeroes_bl, + ceph::real_time::min(), 0, + ++journal_tid); + ObjectExtent extent(oid, 0, zeroes_bl.length()*i, zeroes_bl.length(), 0); + extent.oloc.pool = 0; + extent.buffer_extents.push_back(make_pair(0, 1<<20)); + wr->extents.push_back(extent); + lock.Lock(); + obc.writex(wr, &object_set, &create_finishers[i]); + lock.Unlock(); + } + + // write some 1-valued bits at 256-KB intervals for checking consistency + std::cerr << "Writing some 0xff values" << std::endl; + ceph::buffer::ptr ones(1<<16); + memset(ones.c_str(), 0xff, ones.length()); + ceph::bufferlist ones_bl; + ones_bl.append(ones); + for (int i = 1<<18; i < 1<<22; i+=1<<18) { + ObjectCacher::OSDWrite *wr = obc.prepare_write(snapc, ones_bl, + ceph::real_time::min(), 0, + ++journal_tid); + ObjectExtent extent(oid, 0, i, ones_bl.length(), 0); + extent.oloc.pool = 0; + extent.buffer_extents.push_back(make_pair(0, 1<<16)); + wr->extents.push_back(extent); + lock.Lock(); + obc.writex(wr, &object_set, &create_finishers[i]); + lock.Unlock(); + } + + for (auto i = create_finishers.begin(); i != create_finishers.end(); ++i) { + i->second.wait(); + } + std::cout << "Finished setting up object" << std::endl; + lock.Lock(); + C_SaferCond flushcond; + bool done = obc.flush_all(&flushcond); + if (!done) { + std::cout << "Waiting for flush" << std::endl; + lock.Unlock(); + flushcond.wait(); + lock.Lock(); + } + lock.Unlock(); + + /* now read the back half of the object in, check consistency, + */ + std::cout << "Reading back half of object (1<<21~1<<21)" << std::endl; + bufferlist readbl; + C_SaferCond backreadcond; + ObjectCacher::OSDRead *back_half_rd = obc.prepare_read(CEPH_NOSNAP, &readbl, 0); + ObjectExtent back_half_extent(oid, 0, 1<<21, 1<<21, 0); + back_half_extent.oloc.pool = 0; + back_half_extent.buffer_extents.push_back(make_pair(0, 1<<21)); + back_half_rd->extents.push_back(back_half_extent); + lock.Lock(); + int r = obc.readx(back_half_rd, &object_set, &backreadcond); + lock.Unlock(); + assert(r >= 0); + if (r == 0) { + std::cout << "Waiting to read data into cache" << std::endl; + r = backreadcond.wait(); + } + + assert(r == 1<<21); + + /* Read the whole object in, + * verify we have to wait for it to complete, + * overwrite a small piece, (http://tracker.ceph.com/issues/16002), + * and check consistency */ + + readbl.clear(); + std::cout<< "Reading whole object (0~1<<22)" << std::endl; + C_SaferCond frontreadcond; + ObjectCacher::OSDRead *whole_rd = obc.prepare_read(CEPH_NOSNAP, &readbl, 0); + ObjectExtent whole_extent(oid, 0, 0, 1<<22, 0); + whole_extent.oloc.pool = 0; + whole_extent.buffer_extents.push_back(make_pair(0, 1<<22)); + whole_rd->extents.push_back(whole_extent); + lock.Lock(); + r = obc.readx(whole_rd, &object_set, &frontreadcond); + // we cleared out the cache by reading back half, it shouldn't pass immediately! + assert(r == 0); + std::cout << "Data (correctly) not available without fetching" << std::endl; + + ObjectCacher::OSDWrite *verify_wr = obc.prepare_write(snapc, ones_bl, + ceph::real_time::min(), 0, + ++journal_tid); + ObjectExtent verify_extent(oid, 0, (1<<18)+(1<<16), ones_bl.length(), 0); + verify_extent.oloc.pool = 0; + verify_extent.buffer_extents.push_back(make_pair(0, 1<<16)); + verify_wr->extents.push_back(verify_extent); + C_SaferCond verify_finisher; + obc.writex(verify_wr, &object_set, &verify_finisher); + lock.Unlock(); + std::cout << "wrote dirtying data" << std::endl; + + std::cout << "Waiting to read data into cache" << std::endl; + r = frontreadcond.wait(); + verify_finisher.wait(); + + std::cout << "Validating data" << std::endl; + + for (int i = 1<<18; i < 1<<22; i+=1<<18) { + bufferlist ones_maybe; + ones_maybe.substr_of(readbl, i, ones_bl.length()); + assert(0 == memcmp(ones_maybe.c_str(), ones_bl.c_str(), ones_bl.length())); + } + bufferlist ones_maybe; + ones_maybe.substr_of(readbl, (1<<18)+(1<<16), ones_bl.length()); + assert(0 == memcmp(ones_maybe.c_str(), ones_bl.c_str(), ones_bl.length())); + + std::cout << "validated that data is 0xff where it should be" << std::endl; + + lock.Lock(); + C_SaferCond flushcond2; + done = obc.flush_all(&flushcond2); + if (!done) { + std::cout << "Waiting for final write flush" << std::endl; + lock.Unlock(); + flushcond2.wait(); + lock.Lock(); + } + + bool unclean = obc.release_set(&object_set); + if (unclean) { + std::cout << "unclean buffers left over!" << std::endl; + vector discard_extents; + int i = 0; + for (auto oi = object_set.objects.begin(); !oi.end(); ++oi) { + discard_extents.emplace_back(oid, i++, 0, 1<<22, 0); + } + obc.discard_set(&object_set, discard_extents); + lock.Unlock(); + obc.stop(); + goto fail; + } + lock.Unlock(); + + obc.stop(); + + std::cout << "Testing ObjectCacher correctness complete" << std::endl; + return EXIT_SUCCESS; - std::cout << "Testing ObjectCacher correctness" << std::endl; - return 0; + fail: + return EXIT_FAILURE; } int main(int argc, const char **argv)