From: Igor Fedotov Date: Thu, 3 Dec 2020 19:05:45 +0000 (+0300) Subject: test/objectstore: bmap_allocator_replay_test: rename .cc file X-Git-Tag: v16.1.0~340^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=03341ee13b06eba0077fd4649d93bb67484a768d;p=ceph.git test/objectstore: bmap_allocator_replay_test: rename .cc file Signed-off-by: Igor Fedotov --- diff --git a/src/test/objectstore/CMakeLists.txt b/src/test/objectstore/CMakeLists.txt index 233baa46ebf7..340855657c4f 100644 --- a/src/test/objectstore/CMakeLists.txt +++ b/src/test/objectstore/CMakeLists.txt @@ -163,9 +163,9 @@ add_ceph_unittest(unittest_memstore_clone) target_link_libraries(unittest_memstore_clone os global) if(WITH_BLUESTORE) - add_executable(ceph_test_bmap_alloc_replay - bmap_allocator_replay_test.cc) - target_link_libraries(ceph_test_bmap_alloc_replay os global ${UNITTEST_LIBS}) - install(TARGETS ceph_test_bmap_alloc_replay + add_executable(ceph_test_alloc_replay + allocator_replay_test.cc) + target_link_libraries(ceph_test_alloc_replay os global ${UNITTEST_LIBS}) + install(TARGETS ceph_test_alloc_replay DESTINATION bin) endif() diff --git a/src/test/objectstore/allocator_replay_test.cc b/src/test/objectstore/allocator_replay_test.cc new file mode 100644 index 000000000000..8f4fe18f017e --- /dev/null +++ b/src/test/objectstore/allocator_replay_test.cc @@ -0,0 +1,343 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Allocator replay tool. + * Author: Igor Fedotov, ifedotov@suse.com + */ +#include + +#include "common/ceph_argparse.h" +#include "common/debug.h" +#include "common/Cycles.h" +#include "common/errno.h" +#include "common/ceph_json.h" +#include "common/admin_socket.h" +#include "global/global_init.h" +#include "os/bluestore/Allocator.h" + + +void usage(const string &name) { + cerr << "Usage: " << name << " " + << std::endl; +} + +int replay_and_check_for_duplicate(char* fname) +{ + unique_ptr alloc; + + FILE* f = fopen(fname, "r"); + if (!f) { + std::cerr << "error: unable to open " << fname << std::endl; + return -1; + } + + PExtentVector tmp; + bool init_done = false; + char s[4096]; + char* sp, *token; + interval_set owned_by_app; + while (true) { + if (fgets(s, sizeof(s), f) == nullptr) { + break; + } + sp = strstr(s, "init_add_free"); + if (!sp) { + sp = strstr(s, "release"); + } + if (sp) { + //2019-05-30 03:23:46.780 7f889a5edf00 10 fbmap_alloc 0x5642ed370600 init_add_free 0x100000~680000000 + // or + //2019-05-30 03:23:46.780 7f889a5edf00 10 fbmap_alloc 0x5642ed370600 init_add_free done + // or + // 2019 - 10 - 08T16:19 : 32.257 + 0300 7f5679f3fe80 10 fbmap_alloc 0x564fab96f100 release 0x450000~10000 + // or + // 2019 - 10 - 08T16 : 19 : 32.257 + 0300 7f5679f3fe80 10 fbmap_alloc 0x564fab96f100 release done + if (strstr(sp, "done") != nullptr) { + continue; + } + std::cout << s << std::endl; + if (!init_done) { + std::cerr << "error: no allocator init before: " << s << std::endl; + return -1; + } + uint64_t offs, len; + strtok(sp, " ~"); + token = strtok(nullptr, " ~"); + ceph_assert(token); + offs = strtoul(token, nullptr, 16); + token = strtok(nullptr, " ~"); + ceph_assert(token); + len = strtoul(token, nullptr, 16); + if (len == 0) { + std::cerr << "error: " << sp <<": " << s << std::endl; + return -1; + } + if (!owned_by_app.contains(offs, len)) { + std::cerr << "error: unexpected return to allocator, not owned by app: " + << s << std::endl; + return -1; + } + owned_by_app.erase(offs, len); + if (strstr(sp, "init_add_free") != nullptr) { + alloc->init_add_free(offs, len); + } else { + PExtentVector release_set; + release_set.emplace_back(offs, len); + alloc->release(release_set); + } + continue; + } + sp = strstr(s, "init_rm_free"); + if (sp) { + //2019-05-30 03:23:46.912 7f889a5edf00 10 fbmap_alloc 0x5642ed370600 init_rm_free 0x100000~680000000 + // or + // 2019-05-30 03:23:46.916 7f889a5edf00 10 fbmap_alloc 0x5642ed370600 init_rm_free done + + if (strstr(sp, "done") != nullptr) { + continue; + } + std::cout << s << std::endl; + if (!init_done) { + std::cerr << "error: no allocator init before: " << s << std::endl; + return -1; + } + uint64_t offs, len; + strtok(sp, " ~"); + token = strtok(nullptr, " ~"); + ceph_assert(token); + offs = strtoul(token, nullptr, 16); + token = strtok(nullptr, " ~"); + ceph_assert(token); + len = strtoul(token, nullptr, 16); + if (len == 0) { + std::cerr << "error: " << sp <<": " << s << std::endl; + return -1; + } + alloc->init_rm_free(offs, len); + + if (owned_by_app.intersects(offs, len)) { + std::cerr + << "error: unexpected takeover from allocator, already owned by app: " + << s << std::endl; + return -1; + } else { + owned_by_app.insert(offs, len); + } + + continue; + } + sp = strstr(s, "allocate"); + if (sp) { + //2019-05-30 03:23:48.780 7f889a5edf00 10 fbmap_alloc 0x5642ed370600 allocate 0x80000000/100000,0,0 + // and need to bypass + // 2019-05-30 03:23:48.780 7f889a5edf00 10 fbmap_alloc 0x5642ed370600 allocate 0x69d400000~200000/100000,0,0 + + // Very simple and stupid check to bypass actual allocations + if (strstr(sp, "~") != nullptr) { + continue; + } + + std::cout << s << std::endl; + if (!init_done) { + std::cerr << "error: no allocator init before: " << s << std::endl; + return -1; + } + uint64_t want, alloc_unit; + strtok(sp, " /"); + token = strtok(nullptr, " /"); + ceph_assert(token); + want = strtoul(token, nullptr, 16); + token = strtok(nullptr, " ~"); + ceph_assert(token); + alloc_unit = strtoul(token, nullptr, 16); + if (want == 0 || alloc_unit == 0) { + std::cerr << "error: allocate: " << s << std::endl; + return -1; + } + tmp.clear(); + auto allocated = alloc->allocate(want, alloc_unit, 0, 0, &tmp); + std::cout << "allocated TOTAL: " << allocated << std::endl; + for (auto& ee : tmp) { + std::cerr << "dump extent: " << std::hex + << ee.offset << "~" << ee.length + << std::dec << std::endl; + } + std::cerr << "dump completed." << std::endl; + for (auto& e : tmp) { + if (owned_by_app.intersects(e.offset, e.length)) { + std::cerr << "error: unexpected allocated extent: " << std::hex + << e.offset << "~" << e.length + << " dumping all allocations:" << std::dec << std::endl; + for (auto& ee : tmp) { + std::cerr <<"dump extent: " << std::hex + << ee.offset << "~" << ee.length + << std::dec << std::endl; + } + std::cerr <<"dump completed." << std::endl; + return -1; + } else { + owned_by_app.insert(e.offset, e.length); + } + } + continue; + } + + sp = strstr(s, "BitmapAllocator"); + if (sp) { + // 2019-05-30 03:23:43.460 7f889a5edf00 10 fbmap_alloc 0x5642ed36e900 BitmapAllocator 0x15940000000/100000 + std::cout << s << std::endl; + if (init_done) { + std::cerr << "error: duplicate init: " << s << std::endl; + return -1; + } + uint64_t total, alloc_unit; + strtok(sp, " /"); + token = strtok(nullptr, " /"); + ceph_assert(token); + total = strtoul(token, nullptr, 16); + token = strtok(nullptr, " /"); + ceph_assert(token); + alloc_unit = strtoul(token, nullptr, 16); + if (total == 0 || alloc_unit == 0) { + std::cerr << "error: invalid init: " << s << std::endl; + return -1; + } + alloc.reset(Allocator::create(g_ceph_context, string("bitmap"), total, + alloc_unit)); + owned_by_app.insert(0, total); + + init_done = true; + continue; + } + } + fclose(f); + return 0; +} + +/* +* This replays allocator dump (in JSON) reported by + "ceph daemon bluestore allocator dump " + command and applies custom method to it +*/ +int replay_free_dump_and_apply(char* fname, + std::function fn) +{ + string alloc_type; + string alloc_name; + uint64_t capacity = 0; + uint64_t alloc_unit = 0; + + JSONParser p; + std::cout << "parsing..." << std::endl; + bool b = p.parse(fname); + if (!b) { + std::cerr << "Failed to parse json: " << fname << std::endl; + return -1; + } + + JSONObj::data_val v; + ceph_assert(p.is_object()); + + auto *o = p.find_obj("allocator_type"); + ceph_assert(o); + alloc_type = o->get_data_val().str; + + o = p.find_obj("allocator_name"); + ceph_assert(o); + alloc_name = o->get_data_val().str; + + o = p.find_obj("capacity"); + ceph_assert(o); + decode_json_obj(capacity, o); + o = p.find_obj("alloc_unit"); + ceph_assert(o); + decode_json_obj(alloc_unit, o); + + o = p.find_obj("extents"); + ceph_assert(o); + ceph_assert(o->is_array()); + std::cout << "parsing completed!" << std::endl; + + unique_ptr alloc; + alloc.reset(Allocator::create(g_ceph_context, alloc_type, + capacity, alloc_unit, alloc_name)); + + auto it = o->find_first(); + while (!it.end()) { + auto *item_obj = *it; + uint64_t offset = 0; + uint64_t length = 0; + string offset_str, length_str; + + bool b = JSONDecoder::decode_json("offset", offset_str, item_obj); + ceph_assert(b); + b = JSONDecoder::decode_json("length", length_str, item_obj); + ceph_assert(b); + + char* p; + offset = strtol(offset_str.c_str(), &p, 16); + length = strtol(length_str.c_str(), &p, 16); + + alloc->init_add_free(offset, length); + + ++it; + } + + int r = fn(alloc.get(), alloc_name); + + return r; +} + +void dump_alloc(Allocator* alloc, const string& aname) +{ + AdminSocket* admin_socket = g_ceph_context->get_admin_socket(); + ceph_assert(admin_socket); + + ceph::bufferlist in, out; + ostringstream err; + + string cmd = "{\"prefix\": \"bluestore allocator dump " + aname + "\"}"; + auto r = admin_socket->execute_command( + { cmd }, + in, err, &out); + if (r != 0) { + cerr << "failure querying: " << cpp_strerror(r) << std::endl; + } + else { + std::cout << std::string(out.c_str(), out.length()) << std::endl; + } +} + +int main(int argc, char **argv) +{ + vector args; + auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + g_ceph_context->_conf.apply_changes(nullptr); + + if (argc < 3) { + usage(argv[0]); + return 1; + } + if (strcmp(argv[2], "raw_duplicate") == 0) { + return replay_and_check_for_duplicate(argv[1]); + } else if (strcmp(argv[2], "free_dump") == 0) { + return replay_free_dump_and_apply(argv[1], + [&](Allocator* a, const string& aname) { + ceph_assert(a); + std::cout << "Fragmentation:" << a->get_fragmentation() + << std::endl; + std::cout << "Fragmentation score:" << a->get_fragmentation_score() + << std::endl; + std::cout << "Free:" << std::hex << a->get_free() << std::dec + << std::endl; + { + // stub to implement various testing stuff on properly initialized allocator + // e.g. one can dump allocator back via dump_alloc(a, aname); + } + return 0; + }); + } +} diff --git a/src/test/objectstore/bmap_allocator_replay_test.cc b/src/test/objectstore/bmap_allocator_replay_test.cc deleted file mode 100644 index 8f4fe18f017e..000000000000 --- a/src/test/objectstore/bmap_allocator_replay_test.cc +++ /dev/null @@ -1,343 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Allocator replay tool. - * Author: Igor Fedotov, ifedotov@suse.com - */ -#include - -#include "common/ceph_argparse.h" -#include "common/debug.h" -#include "common/Cycles.h" -#include "common/errno.h" -#include "common/ceph_json.h" -#include "common/admin_socket.h" -#include "global/global_init.h" -#include "os/bluestore/Allocator.h" - - -void usage(const string &name) { - cerr << "Usage: " << name << " " - << std::endl; -} - -int replay_and_check_for_duplicate(char* fname) -{ - unique_ptr alloc; - - FILE* f = fopen(fname, "r"); - if (!f) { - std::cerr << "error: unable to open " << fname << std::endl; - return -1; - } - - PExtentVector tmp; - bool init_done = false; - char s[4096]; - char* sp, *token; - interval_set owned_by_app; - while (true) { - if (fgets(s, sizeof(s), f) == nullptr) { - break; - } - sp = strstr(s, "init_add_free"); - if (!sp) { - sp = strstr(s, "release"); - } - if (sp) { - //2019-05-30 03:23:46.780 7f889a5edf00 10 fbmap_alloc 0x5642ed370600 init_add_free 0x100000~680000000 - // or - //2019-05-30 03:23:46.780 7f889a5edf00 10 fbmap_alloc 0x5642ed370600 init_add_free done - // or - // 2019 - 10 - 08T16:19 : 32.257 + 0300 7f5679f3fe80 10 fbmap_alloc 0x564fab96f100 release 0x450000~10000 - // or - // 2019 - 10 - 08T16 : 19 : 32.257 + 0300 7f5679f3fe80 10 fbmap_alloc 0x564fab96f100 release done - if (strstr(sp, "done") != nullptr) { - continue; - } - std::cout << s << std::endl; - if (!init_done) { - std::cerr << "error: no allocator init before: " << s << std::endl; - return -1; - } - uint64_t offs, len; - strtok(sp, " ~"); - token = strtok(nullptr, " ~"); - ceph_assert(token); - offs = strtoul(token, nullptr, 16); - token = strtok(nullptr, " ~"); - ceph_assert(token); - len = strtoul(token, nullptr, 16); - if (len == 0) { - std::cerr << "error: " << sp <<": " << s << std::endl; - return -1; - } - if (!owned_by_app.contains(offs, len)) { - std::cerr << "error: unexpected return to allocator, not owned by app: " - << s << std::endl; - return -1; - } - owned_by_app.erase(offs, len); - if (strstr(sp, "init_add_free") != nullptr) { - alloc->init_add_free(offs, len); - } else { - PExtentVector release_set; - release_set.emplace_back(offs, len); - alloc->release(release_set); - } - continue; - } - sp = strstr(s, "init_rm_free"); - if (sp) { - //2019-05-30 03:23:46.912 7f889a5edf00 10 fbmap_alloc 0x5642ed370600 init_rm_free 0x100000~680000000 - // or - // 2019-05-30 03:23:46.916 7f889a5edf00 10 fbmap_alloc 0x5642ed370600 init_rm_free done - - if (strstr(sp, "done") != nullptr) { - continue; - } - std::cout << s << std::endl; - if (!init_done) { - std::cerr << "error: no allocator init before: " << s << std::endl; - return -1; - } - uint64_t offs, len; - strtok(sp, " ~"); - token = strtok(nullptr, " ~"); - ceph_assert(token); - offs = strtoul(token, nullptr, 16); - token = strtok(nullptr, " ~"); - ceph_assert(token); - len = strtoul(token, nullptr, 16); - if (len == 0) { - std::cerr << "error: " << sp <<": " << s << std::endl; - return -1; - } - alloc->init_rm_free(offs, len); - - if (owned_by_app.intersects(offs, len)) { - std::cerr - << "error: unexpected takeover from allocator, already owned by app: " - << s << std::endl; - return -1; - } else { - owned_by_app.insert(offs, len); - } - - continue; - } - sp = strstr(s, "allocate"); - if (sp) { - //2019-05-30 03:23:48.780 7f889a5edf00 10 fbmap_alloc 0x5642ed370600 allocate 0x80000000/100000,0,0 - // and need to bypass - // 2019-05-30 03:23:48.780 7f889a5edf00 10 fbmap_alloc 0x5642ed370600 allocate 0x69d400000~200000/100000,0,0 - - // Very simple and stupid check to bypass actual allocations - if (strstr(sp, "~") != nullptr) { - continue; - } - - std::cout << s << std::endl; - if (!init_done) { - std::cerr << "error: no allocator init before: " << s << std::endl; - return -1; - } - uint64_t want, alloc_unit; - strtok(sp, " /"); - token = strtok(nullptr, " /"); - ceph_assert(token); - want = strtoul(token, nullptr, 16); - token = strtok(nullptr, " ~"); - ceph_assert(token); - alloc_unit = strtoul(token, nullptr, 16); - if (want == 0 || alloc_unit == 0) { - std::cerr << "error: allocate: " << s << std::endl; - return -1; - } - tmp.clear(); - auto allocated = alloc->allocate(want, alloc_unit, 0, 0, &tmp); - std::cout << "allocated TOTAL: " << allocated << std::endl; - for (auto& ee : tmp) { - std::cerr << "dump extent: " << std::hex - << ee.offset << "~" << ee.length - << std::dec << std::endl; - } - std::cerr << "dump completed." << std::endl; - for (auto& e : tmp) { - if (owned_by_app.intersects(e.offset, e.length)) { - std::cerr << "error: unexpected allocated extent: " << std::hex - << e.offset << "~" << e.length - << " dumping all allocations:" << std::dec << std::endl; - for (auto& ee : tmp) { - std::cerr <<"dump extent: " << std::hex - << ee.offset << "~" << ee.length - << std::dec << std::endl; - } - std::cerr <<"dump completed." << std::endl; - return -1; - } else { - owned_by_app.insert(e.offset, e.length); - } - } - continue; - } - - sp = strstr(s, "BitmapAllocator"); - if (sp) { - // 2019-05-30 03:23:43.460 7f889a5edf00 10 fbmap_alloc 0x5642ed36e900 BitmapAllocator 0x15940000000/100000 - std::cout << s << std::endl; - if (init_done) { - std::cerr << "error: duplicate init: " << s << std::endl; - return -1; - } - uint64_t total, alloc_unit; - strtok(sp, " /"); - token = strtok(nullptr, " /"); - ceph_assert(token); - total = strtoul(token, nullptr, 16); - token = strtok(nullptr, " /"); - ceph_assert(token); - alloc_unit = strtoul(token, nullptr, 16); - if (total == 0 || alloc_unit == 0) { - std::cerr << "error: invalid init: " << s << std::endl; - return -1; - } - alloc.reset(Allocator::create(g_ceph_context, string("bitmap"), total, - alloc_unit)); - owned_by_app.insert(0, total); - - init_done = true; - continue; - } - } - fclose(f); - return 0; -} - -/* -* This replays allocator dump (in JSON) reported by - "ceph daemon bluestore allocator dump " - command and applies custom method to it -*/ -int replay_free_dump_and_apply(char* fname, - std::function fn) -{ - string alloc_type; - string alloc_name; - uint64_t capacity = 0; - uint64_t alloc_unit = 0; - - JSONParser p; - std::cout << "parsing..." << std::endl; - bool b = p.parse(fname); - if (!b) { - std::cerr << "Failed to parse json: " << fname << std::endl; - return -1; - } - - JSONObj::data_val v; - ceph_assert(p.is_object()); - - auto *o = p.find_obj("allocator_type"); - ceph_assert(o); - alloc_type = o->get_data_val().str; - - o = p.find_obj("allocator_name"); - ceph_assert(o); - alloc_name = o->get_data_val().str; - - o = p.find_obj("capacity"); - ceph_assert(o); - decode_json_obj(capacity, o); - o = p.find_obj("alloc_unit"); - ceph_assert(o); - decode_json_obj(alloc_unit, o); - - o = p.find_obj("extents"); - ceph_assert(o); - ceph_assert(o->is_array()); - std::cout << "parsing completed!" << std::endl; - - unique_ptr alloc; - alloc.reset(Allocator::create(g_ceph_context, alloc_type, - capacity, alloc_unit, alloc_name)); - - auto it = o->find_first(); - while (!it.end()) { - auto *item_obj = *it; - uint64_t offset = 0; - uint64_t length = 0; - string offset_str, length_str; - - bool b = JSONDecoder::decode_json("offset", offset_str, item_obj); - ceph_assert(b); - b = JSONDecoder::decode_json("length", length_str, item_obj); - ceph_assert(b); - - char* p; - offset = strtol(offset_str.c_str(), &p, 16); - length = strtol(length_str.c_str(), &p, 16); - - alloc->init_add_free(offset, length); - - ++it; - } - - int r = fn(alloc.get(), alloc_name); - - return r; -} - -void dump_alloc(Allocator* alloc, const string& aname) -{ - AdminSocket* admin_socket = g_ceph_context->get_admin_socket(); - ceph_assert(admin_socket); - - ceph::bufferlist in, out; - ostringstream err; - - string cmd = "{\"prefix\": \"bluestore allocator dump " + aname + "\"}"; - auto r = admin_socket->execute_command( - { cmd }, - in, err, &out); - if (r != 0) { - cerr << "failure querying: " << cpp_strerror(r) << std::endl; - } - else { - std::cout << std::string(out.c_str(), out.length()) << std::endl; - } -} - -int main(int argc, char **argv) -{ - vector args; - auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, - CODE_ENVIRONMENT_UTILITY, - CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); - common_init_finish(g_ceph_context); - g_ceph_context->_conf.apply_changes(nullptr); - - if (argc < 3) { - usage(argv[0]); - return 1; - } - if (strcmp(argv[2], "raw_duplicate") == 0) { - return replay_and_check_for_duplicate(argv[1]); - } else if (strcmp(argv[2], "free_dump") == 0) { - return replay_free_dump_and_apply(argv[1], - [&](Allocator* a, const string& aname) { - ceph_assert(a); - std::cout << "Fragmentation:" << a->get_fragmentation() - << std::endl; - std::cout << "Fragmentation score:" << a->get_fragmentation_score() - << std::endl; - std::cout << "Free:" << std::hex << a->get_free() << std::dec - << std::endl; - { - // stub to implement various testing stuff on properly initialized allocator - // e.g. one can dump allocator back via dump_alloc(a, aname); - } - return 0; - }); - } -}