From 301a64212f0a38a3b5db4bd1bd0f15e26ff055cf Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Sun, 20 Jan 2019 00:01:55 +0800 Subject: [PATCH] tools/ceph_kvstore_tool: extract StoreTool into kvstore_tool.cc to workaround n bug in GCC 7.3, which leads to ICE like build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/14.0.1-2751-gb268822/rpm/el7/BUILD/ceph-14.0.1-2751-gb268822/src/tools/ceph_kvstore_tool.cc:625:1: internal compiler error: Segmentation fault } ^ Please submit a full bug report, with preprocessed source if appropriate. See for instructions. Signed-off-by: Kefu Chai --- src/tools/CMakeLists.txt | 4 +- src/tools/ceph_kvstore_tool.cc | 279 +-------------------------------- src/tools/kvstore_tool.cc | 272 ++++++++++++++++++++++++++++++++ src/tools/kvstore_tool.h | 73 +++++++++ 4 files changed, 351 insertions(+), 277 deletions(-) create mode 100644 src/tools/kvstore_tool.cc create mode 100644 src/tools/kvstore_tool.h diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 6579d5f6d9126..a8688234f79b2 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -62,7 +62,9 @@ if(WITH_TESTS) endif(WITH_TESTS) endif(WITH_LIBCEPHFS) -add_executable(ceph-kvstore-tool ceph_kvstore_tool.cc) +add_executable(ceph-kvstore-tool + kvstore_tool.cc + ceph_kvstore_tool.cc) target_link_libraries(ceph-kvstore-tool os global) install(TARGETS ceph-kvstore-tool DESTINATION bin) diff --git a/src/tools/ceph_kvstore_tool.cc b/src/tools/ceph_kvstore_tool.cc index cfb4304391403..6f90876910de2 100644 --- a/src/tools/ceph_kvstore_tool.cc +++ b/src/tools/ceph_kvstore_tool.cc @@ -15,289 +15,16 @@ #include #include -#include - #include "common/ceph_argparse.h" #include "common/config.h" #include "common/errno.h" #include "common/strtol.h" -#include "global/global_context.h" -#include "global/global_init.h" -#include "include/stringify.h" -#include "common/Clock.h" -#include "kv/KeyValueDB.h" #include "common/url_escape.h" -#ifdef WITH_BLUESTORE -#include "os/bluestore/BlueStore.h" -#endif - - -class StoreTool -{ -#ifdef WITH_BLUESTORE - struct Deleter { - BlueStore *bluestore; - Deleter() - : bluestore(nullptr) {} - Deleter(BlueStore *store) - : bluestore(store) {} - void operator()(KeyValueDB *db) { - if (bluestore) { - bluestore->umount(); - delete bluestore; - } else { - delete db; - } - } - }; - std::unique_ptr db; -#else - std::unique_ptr db; -#endif - - string store_path; - - public: - StoreTool(string type, const string &path, bool need_open_db=true) : store_path(path) { - if (type == "bluestore-kv") { -#ifdef WITH_BLUESTORE - auto bluestore = new BlueStore(g_ceph_context, path); - KeyValueDB *db_ptr; - int r = bluestore->start_kv_only(&db_ptr, need_open_db); - if (r < 0) { - exit(1); - } - db = decltype(db){db_ptr, Deleter(bluestore)}; -#else - cerr << "bluestore not compiled in" << std::endl; - exit(1); -#endif - } else { - auto db_ptr = KeyValueDB::create(g_ceph_context, type, path); - if (need_open_db) { - int r = db_ptr->open(std::cerr); - if (r < 0) { - cerr << "failed to open type " << type << " path " << path << ": " - << cpp_strerror(r) << std::endl; - exit(1); - } - db.reset(db_ptr); - } - } - } - - uint32_t traverse(const string &prefix, - const bool do_crc, - const bool do_value_dump, - ostream *out) { - KeyValueDB::WholeSpaceIterator iter = db->get_wholespace_iterator(); - - if (prefix.empty()) - iter->seek_to_first(); - else - iter->seek_to_first(prefix); - - uint32_t crc = -1; - - while (iter->valid()) { - pair rk = iter->raw_key(); - if (!prefix.empty() && (rk.first != prefix)) - break; - - if (out) - *out << url_escape(rk.first) << "\t" << url_escape(rk.second); - if (do_crc) { - bufferlist bl; - bl.append(rk.first); - bl.append(rk.second); - bl.append(iter->value()); - - crc = bl.crc32c(crc); - if (out) { - *out << "\t" << bl.crc32c(0); - } - } - if (out) - *out << std::endl; - if (out && do_value_dump) { - bufferptr bp = iter->value_as_ptr(); - bufferlist value; - value.append(bp); - ostringstream os; - value.hexdump(os); - std::cout << os.str() << std::endl; - } - iter->next(); - } - - return crc; - } - - void list(const string &prefix, const bool do_crc, const bool do_value_dump) { - traverse(prefix, do_crc, do_value_dump, &std::cout); - } - - bool exists(const string &prefix) { - ceph_assert(!prefix.empty()); - KeyValueDB::WholeSpaceIterator iter = db->get_wholespace_iterator(); - iter->seek_to_first(prefix); - return (iter->valid() && (iter->raw_key().first == prefix)); - } - - bool exists(const string &prefix, const string &key) { - ceph_assert(!prefix.empty()); - - if (key.empty()) { - return exists(prefix); - } - - bool exists = false; - get(prefix, key, exists); - return exists; - } - - bufferlist get(const string &prefix, const string &key, bool &exists) { - ceph_assert(!prefix.empty() && !key.empty()); - - map result; - std::set keys; - keys.insert(key); - db->get(prefix, keys, &result); - - if (result.count(key) > 0) { - exists = true; - return result[key]; - } - exists = false; - return bufferlist(); - } - - uint64_t get_size() { - map extras; - uint64_t s = db->get_estimated_size(extras); - for (map::iterator p = extras.begin(); - p != extras.end(); ++p) { - std::cout << p->first << " - " << p->second << std::endl; - } - std::cout << "total: " << s << std::endl; - return s; - } - - bool set(const string &prefix, const string &key, bufferlist &val) { - ceph_assert(!prefix.empty()); - ceph_assert(!key.empty()); - ceph_assert(val.length() > 0); - - KeyValueDB::Transaction tx = db->get_transaction(); - tx->set(prefix, key, val); - int ret = db->submit_transaction_sync(tx); - - return (ret == 0); - } - - bool rm(const string& prefix, const string& key) { - ceph_assert(!prefix.empty()); - ceph_assert(!key.empty()); - - KeyValueDB::Transaction tx = db->get_transaction(); - tx->rmkey(prefix, key); - int ret = db->submit_transaction_sync(tx); - - return (ret == 0); - } - - bool rm_prefix(const string& prefix) { - ceph_assert(!prefix.empty()); - - KeyValueDB::Transaction tx = db->get_transaction(); - tx->rmkeys_by_prefix(prefix); - int ret = db->submit_transaction_sync(tx); - - return (ret == 0); - } - - int copy_store_to(string type, const string &other_path, - const int num_keys_per_tx, const string &other_type) { - - if (num_keys_per_tx <= 0) { - std::cerr << "must specify a number of keys/tx > 0" << std::endl; - return -EINVAL; - } - - // open or create a leveldb store at @p other_path - boost::scoped_ptr other; - KeyValueDB *other_ptr = KeyValueDB::create(g_ceph_context, other_type, other_path); - int err = other_ptr->create_and_open(std::cerr); - if (err < 0) - return err; - other.reset(other_ptr); - - KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); - it->seek_to_first(); - uint64_t total_keys = 0; - uint64_t total_size = 0; - uint64_t total_txs = 0; - - auto started_at = coarse_mono_clock::now(); - - do { - int num_keys = 0; - - KeyValueDB::Transaction tx = other->get_transaction(); - - - while (it->valid() && num_keys < num_keys_per_tx) { - pair k = it->raw_key(); - bufferlist v = it->value(); - tx->set(k.first, k.second, v); - - num_keys ++; - total_size += v.length(); - - it->next(); - } - - total_txs ++; - total_keys += num_keys; - - if (num_keys > 0) - other->submit_transaction_sync(tx); - - auto cur_duration = std::chrono::duration(coarse_mono_clock::now() - started_at); - std::cout << "ts = " << cur_duration.count() << "s, copied " << total_keys - << " keys so far (" << stringify(byte_u_t(total_size)) << ")" - << std::endl; - - } while (it->valid()); - - auto time_taken = std::chrono::duration(coarse_mono_clock::now() - started_at); - - std::cout << "summary:" << std::endl; - std::cout << " copied " << total_keys << " keys" << std::endl; - std::cout << " used " << total_txs << " transactions" << std::endl; - std::cout << " total size " << stringify(byte_u_t(total_size)) << std::endl; - std::cout << " from '" << store_path << "' to '" << other_path << "'" - << std::endl; - std::cout << " duration " << time_taken.count() << " seconds" << std::endl; - - return 0; - } - - void compact() { - db->compact(); - } - void compact_prefix(string prefix) { - db->compact_prefix(prefix); - } - void compact_range(string prefix, string start, string end) { - db->compact_range(prefix, start, end); - } +#include "global/global_context.h" +#include "global/global_init.h" - int destructive_repair() { - return db->repair(std::cout); - } -}; +#include "kvstore_tool.h" void usage(const char *pname) { diff --git a/src/tools/kvstore_tool.cc b/src/tools/kvstore_tool.cc new file mode 100644 index 0000000000000..ee2b3d7864a34 --- /dev/null +++ b/src/tools/kvstore_tool.cc @@ -0,0 +1,272 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "kvstore_tool.h" + +#include + +#include "common/errno.h" +#include "common/url_escape.h" +#include "include/buffer.h" +#include "kv/KeyValueDB.h" + +StoreTool::StoreTool(const string& type, const string& path, bool need_open_db) + : store_path(path) +{ + if (type == "bluestore-kv") { +#ifdef WITH_BLUESTORE + auto bluestore = new BlueStore(g_ceph_context, path); + KeyValueDB *db_ptr; + if (int r = bluestore->start_kv_only(&db_ptr, need_open_db); r < 0) { + exit(1); + } + db = decltype(db){db_ptr, Deleter(bluestore)}; +#else + cerr << "bluestore not compiled in" << std::endl; + exit(1); +#endif + } else { + auto db_ptr = KeyValueDB::create(g_ceph_context, type, path); + if (need_open_db) { + if (int r = db_ptr->open(std::cerr); r < 0) { + cerr << "failed to open type " << type << " path " << path << ": " + << cpp_strerror(r) << std::endl; + exit(1); + } + db.reset(db_ptr); + } + } +} + +uint32_t StoreTool::traverse(const string& prefix, + const bool do_crc, + const bool do_value_dump, + ostream *out) +{ + KeyValueDB::WholeSpaceIterator iter = db->get_wholespace_iterator(); + + if (prefix.empty()) + iter->seek_to_first(); + else + iter->seek_to_first(prefix); + + uint32_t crc = -1; + + while (iter->valid()) { + pair rk = iter->raw_key(); + if (!prefix.empty() && (rk.first != prefix)) + break; + + if (out) + *out << url_escape(rk.first) << "\t" << url_escape(rk.second); + if (do_crc) { + bufferlist bl; + bl.append(rk.first); + bl.append(rk.second); + bl.append(iter->value()); + + crc = bl.crc32c(crc); + if (out) { + *out << "\t" << bl.crc32c(0); + } + } + if (out) + *out << std::endl; + if (out && do_value_dump) { + bufferptr bp = iter->value_as_ptr(); + bufferlist value; + value.append(bp); + ostringstream os; + value.hexdump(os); + std::cout << os.str() << std::endl; + } + iter->next(); + } + + return crc; +} + +void StoreTool::list(const string& prefix, const bool do_crc, + const bool do_value_dump) +{ + traverse(prefix, do_crc, do_value_dump,& std::cout); +} + +bool StoreTool::exists(const string& prefix) +{ + ceph_assert(!prefix.empty()); + KeyValueDB::WholeSpaceIterator iter = db->get_wholespace_iterator(); + iter->seek_to_first(prefix); + return (iter->valid() && (iter->raw_key().first == prefix)); +} + +bool StoreTool::exists(const string& prefix, const string& key) +{ + ceph_assert(!prefix.empty()); + + if (key.empty()) { + return exists(prefix); + } + bool exists = false; + get(prefix, key, exists); + return exists; +} + +bufferlist StoreTool::get(const string& prefix, + const string& key, + bool& exists) +{ + ceph_assert(!prefix.empty() && !key.empty()); + + map result; + std::set keys; + keys.insert(key); + db->get(prefix, keys, &result); + + if (result.count(key) > 0) { + exists = true; + return result[key]; + } else { + exists = false; + return bufferlist(); + } +} + +uint64_t StoreTool::get_size() +{ + map extras; + uint64_t s = db->get_estimated_size(extras); + for (auto& [name, size] : extras) { + std::cout << name << " - " << size << std::endl; + } + std::cout << "total: " << s << std::endl; + return s; +} + +bool StoreTool::set(const string &prefix, const string &key, bufferlist &val) +{ + ceph_assert(!prefix.empty()); + ceph_assert(!key.empty()); + ceph_assert(val.length() > 0); + + KeyValueDB::Transaction tx = db->get_transaction(); + tx->set(prefix, key, val); + int ret = db->submit_transaction_sync(tx); + + return (ret == 0); +} + +bool StoreTool::rm(const string& prefix, const string& key) +{ + ceph_assert(!prefix.empty()); + ceph_assert(!key.empty()); + + KeyValueDB::Transaction tx = db->get_transaction(); + tx->rmkey(prefix, key); + int ret = db->submit_transaction_sync(tx); + + return (ret == 0); +} + +bool StoreTool::rm_prefix(const string& prefix) +{ + ceph_assert(!prefix.empty()); + + KeyValueDB::Transaction tx = db->get_transaction(); + tx->rmkeys_by_prefix(prefix); + int ret = db->submit_transaction_sync(tx); + + return (ret == 0); +} + +int StoreTool::copy_store_to(const string& type, const string& other_path, + const int num_keys_per_tx, + const string& other_type) +{ + if (num_keys_per_tx <= 0) { + std::cerr << "must specify a number of keys/tx > 0" << std::endl; + return -EINVAL; + } + + // open or create a leveldb store at @p other_path + boost::scoped_ptr other; + KeyValueDB *other_ptr = KeyValueDB::create(g_ceph_context, + other_type, + other_path); + if (int err = other_ptr->create_and_open(std::cerr); err < 0) { + return err; + } + other.reset(other_ptr); + + KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); + it->seek_to_first(); + uint64_t total_keys = 0; + uint64_t total_size = 0; + uint64_t total_txs = 0; + + auto started_at = coarse_mono_clock::now(); + + do { + int num_keys = 0; + + KeyValueDB::Transaction tx = other->get_transaction(); + + while (it->valid() && num_keys < num_keys_per_tx) { + auto [prefix, key] = it->raw_key(); + bufferlist v = it->value(); + tx->set(prefix, key, v); + + num_keys++; + total_size += v.length(); + + it->next(); + } + + total_txs++; + total_keys += num_keys; + + if (num_keys > 0) + other->submit_transaction_sync(tx); + + auto cur_duration = std::chrono::duration(coarse_mono_clock::now() - + started_at); + std::cout << "ts = " << cur_duration.count() << "s, copied " << total_keys + << " keys so far (" << byte_u_t(total_size) << ")" + << std::endl; + + } while (it->valid()); + + auto time_taken = std::chrono::duration(coarse_mono_clock::now() - started_at); + + std::cout << "summary:" << std::endl; + std::cout << " copied " << total_keys << " keys" << std::endl; + std::cout << " used " << total_txs << " transactions" << std::endl; + std::cout << " total size " << byte_u_t(total_size) << std::endl; + std::cout << " from '" << store_path << "' to '" << other_path << "'" + << std::endl; + std::cout << " duration " << time_taken.count() << " seconds" << std::endl; + + return 0; +} + +void StoreTool::compact() +{ + db->compact(); +} + +void StoreTool::compact_prefix(const string& prefix) +{ + db->compact_prefix(prefix); +} + +void StoreTool::compact_range(const string& prefix, + const string& start, + const string& end) +{ + db->compact_range(prefix, start, end); +} + +int StoreTool::destructive_repair() +{ + return db->repair(std::cout); +} diff --git a/src/tools/kvstore_tool.h b/src/tools/kvstore_tool.h new file mode 100644 index 0000000000000..751d35b40d620 --- /dev/null +++ b/src/tools/kvstore_tool.h @@ -0,0 +1,73 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +#include +#include +#include + +#include "acconfig.h" +#include "include/buffer_fwd.h" +#ifdef WITH_BLUESTORE +#include "os/bluestore/BlueStore.h" +#endif + +class KeyValueDB; + +class StoreTool +{ +#ifdef WITH_BLUESTORE + struct Deleter { + BlueStore *bluestore; + Deleter() + : bluestore(nullptr) {} + Deleter(BlueStore *store) + : bluestore(store) {} + void operator()(KeyValueDB *db) { + if (bluestore) { + bluestore->umount(); + delete bluestore; + } else { + delete db; + } + } + }; + std::unique_ptr db; +#else + std::unique_ptr db; +#endif + + const std::string store_path; + +public: + StoreTool(const std::string& type, + const std::string& path, + bool need_open_db=true); + uint32_t traverse(const std::string& prefix, + const bool do_crc, + const bool do_value_dump, + ostream *out); + void list(const std::string& prefix, + const bool do_crc, + const bool do_value_dump); + bool exists(const std::string& prefix); + bool exists(const std::string& prefix, const std::string& key); + ceph::bufferlist get(const std::string& prefix, + const std::string& key, + bool& exists); + uint64_t get_size(); + bool set(const std::string& prefix, + const std::string& key, + ceph::bufferlist& val); + bool rm(const std::string& prefix, const std::string& key); + bool rm_prefix(const std::string& prefix); + int copy_store_to(const std::string& type, const std::string& other_path, + const int num_keys_per_tx, const std::string& other_type); + void compact(); + void compact_prefix(const std::string& prefix); + void compact_range(const std::string& prefix, + const std::string& start, + const std::string& end); + int destructive_repair(); +}; -- 2.39.5