endif(WITH_TESTS)
endif(WITH_LIBCEPHFS)
-add_executable(ceph-kvstore-tool ceph_kvstore_tool.cc)
+add_executable(ceph-kvstore-tool
+ kvstore_tool.cc
+ ceph_kvstore_tool.cc)
target_link_libraries(ceph-kvstore-tool os global)
install(TARGETS ceph-kvstore-tool DESTINATION bin)
#include <string>
#include <fstream>
-#include <boost/scoped_ptr.hpp>
-
#include "common/ceph_argparse.h"
#include "common/config.h"
#include "common/errno.h"
#include "common/strtol.h"
-#include "global/global_context.h"
-#include "global/global_init.h"
-#include "include/stringify.h"
-#include "common/Clock.h"
-#include "kv/KeyValueDB.h"
#include "common/url_escape.h"
-#ifdef WITH_BLUESTORE
-#include "os/bluestore/BlueStore.h"
-#endif
-
-
-class StoreTool
-{
-#ifdef WITH_BLUESTORE
- struct Deleter {
- BlueStore *bluestore;
- Deleter()
- : bluestore(nullptr) {}
- Deleter(BlueStore *store)
- : bluestore(store) {}
- void operator()(KeyValueDB *db) {
- if (bluestore) {
- bluestore->umount();
- delete bluestore;
- } else {
- delete db;
- }
- }
- };
- std::unique_ptr<KeyValueDB, Deleter> db;
-#else
- std::unique_ptr<KeyValueDB> db;
-#endif
-
- string store_path;
-
- public:
- StoreTool(string type, const string &path, bool need_open_db=true) : store_path(path) {
- if (type == "bluestore-kv") {
-#ifdef WITH_BLUESTORE
- auto bluestore = new BlueStore(g_ceph_context, path);
- KeyValueDB *db_ptr;
- int r = bluestore->start_kv_only(&db_ptr, need_open_db);
- if (r < 0) {
- exit(1);
- }
- db = decltype(db){db_ptr, Deleter(bluestore)};
-#else
- cerr << "bluestore not compiled in" << std::endl;
- exit(1);
-#endif
- } else {
- auto db_ptr = KeyValueDB::create(g_ceph_context, type, path);
- if (need_open_db) {
- int r = db_ptr->open(std::cerr);
- if (r < 0) {
- cerr << "failed to open type " << type << " path " << path << ": "
- << cpp_strerror(r) << std::endl;
- exit(1);
- }
- db.reset(db_ptr);
- }
- }
- }
-
- uint32_t traverse(const string &prefix,
- const bool do_crc,
- const bool do_value_dump,
- ostream *out) {
- KeyValueDB::WholeSpaceIterator iter = db->get_wholespace_iterator();
-
- if (prefix.empty())
- iter->seek_to_first();
- else
- iter->seek_to_first(prefix);
-
- uint32_t crc = -1;
-
- while (iter->valid()) {
- pair<string,string> rk = iter->raw_key();
- if (!prefix.empty() && (rk.first != prefix))
- break;
-
- if (out)
- *out << url_escape(rk.first) << "\t" << url_escape(rk.second);
- if (do_crc) {
- bufferlist bl;
- bl.append(rk.first);
- bl.append(rk.second);
- bl.append(iter->value());
-
- crc = bl.crc32c(crc);
- if (out) {
- *out << "\t" << bl.crc32c(0);
- }
- }
- if (out)
- *out << std::endl;
- if (out && do_value_dump) {
- bufferptr bp = iter->value_as_ptr();
- bufferlist value;
- value.append(bp);
- ostringstream os;
- value.hexdump(os);
- std::cout << os.str() << std::endl;
- }
- iter->next();
- }
-
- return crc;
- }
-
- void list(const string &prefix, const bool do_crc, const bool do_value_dump) {
- traverse(prefix, do_crc, do_value_dump, &std::cout);
- }
-
- bool exists(const string &prefix) {
- ceph_assert(!prefix.empty());
- KeyValueDB::WholeSpaceIterator iter = db->get_wholespace_iterator();
- iter->seek_to_first(prefix);
- return (iter->valid() && (iter->raw_key().first == prefix));
- }
-
- bool exists(const string &prefix, const string &key) {
- ceph_assert(!prefix.empty());
-
- if (key.empty()) {
- return exists(prefix);
- }
-
- bool exists = false;
- get(prefix, key, exists);
- return exists;
- }
-
- bufferlist get(const string &prefix, const string &key, bool &exists) {
- ceph_assert(!prefix.empty() && !key.empty());
-
- map<string,bufferlist> result;
- std::set<std::string> keys;
- keys.insert(key);
- db->get(prefix, keys, &result);
-
- if (result.count(key) > 0) {
- exists = true;
- return result[key];
- }
- exists = false;
- return bufferlist();
- }
-
- uint64_t get_size() {
- map<string,uint64_t> extras;
- uint64_t s = db->get_estimated_size(extras);
- for (map<string,uint64_t>::iterator p = extras.begin();
- p != extras.end(); ++p) {
- std::cout << p->first << " - " << p->second << std::endl;
- }
- std::cout << "total: " << s << std::endl;
- return s;
- }
-
- bool set(const string &prefix, const string &key, bufferlist &val) {
- ceph_assert(!prefix.empty());
- ceph_assert(!key.empty());
- ceph_assert(val.length() > 0);
-
- KeyValueDB::Transaction tx = db->get_transaction();
- tx->set(prefix, key, val);
- int ret = db->submit_transaction_sync(tx);
-
- return (ret == 0);
- }
-
- bool rm(const string& prefix, const string& key) {
- ceph_assert(!prefix.empty());
- ceph_assert(!key.empty());
-
- KeyValueDB::Transaction tx = db->get_transaction();
- tx->rmkey(prefix, key);
- int ret = db->submit_transaction_sync(tx);
-
- return (ret == 0);
- }
-
- bool rm_prefix(const string& prefix) {
- ceph_assert(!prefix.empty());
-
- KeyValueDB::Transaction tx = db->get_transaction();
- tx->rmkeys_by_prefix(prefix);
- int ret = db->submit_transaction_sync(tx);
-
- return (ret == 0);
- }
-
- int copy_store_to(string type, const string &other_path,
- const int num_keys_per_tx, const string &other_type) {
-
- if (num_keys_per_tx <= 0) {
- std::cerr << "must specify a number of keys/tx > 0" << std::endl;
- return -EINVAL;
- }
-
- // open or create a leveldb store at @p other_path
- boost::scoped_ptr<KeyValueDB> other;
- KeyValueDB *other_ptr = KeyValueDB::create(g_ceph_context, other_type, other_path);
- int err = other_ptr->create_and_open(std::cerr);
- if (err < 0)
- return err;
- other.reset(other_ptr);
-
- KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator();
- it->seek_to_first();
- uint64_t total_keys = 0;
- uint64_t total_size = 0;
- uint64_t total_txs = 0;
-
- auto started_at = coarse_mono_clock::now();
-
- do {
- int num_keys = 0;
-
- KeyValueDB::Transaction tx = other->get_transaction();
-
-
- while (it->valid() && num_keys < num_keys_per_tx) {
- pair<string,string> k = it->raw_key();
- bufferlist v = it->value();
- tx->set(k.first, k.second, v);
-
- num_keys ++;
- total_size += v.length();
-
- it->next();
- }
-
- total_txs ++;
- total_keys += num_keys;
-
- if (num_keys > 0)
- other->submit_transaction_sync(tx);
-
- auto cur_duration = std::chrono::duration<double>(coarse_mono_clock::now() - started_at);
- std::cout << "ts = " << cur_duration.count() << "s, copied " << total_keys
- << " keys so far (" << stringify(byte_u_t(total_size)) << ")"
- << std::endl;
-
- } while (it->valid());
-
- auto time_taken = std::chrono::duration<double>(coarse_mono_clock::now() - started_at);
-
- std::cout << "summary:" << std::endl;
- std::cout << " copied " << total_keys << " keys" << std::endl;
- std::cout << " used " << total_txs << " transactions" << std::endl;
- std::cout << " total size " << stringify(byte_u_t(total_size)) << std::endl;
- std::cout << " from '" << store_path << "' to '" << other_path << "'"
- << std::endl;
- std::cout << " duration " << time_taken.count() << " seconds" << std::endl;
-
- return 0;
- }
-
- void compact() {
- db->compact();
- }
- void compact_prefix(string prefix) {
- db->compact_prefix(prefix);
- }
- void compact_range(string prefix, string start, string end) {
- db->compact_range(prefix, start, end);
- }
+#include "global/global_context.h"
+#include "global/global_init.h"
- int destructive_repair() {
- return db->repair(std::cout);
- }
-};
+#include "kvstore_tool.h"
void usage(const char *pname)
{
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "kvstore_tool.h"
+
+#include <iostream>
+
+#include "common/errno.h"
+#include "common/url_escape.h"
+#include "include/buffer.h"
+#include "kv/KeyValueDB.h"
+
+StoreTool::StoreTool(const string& type, const string& path, bool need_open_db)
+ : store_path(path)
+{
+ if (type == "bluestore-kv") {
+#ifdef WITH_BLUESTORE
+ auto bluestore = new BlueStore(g_ceph_context, path);
+ KeyValueDB *db_ptr;
+ if (int r = bluestore->start_kv_only(&db_ptr, need_open_db); r < 0) {
+ exit(1);
+ }
+ db = decltype(db){db_ptr, Deleter(bluestore)};
+#else
+ cerr << "bluestore not compiled in" << std::endl;
+ exit(1);
+#endif
+ } else {
+ auto db_ptr = KeyValueDB::create(g_ceph_context, type, path);
+ if (need_open_db) {
+ if (int r = db_ptr->open(std::cerr); r < 0) {
+ cerr << "failed to open type " << type << " path " << path << ": "
+ << cpp_strerror(r) << std::endl;
+ exit(1);
+ }
+ db.reset(db_ptr);
+ }
+ }
+}
+
+uint32_t StoreTool::traverse(const string& prefix,
+ const bool do_crc,
+ const bool do_value_dump,
+ ostream *out)
+{
+ KeyValueDB::WholeSpaceIterator iter = db->get_wholespace_iterator();
+
+ if (prefix.empty())
+ iter->seek_to_first();
+ else
+ iter->seek_to_first(prefix);
+
+ uint32_t crc = -1;
+
+ while (iter->valid()) {
+ pair<string,string> rk = iter->raw_key();
+ if (!prefix.empty() && (rk.first != prefix))
+ break;
+
+ if (out)
+ *out << url_escape(rk.first) << "\t" << url_escape(rk.second);
+ if (do_crc) {
+ bufferlist bl;
+ bl.append(rk.first);
+ bl.append(rk.second);
+ bl.append(iter->value());
+
+ crc = bl.crc32c(crc);
+ if (out) {
+ *out << "\t" << bl.crc32c(0);
+ }
+ }
+ if (out)
+ *out << std::endl;
+ if (out && do_value_dump) {
+ bufferptr bp = iter->value_as_ptr();
+ bufferlist value;
+ value.append(bp);
+ ostringstream os;
+ value.hexdump(os);
+ std::cout << os.str() << std::endl;
+ }
+ iter->next();
+ }
+
+ return crc;
+}
+
+void StoreTool::list(const string& prefix, const bool do_crc,
+ const bool do_value_dump)
+{
+ traverse(prefix, do_crc, do_value_dump,& std::cout);
+}
+
+bool StoreTool::exists(const string& prefix)
+{
+ ceph_assert(!prefix.empty());
+ KeyValueDB::WholeSpaceIterator iter = db->get_wholespace_iterator();
+ iter->seek_to_first(prefix);
+ return (iter->valid() && (iter->raw_key().first == prefix));
+}
+
+bool StoreTool::exists(const string& prefix, const string& key)
+{
+ ceph_assert(!prefix.empty());
+
+ if (key.empty()) {
+ return exists(prefix);
+ }
+ bool exists = false;
+ get(prefix, key, exists);
+ return exists;
+}
+
+bufferlist StoreTool::get(const string& prefix,
+ const string& key,
+ bool& exists)
+{
+ ceph_assert(!prefix.empty() && !key.empty());
+
+ map<string,bufferlist> result;
+ std::set<std::string> keys;
+ keys.insert(key);
+ db->get(prefix, keys, &result);
+
+ if (result.count(key) > 0) {
+ exists = true;
+ return result[key];
+ } else {
+ exists = false;
+ return bufferlist();
+ }
+}
+
+uint64_t StoreTool::get_size()
+{
+ map<string,uint64_t> extras;
+ uint64_t s = db->get_estimated_size(extras);
+ for (auto& [name, size] : extras) {
+ std::cout << name << " - " << size << std::endl;
+ }
+ std::cout << "total: " << s << std::endl;
+ return s;
+}
+
+bool StoreTool::set(const string &prefix, const string &key, bufferlist &val)
+{
+ ceph_assert(!prefix.empty());
+ ceph_assert(!key.empty());
+ ceph_assert(val.length() > 0);
+
+ KeyValueDB::Transaction tx = db->get_transaction();
+ tx->set(prefix, key, val);
+ int ret = db->submit_transaction_sync(tx);
+
+ return (ret == 0);
+}
+
+bool StoreTool::rm(const string& prefix, const string& key)
+{
+ ceph_assert(!prefix.empty());
+ ceph_assert(!key.empty());
+
+ KeyValueDB::Transaction tx = db->get_transaction();
+ tx->rmkey(prefix, key);
+ int ret = db->submit_transaction_sync(tx);
+
+ return (ret == 0);
+}
+
+bool StoreTool::rm_prefix(const string& prefix)
+{
+ ceph_assert(!prefix.empty());
+
+ KeyValueDB::Transaction tx = db->get_transaction();
+ tx->rmkeys_by_prefix(prefix);
+ int ret = db->submit_transaction_sync(tx);
+
+ return (ret == 0);
+}
+
+int StoreTool::copy_store_to(const string& type, const string& other_path,
+ const int num_keys_per_tx,
+ const string& other_type)
+{
+ if (num_keys_per_tx <= 0) {
+ std::cerr << "must specify a number of keys/tx > 0" << std::endl;
+ return -EINVAL;
+ }
+
+ // open or create a leveldb store at @p other_path
+ boost::scoped_ptr<KeyValueDB> other;
+ KeyValueDB *other_ptr = KeyValueDB::create(g_ceph_context,
+ other_type,
+ other_path);
+ if (int err = other_ptr->create_and_open(std::cerr); err < 0) {
+ return err;
+ }
+ other.reset(other_ptr);
+
+ KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator();
+ it->seek_to_first();
+ uint64_t total_keys = 0;
+ uint64_t total_size = 0;
+ uint64_t total_txs = 0;
+
+ auto started_at = coarse_mono_clock::now();
+
+ do {
+ int num_keys = 0;
+
+ KeyValueDB::Transaction tx = other->get_transaction();
+
+ while (it->valid() && num_keys < num_keys_per_tx) {
+ auto [prefix, key] = it->raw_key();
+ bufferlist v = it->value();
+ tx->set(prefix, key, v);
+
+ num_keys++;
+ total_size += v.length();
+
+ it->next();
+ }
+
+ total_txs++;
+ total_keys += num_keys;
+
+ if (num_keys > 0)
+ other->submit_transaction_sync(tx);
+
+ auto cur_duration = std::chrono::duration<double>(coarse_mono_clock::now() -
+ started_at);
+ std::cout << "ts = " << cur_duration.count() << "s, copied " << total_keys
+ << " keys so far (" << byte_u_t(total_size) << ")"
+ << std::endl;
+
+ } while (it->valid());
+
+ auto time_taken = std::chrono::duration<double>(coarse_mono_clock::now() - started_at);
+
+ std::cout << "summary:" << std::endl;
+ std::cout << " copied " << total_keys << " keys" << std::endl;
+ std::cout << " used " << total_txs << " transactions" << std::endl;
+ std::cout << " total size " << byte_u_t(total_size) << std::endl;
+ std::cout << " from '" << store_path << "' to '" << other_path << "'"
+ << std::endl;
+ std::cout << " duration " << time_taken.count() << " seconds" << std::endl;
+
+ return 0;
+}
+
+void StoreTool::compact()
+{
+ db->compact();
+}
+
+void StoreTool::compact_prefix(const string& prefix)
+{
+ db->compact_prefix(prefix);
+}
+
+void StoreTool::compact_range(const string& prefix,
+ const string& start,
+ const string& end)
+{
+ db->compact_range(prefix, start, end);
+}
+
+int StoreTool::destructive_repair()
+{
+ return db->repair(std::cout);
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <memory>
+#include <ostream>
+#include <string>
+
+#include "acconfig.h"
+#include "include/buffer_fwd.h"
+#ifdef WITH_BLUESTORE
+#include "os/bluestore/BlueStore.h"
+#endif
+
+class KeyValueDB;
+
+class StoreTool
+{
+#ifdef WITH_BLUESTORE
+ struct Deleter {
+ BlueStore *bluestore;
+ Deleter()
+ : bluestore(nullptr) {}
+ Deleter(BlueStore *store)
+ : bluestore(store) {}
+ void operator()(KeyValueDB *db) {
+ if (bluestore) {
+ bluestore->umount();
+ delete bluestore;
+ } else {
+ delete db;
+ }
+ }
+ };
+ std::unique_ptr<KeyValueDB, Deleter> db;
+#else
+ std::unique_ptr<KeyValueDB> db;
+#endif
+
+ const std::string store_path;
+
+public:
+ StoreTool(const std::string& type,
+ const std::string& path,
+ bool need_open_db=true);
+ uint32_t traverse(const std::string& prefix,
+ const bool do_crc,
+ const bool do_value_dump,
+ ostream *out);
+ void list(const std::string& prefix,
+ const bool do_crc,
+ const bool do_value_dump);
+ bool exists(const std::string& prefix);
+ bool exists(const std::string& prefix, const std::string& key);
+ ceph::bufferlist get(const std::string& prefix,
+ const std::string& key,
+ bool& exists);
+ uint64_t get_size();
+ bool set(const std::string& prefix,
+ const std::string& key,
+ ceph::bufferlist& val);
+ bool rm(const std::string& prefix, const std::string& key);
+ bool rm_prefix(const std::string& prefix);
+ int copy_store_to(const std::string& type, const std::string& other_path,
+ const int num_keys_per_tx, const std::string& other_type);
+ void compact();
+ void compact_prefix(const std::string& prefix);
+ void compact_range(const std::string& prefix,
+ const std::string& start,
+ const std::string& end);
+ int destructive_repair();
+};
projects / ceph.git / commitdiff