From: Sage Weil Date: Fri, 14 Mar 2014 20:15:22 +0000 (-0700) Subject: tools/: make filenames consistent X-Git-Tag: v0.79~148 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=f2c6ff33b7027ea757c4a6f641ec5d3b03819311;p=ceph.git tools/: make filenames consistent _ for source files, - for (shippable) build targets. Signed-off-by: Sage Weil --- diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am index 0a386cc388388..64cd49482be15 100644 --- a/src/tools/Makefile.am +++ b/src/tools/Makefile.am @@ -1,25 +1,25 @@ -ceph_osdomap_tool_SOURCES = tools/ceph-osdomap-tool.cc +ceph_osdomap_tool_SOURCES = tools/ceph_osdomap_tool.cc ceph_osdomap_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS) bin_DEBUGPROGRAMS += ceph-osdomap-tool -ceph_monstore_tool_SOURCES = tools/ceph-monstore-tool.cc +ceph_monstore_tool_SOURCES = tools/ceph_monstore_tool.cc ceph_monstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS) bin_DEBUGPROGRAMS += ceph-monstore-tool -ceph_kvstore_tool_SOURCES = tools/ceph-kvstore-tool.cc +ceph_kvstore_tool_SOURCES = tools/ceph_kvstore_tool.cc ceph_kvstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) ceph_kvstore_tool_CXXFLAGS = $(UNITTEST_CXXFLAGS) bin_DEBUGPROGRAMS += ceph-kvstore-tool -ceph_filestore_tool_SOURCES = tools/ceph-filestore-tool.cc +ceph_filestore_tool_SOURCES = tools/ceph_filestore_tool.cc ceph_filestore_tool_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options if LINUX ceph_filestore_tool_LDADD += -ldl endif # LINUX bin_PROGRAMS += ceph_filestore_tool -ceph_filestore_dump_SOURCES = tools/ceph-filestore-dump.cc +ceph_filestore_dump_SOURCES = tools/ceph_filestore_dump.cc ceph_filestore_dump_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS) if LINUX ceph_filestore_dump_LDADD += -ldl diff --git a/src/tools/ceph-filestore-dump.cc b/src/tools/ceph-filestore-dump.cc deleted file mode 100644 index 78ac8391fd21b..0000000000000 --- a/src/tools/ceph-filestore-dump.cc +++ /dev/null @@ -1,1405 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Inktank - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/Formatter.h" - -#include "global/global_init.h" -#include "os/ObjectStore.h" -#include "os/FileStore.h" -#include "common/perf_counters.h" -#include "common/errno.h" -#include "osd/PGLog.h" -#include "osd/OSD.h" - -namespace po = boost::program_options; -using namespace std; - -enum { - TYPE_NONE = 0, - TYPE_PG_BEGIN, - TYPE_PG_END, - TYPE_OBJECT_BEGIN, - TYPE_OBJECT_END, - TYPE_DATA, - TYPE_ATTRS, - TYPE_OMAP_HDR, - TYPE_OMAP, - TYPE_PG_METADATA, - END_OF_TYPES, //Keep at the end -}; - -//#define INTERNAL_TEST -//#define INTERNAL_TEST2 - -#ifdef INTERNAL_TEST -CompatSet get_test_compat_set() { - CompatSet::FeatureSet ceph_osd_feature_compat; - CompatSet::FeatureSet ceph_osd_feature_ro_compat; - CompatSet::FeatureSet ceph_osd_feature_incompat; - ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_BASE); - ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_PGINFO); - ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_OLOC); - ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEC); - ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_CATEGORIES); - ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_HOBJECTPOOL); - ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_BIGINFO); - ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEVELDBINFO); - ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEVELDBLOG); -#ifdef INTERNAL_TEST2 - ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SNAPMAPPER); - ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS); -#endif - return CompatSet(ceph_osd_feature_compat, ceph_osd_feature_ro_compat, - ceph_osd_feature_incompat); -} -#endif - -typedef uint8_t sectiontype_t; -typedef uint32_t mymagic_t; -typedef int64_t mysize_t; -const ssize_t max_read = 1024 * 1024; -const uint16_t shortmagic = 0xffce; //goes into stream as "ceff" -//endmagic goes into stream as "ceff ffec" -const mymagic_t endmagic = (0xecff << 16) | shortmagic; -const int fd_none = INT_MIN; - -//The first FIXED_LENGTH bytes are a fixed -//portion of the export output. This includes the overall -//version number, and size of header and footer. -//THIS STRUCTURE CAN ONLY BE APPENDED TO. If it needs to expand, -//the version can be bumped and then anything -//can be added to the export format. -struct super_header { - static const uint32_t super_magic = (shortmagic << 16) | shortmagic; - static const uint32_t super_ver = 2; - static const uint32_t FIXED_LENGTH = 16; - uint32_t magic; - uint32_t version; - uint32_t header_size; - uint32_t footer_size; - - super_header() : magic(0), version(0), header_size(0), footer_size(0) { } - int read_super(); - - void encode(bufferlist& bl) const { - ::encode(magic, bl); - ::encode(version, bl); - ::encode(header_size, bl); - ::encode(footer_size, bl); - } - void decode(bufferlist::iterator& bl) { - ::decode(magic, bl); - ::decode(version, bl); - ::decode(header_size, bl); - ::decode(footer_size, bl); - } -}; - -struct header { - sectiontype_t type; - mysize_t size; - header(sectiontype_t type, mysize_t size) : - type(type), size(size) { } - header(): type(0), size(0) { } - - int get_header(); - - void encode(bufferlist& bl) const { - uint32_t debug_type = (type << 24) | (type << 16) | shortmagic; - ENCODE_START(1, 1, bl); - ::encode(debug_type, bl); - ::encode(size, bl); - ENCODE_FINISH(bl); - } - void decode(bufferlist::iterator& bl) { - uint32_t debug_type; - DECODE_START(1, bl); - ::decode(debug_type, bl); - type = debug_type >> 24; - ::decode(size, bl); - DECODE_FINISH(bl); - } -}; - -struct footer { - mymagic_t magic; - footer() : magic(endmagic) { } - - int get_footer(); - - void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); - ::encode(magic, bl); - ENCODE_FINISH(bl); - } - void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); - ::decode(magic, bl); - DECODE_FINISH(bl); - } -}; - -struct pg_begin { - pg_t pgid; - OSDSuperblock superblock; - - pg_begin(pg_t pg, const OSDSuperblock& sb): - pgid(pg), superblock(sb) { } - pg_begin() { } - - void encode(bufferlist& bl) const { - // New super_ver prevents decode from ver 1 - ENCODE_START(2, 2, bl); - ::encode(pgid, bl); - ::encode(superblock, bl); - ENCODE_FINISH(bl); - } - void decode(bufferlist::iterator& bl) { - DECODE_START(2, bl); - ::decode(pgid, bl); - if (struct_v > 1) { - ::decode(superblock, bl); - } - DECODE_FINISH(bl); - } -}; - -struct object_begin { - hobject_t hoid; - object_begin(const hobject_t &hoid): hoid(hoid) { } - object_begin() { } - - void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); - ::encode(hoid, bl); - ENCODE_FINISH(bl); - } - void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); - ::decode(hoid, bl); - DECODE_FINISH(bl); - } -}; - -struct data_section { - uint64_t offset; - uint64_t len; - bufferlist databl; - data_section(uint64_t offset, uint64_t len, bufferlist bl): - offset(offset), len(len), databl(bl) { } - data_section(): offset(0), len(0) { } - - void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); - ::encode(offset, bl); - ::encode(len, bl); - ::encode(databl, bl); - ENCODE_FINISH(bl); - } - void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); - ::decode(offset, bl); - ::decode(len, bl); - ::decode(databl, bl); - DECODE_FINISH(bl); - } -}; - -struct attr_section { - map data; - attr_section(const map &data) : data(data) { } - attr_section() { } - - void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); - ::encode(data, bl); - ENCODE_FINISH(bl); - } - void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); - ::decode(data, bl); - DECODE_FINISH(bl); - } -}; - -struct omap_hdr_section { - bufferlist hdr; - omap_hdr_section(bufferlist hdr) : hdr(hdr) { } - omap_hdr_section() { } - - void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); - ::encode(hdr, bl); - ENCODE_FINISH(bl); - } - void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); - ::decode(hdr, bl); - DECODE_FINISH(bl); - } -}; - -struct omap_section { - map omap; - omap_section(const map &omap) : - omap(omap) { } - omap_section() { } - - void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); - ::encode(omap, bl); - ENCODE_FINISH(bl); - } - void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); - ::decode(omap, bl); - DECODE_FINISH(bl); - } -}; - -struct metadata_section { - __u8 struct_ver; - epoch_t map_epoch; - pg_info_t info; - pg_log_t log; - - metadata_section(__u8 struct_ver, epoch_t map_epoch, const pg_info_t &info, - const pg_log_t &log) - : struct_ver(struct_ver), - map_epoch(map_epoch), - info(info), - log(log) { } - metadata_section() - : struct_ver(0), - map_epoch(0) { } - - void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); - ::encode(struct_ver, bl); - ::encode(map_epoch, bl); - ::encode(info, bl); - ::encode(log, bl); - ENCODE_FINISH(bl); - } - void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); - ::decode(struct_ver, bl); - ::decode(map_epoch, bl); - ::decode(info, bl); - ::decode(log, bl); - DECODE_FINISH(bl); - } -}; - -hobject_t infos_oid = OSD::make_infos_oid(); -hobject_t biginfo_oid, log_oid; - -int file_fd = fd_none; -bool debug = false; -super_header sh; - -template -int write_section(sectiontype_t type, const T& obj, int fd) { - bufferlist blhdr, bl, blftr; - obj.encode(bl); - header hdr(type, bl.length()); - hdr.encode(blhdr); - footer ft; - ft.encode(blftr); - - int ret = blhdr.write_fd(fd); - if (ret) return ret; - ret = bl.write_fd(fd); - if (ret) return ret; - ret = blftr.write_fd(fd); - return ret; -} - -int write_simple(sectiontype_t type, int fd) -{ - bufferlist hbl; - - header hdr(type, 0); - hdr.encode(hbl); - return hbl.write_fd(fd); -} - -static void invalid_path(string &path) -{ - cout << "Invalid path to osd store specified: " << path << "\n"; - exit(1); -} - -int get_log(ObjectStore *fs, coll_t coll, pg_t pgid, const pg_info_t &info, - PGLog::IndexedLog &log, pg_missing_t &missing) -{ - map divergent_priors; - try { - ostringstream oss; - PGLog::read_log(fs, coll, log_oid, info, divergent_priors, log, missing, oss); - if (debug && oss.str().size()) - cerr << oss.str() << std::endl; - } - catch (const buffer::error &e) { - cout << "read_log threw exception error " << e.what() << std::endl; - return 1; - } - return 0; -} - -//Based on RemoveWQ::_process() -void remove_coll(ObjectStore *store, const coll_t &coll) -{ - spg_t pg; - coll.is_pg_prefix(pg); - OSDriver driver( - store, - coll_t(), - OSD::make_snapmapper_oid()); - SnapMapper mapper(&driver, 0, 0, 0, pg.shard); - - vector objects; - ghobject_t next; - int r = 0; - int64_t num = 0; - ObjectStore::Transaction *t = new ObjectStore::Transaction; - cout << "remove_coll " << coll << std::endl; - while (!next.is_max()) { - r = store->collection_list_partial(coll, next, 200, 300, 0, - &objects, &next); - if (r < 0) - goto out; - for (vector::iterator i = objects.begin(); - i != objects.end(); - ++i, ++num) { - - assert(i->generation == ghobject_t::NO_GEN); - OSDriver::OSTransaction _t(driver.get_transaction(t)); - cout << "remove " << *i << std::endl; - int r = mapper.remove_oid(i->hobj, &_t); - if (r != 0 && r != -ENOENT) { - assert(0); - } - - t->remove(coll, *i); - if (num >= 30) { - store->apply_transaction(*t); - delete t; - t = new ObjectStore::Transaction; - num = 0; - } - } - } - t->remove_collection(coll); - store->apply_transaction(*t); -out: - delete t; -} - -//Based on part of OSD::load_pgs() -int finish_remove_pgs(ObjectStore *store, uint64_t *next_removal_seq) -{ - vector ls; - int r = store->list_collections(ls); - if (r < 0) { - cout << "finish_remove_pgs: failed to list pgs: " << cpp_strerror(-r) - << std::endl; - return r; - } - - for (vector::iterator it = ls.begin(); - it != ls.end(); - ++it) { - spg_t pgid; - snapid_t snap; - - if (it->is_temp(pgid)) { - cout << "finish_remove_pgs " << *it << " clearing temp" << std::endl; - OSD::recursive_remove_collection(store, *it); - continue; - } - - if (it->is_pg(pgid, snap)) { - continue; - } - - uint64_t seq; - if (it->is_removal(&seq, &pgid)) { - if (seq >= *next_removal_seq) - *next_removal_seq = seq + 1; - cout << "finish_remove_pgs removing " << *it << ", seq is " - << seq << " pgid is " << pgid << std::endl; - remove_coll(store, *it); - continue; - } - - //cout << "finish_remove_pgs ignoring unrecognized " << *it << std::endl; - } - return 0; -} - -int initiate_new_remove_pg(ObjectStore *store, pg_t r_pgid, - uint64_t *next_removal_seq) -{ - ObjectStore::Transaction *rmt = new ObjectStore::Transaction; - - if (store->collection_exists(coll_t(spg_t(r_pgid, ghobject_t::no_shard())))) { - coll_t to_remove = coll_t::make_removal_coll((*next_removal_seq)++, - spg_t(r_pgid, ghobject_t::no_shard())); - cout << "collection rename " << coll_t(spg_t(r_pgid, ghobject_t::no_shard())) - << " to " << to_remove - << std::endl; - rmt->collection_rename(coll_t(spg_t(r_pgid, ghobject_t::no_shard())), to_remove); - } else { - delete rmt; - return ENOENT; - } - - cout << "remove " << coll_t::META_COLL << " " << log_oid.oid << std::endl; - rmt->remove(coll_t::META_COLL, log_oid); - cout << "remove " << coll_t::META_COLL << " " << biginfo_oid.oid << std::endl; - rmt->remove(coll_t::META_COLL, biginfo_oid); - - store->apply_transaction(*rmt); - - return 0; -} - -int header::get_header() -{ - bufferlist ebl; - bufferlist::iterator ebliter = ebl.begin(); - ssize_t bytes; - - bytes = ebl.read_fd(file_fd, sh.header_size); - if ((size_t)bytes != sh.header_size) { - cout << "Unexpected EOF" << std::endl; - return EFAULT; - } - - decode(ebliter); - - return 0; -} - -int footer::get_footer() -{ - bufferlist ebl; - bufferlist::iterator ebliter = ebl.begin(); - ssize_t bytes; - - bytes = ebl.read_fd(file_fd, sh.footer_size); - if ((size_t)bytes != sh.footer_size) { - cout << "Unexpected EOF" << std::endl; - return EFAULT; - } - - decode(ebliter); - - if (magic != endmagic) { - cout << "Bad footer magic" << std::endl; - return EFAULT; - } - - return 0; -} - -int write_info(ObjectStore::Transaction &t, epoch_t epoch, pg_info_t &info, - __u8 struct_ver) -{ - //Empty for this - interval_set snap_collections; // obsolete - map past_intervals; - coll_t coll(info.pgid); - - int ret = PG::_write_info(t, epoch, - info, coll, - past_intervals, - snap_collections, - infos_oid, - struct_ver, - true, true); - if (ret < 0) ret = -ret; - if (ret) cout << "Failed to write info" << std::endl; - return ret; -} - -void write_log(ObjectStore::Transaction &t, pg_log_t &log) -{ - map divergent_priors; - PGLog::write_log(t, log, log_oid, divergent_priors); -} - -int write_pg(ObjectStore::Transaction &t, epoch_t epoch, pg_info_t &info, - pg_log_t &log, __u8 struct_ver) -{ - int ret = write_info(t, epoch, info, struct_ver); - if (ret) return ret; - write_log(t, log); - return 0; -} - -int export_file(ObjectStore *store, coll_t cid, hobject_t &obj) -{ - struct stat st; - mysize_t total; - ostringstream objname; - footer ft; - - int ret = store->stat(cid, obj, &st); - if (ret < 0) - return ret; - - objname << obj; - if (debug && file_fd != STDOUT_FILENO) - cout << "objname=" << objname.str() << std::endl; - - total = st.st_size; - if (debug && file_fd != STDOUT_FILENO) - cout << "size=" << total << std::endl; - - object_begin objb(obj); - ret = write_section(TYPE_OBJECT_BEGIN, objb, file_fd); - if (ret < 0) - return ret; - - uint64_t offset = 0; - bufferlist rawdatabl, databl; - while(total > 0) { - rawdatabl.clear(); - databl.clear(); - mysize_t len = max_read; - if (len > total) - len = total; - - ret = store->read(cid, obj, offset, len, rawdatabl); - if (ret < 0) - return ret; - if (ret == 0) - return -EINVAL; - - data_section dblock(offset, len, rawdatabl); - total -= ret; - offset += ret; - - if (debug && file_fd != STDOUT_FILENO) - cout << "data section offset=" << offset << " len=" << len << std::endl; - - ret = write_section(TYPE_DATA, dblock, file_fd); - if (ret) return ret; - } - - //Handle attrs for this object - map aset; - ret = store->getattrs(cid, obj, aset, false); - if (ret) return ret; - attr_section as(aset); - ret = write_section(TYPE_ATTRS, as, file_fd); - if (ret) - return ret; - - if (debug && file_fd != STDOUT_FILENO) { - cout << "attrs size " << aset.size() << std::endl; - } - - //Handle omap information - databl.clear(); - bufferlist hdrbuf; - map out; - ret = store->omap_get(cid, obj, &hdrbuf, &out); - if (ret < 0) - return ret; - - omap_hdr_section ohs(hdrbuf); - ret = write_section(TYPE_OMAP_HDR, ohs, file_fd); - if (ret) - return ret; - - if (!out.empty()) { - omap_section oms(out); - ret = write_section(TYPE_OMAP, oms, file_fd); - if (ret) - return ret; - - if (debug && file_fd != STDOUT_FILENO) - cout << "omap map size " << out.size() << std::endl; - } - - ret = write_simple(TYPE_OBJECT_END, file_fd); - if (ret) - return ret; - - return 0; -} - -int export_files(ObjectStore *store, coll_t coll) -{ - vector objects; - ghobject_t next; - - while (!next.is_max()) { - int r = store->collection_list_partial(coll, next, 200, 300, 0, - &objects, &next); - if (r < 0) - return r; - for (vector::iterator i = objects.begin(); - i != objects.end(); - ++i) { - assert(i->generation == ghobject_t::NO_GEN); - r = export_file(store, coll, i->hobj); - if (r < 0) - return r; - } - } - return 0; -} - -//Write super_header with its fixed 16 byte length -void write_super() -{ - bufferlist superbl; - super_header sh; - footer ft; - - header hdr(TYPE_NONE, 0); - hdr.encode(superbl); - - sh.magic = super_header::super_magic; - sh.version = super_header::super_ver; - sh.header_size = superbl.length(); - superbl.clear(); - ft.encode(superbl); - sh.footer_size = superbl.length(); - superbl.clear(); - - sh.encode(superbl); - assert(super_header::FIXED_LENGTH == superbl.length()); - superbl.write_fd(file_fd); -} - -int do_export(ObjectStore *fs, coll_t coll, pg_t pgid, pg_info_t &info, - epoch_t map_epoch, __u8 struct_ver, const OSDSuperblock& superblock) -{ - PGLog::IndexedLog log; - pg_missing_t missing; - - int ret = get_log(fs, coll, pgid, info, log, missing); - if (ret > 0) - return ret; - - write_super(); - - pg_begin pgb(pgid, superblock); - ret = write_section(TYPE_PG_BEGIN, pgb, file_fd); - if (ret) - return ret; - - export_files(fs, coll); - - metadata_section ms(struct_ver, map_epoch, info, log); - ret = write_section(TYPE_PG_METADATA, ms, file_fd); - if (ret) - return ret; - - ret = write_simple(TYPE_PG_END, file_fd); - if (ret) - return ret; - - return 0; -} - -int super_header::read_super() -{ - bufferlist ebl; - bufferlist::iterator ebliter = ebl.begin(); - ssize_t bytes; - - bytes = ebl.read_fd(file_fd, super_header::FIXED_LENGTH); - if ((size_t)bytes != super_header::FIXED_LENGTH) { - cout << "Unexpected EOF" << std::endl; - return EFAULT; - } - - decode(ebliter); - - return 0; -} - -int read_section(int fd, sectiontype_t *type, bufferlist *bl) -{ - header hdr; - ssize_t bytes; - - int ret = hdr.get_header(); - if (ret) - return ret; - - *type = hdr.type; - - bl->clear(); - bytes = bl->read_fd(fd, hdr.size); - if (bytes != hdr.size) { - cout << "Unexpected EOF" << std::endl; - return EFAULT; - } - - if (hdr.size > 0) { - footer ft; - ret = ft.get_footer(); - if (ret) - return ret; - } - - return 0; -} - -int get_data(ObjectStore *store, coll_t coll, hobject_t hoid, - ObjectStore::Transaction *t, bufferlist &bl) -{ - bufferlist::iterator ebliter = bl.begin(); - data_section ds; - ds.decode(ebliter); - - if (debug) - cout << "\tdata: offset " << ds.offset << " len " << ds.len << std::endl; - t->write(coll, hoid, ds.offset, ds.len, ds.databl); - return 0; -} - -int get_attrs(ObjectStore *store, coll_t coll, hobject_t hoid, - ObjectStore::Transaction *t, bufferlist &bl, - OSDriver &driver, SnapMapper &snap_mapper) -{ - bufferlist::iterator ebliter = bl.begin(); - attr_section as; - as.decode(ebliter); - - if (debug) - cout << "\tattrs: len " << as.data.size() << std::endl; - t->setattrs(coll, hoid, as.data); - - if (hoid.snap < CEPH_MAXSNAP) { - map::iterator mi = as.data.find(OI_ATTR); - if (mi != as.data.end()) { - bufferlist attr_bl; - attr_bl.push_back(mi->second); - object_info_t oi(attr_bl); - - if (debug) - cout << "object_info " << oi << std::endl; - - OSDriver::OSTransaction _t(driver.get_transaction(t)); - set oi_snaps(oi.snaps.begin(), oi.snaps.end()); - snap_mapper.add_oid(hoid, oi_snaps, &_t); - } - } - - return 0; -} - -int get_omap_hdr(ObjectStore *store, coll_t coll, hobject_t hoid, - ObjectStore::Transaction *t, bufferlist &bl) -{ - bufferlist::iterator ebliter = bl.begin(); - omap_hdr_section oh; - oh.decode(ebliter); - - if (debug) - cout << "\tomap header: " << string(oh.hdr.c_str(), oh.hdr.length()) - << std::endl; - t->omap_setheader(coll, hoid, oh.hdr); - return 0; -} - -int get_omap(ObjectStore *store, coll_t coll, hobject_t hoid, - ObjectStore::Transaction *t, bufferlist &bl) -{ - bufferlist::iterator ebliter = bl.begin(); - omap_section os; - os.decode(ebliter); - - if (debug) - cout << "\tomap: size " << os.omap.size() << std::endl; - t->omap_setkeys(coll, hoid, os.omap); - return 0; -} - -int get_object(ObjectStore *store, coll_t coll, bufferlist &bl) -{ - ObjectStore::Transaction tran; - ObjectStore::Transaction *t = &tran; - bufferlist::iterator ebliter = bl.begin(); - object_begin ob; - ob.decode(ebliter); - OSDriver driver( - store, - coll_t(), - OSD::make_snapmapper_oid()); - spg_t pg; - coll.is_pg_prefix(pg); - SnapMapper mapper(&driver, 0, 0, 0, pg.shard); - - t->touch(coll, ob.hoid); - - if (debug) { - ostringstream objname; - objname << ob.hoid.oid; - cout << "name " << objname.str() << " snap " << ob.hoid.snap << std::endl; - } - - bufferlist ebl; - bool done = false; - while(!done) { - sectiontype_t type; - int ret = read_section(file_fd, &type, &ebl); - if (ret) - return ret; - - //cout << "\tdo_object: Section type " << hex << type << dec << std::endl; - //cout << "\t\tsection size " << ebl.length() << std::endl; - if (type >= END_OF_TYPES) { - cout << "Skipping unknown object section type" << std::endl; - continue; - } - switch(type) { - case TYPE_DATA: - ret = get_data(store, coll, ob.hoid, t, ebl); - if (ret) return ret; - break; - case TYPE_ATTRS: - ret = get_attrs(store, coll, ob.hoid, t, ebl, driver, mapper); - if (ret) return ret; - break; - case TYPE_OMAP_HDR: - ret = get_omap_hdr(store, coll, ob.hoid, t, ebl); - if (ret) return ret; - break; - case TYPE_OMAP: - ret = get_omap(store, coll, ob.hoid, t, ebl); - if (ret) return ret; - break; - case TYPE_OBJECT_END: - done = true; - break; - default: - return EFAULT; - } - } - store->apply_transaction(*t); - return 0; -} - -int get_pg_metadata(ObjectStore *store, coll_t coll, bufferlist &bl) -{ - ObjectStore::Transaction tran; - ObjectStore::Transaction *t = &tran; - bufferlist::iterator ebliter = bl.begin(); - metadata_section ms; - ms.decode(ebliter); - -#if DIAGNOSTIC - Formatter *formatter = new JSONFormatter(true); - cout << "struct_v " << (int)ms.struct_ver << std::endl; - cout << "epoch " << ms.map_epoch << std::endl; - formatter->open_object_section("info"); - ms.info.dump(formatter); - formatter->close_section(); - formatter->flush(cout); - cout << std::endl; - - formatter->open_object_section("log"); - ms.log.dump(formatter); - formatter->close_section(); - formatter->flush(cout); - cout << std::endl; -#endif - - coll_t newcoll(ms.info.pgid); - t->collection_rename(coll, newcoll); - - int ret = write_pg(*t, ms.map_epoch, ms.info, ms.log, ms.struct_ver); - if (ret) return ret; - - store->apply_transaction(*t); - - return 0; -} - -int do_import(ObjectStore *store, OSDSuperblock& sb) -{ - bufferlist ebl; - pg_info_t info; - PGLog::IndexedLog log; - - uint64_t next_removal_seq = 0; //My local seq - finish_remove_pgs(store, &next_removal_seq); - - int ret = sh.read_super(); - if (ret) - return ret; - - if (sh.magic != super_header::super_magic) { - cout << "Invalid magic number" << std::endl; - return EFAULT; - } - - if (sh.version > super_header::super_ver) { - cout << "Can't handle export format version=" << sh.version << std::endl; - return EINVAL; - } - - //First section must be TYPE_PG_BEGIN - sectiontype_t type; - ret = read_section(file_fd, &type, &ebl); - if (type != TYPE_PG_BEGIN) { - return EFAULT; - } - - bufferlist::iterator ebliter = ebl.begin(); - pg_begin pgb; - pgb.decode(ebliter); - pg_t pgid = pgb.pgid; - - if (debug) { - cout << "Exported features: " << pgb.superblock.compat_features << std::endl; - } - if (sb.compat_features.compare(pgb.superblock.compat_features) == -1) { - cout << "Export has incompatible features set " - << pgb.superblock.compat_features << std::endl; - return 1; - } - - log_oid = OSD::make_pg_log_oid(spg_t(pgid, ghobject_t::no_shard())); - biginfo_oid = OSD::make_pg_biginfo_oid(spg_t(pgid, ghobject_t::no_shard())); - - //Check for PG already present. - coll_t coll(spg_t(pgid, ghobject_t::no_shard())); - if (store->collection_exists(coll)) { - cout << "pgid " << pgid << " already exists" << std::endl; - return 1; - } - - //Switch to collection which will be removed automatically if - //this program is interupted. - coll_t rmcoll = coll_t::make_removal_coll( - next_removal_seq, spg_t(pgid, ghobject_t::no_shard())); - ObjectStore::Transaction *t = new ObjectStore::Transaction; - t->create_collection(rmcoll); - store->apply_transaction(*t); - delete t; - - cout << "Importing pgid " << pgid << std::endl; - - bool done = false; - bool found_metadata = false; - while(!done) { - ret = read_section(file_fd, &type, &ebl); - if (ret) - return ret; - - //cout << "do_import: Section type " << hex << type << dec << std::endl; - if (type >= END_OF_TYPES) { - cout << "Skipping unknown section type" << std::endl; - continue; - } - switch(type) { - case TYPE_OBJECT_BEGIN: - ret = get_object(store, rmcoll, ebl); - if (ret) return ret; - break; - case TYPE_PG_METADATA: - ret = get_pg_metadata(store, rmcoll, ebl); - if (ret) return ret; - found_metadata = true; - break; - case TYPE_PG_END: - done = true; - break; - default: - return EFAULT; - } - } - - if (!found_metadata) { - cout << "Missing metadata section" << std::endl; - return EFAULT; - } - - return 0; -} - -int main(int argc, char **argv) -{ - string fspath, jpath, pgidstr, type, file; - Formatter *formatter = new JSONFormatter(true); - - po::options_description desc("Allowed options"); - desc.add_options() - ("help", "produce help message") - ("filestore-path", po::value(&fspath), - "path to filestore directory, mandatory") - ("journal-path", po::value(&jpath), - "path to journal, mandatory") - ("pgid", po::value(&pgidstr), - "PG id, mandatory") - ("type", po::value(&type), - "Type one of info, log, remove, export, or import, mandatory") - ("file", po::value(&file), - "path of file to export or import") - ("debug", "Enable diagnostic output to stderr") - ; - - po::variables_map vm; - po::parsed_options parsed = - po::command_line_parser(argc, argv).options(desc).allow_unregistered().run(); - po::store( parsed, vm); - try { - po::notify(vm); - } - catch(...) { - cout << desc << std::endl; - exit(1); - } - - if (vm.count("help")) { - cout << desc << std::endl; - return 1; - } - - if (!vm.count("filestore-path")) { - cout << "Must provide filestore-path" << std::endl - << desc << std::endl; - return 1; - } - if (!vm.count("journal-path")) { - cout << "Must provide journal-path" << std::endl - << desc << std::endl; - return 1; - } - if (!vm.count("type")) { - cout << "Must provide type (info, log, remove, export, import)" - << std::endl << desc << std::endl; - return 1; - } - if (type != "import" && !vm.count("pgid")) { - cout << "Must provide pgid" << std::endl - << desc << std::endl; - return 1; - } - - file_fd = fd_none; - if (type == "export") { - if (!vm.count("file")) { - file_fd = STDOUT_FILENO; - } else { - file_fd = open(file.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666); - } - } else if (type == "import") { - if (!vm.count("file")) { - file_fd = STDIN_FILENO; - } else { - file_fd = open(file.c_str(), O_RDONLY); - } - } - - if (vm.count("file") && file_fd == fd_none) { - cout << "--file option only applies to import or export" << std::endl; - return 1; - } - - if (file_fd != fd_none && file_fd < 0) { - perror("open"); - return 1; - } - - if ((fspath.length() == 0 || jpath.length() == 0) || - (type != "info" && type != "log" && type != "remove" && type != "export" - && type != "import") || - (type != "import" && pgidstr.length() == 0)) { - cerr << "Invalid params" << std::endl; - exit(1); - } - - if (type == "import" && pgidstr.length()) { - cerr << "--pgid option invalid with import" << std::endl; - exit(1); - } - - vector ceph_options, def_args; - vector ceph_option_strings = po::collect_unrecognized( - parsed.options, po::include_positional); - ceph_options.reserve(ceph_option_strings.size()); - for (vector::iterator i = ceph_option_strings.begin(); - i != ceph_option_strings.end(); - ++i) { - ceph_options.push_back(i->c_str()); - } - - //Suppress derr() output to stderr by default - if (!vm.count("debug")) { - close(STDERR_FILENO); - (void)open("/dev/null", O_WRONLY); - debug = false; - } else { - debug = true; - } - - global_init( - &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD, - CODE_ENVIRONMENT_UTILITY, 0); - //CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); - common_init_finish(g_ceph_context); - g_ceph_context->_conf->apply_changes(NULL); - g_conf = g_ceph_context->_conf; - - //Verify that fspath really is an osd store - struct stat st; - if (::stat(fspath.c_str(), &st) == -1) { - perror("fspath"); - invalid_path(fspath); - } - if (!S_ISDIR(st.st_mode)) { - invalid_path(fspath); - } - string check = fspath + "/whoami"; - if (::stat(check.c_str(), &st) == -1) { - perror("whoami"); - invalid_path(fspath); - } - if (!S_ISREG(st.st_mode)) { - invalid_path(fspath); - } - check = fspath + "/current"; - if (::stat(check.c_str(), &st) == -1) { - perror("current"); - invalid_path(fspath); - } - if (!S_ISDIR(st.st_mode)) { - invalid_path(fspath); - } - - pg_t pgid; - if (pgidstr.length() && !pgid.parse(pgidstr.c_str())) { - cout << "Invalid pgid '" << pgidstr << "' specified" << std::endl; - exit(1); - } - - ObjectStore *fs = new FileStore(fspath, jpath); - - int r = fs->mount(); - if (r < 0) { - if (r == -EBUSY) { - cout << "OSD has the store locked" << std::endl; - } else { - cout << "Mount failed with '" << cpp_strerror(-r) << "'" << std::endl; - } - return 1; - } - - bool fs_sharded_objects = fs->get_allow_sharded_objects(); - - int ret = 0; - vector ls; - vector::iterator it; - CompatSet supported; - -#ifdef INTERNAL_TEST - supported = get_test_compat_set(); -#else - supported = OSD::get_osd_compat_set(); -#endif - - bufferlist bl; - OSDSuperblock superblock; - bufferlist::iterator p; - ret = fs->read(coll_t::META_COLL, OSD_SUPERBLOCK_POBJECT, 0, 0, bl); - if (ret < 0) { - cout << "Failure to read OSD superblock error= " << r << std::endl; - goto out; - } - - p = bl.begin(); - ::decode(superblock, p); - -#ifdef INTERNAL_TEST2 - fs->set_allow_sharded_objects(); - assert(fs->get_allow_sharded_objects()); - fs_sharded_objects = true; - superblock.compat_features.incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS); -#endif - - if (debug && file_fd != STDOUT_FILENO) { - cout << "Supported features: " << supported << std::endl; - cout << "On-disk features: " << superblock.compat_features << std::endl; - } - if (supported.compare(superblock.compat_features) == -1) { - cout << "On-disk OSD incompatible features set " - << superblock.compat_features << std::endl; - ret = EINVAL; - goto out; - } - - // If there was a crash as an OSD was transitioning to sharded objects - // and hadn't completed a set_allow_sharded_objects(). - // This utility does not want to attempt to finish that transition. - if (superblock.compat_features.incompat.contains(CEPH_OSD_FEATURE_INCOMPAT_SHARDS) != fs_sharded_objects) { - // An OSD should never have call set_allow_sharded_objects() before - // updating its own OSD features. - if (fs_sharded_objects) - cout << "FileStore sharded but OSD not set, Corruption?" << std::endl; - else - cout << "Found incomplete transition to sharded objects" << std::endl; - ret = EINVAL; - goto out; - } - - if (type == "import") { - - try { - ret = do_import(fs, superblock); - } - catch (const buffer::error &e) { - cout << "do_import threw exception error " << e.what() << std::endl; - ret = EFAULT; - } - if (ret == EFAULT) { - cout << "Corrupt input for import" << std::endl; - } - goto out; - } - - log_oid = OSD::make_pg_log_oid(spg_t(pgid, ghobject_t::no_shard())); - biginfo_oid = OSD::make_pg_biginfo_oid(spg_t(pgid, ghobject_t::no_shard())); - - if (type == "remove") { - uint64_t next_removal_seq = 0; //My local seq - finish_remove_pgs(fs, &next_removal_seq); - int r = initiate_new_remove_pg(fs, pgid, &next_removal_seq); - if (r) { - cout << "PG '" << pgid << "' not found" << std::endl; - ret = 1; - goto out; - } - finish_remove_pgs(fs, &next_removal_seq); - cout << "Remove successful" << std::endl; - goto out; - } - - r = fs->list_collections(ls); - if (r < 0) { - cout << "failed to list pgs: " << cpp_strerror(-r) << std::endl; - exit(1); - } - - for (it = ls.begin(); it != ls.end(); ++it) { - snapid_t snap; - spg_t tmppgid; - - if (!it->is_pg(tmppgid, snap)) { - continue; - } - - if (tmppgid.pgid != pgid) { - continue; - } - if (snap != CEPH_NOSNAP && debug) { - cerr << "skipping snapped dir " << *it - << " (pg " << pgid << " snap " << snap << ")" << std::endl; - continue; - } - - //Found! - break; - } - - epoch_t map_epoch; - if (it != ls.end()) { - - coll_t coll = *it; - - bufferlist bl; - map_epoch = PG::peek_map_epoch(fs, coll, infos_oid, &bl); - if (debug) - cerr << "map_epoch " << map_epoch << std::endl; - - pg_info_t info(spg_t(pgid, ghobject_t::no_shard())); - map past_intervals; - hobject_t biginfo_oid = OSD::make_pg_biginfo_oid( - spg_t(pgid, ghobject_t::no_shard())); - interval_set snap_collections; - - __u8 struct_ver; - r = PG::read_info(fs, coll, bl, info, past_intervals, biginfo_oid, - infos_oid, snap_collections, struct_ver); - if (r < 0) { - cout << "read_info error " << cpp_strerror(-r) << std::endl; - ret = 1; - goto out; - } - if (debug) - cerr << "struct_v " << (int)struct_ver << std::endl; - - if (type == "export") { - ret = do_export(fs, coll, pgid, info, map_epoch, struct_ver, superblock); - } else if (type == "info") { - formatter->open_object_section("info"); - info.dump(formatter); - formatter->close_section(); - formatter->flush(cout); - cout << std::endl; - } else if (type == "log") { - PGLog::IndexedLog log; - pg_missing_t missing; - ret = get_log(fs, coll, pgid, info, log, missing); - if (ret > 0) - goto out; - - formatter->open_object_section("log"); - log.dump(formatter); - formatter->close_section(); - formatter->flush(cout); - cout << std::endl; - formatter->open_object_section("missing"); - missing.dump(formatter); - formatter->close_section(); - formatter->flush(cout); - cout << std::endl; - } - } else { - cout << "PG '" << pgid << "' not found" << std::endl; - ret = 1; - } - -out: - if (fs->umount() < 0) { - cout << "umount failed" << std::endl; - return 1; - } - - return (ret != 0); -} - diff --git a/src/tools/ceph-filestore-tool.cc b/src/tools/ceph-filestore-tool.cc deleted file mode 100644 index eb9f8dac36d05..0000000000000 --- a/src/tools/ceph-filestore-tool.cc +++ /dev/null @@ -1,260 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Inktank - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/Formatter.h" - -#include "global/global_init.h" -#include "os/ObjectStore.h" -#include "os/FileStore.h" -#include "common/perf_counters.h" -#include "common/errno.h" -#include "osd/PGLog.h" -#include "osd/osd_types.h" -#include "osd/OSD.h" - -namespace po = boost::program_options; -using namespace std; - -static void invalid_path(string &path) -{ - cout << "Invalid path to osd store specified: " << path << "\n"; - exit(1); -} - -int main(int argc, char **argv) -{ - string fspath, jpath, pgidstr; - bool list_lost_objects = false; - bool fix_lost_objects = false; - unsigned LIST_AT_A_TIME = 100; - unsigned scanned = 0; - - po::options_description desc("Allowed options"); - desc.add_options() - ("help", "produce help message") - ("filestore-path", po::value(&fspath), - "path to filestore directory, mandatory") - ("journal-path", po::value(&jpath), - "path to journal, mandatory") - ("pgid", po::value(&pgidstr), - "PG id") - ("list-lost-objects", po::value( - &list_lost_objects)->default_value(false), - "list lost objects") - ("fix-lost-objects", po::value( - &fix_lost_objects)->default_value(false), - "fix lost objects") - ; - - po::variables_map vm; - po::parsed_options parsed = - po::command_line_parser(argc, argv).options(desc). - allow_unregistered().run(); - po::store( parsed, vm); - try { - po::notify(vm); - } - catch(...) { - cout << desc << std::endl; - exit(1); - } - - if (vm.count("help")) { - cout << desc << std::endl; - return 1; - } - - if (!vm.count("filestore-path")) { - cerr << "Must provide filestore-path" << std::endl - << desc << std::endl; - return 1; - } - if (!vm.count("journal-path")) { - cerr << "Must provide journal-path" << std::endl - << desc << std::endl; - return 1; - } - - if ((fspath.length() == 0 || jpath.length() == 0)) { - cerr << "Invalid params" << desc << std::endl; - exit(1); - } - - vector ceph_options, def_args; - vector ceph_option_strings = po::collect_unrecognized( - parsed.options, po::include_positional); - ceph_options.reserve(ceph_option_strings.size()); - for (vector::iterator i = ceph_option_strings.begin(); - i != ceph_option_strings.end(); - ++i) { - ceph_options.push_back(i->c_str()); - } - - global_init( - &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD, - CODE_ENVIRONMENT_UTILITY, 0); - //CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); - common_init_finish(g_ceph_context); - g_ceph_context->_conf->apply_changes(NULL); - g_conf = g_ceph_context->_conf; - - //Verify that fspath really is an osd store - struct stat st; - if (::stat(fspath.c_str(), &st) == -1) { - perror("fspath"); - invalid_path(fspath); - } - if (!S_ISDIR(st.st_mode)) { - invalid_path(fspath); - } - string check = fspath + "/whoami"; - if (::stat(check.c_str(), &st) == -1) { - perror("whoami"); - invalid_path(fspath); - } - if (!S_ISREG(st.st_mode)) { - invalid_path(fspath); - } - check = fspath + "/current"; - if (::stat(check.c_str(), &st) == -1) { - perror("current"); - invalid_path(fspath); - } - if (!S_ISDIR(st.st_mode)) { - invalid_path(fspath); - } - - ObjectStore *fs = new FileStore(fspath, jpath); - - int r = fs->mount(); - if (r < 0) { - if (r == -EBUSY) { - cout << "OSD has the store locked" << std::endl; - } else { - cout << "Mount failed with '" << cpp_strerror(-r) << "'" << std::endl; - } - return 1; - } - - vector colls_to_check; - if (pgidstr.length()) { - spg_t pgid; - if (!pgid.parse(pgidstr.c_str())) { - cout << "Invalid pgid '" << pgidstr << "' specified" << std::endl; - exit(1); - } - colls_to_check.push_back(coll_t(pgid)); - } else { - vector candidates; - r = fs->list_collections(candidates); - if (r < 0) { - cerr << "Error listing collections: " << cpp_strerror(r) << std::endl; - goto UMOUNT; - } - for (vector::iterator i = candidates.begin(); - i != candidates.end(); - ++i) { - spg_t pgid; - snapid_t snap; - if (i->is_pg(pgid, snap)) { - colls_to_check.push_back(*i); - } - } - } - - cerr << colls_to_check.size() << " pgs to scan" << std::endl; - for (vector::iterator i = colls_to_check.begin(); - i != colls_to_check.end(); - ++i, ++scanned) { - cerr << "Scanning " << *i << ", " << scanned << "/" - << colls_to_check.size() << " completed" << std::endl; - ghobject_t next; - while (!next.is_max()) { - vector list; - r = fs->collection_list_partial( - *i, - next, - LIST_AT_A_TIME, - LIST_AT_A_TIME, - CEPH_NOSNAP, - &list, - &next); - if (r < 0) { - cerr << "Error listing collection: " << *i << ", " - << cpp_strerror(r) << std::endl; - goto UMOUNT; - } - for (vector::iterator obj = list.begin(); - obj != list.end(); - ++obj) { - bufferlist attr; - r = fs->getattr(*i, *obj, OI_ATTR, attr); - if (r < 0) { - cerr << "Error getting attr on : " << make_pair(*i, *obj) << ", " - << cpp_strerror(r) << std::endl; - goto UMOUNT; - } - object_info_t oi; - bufferlist::iterator bp = attr.begin(); - try { - ::decode(oi, bp); - } catch (...) { - r = -EINVAL; - cerr << "Error getting attr on : " << make_pair(*i, *obj) << ", " - << cpp_strerror(r) << std::endl; - goto UMOUNT; - } - if (oi.is_lost()) { - if (list_lost_objects) { - cout << *i << "/" << *obj << " is lost" << std::endl; - } - if (fix_lost_objects) { - cerr << *i << "/" << *obj << " is lost, fixing" << std::endl; - oi.clear_flag(object_info_t::FLAG_LOST); - bufferlist bl2; - ::encode(oi, bl2); - ObjectStore::Transaction t; - t.setattr(*i, *obj, OI_ATTR, bl2); - r = fs->apply_transaction(t); - if (r < 0) { - cerr << "Error getting fixing attr on : " << make_pair(*i, *obj) - << ", " - << cpp_strerror(r) << std::endl; - goto UMOUNT; - } - } - } - } - } - } - cerr << "Completed" << std::endl; - - UMOUNT: - fs->sync_and_flush(); - fs->umount(); - return r; -} diff --git a/src/tools/ceph-kvstore-tool.cc b/src/tools/ceph-kvstore-tool.cc deleted file mode 100644 index 5a68d98990dfc..0000000000000 --- a/src/tools/ceph-kvstore-tool.cc +++ /dev/null @@ -1,424 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* -* Ceph - scalable distributed file system -* -* Copyright (C) 2012 Inktank, Inc. -* -* This is free software; you can redistribute it and/or -* modify it under the terms of the GNU Lesser General Public -* License version 2.1, as published by the Free Software -* Foundation. See file COPYING. -*/ -#include -#include -#include -#include -#include -#include - -#include "os/LevelDBStore.h" - -#include "common/ceph_argparse.h" -#include "global/global_init.h" -#include "common/errno.h" -#include "common/safe_io.h" -#include "common/config.h" -#include "common/strtol.h" -#include "include/stringify.h" - -using namespace std; - -class StoreTool -{ - boost::scoped_ptr db; - string store_path; - - public: - StoreTool(const string &path) : store_path(path) { - LevelDBStore *db_ptr = new LevelDBStore(g_ceph_context, store_path); - assert(!db_ptr->open(std::cerr)); - db.reset(db_ptr); - } - - uint32_t traverse(const string &prefix, - const bool do_crc, - ostream *out) { - KeyValueDB::WholeSpaceIterator iter = db->get_iterator(); - - if (prefix.empty()) - iter->seek_to_first(); - else - iter->seek_to_first(prefix); - - uint32_t crc = -1; - - while (iter->valid()) { - pair rk = iter->raw_key(); - if (!prefix.empty() && (rk.first != prefix)) - break; - - if (out) - *out << rk.first << ":" << rk.second; - if (do_crc) { - bufferlist bl; - bl.append(rk.first); - bl.append(rk.second); - bl.append(iter->value()); - - crc = bl.crc32c(crc); - if (out) { - *out << " (" << bl.crc32c(0) << ")"; - } - } - if (out) - *out << std::endl; - iter->next(); - } - - return crc; - } - - void list(const string &prefix, const bool do_crc) { - traverse(prefix, do_crc, &std::cout); - } - - bool exists(const string &prefix) { - assert(!prefix.empty()); - KeyValueDB::WholeSpaceIterator iter = db->get_iterator(); - iter->seek_to_first(prefix); - return (iter->valid() && (iter->raw_key().first == prefix)); - } - - bool exists(const string &prefix, const string &key) { - assert(!prefix.empty()); - - if (key.empty()) { - return exists(prefix); - } - - bool exists = false; - get(prefix, key, exists); - return exists; - } - - bufferlist get(const string &prefix, const string &key, bool &exists) { - assert(!prefix.empty() && !key.empty()); - - map result; - std::set keys; - keys.insert(key); - db->get(prefix, keys, &result); - - if (result.count(key) > 0) { - exists = true; - return result[key]; - } - exists = false; - return bufferlist(); - } - - uint64_t get_size() { - map extras; - uint64_t s = db->get_estimated_size(extras); - for (map::iterator p = extras.begin(); - p != extras.end(); ++p) { - std::cout << p->first << " - " << p->second << std::endl; - } - std::cout << "total: " << s << std::endl; - return s; - } - - bool set(const string &prefix, const string &key, bufferlist &val) { - assert(!prefix.empty()); - assert(!key.empty()); - assert(val.length() > 0); - - KeyValueDB::Transaction tx = db->get_transaction(); - tx->set(prefix, key, val); - int ret = db->submit_transaction_sync(tx); - - return (ret == 0); - } - - int copy_store_to(const string &other_path, const int num_keys_per_tx) { - - if (num_keys_per_tx <= 0) { - std::cerr << "must specify a number of keys/tx > 0" << std::endl; - return -EINVAL; - } - - // open or create a leveldb store at @p other_path - LevelDBStore other(g_ceph_context, other_path); - int err = other.create_and_open(std::cerr); - if (err < 0) - return err; - - KeyValueDB::WholeSpaceIterator it = db->get_iterator(); - it->seek_to_first(); - uint64_t total_keys = 0; - uint64_t total_size = 0; - uint64_t total_txs = 0; - - utime_t started_at = ceph_clock_now(g_ceph_context); - - do { - int num_keys = 0; - - KeyValueDB::Transaction tx = other.get_transaction(); - - - while (it->valid() && num_keys < num_keys_per_tx) { - pair k = it->raw_key(); - bufferlist v = it->value(); - tx->set(k.first, k.second, v); - - num_keys ++; - total_size += v.length(); - - it->next(); - } - - total_txs ++; - total_keys += num_keys; - - if (num_keys > 0) - other.submit_transaction_sync(tx); - - utime_t cur_duration = ceph_clock_now(g_ceph_context) - started_at; - std::cout << "ts = " << cur_duration << "s, copied " << total_keys - << " keys so far (" << stringify(si_t(total_size)) << ")" - << std::endl; - - } while (it->valid()); - - utime_t time_taken = ceph_clock_now(g_ceph_context) - started_at; - - std::cout << "summary:" << std::endl; - std::cout << " copied " << total_keys << " keys" << std::endl; - std::cout << " used " << total_txs << " transactions" << std::endl; - std::cout << " total size " << stringify(si_t(total_size)) << std::endl; - std::cout << " from '" << store_path << "' to '" << other_path << "'" - << std::endl; - std::cout << " duration " << time_taken << " seconds" << std::endl; - - return 0; - } -}; - -void usage(const char *pname) -{ - std::cerr << "Usage: " << pname << " command [args...]\n" - << "\n" - << "Commands:\n" - << " list [prefix]\n" - << " list-crc [prefix]\n" - << " exists [key]\n" - << " get [out ]\n" - << " crc \n" - << " get-size [ ]\n" - << " set [ver |in ]\n" - << " store-copy [num-keys-per-tx]\n" - << " store-crc \n" - << std::endl; -} - -int main(int argc, const char *argv[]) -{ - vector args; - argv_to_vec(argc, argv, args); - env_to_vec(args); - - global_init( - NULL, args, - CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); - common_init_finish(g_ceph_context); - - - if (args.size() < 2) { - usage(argv[0]); - return 1; - } - - string path(args[0]); - string cmd(args[1]); - - StoreTool st(path); - - if (cmd == "list" || cmd == "list-crc") { - string prefix; - if (argc > 3) - prefix = argv[3]; - - bool do_crc = (cmd == "list-crc"); - - st.list(prefix, do_crc); - - } else if (cmd == "exists") { - string key; - if (argc < 4) { - usage(argv[0]); - return 1; - } - string prefix(argv[3]); - if (argc > 4) - key = argv[4]; - - bool ret = st.exists(prefix, key); - std::cout << "(" << prefix << ", " << key << ") " - << (ret ? "exists" : "does not exist") - << std::endl; - return (ret ? 0 : 1); - - } else if (cmd == "get") { - if (argc < 5) { - usage(argv[0]); - return 1; - } - string prefix(argv[3]); - string key(argv[4]); - - bool exists = false; - bufferlist bl = st.get(prefix, key, exists); - std::cout << "(" << prefix << ", " << key << ")"; - if (!exists) { - std::cout << " does not exist" << std::endl; - return 1; - } - std::cout << std::endl; - - if (argc >= 6) { - string subcmd(argv[5]); - string out(argv[6]); - - if (subcmd != "out") { - std::cerr << "unrecognized subcmd '" << subcmd << "'" - << std::endl; - return 1; - } - - if (out.empty()) { - std::cerr << "unspecified out file" << std::endl; - return 1; - } - - int err = bl.write_file(argv[6], 0644); - if (err < 0) { - std::cerr << "error writing value to '" << out << "': " - << cpp_strerror(err) << std::endl; - return 1; - } - } else { - ostringstream os; - bl.hexdump(os); - std::cout << os.str() << std::endl; - } - - } else if (cmd == "crc") { - if (argc < 5) { - usage(argv[0]); - return 1; - } - string prefix(argv[3]); - string key(argv[4]); - - bool exists = false; - bufferlist bl = st.get(prefix, key, exists); - std::cout << "(" << prefix << ", " << key << ") "; - if (!exists) { - std::cout << " does not exist" << std::endl; - return 1; - } - std::cout << " crc " << bl.crc32c(0) << std::endl; - - } else if (cmd == "get-size") { - std::cout << "estimated store size: " << st.get_size() << std::endl; - - if (argc < 4) - return 0; - - if (argc < 5) { - usage(argv[0]); - return 1; - } - string prefix(argv[3]); - string key(argv[4]); - - bool exists = false; - bufferlist bl = st.get(prefix, key, exists); - if (!exists) { - std::cerr << "(" << prefix << "," << key - << ") does not exist" << std::endl; - return 1; - } - std::cout << "(" << prefix << "," << key - << ") size " << si_t(bl.length()) << std::endl; - - } else if (cmd == "set") { - if (argc < 7) { - usage(argv[0]); - return 1; - } - string prefix(argv[3]); - string key(argv[4]); - string subcmd(argv[5]); - - bufferlist val; - string errstr; - if (subcmd == "ver") { - version_t v = (version_t) strict_strtoll(argv[6], 10, &errstr); - if (!errstr.empty()) { - std::cerr << "error reading version: " << errstr << std::endl; - return 1; - } - ::encode(v, val); - } else if (subcmd == "in") { - int ret = val.read_file(argv[6], &errstr); - if (ret < 0 || !errstr.empty()) { - std::cerr << "error reading file: " << errstr << std::endl; - return 1; - } - } else { - std::cerr << "unrecognized subcommand '" << subcmd << "'" << std::endl; - usage(argv[0]); - return 1; - } - - bool ret = st.set(prefix, key, val); - if (!ret) { - std::cerr << "error setting (" - << prefix << "," << key << ")" << std::endl; - return 1; - } - } else if (cmd == "store-copy") { - int num_keys_per_tx = 128; // magic number that just feels right. - if (argc < 4) { - usage(argv[0]); - return 1; - } else if (argc > 4) { - string err; - num_keys_per_tx = strict_strtol(argv[4], 10, &err); - if (!err.empty()) { - std::cerr << "invalid num_keys_per_tx: " << err << std::endl; - return 1; - } - } - - int ret = st.copy_store_to(argv[3], num_keys_per_tx); - if (ret < 0) { - std::cerr << "error copying store to path '" << argv[3] - << "': " << cpp_strerror(ret) << std::endl; - return 1; - } - - } else if (cmd == "store-crc") { - uint32_t crc = st.traverse(string(), true, NULL); - std::cout << "store at '" << path << "' crc " << crc << std::endl; - - } else { - std::cerr << "Unrecognized command: " << cmd << std::endl; - return 1; - } - - return 0; -} diff --git a/src/tools/ceph-monstore-tool.cc b/src/tools/ceph-monstore-tool.cc deleted file mode 100644 index 8f294c4a4e341..0000000000000 --- a/src/tools/ceph-monstore-tool.cc +++ /dev/null @@ -1,419 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* -* Ceph - scalable distributed file system -* -* Copyright (C) 2012 Inktank, Inc. -* -* This is free software; you can redistribute it and/or -* modify it under the terms of the GNU Lesser General Public -* License version 2.1, as published by the Free Software -* Foundation. See file COPYING. -*/ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "global/global_init.h" -#include "os/LevelDBStore.h" -#include "mon/MonitorDBStore.h" -#include "mon/Paxos.h" -#include "common/Formatter.h" -#include "include/stringify.h" - -namespace po = boost::program_options; -using namespace std; - -class TraceIter { - int fd; - unsigned idx; - MonitorDBStore::Transaction t; -public: - TraceIter(string fname) : fd(-1), idx(-1) { - fd = ::open(fname.c_str(), O_RDONLY); - } - bool valid() { - return fd != -1; - } - const MonitorDBStore::Transaction &cur() { - assert(valid()); - return t; - } - unsigned num() { return idx; } - void next() { - ++idx; - bufferlist bl; - int r = bl.read_fd(fd, 6); - if (r < 0) { - std::cerr << "Got error: " << cpp_strerror(r) << " on read_fd" - << std::endl; - ::close(fd); - fd = -1; - return; - } else if ((unsigned)r < 6) { - std::cerr << "short read" << std::endl; - ::close(fd); - fd = -1; - return; - } - bufferlist::iterator bliter = bl.begin(); - uint8_t ver, ver2; - ::decode(ver, bliter); - ::decode(ver2, bliter); - uint32_t len; - ::decode(len, bliter); - r = bl.read_fd(fd, len); - if (r < 0) { - std::cerr << "Got error: " << cpp_strerror(r) << " on read_fd" - << std::endl; - ::close(fd); - fd = -1; - return; - } else if ((unsigned)r < len) { - std::cerr << "short read" << std::endl; - ::close(fd); - fd = -1; - return; - } - bliter = bl.begin(); - t.decode(bliter); - } - void init() { - next(); - } - ~TraceIter() { - if (fd != -1) { - ::close(fd); - fd = -1; - } - } -}; - -int main(int argc, char **argv) { - po::options_description desc("Allowed options"); - int version = -1; - string store_path, cmd, out_path, tfile; - unsigned dstart = 0; - unsigned dstop = ~0; - unsigned num_replays = 1; - unsigned tsize = 200; - unsigned tvalsize = 1024; - unsigned ntrans = 100; - desc.add_options() - ("help", "produce help message") - ("mon-store-path", po::value(&store_path), - "path to mon directory, mandatory") - ("out", po::value(&out_path), - "out path") - ("version", po::value(&version), - "version requested") - ("trace-file", po::value(&tfile), - "trace file") - ("dump-start", po::value(&dstart), - "transaction num to start dumping at") - ("dump-end", po::value(&dstop), - "transaction num to stop dumping at") - ("num-replays", po::value(&num_replays), - "number of times to replay") - ("trans-size", po::value(&tsize), - "keys to write in each transaction") - ("trans-val-size", po::value(&tvalsize), - "val to write in each key") - ("num-trans", po::value(&ntrans), - "number of transactions to run") - ("command", po::value(&cmd), - "command") - ; - po::positional_options_description p; - p.add("command", 1); - p.add("version", 1); - - po::variables_map vm; - po::parsed_options parsed = - po::command_line_parser(argc, argv).options(desc).positional(p).run(); - po::store( - parsed, - vm); - try { - po::notify(vm); - } catch (...) { - cout << desc << std::endl; - return 1; - } - - vector ceph_options, def_args; - vector ceph_option_strings = po::collect_unrecognized( - parsed.options, po::include_positional); - ceph_options.reserve(ceph_option_strings.size()); - for (vector::iterator i = ceph_option_strings.begin(); - i != ceph_option_strings.end(); - ++i) { - ceph_options.push_back(i->c_str()); - } - - global_init( - &def_args, ceph_options, CEPH_ENTITY_TYPE_MON, - CODE_ENVIRONMENT_UTILITY, 0); - common_init_finish(g_ceph_context); - g_ceph_context->_conf->apply_changes(NULL); - g_conf = g_ceph_context->_conf; - - if (vm.count("help")) { - std::cerr << desc << std::endl; - return 1; - } - - int fd; - if (vm.count("out")) { - if ((fd = open(out_path.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666)) < 0) { - int _err = errno; - if (_err != EISDIR) { - std::cerr << "Couldn't open " << out_path << ": " << cpp_strerror(_err) << std::endl; - return 1; - } - } - } else { - fd = STDOUT_FILENO; - } - - if (fd < 0 && cmd != "store-copy") { - std::cerr << "error: '" << out_path << "' is a directory!" << std::endl; - return 1; - } - - MonitorDBStore st(store_path); - if (store_path.size()) { - stringstream ss; - int r = st.open(ss); - if (r < 0) { - std::cerr << ss.str() << std::endl; - goto done; - } - } - if (cmd == "dump-keys") { - KeyValueDB::WholeSpaceIterator iter = st.get_iterator(); - while (iter->valid()) { - pair key(iter->raw_key()); - cout << key.first << " / " << key.second << std::endl; - iter->next(); - } - } else if (cmd == "compact") { - st.compact(); - } else if (cmd == "getmonmap") { - assert(fd >= 0); - if (!store_path.size()) { - std::cerr << "need mon store path" << std::endl; - std::cerr << desc << std::endl; - goto done; - } - version_t v; - if (version <= 0) { - v = st.get("monmap", "last_committed"); - } else { - v = version; - } - - bufferlist bl; - /// XXX: this is not ok, osdmap and full should be abstracted somewhere - int r = st.get("monmap", v, bl); - if (r < 0) { - std::cerr << "Error getting map: " << cpp_strerror(r) << std::endl; - goto done; - } - bl.write_fd(fd); - } else if (cmd == "getosdmap") { - if (!store_path.size()) { - std::cerr << "need mon store path" << std::endl; - std::cerr << desc << std::endl; - goto done; - } - version_t v; - if (version == -1) { - v = st.get("osdmap", "last_committed"); - } else { - v = version; - } - - bufferlist bl; - /// XXX: this is not ok, osdmap and full should be abstracted somewhere - int r = st.get("osdmap", st.combine_strings("full", v), bl); - if (r < 0) { - std::cerr << "Error getting map: " << cpp_strerror(r) << std::endl; - goto done; - } - bl.write_fd(fd); - } else if (cmd == "dump-paxos") { - for (version_t v = dstart; v <= dstop; ++v) { - bufferlist bl; - st.get("paxos", v, bl); - if (bl.length() == 0) - break; - cout << "\n--- " << v << " ---" << std::endl; - MonitorDBStore::Transaction tx; - Paxos::decode_append_transaction(tx, bl); - JSONFormatter f(true); - tx.dump(&f); - f.flush(cout); - } - } else if (cmd == "dump-trace") { - if (tfile.empty()) { - std::cerr << "Need trace_file" << std::endl; - std::cerr << desc << std::endl; - goto done; - } - TraceIter iter(tfile.c_str()); - iter.init(); - while (true) { - if (!iter.valid()) - break; - if (iter.num() >= dstop) { - break; - } - if (iter.num() >= dstart) { - JSONFormatter f(true); - iter.cur().dump(&f, false); - f.flush(std::cout); - std::cout << std::endl; - } - iter.next(); - } - std::cerr << "Read up to transaction " << iter.num() << std::endl; - } else if (cmd == "replay-trace") { - if (!store_path.size()) { - std::cerr << "need mon store path" << std::endl; - std::cerr << desc << std::endl; - goto done; - } - if (tfile.empty()) { - std::cerr << "Need trace_file" << std::endl; - std::cerr << desc << std::endl; - goto done; - } - unsigned num = 0; - for (unsigned i = 0; i < num_replays; ++i) { - TraceIter iter(tfile.c_str()); - iter.init(); - while (true) { - if (!iter.valid()) - break; - std::cerr << "Replaying trans num " << num << std::endl; - st.apply_transaction(iter.cur()); - iter.next(); - ++num; - } - std::cerr << "Read up to transaction " << iter.num() << std::endl; - } - } else if (cmd == "random-gen") { - if (!store_path.size()) { - std::cerr << "need mon store path" << std::endl; - std::cerr << desc << std::endl; - goto done; - } - unsigned num = 0; - for (unsigned i = 0; i < ntrans; ++i) { - std::cerr << "Applying trans " << i << std::endl; - MonitorDBStore::Transaction t; - string prefix; - prefix.push_back((i%26)+'a'); - for (unsigned j = 0; j < tsize; ++j) { - stringstream os; - os << num; - bufferlist bl; - for (unsigned k = 0; k < tvalsize; ++k) bl.append(rand()); - t.put(prefix, os.str(), bl); - ++num; - } - t.compact_prefix(prefix); - st.apply_transaction(t); - } - } else if (cmd == "store-copy") { - if (!store_path.size()) { - std::cerr << "need mon store path to copy from" << std::endl; - std::cerr << desc << std::endl; - goto done; - } - if (!out_path.size()) { - std::cerr << "need mon store path to copy to (--out )" - << std::endl; - std::cerr << desc << std::endl; - goto done; - } - if (fd > 0) { - std::cerr << "supplied out path '" << out_path << "' is not a directory" - << std::endl; - goto done; - } - - MonitorDBStore out_store(out_path); - { - stringstream ss; - int r = out_store.create_and_open(ss); - if (r < 0) { - std::cerr << ss.str() << std::endl; - goto done; - } - } - - - KeyValueDB::WholeSpaceIterator it = st.get_iterator(); - uint64_t total_keys = 0; - uint64_t total_size = 0; - uint64_t total_tx = 0; - - do { - uint64_t num_keys = 0; - - MonitorDBStore::Transaction tx; - - while (it->valid() && num_keys < 128) { - pair k = it->raw_key(); - bufferlist v = it->value(); - tx.put(k.first, k.second, v); - - num_keys ++; - total_tx ++; - total_size += v.length(); - - it->next(); - } - - total_keys += num_keys; - - if (!tx.empty()) - out_store.apply_transaction(tx); - - std::cout << "copied " << total_keys << " keys so far (" - << stringify(si_t(total_size)) << ")" << std::endl; - - } while (it->valid()); - - std::cout << "summary: copied " << total_keys << " keys, using " - << total_tx << " transactions, totalling " - << stringify(si_t(total_size)) << std::endl; - std::cout << "from '" << store_path << "' to '" << out_path << "'" - << std::endl; - } else { - std::cerr << "Unrecognized command: " << cmd << std::endl; - goto done; - } - - done: - if (vm.count("out") && fd > 0) { - ::close(fd); - } - return 0; -} diff --git a/src/tools/ceph-osdomap-tool.cc b/src/tools/ceph-osdomap-tool.cc deleted file mode 100644 index bde4b28b45f39..0000000000000 --- a/src/tools/ceph-osdomap-tool.cc +++ /dev/null @@ -1,161 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* -* Ceph - scalable distributed file system -* -* Copyright (C) 2012 Inktank, Inc. -* -* This is free software; you can redistribute it and/or -* modify it under the terms of the GNU Lesser General Public -* License kkjversion 2.1, as published by the Free Software -* Foundation. See file COPYING. -*/ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "global/global_init.h" -#include "os/LevelDBStore.h" -#include "mon/MonitorDBStore.h" -#include "os/DBObjectMap.h" - -namespace po = boost::program_options; -using namespace std; - -int main(int argc, char **argv) { - po::options_description desc("Allowed options"); - string store_path, cmd, out_path; - bool paranoid = false; - desc.add_options() - ("help", "produce help message") - ("omap-path", po::value(&store_path), - "path to mon directory, mandatory (current/omap usually)") - ("paranoid", po::value(¶noid), - "use paranoid checking") - ("command", po::value(&cmd), - "command") - ; - po::positional_options_description p; - p.add("command", 1); - - po::variables_map vm; - po::parsed_options parsed = - po::command_line_parser(argc, argv).options(desc).positional(p).run(); - po::store( - parsed, - vm); - try { - po::notify(vm); - } catch (...) { - cout << desc << std::endl; - return 1; - } - - vector ceph_options, def_args; - vector ceph_option_strings = po::collect_unrecognized( - parsed.options, po::include_positional); - ceph_options.reserve(ceph_option_strings.size()); - for (vector::iterator i = ceph_option_strings.begin(); - i != ceph_option_strings.end(); - ++i) { - ceph_options.push_back(i->c_str()); - } - - global_init( - &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD, - CODE_ENVIRONMENT_UTILITY, 0); - common_init_finish(g_ceph_context); - g_ceph_context->_conf->apply_changes(NULL); - g_conf = g_ceph_context->_conf; - - if (vm.count("help")) { - std::cerr << desc << std::endl; - return 1; - } - - LevelDBStore* store(new LevelDBStore(g_ceph_context, store_path)); - if (paranoid) { - std::cerr << "Enabling paranoid checks" << std::endl; - store->options.paranoid_checks = paranoid; - } - DBObjectMap omap(store); - stringstream out; - int r = store->open(out); - if (r < 0) { - std::cerr << "Store open got: " << cpp_strerror(r) << std::endl; - std::cerr << "Output: " << out.str() << std::endl; - goto done; - } - r = 0; - - - if (cmd == "dump-raw-keys") { - KeyValueDB::WholeSpaceIterator i = store->get_iterator(); - for (i->seek_to_first(); i->valid(); i->next()) { - std::cout << i->raw_key() << std::endl; - } - } else if (cmd == "dump-raw-key-vals") { - KeyValueDB::WholeSpaceIterator i = store->get_iterator(); - for (i->seek_to_first(); i->valid(); i->next()) { - std::cout << i->raw_key() << std::endl; - i->value().hexdump(std::cout); - } - } else if (cmd == "dump-objects") { - vector objects; - r = omap.list_objects(&objects); - if (r < 0) { - std::cerr << "list_objects got: " << cpp_strerror(r) << std::endl; - goto done; - } - for (vector::iterator i = objects.begin(); - i != objects.end(); - ++i) { - std::cout << *i << std::endl; - } - r = 0; - } else if (cmd == "dump-objects-with-keys") { - vector objects; - r = omap.list_objects(&objects); - if (r < 0) { - std::cerr << "list_objects got: " << cpp_strerror(r) << std::endl; - goto done; - } - for (vector::iterator i = objects.begin(); - i != objects.end(); - ++i) { - std::cout << "Object: " << *i << std::endl; - ObjectMap::ObjectMapIterator j = omap.get_iterator(i->hobj); - for (j->seek_to_first(); j->valid(); j->next()) { - std::cout << j->key() << std::endl; - j->value().hexdump(std::cout); - } - } - } else if (cmd == "check") { - r = omap.check(std::cout); - if (!r) { - std::cerr << "check got: " << cpp_strerror(r) << std::endl; - goto done; - } - std::cout << "check succeeded" << std::endl; - } else { - std::cerr << "Did not recognize command " << cmd << std::endl; - goto done; - } - - done: - return r; -} diff --git a/src/tools/ceph_filestore_dump.cc b/src/tools/ceph_filestore_dump.cc new file mode 100644 index 0000000000000..78ac8391fd21b --- /dev/null +++ b/src/tools/ceph_filestore_dump.cc @@ -0,0 +1,1405 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Inktank + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/Formatter.h" + +#include "global/global_init.h" +#include "os/ObjectStore.h" +#include "os/FileStore.h" +#include "common/perf_counters.h" +#include "common/errno.h" +#include "osd/PGLog.h" +#include "osd/OSD.h" + +namespace po = boost::program_options; +using namespace std; + +enum { + TYPE_NONE = 0, + TYPE_PG_BEGIN, + TYPE_PG_END, + TYPE_OBJECT_BEGIN, + TYPE_OBJECT_END, + TYPE_DATA, + TYPE_ATTRS, + TYPE_OMAP_HDR, + TYPE_OMAP, + TYPE_PG_METADATA, + END_OF_TYPES, //Keep at the end +}; + +//#define INTERNAL_TEST +//#define INTERNAL_TEST2 + +#ifdef INTERNAL_TEST +CompatSet get_test_compat_set() { + CompatSet::FeatureSet ceph_osd_feature_compat; + CompatSet::FeatureSet ceph_osd_feature_ro_compat; + CompatSet::FeatureSet ceph_osd_feature_incompat; + ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_BASE); + ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_PGINFO); + ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_OLOC); + ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEC); + ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_CATEGORIES); + ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_HOBJECTPOOL); + ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_BIGINFO); + ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEVELDBINFO); + ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEVELDBLOG); +#ifdef INTERNAL_TEST2 + ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SNAPMAPPER); + ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS); +#endif + return CompatSet(ceph_osd_feature_compat, ceph_osd_feature_ro_compat, + ceph_osd_feature_incompat); +} +#endif + +typedef uint8_t sectiontype_t; +typedef uint32_t mymagic_t; +typedef int64_t mysize_t; +const ssize_t max_read = 1024 * 1024; +const uint16_t shortmagic = 0xffce; //goes into stream as "ceff" +//endmagic goes into stream as "ceff ffec" +const mymagic_t endmagic = (0xecff << 16) | shortmagic; +const int fd_none = INT_MIN; + +//The first FIXED_LENGTH bytes are a fixed +//portion of the export output. This includes the overall +//version number, and size of header and footer. +//THIS STRUCTURE CAN ONLY BE APPENDED TO. If it needs to expand, +//the version can be bumped and then anything +//can be added to the export format. +struct super_header { + static const uint32_t super_magic = (shortmagic << 16) | shortmagic; + static const uint32_t super_ver = 2; + static const uint32_t FIXED_LENGTH = 16; + uint32_t magic; + uint32_t version; + uint32_t header_size; + uint32_t footer_size; + + super_header() : magic(0), version(0), header_size(0), footer_size(0) { } + int read_super(); + + void encode(bufferlist& bl) const { + ::encode(magic, bl); + ::encode(version, bl); + ::encode(header_size, bl); + ::encode(footer_size, bl); + } + void decode(bufferlist::iterator& bl) { + ::decode(magic, bl); + ::decode(version, bl); + ::decode(header_size, bl); + ::decode(footer_size, bl); + } +}; + +struct header { + sectiontype_t type; + mysize_t size; + header(sectiontype_t type, mysize_t size) : + type(type), size(size) { } + header(): type(0), size(0) { } + + int get_header(); + + void encode(bufferlist& bl) const { + uint32_t debug_type = (type << 24) | (type << 16) | shortmagic; + ENCODE_START(1, 1, bl); + ::encode(debug_type, bl); + ::encode(size, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + uint32_t debug_type; + DECODE_START(1, bl); + ::decode(debug_type, bl); + type = debug_type >> 24; + ::decode(size, bl); + DECODE_FINISH(bl); + } +}; + +struct footer { + mymagic_t magic; + footer() : magic(endmagic) { } + + int get_footer(); + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(magic, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(magic, bl); + DECODE_FINISH(bl); + } +}; + +struct pg_begin { + pg_t pgid; + OSDSuperblock superblock; + + pg_begin(pg_t pg, const OSDSuperblock& sb): + pgid(pg), superblock(sb) { } + pg_begin() { } + + void encode(bufferlist& bl) const { + // New super_ver prevents decode from ver 1 + ENCODE_START(2, 2, bl); + ::encode(pgid, bl); + ::encode(superblock, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(2, bl); + ::decode(pgid, bl); + if (struct_v > 1) { + ::decode(superblock, bl); + } + DECODE_FINISH(bl); + } +}; + +struct object_begin { + hobject_t hoid; + object_begin(const hobject_t &hoid): hoid(hoid) { } + object_begin() { } + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(hoid, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(hoid, bl); + DECODE_FINISH(bl); + } +}; + +struct data_section { + uint64_t offset; + uint64_t len; + bufferlist databl; + data_section(uint64_t offset, uint64_t len, bufferlist bl): + offset(offset), len(len), databl(bl) { } + data_section(): offset(0), len(0) { } + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(offset, bl); + ::encode(len, bl); + ::encode(databl, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(offset, bl); + ::decode(len, bl); + ::decode(databl, bl); + DECODE_FINISH(bl); + } +}; + +struct attr_section { + map data; + attr_section(const map &data) : data(data) { } + attr_section() { } + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(data, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(data, bl); + DECODE_FINISH(bl); + } +}; + +struct omap_hdr_section { + bufferlist hdr; + omap_hdr_section(bufferlist hdr) : hdr(hdr) { } + omap_hdr_section() { } + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(hdr, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(hdr, bl); + DECODE_FINISH(bl); + } +}; + +struct omap_section { + map omap; + omap_section(const map &omap) : + omap(omap) { } + omap_section() { } + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(omap, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(omap, bl); + DECODE_FINISH(bl); + } +}; + +struct metadata_section { + __u8 struct_ver; + epoch_t map_epoch; + pg_info_t info; + pg_log_t log; + + metadata_section(__u8 struct_ver, epoch_t map_epoch, const pg_info_t &info, + const pg_log_t &log) + : struct_ver(struct_ver), + map_epoch(map_epoch), + info(info), + log(log) { } + metadata_section() + : struct_ver(0), + map_epoch(0) { } + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(struct_ver, bl); + ::encode(map_epoch, bl); + ::encode(info, bl); + ::encode(log, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(struct_ver, bl); + ::decode(map_epoch, bl); + ::decode(info, bl); + ::decode(log, bl); + DECODE_FINISH(bl); + } +}; + +hobject_t infos_oid = OSD::make_infos_oid(); +hobject_t biginfo_oid, log_oid; + +int file_fd = fd_none; +bool debug = false; +super_header sh; + +template +int write_section(sectiontype_t type, const T& obj, int fd) { + bufferlist blhdr, bl, blftr; + obj.encode(bl); + header hdr(type, bl.length()); + hdr.encode(blhdr); + footer ft; + ft.encode(blftr); + + int ret = blhdr.write_fd(fd); + if (ret) return ret; + ret = bl.write_fd(fd); + if (ret) return ret; + ret = blftr.write_fd(fd); + return ret; +} + +int write_simple(sectiontype_t type, int fd) +{ + bufferlist hbl; + + header hdr(type, 0); + hdr.encode(hbl); + return hbl.write_fd(fd); +} + +static void invalid_path(string &path) +{ + cout << "Invalid path to osd store specified: " << path << "\n"; + exit(1); +} + +int get_log(ObjectStore *fs, coll_t coll, pg_t pgid, const pg_info_t &info, + PGLog::IndexedLog &log, pg_missing_t &missing) +{ + map divergent_priors; + try { + ostringstream oss; + PGLog::read_log(fs, coll, log_oid, info, divergent_priors, log, missing, oss); + if (debug && oss.str().size()) + cerr << oss.str() << std::endl; + } + catch (const buffer::error &e) { + cout << "read_log threw exception error " << e.what() << std::endl; + return 1; + } + return 0; +} + +//Based on RemoveWQ::_process() +void remove_coll(ObjectStore *store, const coll_t &coll) +{ + spg_t pg; + coll.is_pg_prefix(pg); + OSDriver driver( + store, + coll_t(), + OSD::make_snapmapper_oid()); + SnapMapper mapper(&driver, 0, 0, 0, pg.shard); + + vector objects; + ghobject_t next; + int r = 0; + int64_t num = 0; + ObjectStore::Transaction *t = new ObjectStore::Transaction; + cout << "remove_coll " << coll << std::endl; + while (!next.is_max()) { + r = store->collection_list_partial(coll, next, 200, 300, 0, + &objects, &next); + if (r < 0) + goto out; + for (vector::iterator i = objects.begin(); + i != objects.end(); + ++i, ++num) { + + assert(i->generation == ghobject_t::NO_GEN); + OSDriver::OSTransaction _t(driver.get_transaction(t)); + cout << "remove " << *i << std::endl; + int r = mapper.remove_oid(i->hobj, &_t); + if (r != 0 && r != -ENOENT) { + assert(0); + } + + t->remove(coll, *i); + if (num >= 30) { + store->apply_transaction(*t); + delete t; + t = new ObjectStore::Transaction; + num = 0; + } + } + } + t->remove_collection(coll); + store->apply_transaction(*t); +out: + delete t; +} + +//Based on part of OSD::load_pgs() +int finish_remove_pgs(ObjectStore *store, uint64_t *next_removal_seq) +{ + vector ls; + int r = store->list_collections(ls); + if (r < 0) { + cout << "finish_remove_pgs: failed to list pgs: " << cpp_strerror(-r) + << std::endl; + return r; + } + + for (vector::iterator it = ls.begin(); + it != ls.end(); + ++it) { + spg_t pgid; + snapid_t snap; + + if (it->is_temp(pgid)) { + cout << "finish_remove_pgs " << *it << " clearing temp" << std::endl; + OSD::recursive_remove_collection(store, *it); + continue; + } + + if (it->is_pg(pgid, snap)) { + continue; + } + + uint64_t seq; + if (it->is_removal(&seq, &pgid)) { + if (seq >= *next_removal_seq) + *next_removal_seq = seq + 1; + cout << "finish_remove_pgs removing " << *it << ", seq is " + << seq << " pgid is " << pgid << std::endl; + remove_coll(store, *it); + continue; + } + + //cout << "finish_remove_pgs ignoring unrecognized " << *it << std::endl; + } + return 0; +} + +int initiate_new_remove_pg(ObjectStore *store, pg_t r_pgid, + uint64_t *next_removal_seq) +{ + ObjectStore::Transaction *rmt = new ObjectStore::Transaction; + + if (store->collection_exists(coll_t(spg_t(r_pgid, ghobject_t::no_shard())))) { + coll_t to_remove = coll_t::make_removal_coll((*next_removal_seq)++, + spg_t(r_pgid, ghobject_t::no_shard())); + cout << "collection rename " << coll_t(spg_t(r_pgid, ghobject_t::no_shard())) + << " to " << to_remove + << std::endl; + rmt->collection_rename(coll_t(spg_t(r_pgid, ghobject_t::no_shard())), to_remove); + } else { + delete rmt; + return ENOENT; + } + + cout << "remove " << coll_t::META_COLL << " " << log_oid.oid << std::endl; + rmt->remove(coll_t::META_COLL, log_oid); + cout << "remove " << coll_t::META_COLL << " " << biginfo_oid.oid << std::endl; + rmt->remove(coll_t::META_COLL, biginfo_oid); + + store->apply_transaction(*rmt); + + return 0; +} + +int header::get_header() +{ + bufferlist ebl; + bufferlist::iterator ebliter = ebl.begin(); + ssize_t bytes; + + bytes = ebl.read_fd(file_fd, sh.header_size); + if ((size_t)bytes != sh.header_size) { + cout << "Unexpected EOF" << std::endl; + return EFAULT; + } + + decode(ebliter); + + return 0; +} + +int footer::get_footer() +{ + bufferlist ebl; + bufferlist::iterator ebliter = ebl.begin(); + ssize_t bytes; + + bytes = ebl.read_fd(file_fd, sh.footer_size); + if ((size_t)bytes != sh.footer_size) { + cout << "Unexpected EOF" << std::endl; + return EFAULT; + } + + decode(ebliter); + + if (magic != endmagic) { + cout << "Bad footer magic" << std::endl; + return EFAULT; + } + + return 0; +} + +int write_info(ObjectStore::Transaction &t, epoch_t epoch, pg_info_t &info, + __u8 struct_ver) +{ + //Empty for this + interval_set snap_collections; // obsolete + map past_intervals; + coll_t coll(info.pgid); + + int ret = PG::_write_info(t, epoch, + info, coll, + past_intervals, + snap_collections, + infos_oid, + struct_ver, + true, true); + if (ret < 0) ret = -ret; + if (ret) cout << "Failed to write info" << std::endl; + return ret; +} + +void write_log(ObjectStore::Transaction &t, pg_log_t &log) +{ + map divergent_priors; + PGLog::write_log(t, log, log_oid, divergent_priors); +} + +int write_pg(ObjectStore::Transaction &t, epoch_t epoch, pg_info_t &info, + pg_log_t &log, __u8 struct_ver) +{ + int ret = write_info(t, epoch, info, struct_ver); + if (ret) return ret; + write_log(t, log); + return 0; +} + +int export_file(ObjectStore *store, coll_t cid, hobject_t &obj) +{ + struct stat st; + mysize_t total; + ostringstream objname; + footer ft; + + int ret = store->stat(cid, obj, &st); + if (ret < 0) + return ret; + + objname << obj; + if (debug && file_fd != STDOUT_FILENO) + cout << "objname=" << objname.str() << std::endl; + + total = st.st_size; + if (debug && file_fd != STDOUT_FILENO) + cout << "size=" << total << std::endl; + + object_begin objb(obj); + ret = write_section(TYPE_OBJECT_BEGIN, objb, file_fd); + if (ret < 0) + return ret; + + uint64_t offset = 0; + bufferlist rawdatabl, databl; + while(total > 0) { + rawdatabl.clear(); + databl.clear(); + mysize_t len = max_read; + if (len > total) + len = total; + + ret = store->read(cid, obj, offset, len, rawdatabl); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + data_section dblock(offset, len, rawdatabl); + total -= ret; + offset += ret; + + if (debug && file_fd != STDOUT_FILENO) + cout << "data section offset=" << offset << " len=" << len << std::endl; + + ret = write_section(TYPE_DATA, dblock, file_fd); + if (ret) return ret; + } + + //Handle attrs for this object + map aset; + ret = store->getattrs(cid, obj, aset, false); + if (ret) return ret; + attr_section as(aset); + ret = write_section(TYPE_ATTRS, as, file_fd); + if (ret) + return ret; + + if (debug && file_fd != STDOUT_FILENO) { + cout << "attrs size " << aset.size() << std::endl; + } + + //Handle omap information + databl.clear(); + bufferlist hdrbuf; + map out; + ret = store->omap_get(cid, obj, &hdrbuf, &out); + if (ret < 0) + return ret; + + omap_hdr_section ohs(hdrbuf); + ret = write_section(TYPE_OMAP_HDR, ohs, file_fd); + if (ret) + return ret; + + if (!out.empty()) { + omap_section oms(out); + ret = write_section(TYPE_OMAP, oms, file_fd); + if (ret) + return ret; + + if (debug && file_fd != STDOUT_FILENO) + cout << "omap map size " << out.size() << std::endl; + } + + ret = write_simple(TYPE_OBJECT_END, file_fd); + if (ret) + return ret; + + return 0; +} + +int export_files(ObjectStore *store, coll_t coll) +{ + vector objects; + ghobject_t next; + + while (!next.is_max()) { + int r = store->collection_list_partial(coll, next, 200, 300, 0, + &objects, &next); + if (r < 0) + return r; + for (vector::iterator i = objects.begin(); + i != objects.end(); + ++i) { + assert(i->generation == ghobject_t::NO_GEN); + r = export_file(store, coll, i->hobj); + if (r < 0) + return r; + } + } + return 0; +} + +//Write super_header with its fixed 16 byte length +void write_super() +{ + bufferlist superbl; + super_header sh; + footer ft; + + header hdr(TYPE_NONE, 0); + hdr.encode(superbl); + + sh.magic = super_header::super_magic; + sh.version = super_header::super_ver; + sh.header_size = superbl.length(); + superbl.clear(); + ft.encode(superbl); + sh.footer_size = superbl.length(); + superbl.clear(); + + sh.encode(superbl); + assert(super_header::FIXED_LENGTH == superbl.length()); + superbl.write_fd(file_fd); +} + +int do_export(ObjectStore *fs, coll_t coll, pg_t pgid, pg_info_t &info, + epoch_t map_epoch, __u8 struct_ver, const OSDSuperblock& superblock) +{ + PGLog::IndexedLog log; + pg_missing_t missing; + + int ret = get_log(fs, coll, pgid, info, log, missing); + if (ret > 0) + return ret; + + write_super(); + + pg_begin pgb(pgid, superblock); + ret = write_section(TYPE_PG_BEGIN, pgb, file_fd); + if (ret) + return ret; + + export_files(fs, coll); + + metadata_section ms(struct_ver, map_epoch, info, log); + ret = write_section(TYPE_PG_METADATA, ms, file_fd); + if (ret) + return ret; + + ret = write_simple(TYPE_PG_END, file_fd); + if (ret) + return ret; + + return 0; +} + +int super_header::read_super() +{ + bufferlist ebl; + bufferlist::iterator ebliter = ebl.begin(); + ssize_t bytes; + + bytes = ebl.read_fd(file_fd, super_header::FIXED_LENGTH); + if ((size_t)bytes != super_header::FIXED_LENGTH) { + cout << "Unexpected EOF" << std::endl; + return EFAULT; + } + + decode(ebliter); + + return 0; +} + +int read_section(int fd, sectiontype_t *type, bufferlist *bl) +{ + header hdr; + ssize_t bytes; + + int ret = hdr.get_header(); + if (ret) + return ret; + + *type = hdr.type; + + bl->clear(); + bytes = bl->read_fd(fd, hdr.size); + if (bytes != hdr.size) { + cout << "Unexpected EOF" << std::endl; + return EFAULT; + } + + if (hdr.size > 0) { + footer ft; + ret = ft.get_footer(); + if (ret) + return ret; + } + + return 0; +} + +int get_data(ObjectStore *store, coll_t coll, hobject_t hoid, + ObjectStore::Transaction *t, bufferlist &bl) +{ + bufferlist::iterator ebliter = bl.begin(); + data_section ds; + ds.decode(ebliter); + + if (debug) + cout << "\tdata: offset " << ds.offset << " len " << ds.len << std::endl; + t->write(coll, hoid, ds.offset, ds.len, ds.databl); + return 0; +} + +int get_attrs(ObjectStore *store, coll_t coll, hobject_t hoid, + ObjectStore::Transaction *t, bufferlist &bl, + OSDriver &driver, SnapMapper &snap_mapper) +{ + bufferlist::iterator ebliter = bl.begin(); + attr_section as; + as.decode(ebliter); + + if (debug) + cout << "\tattrs: len " << as.data.size() << std::endl; + t->setattrs(coll, hoid, as.data); + + if (hoid.snap < CEPH_MAXSNAP) { + map::iterator mi = as.data.find(OI_ATTR); + if (mi != as.data.end()) { + bufferlist attr_bl; + attr_bl.push_back(mi->second); + object_info_t oi(attr_bl); + + if (debug) + cout << "object_info " << oi << std::endl; + + OSDriver::OSTransaction _t(driver.get_transaction(t)); + set oi_snaps(oi.snaps.begin(), oi.snaps.end()); + snap_mapper.add_oid(hoid, oi_snaps, &_t); + } + } + + return 0; +} + +int get_omap_hdr(ObjectStore *store, coll_t coll, hobject_t hoid, + ObjectStore::Transaction *t, bufferlist &bl) +{ + bufferlist::iterator ebliter = bl.begin(); + omap_hdr_section oh; + oh.decode(ebliter); + + if (debug) + cout << "\tomap header: " << string(oh.hdr.c_str(), oh.hdr.length()) + << std::endl; + t->omap_setheader(coll, hoid, oh.hdr); + return 0; +} + +int get_omap(ObjectStore *store, coll_t coll, hobject_t hoid, + ObjectStore::Transaction *t, bufferlist &bl) +{ + bufferlist::iterator ebliter = bl.begin(); + omap_section os; + os.decode(ebliter); + + if (debug) + cout << "\tomap: size " << os.omap.size() << std::endl; + t->omap_setkeys(coll, hoid, os.omap); + return 0; +} + +int get_object(ObjectStore *store, coll_t coll, bufferlist &bl) +{ + ObjectStore::Transaction tran; + ObjectStore::Transaction *t = &tran; + bufferlist::iterator ebliter = bl.begin(); + object_begin ob; + ob.decode(ebliter); + OSDriver driver( + store, + coll_t(), + OSD::make_snapmapper_oid()); + spg_t pg; + coll.is_pg_prefix(pg); + SnapMapper mapper(&driver, 0, 0, 0, pg.shard); + + t->touch(coll, ob.hoid); + + if (debug) { + ostringstream objname; + objname << ob.hoid.oid; + cout << "name " << objname.str() << " snap " << ob.hoid.snap << std::endl; + } + + bufferlist ebl; + bool done = false; + while(!done) { + sectiontype_t type; + int ret = read_section(file_fd, &type, &ebl); + if (ret) + return ret; + + //cout << "\tdo_object: Section type " << hex << type << dec << std::endl; + //cout << "\t\tsection size " << ebl.length() << std::endl; + if (type >= END_OF_TYPES) { + cout << "Skipping unknown object section type" << std::endl; + continue; + } + switch(type) { + case TYPE_DATA: + ret = get_data(store, coll, ob.hoid, t, ebl); + if (ret) return ret; + break; + case TYPE_ATTRS: + ret = get_attrs(store, coll, ob.hoid, t, ebl, driver, mapper); + if (ret) return ret; + break; + case TYPE_OMAP_HDR: + ret = get_omap_hdr(store, coll, ob.hoid, t, ebl); + if (ret) return ret; + break; + case TYPE_OMAP: + ret = get_omap(store, coll, ob.hoid, t, ebl); + if (ret) return ret; + break; + case TYPE_OBJECT_END: + done = true; + break; + default: + return EFAULT; + } + } + store->apply_transaction(*t); + return 0; +} + +int get_pg_metadata(ObjectStore *store, coll_t coll, bufferlist &bl) +{ + ObjectStore::Transaction tran; + ObjectStore::Transaction *t = &tran; + bufferlist::iterator ebliter = bl.begin(); + metadata_section ms; + ms.decode(ebliter); + +#if DIAGNOSTIC + Formatter *formatter = new JSONFormatter(true); + cout << "struct_v " << (int)ms.struct_ver << std::endl; + cout << "epoch " << ms.map_epoch << std::endl; + formatter->open_object_section("info"); + ms.info.dump(formatter); + formatter->close_section(); + formatter->flush(cout); + cout << std::endl; + + formatter->open_object_section("log"); + ms.log.dump(formatter); + formatter->close_section(); + formatter->flush(cout); + cout << std::endl; +#endif + + coll_t newcoll(ms.info.pgid); + t->collection_rename(coll, newcoll); + + int ret = write_pg(*t, ms.map_epoch, ms.info, ms.log, ms.struct_ver); + if (ret) return ret; + + store->apply_transaction(*t); + + return 0; +} + +int do_import(ObjectStore *store, OSDSuperblock& sb) +{ + bufferlist ebl; + pg_info_t info; + PGLog::IndexedLog log; + + uint64_t next_removal_seq = 0; //My local seq + finish_remove_pgs(store, &next_removal_seq); + + int ret = sh.read_super(); + if (ret) + return ret; + + if (sh.magic != super_header::super_magic) { + cout << "Invalid magic number" << std::endl; + return EFAULT; + } + + if (sh.version > super_header::super_ver) { + cout << "Can't handle export format version=" << sh.version << std::endl; + return EINVAL; + } + + //First section must be TYPE_PG_BEGIN + sectiontype_t type; + ret = read_section(file_fd, &type, &ebl); + if (type != TYPE_PG_BEGIN) { + return EFAULT; + } + + bufferlist::iterator ebliter = ebl.begin(); + pg_begin pgb; + pgb.decode(ebliter); + pg_t pgid = pgb.pgid; + + if (debug) { + cout << "Exported features: " << pgb.superblock.compat_features << std::endl; + } + if (sb.compat_features.compare(pgb.superblock.compat_features) == -1) { + cout << "Export has incompatible features set " + << pgb.superblock.compat_features << std::endl; + return 1; + } + + log_oid = OSD::make_pg_log_oid(spg_t(pgid, ghobject_t::no_shard())); + biginfo_oid = OSD::make_pg_biginfo_oid(spg_t(pgid, ghobject_t::no_shard())); + + //Check for PG already present. + coll_t coll(spg_t(pgid, ghobject_t::no_shard())); + if (store->collection_exists(coll)) { + cout << "pgid " << pgid << " already exists" << std::endl; + return 1; + } + + //Switch to collection which will be removed automatically if + //this program is interupted. + coll_t rmcoll = coll_t::make_removal_coll( + next_removal_seq, spg_t(pgid, ghobject_t::no_shard())); + ObjectStore::Transaction *t = new ObjectStore::Transaction; + t->create_collection(rmcoll); + store->apply_transaction(*t); + delete t; + + cout << "Importing pgid " << pgid << std::endl; + + bool done = false; + bool found_metadata = false; + while(!done) { + ret = read_section(file_fd, &type, &ebl); + if (ret) + return ret; + + //cout << "do_import: Section type " << hex << type << dec << std::endl; + if (type >= END_OF_TYPES) { + cout << "Skipping unknown section type" << std::endl; + continue; + } + switch(type) { + case TYPE_OBJECT_BEGIN: + ret = get_object(store, rmcoll, ebl); + if (ret) return ret; + break; + case TYPE_PG_METADATA: + ret = get_pg_metadata(store, rmcoll, ebl); + if (ret) return ret; + found_metadata = true; + break; + case TYPE_PG_END: + done = true; + break; + default: + return EFAULT; + } + } + + if (!found_metadata) { + cout << "Missing metadata section" << std::endl; + return EFAULT; + } + + return 0; +} + +int main(int argc, char **argv) +{ + string fspath, jpath, pgidstr, type, file; + Formatter *formatter = new JSONFormatter(true); + + po::options_description desc("Allowed options"); + desc.add_options() + ("help", "produce help message") + ("filestore-path", po::value(&fspath), + "path to filestore directory, mandatory") + ("journal-path", po::value(&jpath), + "path to journal, mandatory") + ("pgid", po::value(&pgidstr), + "PG id, mandatory") + ("type", po::value(&type), + "Type one of info, log, remove, export, or import, mandatory") + ("file", po::value(&file), + "path of file to export or import") + ("debug", "Enable diagnostic output to stderr") + ; + + po::variables_map vm; + po::parsed_options parsed = + po::command_line_parser(argc, argv).options(desc).allow_unregistered().run(); + po::store( parsed, vm); + try { + po::notify(vm); + } + catch(...) { + cout << desc << std::endl; + exit(1); + } + + if (vm.count("help")) { + cout << desc << std::endl; + return 1; + } + + if (!vm.count("filestore-path")) { + cout << "Must provide filestore-path" << std::endl + << desc << std::endl; + return 1; + } + if (!vm.count("journal-path")) { + cout << "Must provide journal-path" << std::endl + << desc << std::endl; + return 1; + } + if (!vm.count("type")) { + cout << "Must provide type (info, log, remove, export, import)" + << std::endl << desc << std::endl; + return 1; + } + if (type != "import" && !vm.count("pgid")) { + cout << "Must provide pgid" << std::endl + << desc << std::endl; + return 1; + } + + file_fd = fd_none; + if (type == "export") { + if (!vm.count("file")) { + file_fd = STDOUT_FILENO; + } else { + file_fd = open(file.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666); + } + } else if (type == "import") { + if (!vm.count("file")) { + file_fd = STDIN_FILENO; + } else { + file_fd = open(file.c_str(), O_RDONLY); + } + } + + if (vm.count("file") && file_fd == fd_none) { + cout << "--file option only applies to import or export" << std::endl; + return 1; + } + + if (file_fd != fd_none && file_fd < 0) { + perror("open"); + return 1; + } + + if ((fspath.length() == 0 || jpath.length() == 0) || + (type != "info" && type != "log" && type != "remove" && type != "export" + && type != "import") || + (type != "import" && pgidstr.length() == 0)) { + cerr << "Invalid params" << std::endl; + exit(1); + } + + if (type == "import" && pgidstr.length()) { + cerr << "--pgid option invalid with import" << std::endl; + exit(1); + } + + vector ceph_options, def_args; + vector ceph_option_strings = po::collect_unrecognized( + parsed.options, po::include_positional); + ceph_options.reserve(ceph_option_strings.size()); + for (vector::iterator i = ceph_option_strings.begin(); + i != ceph_option_strings.end(); + ++i) { + ceph_options.push_back(i->c_str()); + } + + //Suppress derr() output to stderr by default + if (!vm.count("debug")) { + close(STDERR_FILENO); + (void)open("/dev/null", O_WRONLY); + debug = false; + } else { + debug = true; + } + + global_init( + &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD, + CODE_ENVIRONMENT_UTILITY, 0); + //CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + g_ceph_context->_conf->apply_changes(NULL); + g_conf = g_ceph_context->_conf; + + //Verify that fspath really is an osd store + struct stat st; + if (::stat(fspath.c_str(), &st) == -1) { + perror("fspath"); + invalid_path(fspath); + } + if (!S_ISDIR(st.st_mode)) { + invalid_path(fspath); + } + string check = fspath + "/whoami"; + if (::stat(check.c_str(), &st) == -1) { + perror("whoami"); + invalid_path(fspath); + } + if (!S_ISREG(st.st_mode)) { + invalid_path(fspath); + } + check = fspath + "/current"; + if (::stat(check.c_str(), &st) == -1) { + perror("current"); + invalid_path(fspath); + } + if (!S_ISDIR(st.st_mode)) { + invalid_path(fspath); + } + + pg_t pgid; + if (pgidstr.length() && !pgid.parse(pgidstr.c_str())) { + cout << "Invalid pgid '" << pgidstr << "' specified" << std::endl; + exit(1); + } + + ObjectStore *fs = new FileStore(fspath, jpath); + + int r = fs->mount(); + if (r < 0) { + if (r == -EBUSY) { + cout << "OSD has the store locked" << std::endl; + } else { + cout << "Mount failed with '" << cpp_strerror(-r) << "'" << std::endl; + } + return 1; + } + + bool fs_sharded_objects = fs->get_allow_sharded_objects(); + + int ret = 0; + vector ls; + vector::iterator it; + CompatSet supported; + +#ifdef INTERNAL_TEST + supported = get_test_compat_set(); +#else + supported = OSD::get_osd_compat_set(); +#endif + + bufferlist bl; + OSDSuperblock superblock; + bufferlist::iterator p; + ret = fs->read(coll_t::META_COLL, OSD_SUPERBLOCK_POBJECT, 0, 0, bl); + if (ret < 0) { + cout << "Failure to read OSD superblock error= " << r << std::endl; + goto out; + } + + p = bl.begin(); + ::decode(superblock, p); + +#ifdef INTERNAL_TEST2 + fs->set_allow_sharded_objects(); + assert(fs->get_allow_sharded_objects()); + fs_sharded_objects = true; + superblock.compat_features.incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS); +#endif + + if (debug && file_fd != STDOUT_FILENO) { + cout << "Supported features: " << supported << std::endl; + cout << "On-disk features: " << superblock.compat_features << std::endl; + } + if (supported.compare(superblock.compat_features) == -1) { + cout << "On-disk OSD incompatible features set " + << superblock.compat_features << std::endl; + ret = EINVAL; + goto out; + } + + // If there was a crash as an OSD was transitioning to sharded objects + // and hadn't completed a set_allow_sharded_objects(). + // This utility does not want to attempt to finish that transition. + if (superblock.compat_features.incompat.contains(CEPH_OSD_FEATURE_INCOMPAT_SHARDS) != fs_sharded_objects) { + // An OSD should never have call set_allow_sharded_objects() before + // updating its own OSD features. + if (fs_sharded_objects) + cout << "FileStore sharded but OSD not set, Corruption?" << std::endl; + else + cout << "Found incomplete transition to sharded objects" << std::endl; + ret = EINVAL; + goto out; + } + + if (type == "import") { + + try { + ret = do_import(fs, superblock); + } + catch (const buffer::error &e) { + cout << "do_import threw exception error " << e.what() << std::endl; + ret = EFAULT; + } + if (ret == EFAULT) { + cout << "Corrupt input for import" << std::endl; + } + goto out; + } + + log_oid = OSD::make_pg_log_oid(spg_t(pgid, ghobject_t::no_shard())); + biginfo_oid = OSD::make_pg_biginfo_oid(spg_t(pgid, ghobject_t::no_shard())); + + if (type == "remove") { + uint64_t next_removal_seq = 0; //My local seq + finish_remove_pgs(fs, &next_removal_seq); + int r = initiate_new_remove_pg(fs, pgid, &next_removal_seq); + if (r) { + cout << "PG '" << pgid << "' not found" << std::endl; + ret = 1; + goto out; + } + finish_remove_pgs(fs, &next_removal_seq); + cout << "Remove successful" << std::endl; + goto out; + } + + r = fs->list_collections(ls); + if (r < 0) { + cout << "failed to list pgs: " << cpp_strerror(-r) << std::endl; + exit(1); + } + + for (it = ls.begin(); it != ls.end(); ++it) { + snapid_t snap; + spg_t tmppgid; + + if (!it->is_pg(tmppgid, snap)) { + continue; + } + + if (tmppgid.pgid != pgid) { + continue; + } + if (snap != CEPH_NOSNAP && debug) { + cerr << "skipping snapped dir " << *it + << " (pg " << pgid << " snap " << snap << ")" << std::endl; + continue; + } + + //Found! + break; + } + + epoch_t map_epoch; + if (it != ls.end()) { + + coll_t coll = *it; + + bufferlist bl; + map_epoch = PG::peek_map_epoch(fs, coll, infos_oid, &bl); + if (debug) + cerr << "map_epoch " << map_epoch << std::endl; + + pg_info_t info(spg_t(pgid, ghobject_t::no_shard())); + map past_intervals; + hobject_t biginfo_oid = OSD::make_pg_biginfo_oid( + spg_t(pgid, ghobject_t::no_shard())); + interval_set snap_collections; + + __u8 struct_ver; + r = PG::read_info(fs, coll, bl, info, past_intervals, biginfo_oid, + infos_oid, snap_collections, struct_ver); + if (r < 0) { + cout << "read_info error " << cpp_strerror(-r) << std::endl; + ret = 1; + goto out; + } + if (debug) + cerr << "struct_v " << (int)struct_ver << std::endl; + + if (type == "export") { + ret = do_export(fs, coll, pgid, info, map_epoch, struct_ver, superblock); + } else if (type == "info") { + formatter->open_object_section("info"); + info.dump(formatter); + formatter->close_section(); + formatter->flush(cout); + cout << std::endl; + } else if (type == "log") { + PGLog::IndexedLog log; + pg_missing_t missing; + ret = get_log(fs, coll, pgid, info, log, missing); + if (ret > 0) + goto out; + + formatter->open_object_section("log"); + log.dump(formatter); + formatter->close_section(); + formatter->flush(cout); + cout << std::endl; + formatter->open_object_section("missing"); + missing.dump(formatter); + formatter->close_section(); + formatter->flush(cout); + cout << std::endl; + } + } else { + cout << "PG '" << pgid << "' not found" << std::endl; + ret = 1; + } + +out: + if (fs->umount() < 0) { + cout << "umount failed" << std::endl; + return 1; + } + + return (ret != 0); +} + diff --git a/src/tools/ceph_filestore_tool.cc b/src/tools/ceph_filestore_tool.cc new file mode 100644 index 0000000000000..eb9f8dac36d05 --- /dev/null +++ b/src/tools/ceph_filestore_tool.cc @@ -0,0 +1,260 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Inktank + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/Formatter.h" + +#include "global/global_init.h" +#include "os/ObjectStore.h" +#include "os/FileStore.h" +#include "common/perf_counters.h" +#include "common/errno.h" +#include "osd/PGLog.h" +#include "osd/osd_types.h" +#include "osd/OSD.h" + +namespace po = boost::program_options; +using namespace std; + +static void invalid_path(string &path) +{ + cout << "Invalid path to osd store specified: " << path << "\n"; + exit(1); +} + +int main(int argc, char **argv) +{ + string fspath, jpath, pgidstr; + bool list_lost_objects = false; + bool fix_lost_objects = false; + unsigned LIST_AT_A_TIME = 100; + unsigned scanned = 0; + + po::options_description desc("Allowed options"); + desc.add_options() + ("help", "produce help message") + ("filestore-path", po::value(&fspath), + "path to filestore directory, mandatory") + ("journal-path", po::value(&jpath), + "path to journal, mandatory") + ("pgid", po::value(&pgidstr), + "PG id") + ("list-lost-objects", po::value( + &list_lost_objects)->default_value(false), + "list lost objects") + ("fix-lost-objects", po::value( + &fix_lost_objects)->default_value(false), + "fix lost objects") + ; + + po::variables_map vm; + po::parsed_options parsed = + po::command_line_parser(argc, argv).options(desc). + allow_unregistered().run(); + po::store( parsed, vm); + try { + po::notify(vm); + } + catch(...) { + cout << desc << std::endl; + exit(1); + } + + if (vm.count("help")) { + cout << desc << std::endl; + return 1; + } + + if (!vm.count("filestore-path")) { + cerr << "Must provide filestore-path" << std::endl + << desc << std::endl; + return 1; + } + if (!vm.count("journal-path")) { + cerr << "Must provide journal-path" << std::endl + << desc << std::endl; + return 1; + } + + if ((fspath.length() == 0 || jpath.length() == 0)) { + cerr << "Invalid params" << desc << std::endl; + exit(1); + } + + vector ceph_options, def_args; + vector ceph_option_strings = po::collect_unrecognized( + parsed.options, po::include_positional); + ceph_options.reserve(ceph_option_strings.size()); + for (vector::iterator i = ceph_option_strings.begin(); + i != ceph_option_strings.end(); + ++i) { + ceph_options.push_back(i->c_str()); + } + + global_init( + &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD, + CODE_ENVIRONMENT_UTILITY, 0); + //CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + g_ceph_context->_conf->apply_changes(NULL); + g_conf = g_ceph_context->_conf; + + //Verify that fspath really is an osd store + struct stat st; + if (::stat(fspath.c_str(), &st) == -1) { + perror("fspath"); + invalid_path(fspath); + } + if (!S_ISDIR(st.st_mode)) { + invalid_path(fspath); + } + string check = fspath + "/whoami"; + if (::stat(check.c_str(), &st) == -1) { + perror("whoami"); + invalid_path(fspath); + } + if (!S_ISREG(st.st_mode)) { + invalid_path(fspath); + } + check = fspath + "/current"; + if (::stat(check.c_str(), &st) == -1) { + perror("current"); + invalid_path(fspath); + } + if (!S_ISDIR(st.st_mode)) { + invalid_path(fspath); + } + + ObjectStore *fs = new FileStore(fspath, jpath); + + int r = fs->mount(); + if (r < 0) { + if (r == -EBUSY) { + cout << "OSD has the store locked" << std::endl; + } else { + cout << "Mount failed with '" << cpp_strerror(-r) << "'" << std::endl; + } + return 1; + } + + vector colls_to_check; + if (pgidstr.length()) { + spg_t pgid; + if (!pgid.parse(pgidstr.c_str())) { + cout << "Invalid pgid '" << pgidstr << "' specified" << std::endl; + exit(1); + } + colls_to_check.push_back(coll_t(pgid)); + } else { + vector candidates; + r = fs->list_collections(candidates); + if (r < 0) { + cerr << "Error listing collections: " << cpp_strerror(r) << std::endl; + goto UMOUNT; + } + for (vector::iterator i = candidates.begin(); + i != candidates.end(); + ++i) { + spg_t pgid; + snapid_t snap; + if (i->is_pg(pgid, snap)) { + colls_to_check.push_back(*i); + } + } + } + + cerr << colls_to_check.size() << " pgs to scan" << std::endl; + for (vector::iterator i = colls_to_check.begin(); + i != colls_to_check.end(); + ++i, ++scanned) { + cerr << "Scanning " << *i << ", " << scanned << "/" + << colls_to_check.size() << " completed" << std::endl; + ghobject_t next; + while (!next.is_max()) { + vector list; + r = fs->collection_list_partial( + *i, + next, + LIST_AT_A_TIME, + LIST_AT_A_TIME, + CEPH_NOSNAP, + &list, + &next); + if (r < 0) { + cerr << "Error listing collection: " << *i << ", " + << cpp_strerror(r) << std::endl; + goto UMOUNT; + } + for (vector::iterator obj = list.begin(); + obj != list.end(); + ++obj) { + bufferlist attr; + r = fs->getattr(*i, *obj, OI_ATTR, attr); + if (r < 0) { + cerr << "Error getting attr on : " << make_pair(*i, *obj) << ", " + << cpp_strerror(r) << std::endl; + goto UMOUNT; + } + object_info_t oi; + bufferlist::iterator bp = attr.begin(); + try { + ::decode(oi, bp); + } catch (...) { + r = -EINVAL; + cerr << "Error getting attr on : " << make_pair(*i, *obj) << ", " + << cpp_strerror(r) << std::endl; + goto UMOUNT; + } + if (oi.is_lost()) { + if (list_lost_objects) { + cout << *i << "/" << *obj << " is lost" << std::endl; + } + if (fix_lost_objects) { + cerr << *i << "/" << *obj << " is lost, fixing" << std::endl; + oi.clear_flag(object_info_t::FLAG_LOST); + bufferlist bl2; + ::encode(oi, bl2); + ObjectStore::Transaction t; + t.setattr(*i, *obj, OI_ATTR, bl2); + r = fs->apply_transaction(t); + if (r < 0) { + cerr << "Error getting fixing attr on : " << make_pair(*i, *obj) + << ", " + << cpp_strerror(r) << std::endl; + goto UMOUNT; + } + } + } + } + } + } + cerr << "Completed" << std::endl; + + UMOUNT: + fs->sync_and_flush(); + fs->umount(); + return r; +} diff --git a/src/tools/ceph_kvstore_tool.cc b/src/tools/ceph_kvstore_tool.cc new file mode 100644 index 0000000000000..5a68d98990dfc --- /dev/null +++ b/src/tools/ceph_kvstore_tool.cc @@ -0,0 +1,424 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* +* Ceph - scalable distributed file system +* +* Copyright (C) 2012 Inktank, Inc. +* +* This is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License version 2.1, as published by the Free Software +* Foundation. See file COPYING. +*/ +#include +#include +#include +#include +#include +#include + +#include "os/LevelDBStore.h" + +#include "common/ceph_argparse.h" +#include "global/global_init.h" +#include "common/errno.h" +#include "common/safe_io.h" +#include "common/config.h" +#include "common/strtol.h" +#include "include/stringify.h" + +using namespace std; + +class StoreTool +{ + boost::scoped_ptr db; + string store_path; + + public: + StoreTool(const string &path) : store_path(path) { + LevelDBStore *db_ptr = new LevelDBStore(g_ceph_context, store_path); + assert(!db_ptr->open(std::cerr)); + db.reset(db_ptr); + } + + uint32_t traverse(const string &prefix, + const bool do_crc, + ostream *out) { + KeyValueDB::WholeSpaceIterator iter = db->get_iterator(); + + if (prefix.empty()) + iter->seek_to_first(); + else + iter->seek_to_first(prefix); + + uint32_t crc = -1; + + while (iter->valid()) { + pair rk = iter->raw_key(); + if (!prefix.empty() && (rk.first != prefix)) + break; + + if (out) + *out << rk.first << ":" << rk.second; + if (do_crc) { + bufferlist bl; + bl.append(rk.first); + bl.append(rk.second); + bl.append(iter->value()); + + crc = bl.crc32c(crc); + if (out) { + *out << " (" << bl.crc32c(0) << ")"; + } + } + if (out) + *out << std::endl; + iter->next(); + } + + return crc; + } + + void list(const string &prefix, const bool do_crc) { + traverse(prefix, do_crc, &std::cout); + } + + bool exists(const string &prefix) { + assert(!prefix.empty()); + KeyValueDB::WholeSpaceIterator iter = db->get_iterator(); + iter->seek_to_first(prefix); + return (iter->valid() && (iter->raw_key().first == prefix)); + } + + bool exists(const string &prefix, const string &key) { + assert(!prefix.empty()); + + if (key.empty()) { + return exists(prefix); + } + + bool exists = false; + get(prefix, key, exists); + return exists; + } + + bufferlist get(const string &prefix, const string &key, bool &exists) { + assert(!prefix.empty() && !key.empty()); + + map result; + std::set keys; + keys.insert(key); + db->get(prefix, keys, &result); + + if (result.count(key) > 0) { + exists = true; + return result[key]; + } + exists = false; + return bufferlist(); + } + + uint64_t get_size() { + map extras; + uint64_t s = db->get_estimated_size(extras); + for (map::iterator p = extras.begin(); + p != extras.end(); ++p) { + std::cout << p->first << " - " << p->second << std::endl; + } + std::cout << "total: " << s << std::endl; + return s; + } + + bool set(const string &prefix, const string &key, bufferlist &val) { + assert(!prefix.empty()); + assert(!key.empty()); + assert(val.length() > 0); + + KeyValueDB::Transaction tx = db->get_transaction(); + tx->set(prefix, key, val); + int ret = db->submit_transaction_sync(tx); + + return (ret == 0); + } + + int copy_store_to(const string &other_path, const int num_keys_per_tx) { + + if (num_keys_per_tx <= 0) { + std::cerr << "must specify a number of keys/tx > 0" << std::endl; + return -EINVAL; + } + + // open or create a leveldb store at @p other_path + LevelDBStore other(g_ceph_context, other_path); + int err = other.create_and_open(std::cerr); + if (err < 0) + return err; + + KeyValueDB::WholeSpaceIterator it = db->get_iterator(); + it->seek_to_first(); + uint64_t total_keys = 0; + uint64_t total_size = 0; + uint64_t total_txs = 0; + + utime_t started_at = ceph_clock_now(g_ceph_context); + + do { + int num_keys = 0; + + KeyValueDB::Transaction tx = other.get_transaction(); + + + while (it->valid() && num_keys < num_keys_per_tx) { + pair k = it->raw_key(); + bufferlist v = it->value(); + tx->set(k.first, k.second, v); + + num_keys ++; + total_size += v.length(); + + it->next(); + } + + total_txs ++; + total_keys += num_keys; + + if (num_keys > 0) + other.submit_transaction_sync(tx); + + utime_t cur_duration = ceph_clock_now(g_ceph_context) - started_at; + std::cout << "ts = " << cur_duration << "s, copied " << total_keys + << " keys so far (" << stringify(si_t(total_size)) << ")" + << std::endl; + + } while (it->valid()); + + utime_t time_taken = ceph_clock_now(g_ceph_context) - started_at; + + std::cout << "summary:" << std::endl; + std::cout << " copied " << total_keys << " keys" << std::endl; + std::cout << " used " << total_txs << " transactions" << std::endl; + std::cout << " total size " << stringify(si_t(total_size)) << std::endl; + std::cout << " from '" << store_path << "' to '" << other_path << "'" + << std::endl; + std::cout << " duration " << time_taken << " seconds" << std::endl; + + return 0; + } +}; + +void usage(const char *pname) +{ + std::cerr << "Usage: " << pname << " command [args...]\n" + << "\n" + << "Commands:\n" + << " list [prefix]\n" + << " list-crc [prefix]\n" + << " exists [key]\n" + << " get [out ]\n" + << " crc \n" + << " get-size [ ]\n" + << " set [ver |in ]\n" + << " store-copy [num-keys-per-tx]\n" + << " store-crc \n" + << std::endl; +} + +int main(int argc, const char *argv[]) +{ + vector args; + argv_to_vec(argc, argv, args); + env_to_vec(args); + + global_init( + NULL, args, + CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + + + if (args.size() < 2) { + usage(argv[0]); + return 1; + } + + string path(args[0]); + string cmd(args[1]); + + StoreTool st(path); + + if (cmd == "list" || cmd == "list-crc") { + string prefix; + if (argc > 3) + prefix = argv[3]; + + bool do_crc = (cmd == "list-crc"); + + st.list(prefix, do_crc); + + } else if (cmd == "exists") { + string key; + if (argc < 4) { + usage(argv[0]); + return 1; + } + string prefix(argv[3]); + if (argc > 4) + key = argv[4]; + + bool ret = st.exists(prefix, key); + std::cout << "(" << prefix << ", " << key << ") " + << (ret ? "exists" : "does not exist") + << std::endl; + return (ret ? 0 : 1); + + } else if (cmd == "get") { + if (argc < 5) { + usage(argv[0]); + return 1; + } + string prefix(argv[3]); + string key(argv[4]); + + bool exists = false; + bufferlist bl = st.get(prefix, key, exists); + std::cout << "(" << prefix << ", " << key << ")"; + if (!exists) { + std::cout << " does not exist" << std::endl; + return 1; + } + std::cout << std::endl; + + if (argc >= 6) { + string subcmd(argv[5]); + string out(argv[6]); + + if (subcmd != "out") { + std::cerr << "unrecognized subcmd '" << subcmd << "'" + << std::endl; + return 1; + } + + if (out.empty()) { + std::cerr << "unspecified out file" << std::endl; + return 1; + } + + int err = bl.write_file(argv[6], 0644); + if (err < 0) { + std::cerr << "error writing value to '" << out << "': " + << cpp_strerror(err) << std::endl; + return 1; + } + } else { + ostringstream os; + bl.hexdump(os); + std::cout << os.str() << std::endl; + } + + } else if (cmd == "crc") { + if (argc < 5) { + usage(argv[0]); + return 1; + } + string prefix(argv[3]); + string key(argv[4]); + + bool exists = false; + bufferlist bl = st.get(prefix, key, exists); + std::cout << "(" << prefix << ", " << key << ") "; + if (!exists) { + std::cout << " does not exist" << std::endl; + return 1; + } + std::cout << " crc " << bl.crc32c(0) << std::endl; + + } else if (cmd == "get-size") { + std::cout << "estimated store size: " << st.get_size() << std::endl; + + if (argc < 4) + return 0; + + if (argc < 5) { + usage(argv[0]); + return 1; + } + string prefix(argv[3]); + string key(argv[4]); + + bool exists = false; + bufferlist bl = st.get(prefix, key, exists); + if (!exists) { + std::cerr << "(" << prefix << "," << key + << ") does not exist" << std::endl; + return 1; + } + std::cout << "(" << prefix << "," << key + << ") size " << si_t(bl.length()) << std::endl; + + } else if (cmd == "set") { + if (argc < 7) { + usage(argv[0]); + return 1; + } + string prefix(argv[3]); + string key(argv[4]); + string subcmd(argv[5]); + + bufferlist val; + string errstr; + if (subcmd == "ver") { + version_t v = (version_t) strict_strtoll(argv[6], 10, &errstr); + if (!errstr.empty()) { + std::cerr << "error reading version: " << errstr << std::endl; + return 1; + } + ::encode(v, val); + } else if (subcmd == "in") { + int ret = val.read_file(argv[6], &errstr); + if (ret < 0 || !errstr.empty()) { + std::cerr << "error reading file: " << errstr << std::endl; + return 1; + } + } else { + std::cerr << "unrecognized subcommand '" << subcmd << "'" << std::endl; + usage(argv[0]); + return 1; + } + + bool ret = st.set(prefix, key, val); + if (!ret) { + std::cerr << "error setting (" + << prefix << "," << key << ")" << std::endl; + return 1; + } + } else if (cmd == "store-copy") { + int num_keys_per_tx = 128; // magic number that just feels right. + if (argc < 4) { + usage(argv[0]); + return 1; + } else if (argc > 4) { + string err; + num_keys_per_tx = strict_strtol(argv[4], 10, &err); + if (!err.empty()) { + std::cerr << "invalid num_keys_per_tx: " << err << std::endl; + return 1; + } + } + + int ret = st.copy_store_to(argv[3], num_keys_per_tx); + if (ret < 0) { + std::cerr << "error copying store to path '" << argv[3] + << "': " << cpp_strerror(ret) << std::endl; + return 1; + } + + } else if (cmd == "store-crc") { + uint32_t crc = st.traverse(string(), true, NULL); + std::cout << "store at '" << path << "' crc " << crc << std::endl; + + } else { + std::cerr << "Unrecognized command: " << cmd << std::endl; + return 1; + } + + return 0; +} diff --git a/src/tools/ceph_monstore_tool.cc b/src/tools/ceph_monstore_tool.cc new file mode 100644 index 0000000000000..8f294c4a4e341 --- /dev/null +++ b/src/tools/ceph_monstore_tool.cc @@ -0,0 +1,419 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* +* Ceph - scalable distributed file system +* +* Copyright (C) 2012 Inktank, Inc. +* +* This is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License version 2.1, as published by the Free Software +* Foundation. See file COPYING. +*/ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "global/global_init.h" +#include "os/LevelDBStore.h" +#include "mon/MonitorDBStore.h" +#include "mon/Paxos.h" +#include "common/Formatter.h" +#include "include/stringify.h" + +namespace po = boost::program_options; +using namespace std; + +class TraceIter { + int fd; + unsigned idx; + MonitorDBStore::Transaction t; +public: + TraceIter(string fname) : fd(-1), idx(-1) { + fd = ::open(fname.c_str(), O_RDONLY); + } + bool valid() { + return fd != -1; + } + const MonitorDBStore::Transaction &cur() { + assert(valid()); + return t; + } + unsigned num() { return idx; } + void next() { + ++idx; + bufferlist bl; + int r = bl.read_fd(fd, 6); + if (r < 0) { + std::cerr << "Got error: " << cpp_strerror(r) << " on read_fd" + << std::endl; + ::close(fd); + fd = -1; + return; + } else if ((unsigned)r < 6) { + std::cerr << "short read" << std::endl; + ::close(fd); + fd = -1; + return; + } + bufferlist::iterator bliter = bl.begin(); + uint8_t ver, ver2; + ::decode(ver, bliter); + ::decode(ver2, bliter); + uint32_t len; + ::decode(len, bliter); + r = bl.read_fd(fd, len); + if (r < 0) { + std::cerr << "Got error: " << cpp_strerror(r) << " on read_fd" + << std::endl; + ::close(fd); + fd = -1; + return; + } else if ((unsigned)r < len) { + std::cerr << "short read" << std::endl; + ::close(fd); + fd = -1; + return; + } + bliter = bl.begin(); + t.decode(bliter); + } + void init() { + next(); + } + ~TraceIter() { + if (fd != -1) { + ::close(fd); + fd = -1; + } + } +}; + +int main(int argc, char **argv) { + po::options_description desc("Allowed options"); + int version = -1; + string store_path, cmd, out_path, tfile; + unsigned dstart = 0; + unsigned dstop = ~0; + unsigned num_replays = 1; + unsigned tsize = 200; + unsigned tvalsize = 1024; + unsigned ntrans = 100; + desc.add_options() + ("help", "produce help message") + ("mon-store-path", po::value(&store_path), + "path to mon directory, mandatory") + ("out", po::value(&out_path), + "out path") + ("version", po::value(&version), + "version requested") + ("trace-file", po::value(&tfile), + "trace file") + ("dump-start", po::value(&dstart), + "transaction num to start dumping at") + ("dump-end", po::value(&dstop), + "transaction num to stop dumping at") + ("num-replays", po::value(&num_replays), + "number of times to replay") + ("trans-size", po::value(&tsize), + "keys to write in each transaction") + ("trans-val-size", po::value(&tvalsize), + "val to write in each key") + ("num-trans", po::value(&ntrans), + "number of transactions to run") + ("command", po::value(&cmd), + "command") + ; + po::positional_options_description p; + p.add("command", 1); + p.add("version", 1); + + po::variables_map vm; + po::parsed_options parsed = + po::command_line_parser(argc, argv).options(desc).positional(p).run(); + po::store( + parsed, + vm); + try { + po::notify(vm); + } catch (...) { + cout << desc << std::endl; + return 1; + } + + vector ceph_options, def_args; + vector ceph_option_strings = po::collect_unrecognized( + parsed.options, po::include_positional); + ceph_options.reserve(ceph_option_strings.size()); + for (vector::iterator i = ceph_option_strings.begin(); + i != ceph_option_strings.end(); + ++i) { + ceph_options.push_back(i->c_str()); + } + + global_init( + &def_args, ceph_options, CEPH_ENTITY_TYPE_MON, + CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + g_ceph_context->_conf->apply_changes(NULL); + g_conf = g_ceph_context->_conf; + + if (vm.count("help")) { + std::cerr << desc << std::endl; + return 1; + } + + int fd; + if (vm.count("out")) { + if ((fd = open(out_path.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666)) < 0) { + int _err = errno; + if (_err != EISDIR) { + std::cerr << "Couldn't open " << out_path << ": " << cpp_strerror(_err) << std::endl; + return 1; + } + } + } else { + fd = STDOUT_FILENO; + } + + if (fd < 0 && cmd != "store-copy") { + std::cerr << "error: '" << out_path << "' is a directory!" << std::endl; + return 1; + } + + MonitorDBStore st(store_path); + if (store_path.size()) { + stringstream ss; + int r = st.open(ss); + if (r < 0) { + std::cerr << ss.str() << std::endl; + goto done; + } + } + if (cmd == "dump-keys") { + KeyValueDB::WholeSpaceIterator iter = st.get_iterator(); + while (iter->valid()) { + pair key(iter->raw_key()); + cout << key.first << " / " << key.second << std::endl; + iter->next(); + } + } else if (cmd == "compact") { + st.compact(); + } else if (cmd == "getmonmap") { + assert(fd >= 0); + if (!store_path.size()) { + std::cerr << "need mon store path" << std::endl; + std::cerr << desc << std::endl; + goto done; + } + version_t v; + if (version <= 0) { + v = st.get("monmap", "last_committed"); + } else { + v = version; + } + + bufferlist bl; + /// XXX: this is not ok, osdmap and full should be abstracted somewhere + int r = st.get("monmap", v, bl); + if (r < 0) { + std::cerr << "Error getting map: " << cpp_strerror(r) << std::endl; + goto done; + } + bl.write_fd(fd); + } else if (cmd == "getosdmap") { + if (!store_path.size()) { + std::cerr << "need mon store path" << std::endl; + std::cerr << desc << std::endl; + goto done; + } + version_t v; + if (version == -1) { + v = st.get("osdmap", "last_committed"); + } else { + v = version; + } + + bufferlist bl; + /// XXX: this is not ok, osdmap and full should be abstracted somewhere + int r = st.get("osdmap", st.combine_strings("full", v), bl); + if (r < 0) { + std::cerr << "Error getting map: " << cpp_strerror(r) << std::endl; + goto done; + } + bl.write_fd(fd); + } else if (cmd == "dump-paxos") { + for (version_t v = dstart; v <= dstop; ++v) { + bufferlist bl; + st.get("paxos", v, bl); + if (bl.length() == 0) + break; + cout << "\n--- " << v << " ---" << std::endl; + MonitorDBStore::Transaction tx; + Paxos::decode_append_transaction(tx, bl); + JSONFormatter f(true); + tx.dump(&f); + f.flush(cout); + } + } else if (cmd == "dump-trace") { + if (tfile.empty()) { + std::cerr << "Need trace_file" << std::endl; + std::cerr << desc << std::endl; + goto done; + } + TraceIter iter(tfile.c_str()); + iter.init(); + while (true) { + if (!iter.valid()) + break; + if (iter.num() >= dstop) { + break; + } + if (iter.num() >= dstart) { + JSONFormatter f(true); + iter.cur().dump(&f, false); + f.flush(std::cout); + std::cout << std::endl; + } + iter.next(); + } + std::cerr << "Read up to transaction " << iter.num() << std::endl; + } else if (cmd == "replay-trace") { + if (!store_path.size()) { + std::cerr << "need mon store path" << std::endl; + std::cerr << desc << std::endl; + goto done; + } + if (tfile.empty()) { + std::cerr << "Need trace_file" << std::endl; + std::cerr << desc << std::endl; + goto done; + } + unsigned num = 0; + for (unsigned i = 0; i < num_replays; ++i) { + TraceIter iter(tfile.c_str()); + iter.init(); + while (true) { + if (!iter.valid()) + break; + std::cerr << "Replaying trans num " << num << std::endl; + st.apply_transaction(iter.cur()); + iter.next(); + ++num; + } + std::cerr << "Read up to transaction " << iter.num() << std::endl; + } + } else if (cmd == "random-gen") { + if (!store_path.size()) { + std::cerr << "need mon store path" << std::endl; + std::cerr << desc << std::endl; + goto done; + } + unsigned num = 0; + for (unsigned i = 0; i < ntrans; ++i) { + std::cerr << "Applying trans " << i << std::endl; + MonitorDBStore::Transaction t; + string prefix; + prefix.push_back((i%26)+'a'); + for (unsigned j = 0; j < tsize; ++j) { + stringstream os; + os << num; + bufferlist bl; + for (unsigned k = 0; k < tvalsize; ++k) bl.append(rand()); + t.put(prefix, os.str(), bl); + ++num; + } + t.compact_prefix(prefix); + st.apply_transaction(t); + } + } else if (cmd == "store-copy") { + if (!store_path.size()) { + std::cerr << "need mon store path to copy from" << std::endl; + std::cerr << desc << std::endl; + goto done; + } + if (!out_path.size()) { + std::cerr << "need mon store path to copy to (--out )" + << std::endl; + std::cerr << desc << std::endl; + goto done; + } + if (fd > 0) { + std::cerr << "supplied out path '" << out_path << "' is not a directory" + << std::endl; + goto done; + } + + MonitorDBStore out_store(out_path); + { + stringstream ss; + int r = out_store.create_and_open(ss); + if (r < 0) { + std::cerr << ss.str() << std::endl; + goto done; + } + } + + + KeyValueDB::WholeSpaceIterator it = st.get_iterator(); + uint64_t total_keys = 0; + uint64_t total_size = 0; + uint64_t total_tx = 0; + + do { + uint64_t num_keys = 0; + + MonitorDBStore::Transaction tx; + + while (it->valid() && num_keys < 128) { + pair k = it->raw_key(); + bufferlist v = it->value(); + tx.put(k.first, k.second, v); + + num_keys ++; + total_tx ++; + total_size += v.length(); + + it->next(); + } + + total_keys += num_keys; + + if (!tx.empty()) + out_store.apply_transaction(tx); + + std::cout << "copied " << total_keys << " keys so far (" + << stringify(si_t(total_size)) << ")" << std::endl; + + } while (it->valid()); + + std::cout << "summary: copied " << total_keys << " keys, using " + << total_tx << " transactions, totalling " + << stringify(si_t(total_size)) << std::endl; + std::cout << "from '" << store_path << "' to '" << out_path << "'" + << std::endl; + } else { + std::cerr << "Unrecognized command: " << cmd << std::endl; + goto done; + } + + done: + if (vm.count("out") && fd > 0) { + ::close(fd); + } + return 0; +} diff --git a/src/tools/ceph_osdomap_tool.cc b/src/tools/ceph_osdomap_tool.cc new file mode 100644 index 0000000000000..bde4b28b45f39 --- /dev/null +++ b/src/tools/ceph_osdomap_tool.cc @@ -0,0 +1,161 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* +* Ceph - scalable distributed file system +* +* Copyright (C) 2012 Inktank, Inc. +* +* This is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License kkjversion 2.1, as published by the Free Software +* Foundation. See file COPYING. +*/ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "global/global_init.h" +#include "os/LevelDBStore.h" +#include "mon/MonitorDBStore.h" +#include "os/DBObjectMap.h" + +namespace po = boost::program_options; +using namespace std; + +int main(int argc, char **argv) { + po::options_description desc("Allowed options"); + string store_path, cmd, out_path; + bool paranoid = false; + desc.add_options() + ("help", "produce help message") + ("omap-path", po::value(&store_path), + "path to mon directory, mandatory (current/omap usually)") + ("paranoid", po::value(¶noid), + "use paranoid checking") + ("command", po::value(&cmd), + "command") + ; + po::positional_options_description p; + p.add("command", 1); + + po::variables_map vm; + po::parsed_options parsed = + po::command_line_parser(argc, argv).options(desc).positional(p).run(); + po::store( + parsed, + vm); + try { + po::notify(vm); + } catch (...) { + cout << desc << std::endl; + return 1; + } + + vector ceph_options, def_args; + vector ceph_option_strings = po::collect_unrecognized( + parsed.options, po::include_positional); + ceph_options.reserve(ceph_option_strings.size()); + for (vector::iterator i = ceph_option_strings.begin(); + i != ceph_option_strings.end(); + ++i) { + ceph_options.push_back(i->c_str()); + } + + global_init( + &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD, + CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + g_ceph_context->_conf->apply_changes(NULL); + g_conf = g_ceph_context->_conf; + + if (vm.count("help")) { + std::cerr << desc << std::endl; + return 1; + } + + LevelDBStore* store(new LevelDBStore(g_ceph_context, store_path)); + if (paranoid) { + std::cerr << "Enabling paranoid checks" << std::endl; + store->options.paranoid_checks = paranoid; + } + DBObjectMap omap(store); + stringstream out; + int r = store->open(out); + if (r < 0) { + std::cerr << "Store open got: " << cpp_strerror(r) << std::endl; + std::cerr << "Output: " << out.str() << std::endl; + goto done; + } + r = 0; + + + if (cmd == "dump-raw-keys") { + KeyValueDB::WholeSpaceIterator i = store->get_iterator(); + for (i->seek_to_first(); i->valid(); i->next()) { + std::cout << i->raw_key() << std::endl; + } + } else if (cmd == "dump-raw-key-vals") { + KeyValueDB::WholeSpaceIterator i = store->get_iterator(); + for (i->seek_to_first(); i->valid(); i->next()) { + std::cout << i->raw_key() << std::endl; + i->value().hexdump(std::cout); + } + } else if (cmd == "dump-objects") { + vector objects; + r = omap.list_objects(&objects); + if (r < 0) { + std::cerr << "list_objects got: " << cpp_strerror(r) << std::endl; + goto done; + } + for (vector::iterator i = objects.begin(); + i != objects.end(); + ++i) { + std::cout << *i << std::endl; + } + r = 0; + } else if (cmd == "dump-objects-with-keys") { + vector objects; + r = omap.list_objects(&objects); + if (r < 0) { + std::cerr << "list_objects got: " << cpp_strerror(r) << std::endl; + goto done; + } + for (vector::iterator i = objects.begin(); + i != objects.end(); + ++i) { + std::cout << "Object: " << *i << std::endl; + ObjectMap::ObjectMapIterator j = omap.get_iterator(i->hobj); + for (j->seek_to_first(); j->valid(); j->next()) { + std::cout << j->key() << std::endl; + j->value().hexdump(std::cout); + } + } + } else if (cmd == "check") { + r = omap.check(std::cout); + if (!r) { + std::cerr << "check got: " << cpp_strerror(r) << std::endl; + goto done; + } + std::cout << "check succeeded" << std::endl; + } else { + std::cerr << "Did not recognize command " << cmd << std::endl; + goto done; + } + + done: + return r; +}