_ for source files, - for (shippable) build targets.
Signed-off-by: Sage Weil <sage@inktank.com>
-ceph_osdomap_tool_SOURCES = tools/ceph-osdomap-tool.cc
+ceph_osdomap_tool_SOURCES = tools/ceph_osdomap_tool.cc
ceph_osdomap_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS)
bin_DEBUGPROGRAMS += ceph-osdomap-tool
-ceph_monstore_tool_SOURCES = tools/ceph-monstore-tool.cc
+ceph_monstore_tool_SOURCES = tools/ceph_monstore_tool.cc
ceph_monstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS)
bin_DEBUGPROGRAMS += ceph-monstore-tool
-ceph_kvstore_tool_SOURCES = tools/ceph-kvstore-tool.cc
+ceph_kvstore_tool_SOURCES = tools/ceph_kvstore_tool.cc
ceph_kvstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL)
ceph_kvstore_tool_CXXFLAGS = $(UNITTEST_CXXFLAGS)
bin_DEBUGPROGRAMS += ceph-kvstore-tool
-ceph_filestore_tool_SOURCES = tools/ceph-filestore-tool.cc
+ceph_filestore_tool_SOURCES = tools/ceph_filestore_tool.cc
ceph_filestore_tool_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options
if LINUX
ceph_filestore_tool_LDADD += -ldl
endif # LINUX
bin_PROGRAMS += ceph_filestore_tool
-ceph_filestore_dump_SOURCES = tools/ceph-filestore-dump.cc
+ceph_filestore_dump_SOURCES = tools/ceph_filestore_dump.cc
ceph_filestore_dump_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS)
if LINUX
ceph_filestore_dump_LDADD += -ldl
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Inktank
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation. See file COPYING.
- *
- */
-
-#include <boost/scoped_ptr.hpp>
-#include <boost/lexical_cast.hpp>
-#include <boost/program_options/option.hpp>
-#include <boost/program_options/options_description.hpp>
-#include <boost/program_options/variables_map.hpp>
-#include <boost/program_options/cmdline.hpp>
-#include <boost/program_options/parsers.hpp>
-#include <iostream>
-#include <set>
-#include <sstream>
-#include <stdlib.h>
-#include <fstream>
-
-#include "common/Formatter.h"
-
-#include "global/global_init.h"
-#include "os/ObjectStore.h"
-#include "os/FileStore.h"
-#include "common/perf_counters.h"
-#include "common/errno.h"
-#include "osd/PGLog.h"
-#include "osd/OSD.h"
-
-namespace po = boost::program_options;
-using namespace std;
-
-enum {
- TYPE_NONE = 0,
- TYPE_PG_BEGIN,
- TYPE_PG_END,
- TYPE_OBJECT_BEGIN,
- TYPE_OBJECT_END,
- TYPE_DATA,
- TYPE_ATTRS,
- TYPE_OMAP_HDR,
- TYPE_OMAP,
- TYPE_PG_METADATA,
- END_OF_TYPES, //Keep at the end
-};
-
-//#define INTERNAL_TEST
-//#define INTERNAL_TEST2
-
-#ifdef INTERNAL_TEST
-CompatSet get_test_compat_set() {
- CompatSet::FeatureSet ceph_osd_feature_compat;
- CompatSet::FeatureSet ceph_osd_feature_ro_compat;
- CompatSet::FeatureSet ceph_osd_feature_incompat;
- ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_BASE);
- ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_PGINFO);
- ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_OLOC);
- ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEC);
- ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_CATEGORIES);
- ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_HOBJECTPOOL);
- ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_BIGINFO);
- ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEVELDBINFO);
- ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEVELDBLOG);
-#ifdef INTERNAL_TEST2
- ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SNAPMAPPER);
- ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS);
-#endif
- return CompatSet(ceph_osd_feature_compat, ceph_osd_feature_ro_compat,
- ceph_osd_feature_incompat);
-}
-#endif
-
-typedef uint8_t sectiontype_t;
-typedef uint32_t mymagic_t;
-typedef int64_t mysize_t;
-const ssize_t max_read = 1024 * 1024;
-const uint16_t shortmagic = 0xffce; //goes into stream as "ceff"
-//endmagic goes into stream as "ceff ffec"
-const mymagic_t endmagic = (0xecff << 16) | shortmagic;
-const int fd_none = INT_MIN;
-
-//The first FIXED_LENGTH bytes are a fixed
-//portion of the export output. This includes the overall
-//version number, and size of header and footer.
-//THIS STRUCTURE CAN ONLY BE APPENDED TO. If it needs to expand,
-//the version can be bumped and then anything
-//can be added to the export format.
-struct super_header {
- static const uint32_t super_magic = (shortmagic << 16) | shortmagic;
- static const uint32_t super_ver = 2;
- static const uint32_t FIXED_LENGTH = 16;
- uint32_t magic;
- uint32_t version;
- uint32_t header_size;
- uint32_t footer_size;
-
- super_header() : magic(0), version(0), header_size(0), footer_size(0) { }
- int read_super();
-
- void encode(bufferlist& bl) const {
- ::encode(magic, bl);
- ::encode(version, bl);
- ::encode(header_size, bl);
- ::encode(footer_size, bl);
- }
- void decode(bufferlist::iterator& bl) {
- ::decode(magic, bl);
- ::decode(version, bl);
- ::decode(header_size, bl);
- ::decode(footer_size, bl);
- }
-};
-
-struct header {
- sectiontype_t type;
- mysize_t size;
- header(sectiontype_t type, mysize_t size) :
- type(type), size(size) { }
- header(): type(0), size(0) { }
-
- int get_header();
-
- void encode(bufferlist& bl) const {
- uint32_t debug_type = (type << 24) | (type << 16) | shortmagic;
- ENCODE_START(1, 1, bl);
- ::encode(debug_type, bl);
- ::encode(size, bl);
- ENCODE_FINISH(bl);
- }
- void decode(bufferlist::iterator& bl) {
- uint32_t debug_type;
- DECODE_START(1, bl);
- ::decode(debug_type, bl);
- type = debug_type >> 24;
- ::decode(size, bl);
- DECODE_FINISH(bl);
- }
-};
-
-struct footer {
- mymagic_t magic;
- footer() : magic(endmagic) { }
-
- int get_footer();
-
- void encode(bufferlist& bl) const {
- ENCODE_START(1, 1, bl);
- ::encode(magic, bl);
- ENCODE_FINISH(bl);
- }
- void decode(bufferlist::iterator& bl) {
- DECODE_START(1, bl);
- ::decode(magic, bl);
- DECODE_FINISH(bl);
- }
-};
-
-struct pg_begin {
- pg_t pgid;
- OSDSuperblock superblock;
-
- pg_begin(pg_t pg, const OSDSuperblock& sb):
- pgid(pg), superblock(sb) { }
- pg_begin() { }
-
- void encode(bufferlist& bl) const {
- // New super_ver prevents decode from ver 1
- ENCODE_START(2, 2, bl);
- ::encode(pgid, bl);
- ::encode(superblock, bl);
- ENCODE_FINISH(bl);
- }
- void decode(bufferlist::iterator& bl) {
- DECODE_START(2, bl);
- ::decode(pgid, bl);
- if (struct_v > 1) {
- ::decode(superblock, bl);
- }
- DECODE_FINISH(bl);
- }
-};
-
-struct object_begin {
- hobject_t hoid;
- object_begin(const hobject_t &hoid): hoid(hoid) { }
- object_begin() { }
-
- void encode(bufferlist& bl) const {
- ENCODE_START(1, 1, bl);
- ::encode(hoid, bl);
- ENCODE_FINISH(bl);
- }
- void decode(bufferlist::iterator& bl) {
- DECODE_START(1, bl);
- ::decode(hoid, bl);
- DECODE_FINISH(bl);
- }
-};
-
-struct data_section {
- uint64_t offset;
- uint64_t len;
- bufferlist databl;
- data_section(uint64_t offset, uint64_t len, bufferlist bl):
- offset(offset), len(len), databl(bl) { }
- data_section(): offset(0), len(0) { }
-
- void encode(bufferlist& bl) const {
- ENCODE_START(1, 1, bl);
- ::encode(offset, bl);
- ::encode(len, bl);
- ::encode(databl, bl);
- ENCODE_FINISH(bl);
- }
- void decode(bufferlist::iterator& bl) {
- DECODE_START(1, bl);
- ::decode(offset, bl);
- ::decode(len, bl);
- ::decode(databl, bl);
- DECODE_FINISH(bl);
- }
-};
-
-struct attr_section {
- map<string,bufferptr> data;
- attr_section(const map<string,bufferptr> &data) : data(data) { }
- attr_section() { }
-
- void encode(bufferlist& bl) const {
- ENCODE_START(1, 1, bl);
- ::encode(data, bl);
- ENCODE_FINISH(bl);
- }
- void decode(bufferlist::iterator& bl) {
- DECODE_START(1, bl);
- ::decode(data, bl);
- DECODE_FINISH(bl);
- }
-};
-
-struct omap_hdr_section {
- bufferlist hdr;
- omap_hdr_section(bufferlist hdr) : hdr(hdr) { }
- omap_hdr_section() { }
-
- void encode(bufferlist& bl) const {
- ENCODE_START(1, 1, bl);
- ::encode(hdr, bl);
- ENCODE_FINISH(bl);
- }
- void decode(bufferlist::iterator& bl) {
- DECODE_START(1, bl);
- ::decode(hdr, bl);
- DECODE_FINISH(bl);
- }
-};
-
-struct omap_section {
- map<string, bufferlist> omap;
- omap_section(const map<string, bufferlist> &omap) :
- omap(omap) { }
- omap_section() { }
-
- void encode(bufferlist& bl) const {
- ENCODE_START(1, 1, bl);
- ::encode(omap, bl);
- ENCODE_FINISH(bl);
- }
- void decode(bufferlist::iterator& bl) {
- DECODE_START(1, bl);
- ::decode(omap, bl);
- DECODE_FINISH(bl);
- }
-};
-
-struct metadata_section {
- __u8 struct_ver;
- epoch_t map_epoch;
- pg_info_t info;
- pg_log_t log;
-
- metadata_section(__u8 struct_ver, epoch_t map_epoch, const pg_info_t &info,
- const pg_log_t &log)
- : struct_ver(struct_ver),
- map_epoch(map_epoch),
- info(info),
- log(log) { }
- metadata_section()
- : struct_ver(0),
- map_epoch(0) { }
-
- void encode(bufferlist& bl) const {
- ENCODE_START(1, 1, bl);
- ::encode(struct_ver, bl);
- ::encode(map_epoch, bl);
- ::encode(info, bl);
- ::encode(log, bl);
- ENCODE_FINISH(bl);
- }
- void decode(bufferlist::iterator& bl) {
- DECODE_START(1, bl);
- ::decode(struct_ver, bl);
- ::decode(map_epoch, bl);
- ::decode(info, bl);
- ::decode(log, bl);
- DECODE_FINISH(bl);
- }
-};
-
-hobject_t infos_oid = OSD::make_infos_oid();
-hobject_t biginfo_oid, log_oid;
-
-int file_fd = fd_none;
-bool debug = false;
-super_header sh;
-
-template <typename T>
-int write_section(sectiontype_t type, const T& obj, int fd) {
- bufferlist blhdr, bl, blftr;
- obj.encode(bl);
- header hdr(type, bl.length());
- hdr.encode(blhdr);
- footer ft;
- ft.encode(blftr);
-
- int ret = blhdr.write_fd(fd);
- if (ret) return ret;
- ret = bl.write_fd(fd);
- if (ret) return ret;
- ret = blftr.write_fd(fd);
- return ret;
-}
-
-int write_simple(sectiontype_t type, int fd)
-{
- bufferlist hbl;
-
- header hdr(type, 0);
- hdr.encode(hbl);
- return hbl.write_fd(fd);
-}
-
-static void invalid_path(string &path)
-{
- cout << "Invalid path to osd store specified: " << path << "\n";
- exit(1);
-}
-
-int get_log(ObjectStore *fs, coll_t coll, pg_t pgid, const pg_info_t &info,
- PGLog::IndexedLog &log, pg_missing_t &missing)
-{
- map<eversion_t, hobject_t> divergent_priors;
- try {
- ostringstream oss;
- PGLog::read_log(fs, coll, log_oid, info, divergent_priors, log, missing, oss);
- if (debug && oss.str().size())
- cerr << oss.str() << std::endl;
- }
- catch (const buffer::error &e) {
- cout << "read_log threw exception error " << e.what() << std::endl;
- return 1;
- }
- return 0;
-}
-
-//Based on RemoveWQ::_process()
-void remove_coll(ObjectStore *store, const coll_t &coll)
-{
- spg_t pg;
- coll.is_pg_prefix(pg);
- OSDriver driver(
- store,
- coll_t(),
- OSD::make_snapmapper_oid());
- SnapMapper mapper(&driver, 0, 0, 0, pg.shard);
-
- vector<ghobject_t> objects;
- ghobject_t next;
- int r = 0;
- int64_t num = 0;
- ObjectStore::Transaction *t = new ObjectStore::Transaction;
- cout << "remove_coll " << coll << std::endl;
- while (!next.is_max()) {
- r = store->collection_list_partial(coll, next, 200, 300, 0,
- &objects, &next);
- if (r < 0)
- goto out;
- for (vector<ghobject_t>::iterator i = objects.begin();
- i != objects.end();
- ++i, ++num) {
-
- assert(i->generation == ghobject_t::NO_GEN);
- OSDriver::OSTransaction _t(driver.get_transaction(t));
- cout << "remove " << *i << std::endl;
- int r = mapper.remove_oid(i->hobj, &_t);
- if (r != 0 && r != -ENOENT) {
- assert(0);
- }
-
- t->remove(coll, *i);
- if (num >= 30) {
- store->apply_transaction(*t);
- delete t;
- t = new ObjectStore::Transaction;
- num = 0;
- }
- }
- }
- t->remove_collection(coll);
- store->apply_transaction(*t);
-out:
- delete t;
-}
-
-//Based on part of OSD::load_pgs()
-int finish_remove_pgs(ObjectStore *store, uint64_t *next_removal_seq)
-{
- vector<coll_t> ls;
- int r = store->list_collections(ls);
- if (r < 0) {
- cout << "finish_remove_pgs: failed to list pgs: " << cpp_strerror(-r)
- << std::endl;
- return r;
- }
-
- for (vector<coll_t>::iterator it = ls.begin();
- it != ls.end();
- ++it) {
- spg_t pgid;
- snapid_t snap;
-
- if (it->is_temp(pgid)) {
- cout << "finish_remove_pgs " << *it << " clearing temp" << std::endl;
- OSD::recursive_remove_collection(store, *it);
- continue;
- }
-
- if (it->is_pg(pgid, snap)) {
- continue;
- }
-
- uint64_t seq;
- if (it->is_removal(&seq, &pgid)) {
- if (seq >= *next_removal_seq)
- *next_removal_seq = seq + 1;
- cout << "finish_remove_pgs removing " << *it << ", seq is "
- << seq << " pgid is " << pgid << std::endl;
- remove_coll(store, *it);
- continue;
- }
-
- //cout << "finish_remove_pgs ignoring unrecognized " << *it << std::endl;
- }
- return 0;
-}
-
-int initiate_new_remove_pg(ObjectStore *store, pg_t r_pgid,
- uint64_t *next_removal_seq)
-{
- ObjectStore::Transaction *rmt = new ObjectStore::Transaction;
-
- if (store->collection_exists(coll_t(spg_t(r_pgid, ghobject_t::no_shard())))) {
- coll_t to_remove = coll_t::make_removal_coll((*next_removal_seq)++,
- spg_t(r_pgid, ghobject_t::no_shard()));
- cout << "collection rename " << coll_t(spg_t(r_pgid, ghobject_t::no_shard()))
- << " to " << to_remove
- << std::endl;
- rmt->collection_rename(coll_t(spg_t(r_pgid, ghobject_t::no_shard())), to_remove);
- } else {
- delete rmt;
- return ENOENT;
- }
-
- cout << "remove " << coll_t::META_COLL << " " << log_oid.oid << std::endl;
- rmt->remove(coll_t::META_COLL, log_oid);
- cout << "remove " << coll_t::META_COLL << " " << biginfo_oid.oid << std::endl;
- rmt->remove(coll_t::META_COLL, biginfo_oid);
-
- store->apply_transaction(*rmt);
-
- return 0;
-}
-
-int header::get_header()
-{
- bufferlist ebl;
- bufferlist::iterator ebliter = ebl.begin();
- ssize_t bytes;
-
- bytes = ebl.read_fd(file_fd, sh.header_size);
- if ((size_t)bytes != sh.header_size) {
- cout << "Unexpected EOF" << std::endl;
- return EFAULT;
- }
-
- decode(ebliter);
-
- return 0;
-}
-
-int footer::get_footer()
-{
- bufferlist ebl;
- bufferlist::iterator ebliter = ebl.begin();
- ssize_t bytes;
-
- bytes = ebl.read_fd(file_fd, sh.footer_size);
- if ((size_t)bytes != sh.footer_size) {
- cout << "Unexpected EOF" << std::endl;
- return EFAULT;
- }
-
- decode(ebliter);
-
- if (magic != endmagic) {
- cout << "Bad footer magic" << std::endl;
- return EFAULT;
- }
-
- return 0;
-}
-
-int write_info(ObjectStore::Transaction &t, epoch_t epoch, pg_info_t &info,
- __u8 struct_ver)
-{
- //Empty for this
- interval_set<snapid_t> snap_collections; // obsolete
- map<epoch_t,pg_interval_t> past_intervals;
- coll_t coll(info.pgid);
-
- int ret = PG::_write_info(t, epoch,
- info, coll,
- past_intervals,
- snap_collections,
- infos_oid,
- struct_ver,
- true, true);
- if (ret < 0) ret = -ret;
- if (ret) cout << "Failed to write info" << std::endl;
- return ret;
-}
-
-void write_log(ObjectStore::Transaction &t, pg_log_t &log)
-{
- map<eversion_t, hobject_t> divergent_priors;
- PGLog::write_log(t, log, log_oid, divergent_priors);
-}
-
-int write_pg(ObjectStore::Transaction &t, epoch_t epoch, pg_info_t &info,
- pg_log_t &log, __u8 struct_ver)
-{
- int ret = write_info(t, epoch, info, struct_ver);
- if (ret) return ret;
- write_log(t, log);
- return 0;
-}
-
-int export_file(ObjectStore *store, coll_t cid, hobject_t &obj)
-{
- struct stat st;
- mysize_t total;
- ostringstream objname;
- footer ft;
-
- int ret = store->stat(cid, obj, &st);
- if (ret < 0)
- return ret;
-
- objname << obj;
- if (debug && file_fd != STDOUT_FILENO)
- cout << "objname=" << objname.str() << std::endl;
-
- total = st.st_size;
- if (debug && file_fd != STDOUT_FILENO)
- cout << "size=" << total << std::endl;
-
- object_begin objb(obj);
- ret = write_section(TYPE_OBJECT_BEGIN, objb, file_fd);
- if (ret < 0)
- return ret;
-
- uint64_t offset = 0;
- bufferlist rawdatabl, databl;
- while(total > 0) {
- rawdatabl.clear();
- databl.clear();
- mysize_t len = max_read;
- if (len > total)
- len = total;
-
- ret = store->read(cid, obj, offset, len, rawdatabl);
- if (ret < 0)
- return ret;
- if (ret == 0)
- return -EINVAL;
-
- data_section dblock(offset, len, rawdatabl);
- total -= ret;
- offset += ret;
-
- if (debug && file_fd != STDOUT_FILENO)
- cout << "data section offset=" << offset << " len=" << len << std::endl;
-
- ret = write_section(TYPE_DATA, dblock, file_fd);
- if (ret) return ret;
- }
-
- //Handle attrs for this object
- map<string,bufferptr> aset;
- ret = store->getattrs(cid, obj, aset, false);
- if (ret) return ret;
- attr_section as(aset);
- ret = write_section(TYPE_ATTRS, as, file_fd);
- if (ret)
- return ret;
-
- if (debug && file_fd != STDOUT_FILENO) {
- cout << "attrs size " << aset.size() << std::endl;
- }
-
- //Handle omap information
- databl.clear();
- bufferlist hdrbuf;
- map<string, bufferlist> out;
- ret = store->omap_get(cid, obj, &hdrbuf, &out);
- if (ret < 0)
- return ret;
-
- omap_hdr_section ohs(hdrbuf);
- ret = write_section(TYPE_OMAP_HDR, ohs, file_fd);
- if (ret)
- return ret;
-
- if (!out.empty()) {
- omap_section oms(out);
- ret = write_section(TYPE_OMAP, oms, file_fd);
- if (ret)
- return ret;
-
- if (debug && file_fd != STDOUT_FILENO)
- cout << "omap map size " << out.size() << std::endl;
- }
-
- ret = write_simple(TYPE_OBJECT_END, file_fd);
- if (ret)
- return ret;
-
- return 0;
-}
-
-int export_files(ObjectStore *store, coll_t coll)
-{
- vector<ghobject_t> objects;
- ghobject_t next;
-
- while (!next.is_max()) {
- int r = store->collection_list_partial(coll, next, 200, 300, 0,
- &objects, &next);
- if (r < 0)
- return r;
- for (vector<ghobject_t>::iterator i = objects.begin();
- i != objects.end();
- ++i) {
- assert(i->generation == ghobject_t::NO_GEN);
- r = export_file(store, coll, i->hobj);
- if (r < 0)
- return r;
- }
- }
- return 0;
-}
-
-//Write super_header with its fixed 16 byte length
-void write_super()
-{
- bufferlist superbl;
- super_header sh;
- footer ft;
-
- header hdr(TYPE_NONE, 0);
- hdr.encode(superbl);
-
- sh.magic = super_header::super_magic;
- sh.version = super_header::super_ver;
- sh.header_size = superbl.length();
- superbl.clear();
- ft.encode(superbl);
- sh.footer_size = superbl.length();
- superbl.clear();
-
- sh.encode(superbl);
- assert(super_header::FIXED_LENGTH == superbl.length());
- superbl.write_fd(file_fd);
-}
-
-int do_export(ObjectStore *fs, coll_t coll, pg_t pgid, pg_info_t &info,
- epoch_t map_epoch, __u8 struct_ver, const OSDSuperblock& superblock)
-{
- PGLog::IndexedLog log;
- pg_missing_t missing;
-
- int ret = get_log(fs, coll, pgid, info, log, missing);
- if (ret > 0)
- return ret;
-
- write_super();
-
- pg_begin pgb(pgid, superblock);
- ret = write_section(TYPE_PG_BEGIN, pgb, file_fd);
- if (ret)
- return ret;
-
- export_files(fs, coll);
-
- metadata_section ms(struct_ver, map_epoch, info, log);
- ret = write_section(TYPE_PG_METADATA, ms, file_fd);
- if (ret)
- return ret;
-
- ret = write_simple(TYPE_PG_END, file_fd);
- if (ret)
- return ret;
-
- return 0;
-}
-
-int super_header::read_super()
-{
- bufferlist ebl;
- bufferlist::iterator ebliter = ebl.begin();
- ssize_t bytes;
-
- bytes = ebl.read_fd(file_fd, super_header::FIXED_LENGTH);
- if ((size_t)bytes != super_header::FIXED_LENGTH) {
- cout << "Unexpected EOF" << std::endl;
- return EFAULT;
- }
-
- decode(ebliter);
-
- return 0;
-}
-
-int read_section(int fd, sectiontype_t *type, bufferlist *bl)
-{
- header hdr;
- ssize_t bytes;
-
- int ret = hdr.get_header();
- if (ret)
- return ret;
-
- *type = hdr.type;
-
- bl->clear();
- bytes = bl->read_fd(fd, hdr.size);
- if (bytes != hdr.size) {
- cout << "Unexpected EOF" << std::endl;
- return EFAULT;
- }
-
- if (hdr.size > 0) {
- footer ft;
- ret = ft.get_footer();
- if (ret)
- return ret;
- }
-
- return 0;
-}
-
-int get_data(ObjectStore *store, coll_t coll, hobject_t hoid,
- ObjectStore::Transaction *t, bufferlist &bl)
-{
- bufferlist::iterator ebliter = bl.begin();
- data_section ds;
- ds.decode(ebliter);
-
- if (debug)
- cout << "\tdata: offset " << ds.offset << " len " << ds.len << std::endl;
- t->write(coll, hoid, ds.offset, ds.len, ds.databl);
- return 0;
-}
-
-int get_attrs(ObjectStore *store, coll_t coll, hobject_t hoid,
- ObjectStore::Transaction *t, bufferlist &bl,
- OSDriver &driver, SnapMapper &snap_mapper)
-{
- bufferlist::iterator ebliter = bl.begin();
- attr_section as;
- as.decode(ebliter);
-
- if (debug)
- cout << "\tattrs: len " << as.data.size() << std::endl;
- t->setattrs(coll, hoid, as.data);
-
- if (hoid.snap < CEPH_MAXSNAP) {
- map<string,bufferptr>::iterator mi = as.data.find(OI_ATTR);
- if (mi != as.data.end()) {
- bufferlist attr_bl;
- attr_bl.push_back(mi->second);
- object_info_t oi(attr_bl);
-
- if (debug)
- cout << "object_info " << oi << std::endl;
-
- OSDriver::OSTransaction _t(driver.get_transaction(t));
- set<snapid_t> oi_snaps(oi.snaps.begin(), oi.snaps.end());
- snap_mapper.add_oid(hoid, oi_snaps, &_t);
- }
- }
-
- return 0;
-}
-
-int get_omap_hdr(ObjectStore *store, coll_t coll, hobject_t hoid,
- ObjectStore::Transaction *t, bufferlist &bl)
-{
- bufferlist::iterator ebliter = bl.begin();
- omap_hdr_section oh;
- oh.decode(ebliter);
-
- if (debug)
- cout << "\tomap header: " << string(oh.hdr.c_str(), oh.hdr.length())
- << std::endl;
- t->omap_setheader(coll, hoid, oh.hdr);
- return 0;
-}
-
-int get_omap(ObjectStore *store, coll_t coll, hobject_t hoid,
- ObjectStore::Transaction *t, bufferlist &bl)
-{
- bufferlist::iterator ebliter = bl.begin();
- omap_section os;
- os.decode(ebliter);
-
- if (debug)
- cout << "\tomap: size " << os.omap.size() << std::endl;
- t->omap_setkeys(coll, hoid, os.omap);
- return 0;
-}
-
-int get_object(ObjectStore *store, coll_t coll, bufferlist &bl)
-{
- ObjectStore::Transaction tran;
- ObjectStore::Transaction *t = &tran;
- bufferlist::iterator ebliter = bl.begin();
- object_begin ob;
- ob.decode(ebliter);
- OSDriver driver(
- store,
- coll_t(),
- OSD::make_snapmapper_oid());
- spg_t pg;
- coll.is_pg_prefix(pg);
- SnapMapper mapper(&driver, 0, 0, 0, pg.shard);
-
- t->touch(coll, ob.hoid);
-
- if (debug) {
- ostringstream objname;
- objname << ob.hoid.oid;
- cout << "name " << objname.str() << " snap " << ob.hoid.snap << std::endl;
- }
-
- bufferlist ebl;
- bool done = false;
- while(!done) {
- sectiontype_t type;
- int ret = read_section(file_fd, &type, &ebl);
- if (ret)
- return ret;
-
- //cout << "\tdo_object: Section type " << hex << type << dec << std::endl;
- //cout << "\t\tsection size " << ebl.length() << std::endl;
- if (type >= END_OF_TYPES) {
- cout << "Skipping unknown object section type" << std::endl;
- continue;
- }
- switch(type) {
- case TYPE_DATA:
- ret = get_data(store, coll, ob.hoid, t, ebl);
- if (ret) return ret;
- break;
- case TYPE_ATTRS:
- ret = get_attrs(store, coll, ob.hoid, t, ebl, driver, mapper);
- if (ret) return ret;
- break;
- case TYPE_OMAP_HDR:
- ret = get_omap_hdr(store, coll, ob.hoid, t, ebl);
- if (ret) return ret;
- break;
- case TYPE_OMAP:
- ret = get_omap(store, coll, ob.hoid, t, ebl);
- if (ret) return ret;
- break;
- case TYPE_OBJECT_END:
- done = true;
- break;
- default:
- return EFAULT;
- }
- }
- store->apply_transaction(*t);
- return 0;
-}
-
-int get_pg_metadata(ObjectStore *store, coll_t coll, bufferlist &bl)
-{
- ObjectStore::Transaction tran;
- ObjectStore::Transaction *t = &tran;
- bufferlist::iterator ebliter = bl.begin();
- metadata_section ms;
- ms.decode(ebliter);
-
-#if DIAGNOSTIC
- Formatter *formatter = new JSONFormatter(true);
- cout << "struct_v " << (int)ms.struct_ver << std::endl;
- cout << "epoch " << ms.map_epoch << std::endl;
- formatter->open_object_section("info");
- ms.info.dump(formatter);
- formatter->close_section();
- formatter->flush(cout);
- cout << std::endl;
-
- formatter->open_object_section("log");
- ms.log.dump(formatter);
- formatter->close_section();
- formatter->flush(cout);
- cout << std::endl;
-#endif
-
- coll_t newcoll(ms.info.pgid);
- t->collection_rename(coll, newcoll);
-
- int ret = write_pg(*t, ms.map_epoch, ms.info, ms.log, ms.struct_ver);
- if (ret) return ret;
-
- store->apply_transaction(*t);
-
- return 0;
-}
-
-int do_import(ObjectStore *store, OSDSuperblock& sb)
-{
- bufferlist ebl;
- pg_info_t info;
- PGLog::IndexedLog log;
-
- uint64_t next_removal_seq = 0; //My local seq
- finish_remove_pgs(store, &next_removal_seq);
-
- int ret = sh.read_super();
- if (ret)
- return ret;
-
- if (sh.magic != super_header::super_magic) {
- cout << "Invalid magic number" << std::endl;
- return EFAULT;
- }
-
- if (sh.version > super_header::super_ver) {
- cout << "Can't handle export format version=" << sh.version << std::endl;
- return EINVAL;
- }
-
- //First section must be TYPE_PG_BEGIN
- sectiontype_t type;
- ret = read_section(file_fd, &type, &ebl);
- if (type != TYPE_PG_BEGIN) {
- return EFAULT;
- }
-
- bufferlist::iterator ebliter = ebl.begin();
- pg_begin pgb;
- pgb.decode(ebliter);
- pg_t pgid = pgb.pgid;
-
- if (debug) {
- cout << "Exported features: " << pgb.superblock.compat_features << std::endl;
- }
- if (sb.compat_features.compare(pgb.superblock.compat_features) == -1) {
- cout << "Export has incompatible features set "
- << pgb.superblock.compat_features << std::endl;
- return 1;
- }
-
- log_oid = OSD::make_pg_log_oid(spg_t(pgid, ghobject_t::no_shard()));
- biginfo_oid = OSD::make_pg_biginfo_oid(spg_t(pgid, ghobject_t::no_shard()));
-
- //Check for PG already present.
- coll_t coll(spg_t(pgid, ghobject_t::no_shard()));
- if (store->collection_exists(coll)) {
- cout << "pgid " << pgid << " already exists" << std::endl;
- return 1;
- }
-
- //Switch to collection which will be removed automatically if
- //this program is interupted.
- coll_t rmcoll = coll_t::make_removal_coll(
- next_removal_seq, spg_t(pgid, ghobject_t::no_shard()));
- ObjectStore::Transaction *t = new ObjectStore::Transaction;
- t->create_collection(rmcoll);
- store->apply_transaction(*t);
- delete t;
-
- cout << "Importing pgid " << pgid << std::endl;
-
- bool done = false;
- bool found_metadata = false;
- while(!done) {
- ret = read_section(file_fd, &type, &ebl);
- if (ret)
- return ret;
-
- //cout << "do_import: Section type " << hex << type << dec << std::endl;
- if (type >= END_OF_TYPES) {
- cout << "Skipping unknown section type" << std::endl;
- continue;
- }
- switch(type) {
- case TYPE_OBJECT_BEGIN:
- ret = get_object(store, rmcoll, ebl);
- if (ret) return ret;
- break;
- case TYPE_PG_METADATA:
- ret = get_pg_metadata(store, rmcoll, ebl);
- if (ret) return ret;
- found_metadata = true;
- break;
- case TYPE_PG_END:
- done = true;
- break;
- default:
- return EFAULT;
- }
- }
-
- if (!found_metadata) {
- cout << "Missing metadata section" << std::endl;
- return EFAULT;
- }
-
- return 0;
-}
-
-int main(int argc, char **argv)
-{
- string fspath, jpath, pgidstr, type, file;
- Formatter *formatter = new JSONFormatter(true);
-
- po::options_description desc("Allowed options");
- desc.add_options()
- ("help", "produce help message")
- ("filestore-path", po::value<string>(&fspath),
- "path to filestore directory, mandatory")
- ("journal-path", po::value<string>(&jpath),
- "path to journal, mandatory")
- ("pgid", po::value<string>(&pgidstr),
- "PG id, mandatory")
- ("type", po::value<string>(&type),
- "Type one of info, log, remove, export, or import, mandatory")
- ("file", po::value<string>(&file),
- "path of file to export or import")
- ("debug", "Enable diagnostic output to stderr")
- ;
-
- po::variables_map vm;
- po::parsed_options parsed =
- po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
- po::store( parsed, vm);
- try {
- po::notify(vm);
- }
- catch(...) {
- cout << desc << std::endl;
- exit(1);
- }
-
- if (vm.count("help")) {
- cout << desc << std::endl;
- return 1;
- }
-
- if (!vm.count("filestore-path")) {
- cout << "Must provide filestore-path" << std::endl
- << desc << std::endl;
- return 1;
- }
- if (!vm.count("journal-path")) {
- cout << "Must provide journal-path" << std::endl
- << desc << std::endl;
- return 1;
- }
- if (!vm.count("type")) {
- cout << "Must provide type (info, log, remove, export, import)"
- << std::endl << desc << std::endl;
- return 1;
- }
- if (type != "import" && !vm.count("pgid")) {
- cout << "Must provide pgid" << std::endl
- << desc << std::endl;
- return 1;
- }
-
- file_fd = fd_none;
- if (type == "export") {
- if (!vm.count("file")) {
- file_fd = STDOUT_FILENO;
- } else {
- file_fd = open(file.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666);
- }
- } else if (type == "import") {
- if (!vm.count("file")) {
- file_fd = STDIN_FILENO;
- } else {
- file_fd = open(file.c_str(), O_RDONLY);
- }
- }
-
- if (vm.count("file") && file_fd == fd_none) {
- cout << "--file option only applies to import or export" << std::endl;
- return 1;
- }
-
- if (file_fd != fd_none && file_fd < 0) {
- perror("open");
- return 1;
- }
-
- if ((fspath.length() == 0 || jpath.length() == 0) ||
- (type != "info" && type != "log" && type != "remove" && type != "export"
- && type != "import") ||
- (type != "import" && pgidstr.length() == 0)) {
- cerr << "Invalid params" << std::endl;
- exit(1);
- }
-
- if (type == "import" && pgidstr.length()) {
- cerr << "--pgid option invalid with import" << std::endl;
- exit(1);
- }
-
- vector<const char *> ceph_options, def_args;
- vector<string> ceph_option_strings = po::collect_unrecognized(
- parsed.options, po::include_positional);
- ceph_options.reserve(ceph_option_strings.size());
- for (vector<string>::iterator i = ceph_option_strings.begin();
- i != ceph_option_strings.end();
- ++i) {
- ceph_options.push_back(i->c_str());
- }
-
- //Suppress derr() output to stderr by default
- if (!vm.count("debug")) {
- close(STDERR_FILENO);
- (void)open("/dev/null", O_WRONLY);
- debug = false;
- } else {
- debug = true;
- }
-
- global_init(
- &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD,
- CODE_ENVIRONMENT_UTILITY, 0);
- //CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
- common_init_finish(g_ceph_context);
- g_ceph_context->_conf->apply_changes(NULL);
- g_conf = g_ceph_context->_conf;
-
- //Verify that fspath really is an osd store
- struct stat st;
- if (::stat(fspath.c_str(), &st) == -1) {
- perror("fspath");
- invalid_path(fspath);
- }
- if (!S_ISDIR(st.st_mode)) {
- invalid_path(fspath);
- }
- string check = fspath + "/whoami";
- if (::stat(check.c_str(), &st) == -1) {
- perror("whoami");
- invalid_path(fspath);
- }
- if (!S_ISREG(st.st_mode)) {
- invalid_path(fspath);
- }
- check = fspath + "/current";
- if (::stat(check.c_str(), &st) == -1) {
- perror("current");
- invalid_path(fspath);
- }
- if (!S_ISDIR(st.st_mode)) {
- invalid_path(fspath);
- }
-
- pg_t pgid;
- if (pgidstr.length() && !pgid.parse(pgidstr.c_str())) {
- cout << "Invalid pgid '" << pgidstr << "' specified" << std::endl;
- exit(1);
- }
-
- ObjectStore *fs = new FileStore(fspath, jpath);
-
- int r = fs->mount();
- if (r < 0) {
- if (r == -EBUSY) {
- cout << "OSD has the store locked" << std::endl;
- } else {
- cout << "Mount failed with '" << cpp_strerror(-r) << "'" << std::endl;
- }
- return 1;
- }
-
- bool fs_sharded_objects = fs->get_allow_sharded_objects();
-
- int ret = 0;
- vector<coll_t> ls;
- vector<coll_t>::iterator it;
- CompatSet supported;
-
-#ifdef INTERNAL_TEST
- supported = get_test_compat_set();
-#else
- supported = OSD::get_osd_compat_set();
-#endif
-
- bufferlist bl;
- OSDSuperblock superblock;
- bufferlist::iterator p;
- ret = fs->read(coll_t::META_COLL, OSD_SUPERBLOCK_POBJECT, 0, 0, bl);
- if (ret < 0) {
- cout << "Failure to read OSD superblock error= " << r << std::endl;
- goto out;
- }
-
- p = bl.begin();
- ::decode(superblock, p);
-
-#ifdef INTERNAL_TEST2
- fs->set_allow_sharded_objects();
- assert(fs->get_allow_sharded_objects());
- fs_sharded_objects = true;
- superblock.compat_features.incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS);
-#endif
-
- if (debug && file_fd != STDOUT_FILENO) {
- cout << "Supported features: " << supported << std::endl;
- cout << "On-disk features: " << superblock.compat_features << std::endl;
- }
- if (supported.compare(superblock.compat_features) == -1) {
- cout << "On-disk OSD incompatible features set "
- << superblock.compat_features << std::endl;
- ret = EINVAL;
- goto out;
- }
-
- // If there was a crash as an OSD was transitioning to sharded objects
- // and hadn't completed a set_allow_sharded_objects().
- // This utility does not want to attempt to finish that transition.
- if (superblock.compat_features.incompat.contains(CEPH_OSD_FEATURE_INCOMPAT_SHARDS) != fs_sharded_objects) {
- // An OSD should never have call set_allow_sharded_objects() before
- // updating its own OSD features.
- if (fs_sharded_objects)
- cout << "FileStore sharded but OSD not set, Corruption?" << std::endl;
- else
- cout << "Found incomplete transition to sharded objects" << std::endl;
- ret = EINVAL;
- goto out;
- }
-
- if (type == "import") {
-
- try {
- ret = do_import(fs, superblock);
- }
- catch (const buffer::error &e) {
- cout << "do_import threw exception error " << e.what() << std::endl;
- ret = EFAULT;
- }
- if (ret == EFAULT) {
- cout << "Corrupt input for import" << std::endl;
- }
- goto out;
- }
-
- log_oid = OSD::make_pg_log_oid(spg_t(pgid, ghobject_t::no_shard()));
- biginfo_oid = OSD::make_pg_biginfo_oid(spg_t(pgid, ghobject_t::no_shard()));
-
- if (type == "remove") {
- uint64_t next_removal_seq = 0; //My local seq
- finish_remove_pgs(fs, &next_removal_seq);
- int r = initiate_new_remove_pg(fs, pgid, &next_removal_seq);
- if (r) {
- cout << "PG '" << pgid << "' not found" << std::endl;
- ret = 1;
- goto out;
- }
- finish_remove_pgs(fs, &next_removal_seq);
- cout << "Remove successful" << std::endl;
- goto out;
- }
-
- r = fs->list_collections(ls);
- if (r < 0) {
- cout << "failed to list pgs: " << cpp_strerror(-r) << std::endl;
- exit(1);
- }
-
- for (it = ls.begin(); it != ls.end(); ++it) {
- snapid_t snap;
- spg_t tmppgid;
-
- if (!it->is_pg(tmppgid, snap)) {
- continue;
- }
-
- if (tmppgid.pgid != pgid) {
- continue;
- }
- if (snap != CEPH_NOSNAP && debug) {
- cerr << "skipping snapped dir " << *it
- << " (pg " << pgid << " snap " << snap << ")" << std::endl;
- continue;
- }
-
- //Found!
- break;
- }
-
- epoch_t map_epoch;
- if (it != ls.end()) {
-
- coll_t coll = *it;
-
- bufferlist bl;
- map_epoch = PG::peek_map_epoch(fs, coll, infos_oid, &bl);
- if (debug)
- cerr << "map_epoch " << map_epoch << std::endl;
-
- pg_info_t info(spg_t(pgid, ghobject_t::no_shard()));
- map<epoch_t,pg_interval_t> past_intervals;
- hobject_t biginfo_oid = OSD::make_pg_biginfo_oid(
- spg_t(pgid, ghobject_t::no_shard()));
- interval_set<snapid_t> snap_collections;
-
- __u8 struct_ver;
- r = PG::read_info(fs, coll, bl, info, past_intervals, biginfo_oid,
- infos_oid, snap_collections, struct_ver);
- if (r < 0) {
- cout << "read_info error " << cpp_strerror(-r) << std::endl;
- ret = 1;
- goto out;
- }
- if (debug)
- cerr << "struct_v " << (int)struct_ver << std::endl;
-
- if (type == "export") {
- ret = do_export(fs, coll, pgid, info, map_epoch, struct_ver, superblock);
- } else if (type == "info") {
- formatter->open_object_section("info");
- info.dump(formatter);
- formatter->close_section();
- formatter->flush(cout);
- cout << std::endl;
- } else if (type == "log") {
- PGLog::IndexedLog log;
- pg_missing_t missing;
- ret = get_log(fs, coll, pgid, info, log, missing);
- if (ret > 0)
- goto out;
-
- formatter->open_object_section("log");
- log.dump(formatter);
- formatter->close_section();
- formatter->flush(cout);
- cout << std::endl;
- formatter->open_object_section("missing");
- missing.dump(formatter);
- formatter->close_section();
- formatter->flush(cout);
- cout << std::endl;
- }
- } else {
- cout << "PG '" << pgid << "' not found" << std::endl;
- ret = 1;
- }
-
-out:
- if (fs->umount() < 0) {
- cout << "umount failed" << std::endl;
- return 1;
- }
-
- return (ret != 0);
-}
-
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Inktank
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation. See file COPYING.
- *
- */
-
-#include <boost/scoped_ptr.hpp>
-#include <boost/lexical_cast.hpp>
-#include <boost/program_options/option.hpp>
-#include <boost/program_options/options_description.hpp>
-#include <boost/program_options/variables_map.hpp>
-#include <boost/program_options/cmdline.hpp>
-#include <boost/program_options/parsers.hpp>
-#include <iostream>
-#include <set>
-#include <sstream>
-#include <stdlib.h>
-#include <fstream>
-
-#include "common/Formatter.h"
-
-#include "global/global_init.h"
-#include "os/ObjectStore.h"
-#include "os/FileStore.h"
-#include "common/perf_counters.h"
-#include "common/errno.h"
-#include "osd/PGLog.h"
-#include "osd/osd_types.h"
-#include "osd/OSD.h"
-
-namespace po = boost::program_options;
-using namespace std;
-
-static void invalid_path(string &path)
-{
- cout << "Invalid path to osd store specified: " << path << "\n";
- exit(1);
-}
-
-int main(int argc, char **argv)
-{
- string fspath, jpath, pgidstr;
- bool list_lost_objects = false;
- bool fix_lost_objects = false;
- unsigned LIST_AT_A_TIME = 100;
- unsigned scanned = 0;
-
- po::options_description desc("Allowed options");
- desc.add_options()
- ("help", "produce help message")
- ("filestore-path", po::value<string>(&fspath),
- "path to filestore directory, mandatory")
- ("journal-path", po::value<string>(&jpath),
- "path to journal, mandatory")
- ("pgid", po::value<string>(&pgidstr),
- "PG id")
- ("list-lost-objects", po::value<bool>(
- &list_lost_objects)->default_value(false),
- "list lost objects")
- ("fix-lost-objects", po::value<bool>(
- &fix_lost_objects)->default_value(false),
- "fix lost objects")
- ;
-
- po::variables_map vm;
- po::parsed_options parsed =
- po::command_line_parser(argc, argv).options(desc).
- allow_unregistered().run();
- po::store( parsed, vm);
- try {
- po::notify(vm);
- }
- catch(...) {
- cout << desc << std::endl;
- exit(1);
- }
-
- if (vm.count("help")) {
- cout << desc << std::endl;
- return 1;
- }
-
- if (!vm.count("filestore-path")) {
- cerr << "Must provide filestore-path" << std::endl
- << desc << std::endl;
- return 1;
- }
- if (!vm.count("journal-path")) {
- cerr << "Must provide journal-path" << std::endl
- << desc << std::endl;
- return 1;
- }
-
- if ((fspath.length() == 0 || jpath.length() == 0)) {
- cerr << "Invalid params" << desc << std::endl;
- exit(1);
- }
-
- vector<const char *> ceph_options, def_args;
- vector<string> ceph_option_strings = po::collect_unrecognized(
- parsed.options, po::include_positional);
- ceph_options.reserve(ceph_option_strings.size());
- for (vector<string>::iterator i = ceph_option_strings.begin();
- i != ceph_option_strings.end();
- ++i) {
- ceph_options.push_back(i->c_str());
- }
-
- global_init(
- &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD,
- CODE_ENVIRONMENT_UTILITY, 0);
- //CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
- common_init_finish(g_ceph_context);
- g_ceph_context->_conf->apply_changes(NULL);
- g_conf = g_ceph_context->_conf;
-
- //Verify that fspath really is an osd store
- struct stat st;
- if (::stat(fspath.c_str(), &st) == -1) {
- perror("fspath");
- invalid_path(fspath);
- }
- if (!S_ISDIR(st.st_mode)) {
- invalid_path(fspath);
- }
- string check = fspath + "/whoami";
- if (::stat(check.c_str(), &st) == -1) {
- perror("whoami");
- invalid_path(fspath);
- }
- if (!S_ISREG(st.st_mode)) {
- invalid_path(fspath);
- }
- check = fspath + "/current";
- if (::stat(check.c_str(), &st) == -1) {
- perror("current");
- invalid_path(fspath);
- }
- if (!S_ISDIR(st.st_mode)) {
- invalid_path(fspath);
- }
-
- ObjectStore *fs = new FileStore(fspath, jpath);
-
- int r = fs->mount();
- if (r < 0) {
- if (r == -EBUSY) {
- cout << "OSD has the store locked" << std::endl;
- } else {
- cout << "Mount failed with '" << cpp_strerror(-r) << "'" << std::endl;
- }
- return 1;
- }
-
- vector<coll_t> colls_to_check;
- if (pgidstr.length()) {
- spg_t pgid;
- if (!pgid.parse(pgidstr.c_str())) {
- cout << "Invalid pgid '" << pgidstr << "' specified" << std::endl;
- exit(1);
- }
- colls_to_check.push_back(coll_t(pgid));
- } else {
- vector<coll_t> candidates;
- r = fs->list_collections(candidates);
- if (r < 0) {
- cerr << "Error listing collections: " << cpp_strerror(r) << std::endl;
- goto UMOUNT;
- }
- for (vector<coll_t>::iterator i = candidates.begin();
- i != candidates.end();
- ++i) {
- spg_t pgid;
- snapid_t snap;
- if (i->is_pg(pgid, snap)) {
- colls_to_check.push_back(*i);
- }
- }
- }
-
- cerr << colls_to_check.size() << " pgs to scan" << std::endl;
- for (vector<coll_t>::iterator i = colls_to_check.begin();
- i != colls_to_check.end();
- ++i, ++scanned) {
- cerr << "Scanning " << *i << ", " << scanned << "/"
- << colls_to_check.size() << " completed" << std::endl;
- ghobject_t next;
- while (!next.is_max()) {
- vector<ghobject_t> list;
- r = fs->collection_list_partial(
- *i,
- next,
- LIST_AT_A_TIME,
- LIST_AT_A_TIME,
- CEPH_NOSNAP,
- &list,
- &next);
- if (r < 0) {
- cerr << "Error listing collection: " << *i << ", "
- << cpp_strerror(r) << std::endl;
- goto UMOUNT;
- }
- for (vector<ghobject_t>::iterator obj = list.begin();
- obj != list.end();
- ++obj) {
- bufferlist attr;
- r = fs->getattr(*i, *obj, OI_ATTR, attr);
- if (r < 0) {
- cerr << "Error getting attr on : " << make_pair(*i, *obj) << ", "
- << cpp_strerror(r) << std::endl;
- goto UMOUNT;
- }
- object_info_t oi;
- bufferlist::iterator bp = attr.begin();
- try {
- ::decode(oi, bp);
- } catch (...) {
- r = -EINVAL;
- cerr << "Error getting attr on : " << make_pair(*i, *obj) << ", "
- << cpp_strerror(r) << std::endl;
- goto UMOUNT;
- }
- if (oi.is_lost()) {
- if (list_lost_objects) {
- cout << *i << "/" << *obj << " is lost" << std::endl;
- }
- if (fix_lost_objects) {
- cerr << *i << "/" << *obj << " is lost, fixing" << std::endl;
- oi.clear_flag(object_info_t::FLAG_LOST);
- bufferlist bl2;
- ::encode(oi, bl2);
- ObjectStore::Transaction t;
- t.setattr(*i, *obj, OI_ATTR, bl2);
- r = fs->apply_transaction(t);
- if (r < 0) {
- cerr << "Error getting fixing attr on : " << make_pair(*i, *obj)
- << ", "
- << cpp_strerror(r) << std::endl;
- goto UMOUNT;
- }
- }
- }
- }
- }
- }
- cerr << "Completed" << std::endl;
-
- UMOUNT:
- fs->sync_and_flush();
- fs->umount();
- return r;
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 Inktank, Inc.
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License version 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#include <iostream>
-#include <string>
-#include <sstream>
-#include <map>
-#include <set>
-#include <boost/scoped_ptr.hpp>
-
-#include "os/LevelDBStore.h"
-
-#include "common/ceph_argparse.h"
-#include "global/global_init.h"
-#include "common/errno.h"
-#include "common/safe_io.h"
-#include "common/config.h"
-#include "common/strtol.h"
-#include "include/stringify.h"
-
-using namespace std;
-
-class StoreTool
-{
- boost::scoped_ptr<KeyValueDB> db;
- string store_path;
-
- public:
- StoreTool(const string &path) : store_path(path) {
- LevelDBStore *db_ptr = new LevelDBStore(g_ceph_context, store_path);
- assert(!db_ptr->open(std::cerr));
- db.reset(db_ptr);
- }
-
- uint32_t traverse(const string &prefix,
- const bool do_crc,
- ostream *out) {
- KeyValueDB::WholeSpaceIterator iter = db->get_iterator();
-
- if (prefix.empty())
- iter->seek_to_first();
- else
- iter->seek_to_first(prefix);
-
- uint32_t crc = -1;
-
- while (iter->valid()) {
- pair<string,string> rk = iter->raw_key();
- if (!prefix.empty() && (rk.first != prefix))
- break;
-
- if (out)
- *out << rk.first << ":" << rk.second;
- if (do_crc) {
- bufferlist bl;
- bl.append(rk.first);
- bl.append(rk.second);
- bl.append(iter->value());
-
- crc = bl.crc32c(crc);
- if (out) {
- *out << " (" << bl.crc32c(0) << ")";
- }
- }
- if (out)
- *out << std::endl;
- iter->next();
- }
-
- return crc;
- }
-
- void list(const string &prefix, const bool do_crc) {
- traverse(prefix, do_crc, &std::cout);
- }
-
- bool exists(const string &prefix) {
- assert(!prefix.empty());
- KeyValueDB::WholeSpaceIterator iter = db->get_iterator();
- iter->seek_to_first(prefix);
- return (iter->valid() && (iter->raw_key().first == prefix));
- }
-
- bool exists(const string &prefix, const string &key) {
- assert(!prefix.empty());
-
- if (key.empty()) {
- return exists(prefix);
- }
-
- bool exists = false;
- get(prefix, key, exists);
- return exists;
- }
-
- bufferlist get(const string &prefix, const string &key, bool &exists) {
- assert(!prefix.empty() && !key.empty());
-
- map<string,bufferlist> result;
- std::set<std::string> keys;
- keys.insert(key);
- db->get(prefix, keys, &result);
-
- if (result.count(key) > 0) {
- exists = true;
- return result[key];
- }
- exists = false;
- return bufferlist();
- }
-
- uint64_t get_size() {
- map<string,uint64_t> extras;
- uint64_t s = db->get_estimated_size(extras);
- for (map<string,uint64_t>::iterator p = extras.begin();
- p != extras.end(); ++p) {
- std::cout << p->first << " - " << p->second << std::endl;
- }
- std::cout << "total: " << s << std::endl;
- return s;
- }
-
- bool set(const string &prefix, const string &key, bufferlist &val) {
- assert(!prefix.empty());
- assert(!key.empty());
- assert(val.length() > 0);
-
- KeyValueDB::Transaction tx = db->get_transaction();
- tx->set(prefix, key, val);
- int ret = db->submit_transaction_sync(tx);
-
- return (ret == 0);
- }
-
- int copy_store_to(const string &other_path, const int num_keys_per_tx) {
-
- if (num_keys_per_tx <= 0) {
- std::cerr << "must specify a number of keys/tx > 0" << std::endl;
- return -EINVAL;
- }
-
- // open or create a leveldb store at @p other_path
- LevelDBStore other(g_ceph_context, other_path);
- int err = other.create_and_open(std::cerr);
- if (err < 0)
- return err;
-
- KeyValueDB::WholeSpaceIterator it = db->get_iterator();
- it->seek_to_first();
- uint64_t total_keys = 0;
- uint64_t total_size = 0;
- uint64_t total_txs = 0;
-
- utime_t started_at = ceph_clock_now(g_ceph_context);
-
- do {
- int num_keys = 0;
-
- KeyValueDB::Transaction tx = other.get_transaction();
-
-
- while (it->valid() && num_keys < num_keys_per_tx) {
- pair<string,string> k = it->raw_key();
- bufferlist v = it->value();
- tx->set(k.first, k.second, v);
-
- num_keys ++;
- total_size += v.length();
-
- it->next();
- }
-
- total_txs ++;
- total_keys += num_keys;
-
- if (num_keys > 0)
- other.submit_transaction_sync(tx);
-
- utime_t cur_duration = ceph_clock_now(g_ceph_context) - started_at;
- std::cout << "ts = " << cur_duration << "s, copied " << total_keys
- << " keys so far (" << stringify(si_t(total_size)) << ")"
- << std::endl;
-
- } while (it->valid());
-
- utime_t time_taken = ceph_clock_now(g_ceph_context) - started_at;
-
- std::cout << "summary:" << std::endl;
- std::cout << " copied " << total_keys << " keys" << std::endl;
- std::cout << " used " << total_txs << " transactions" << std::endl;
- std::cout << " total size " << stringify(si_t(total_size)) << std::endl;
- std::cout << " from '" << store_path << "' to '" << other_path << "'"
- << std::endl;
- std::cout << " duration " << time_taken << " seconds" << std::endl;
-
- return 0;
- }
-};
-
-void usage(const char *pname)
-{
- std::cerr << "Usage: " << pname << " <store path> command [args...]\n"
- << "\n"
- << "Commands:\n"
- << " list [prefix]\n"
- << " list-crc [prefix]\n"
- << " exists <prefix> [key]\n"
- << " get <prefix> <key> [out <file>]\n"
- << " crc <prefix> <key>\n"
- << " get-size [<prefix> <key>]\n"
- << " set <prefix> <key> [ver <N>|in <file>]\n"
- << " store-copy <path> [num-keys-per-tx]\n"
- << " store-crc <path>\n"
- << std::endl;
-}
-
-int main(int argc, const char *argv[])
-{
- vector<const char*> args;
- argv_to_vec(argc, argv, args);
- env_to_vec(args);
-
- global_init(
- NULL, args,
- CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
- common_init_finish(g_ceph_context);
-
-
- if (args.size() < 2) {
- usage(argv[0]);
- return 1;
- }
-
- string path(args[0]);
- string cmd(args[1]);
-
- StoreTool st(path);
-
- if (cmd == "list" || cmd == "list-crc") {
- string prefix;
- if (argc > 3)
- prefix = argv[3];
-
- bool do_crc = (cmd == "list-crc");
-
- st.list(prefix, do_crc);
-
- } else if (cmd == "exists") {
- string key;
- if (argc < 4) {
- usage(argv[0]);
- return 1;
- }
- string prefix(argv[3]);
- if (argc > 4)
- key = argv[4];
-
- bool ret = st.exists(prefix, key);
- std::cout << "(" << prefix << ", " << key << ") "
- << (ret ? "exists" : "does not exist")
- << std::endl;
- return (ret ? 0 : 1);
-
- } else if (cmd == "get") {
- if (argc < 5) {
- usage(argv[0]);
- return 1;
- }
- string prefix(argv[3]);
- string key(argv[4]);
-
- bool exists = false;
- bufferlist bl = st.get(prefix, key, exists);
- std::cout << "(" << prefix << ", " << key << ")";
- if (!exists) {
- std::cout << " does not exist" << std::endl;
- return 1;
- }
- std::cout << std::endl;
-
- if (argc >= 6) {
- string subcmd(argv[5]);
- string out(argv[6]);
-
- if (subcmd != "out") {
- std::cerr << "unrecognized subcmd '" << subcmd << "'"
- << std::endl;
- return 1;
- }
-
- if (out.empty()) {
- std::cerr << "unspecified out file" << std::endl;
- return 1;
- }
-
- int err = bl.write_file(argv[6], 0644);
- if (err < 0) {
- std::cerr << "error writing value to '" << out << "': "
- << cpp_strerror(err) << std::endl;
- return 1;
- }
- } else {
- ostringstream os;
- bl.hexdump(os);
- std::cout << os.str() << std::endl;
- }
-
- } else if (cmd == "crc") {
- if (argc < 5) {
- usage(argv[0]);
- return 1;
- }
- string prefix(argv[3]);
- string key(argv[4]);
-
- bool exists = false;
- bufferlist bl = st.get(prefix, key, exists);
- std::cout << "(" << prefix << ", " << key << ") ";
- if (!exists) {
- std::cout << " does not exist" << std::endl;
- return 1;
- }
- std::cout << " crc " << bl.crc32c(0) << std::endl;
-
- } else if (cmd == "get-size") {
- std::cout << "estimated store size: " << st.get_size() << std::endl;
-
- if (argc < 4)
- return 0;
-
- if (argc < 5) {
- usage(argv[0]);
- return 1;
- }
- string prefix(argv[3]);
- string key(argv[4]);
-
- bool exists = false;
- bufferlist bl = st.get(prefix, key, exists);
- if (!exists) {
- std::cerr << "(" << prefix << "," << key
- << ") does not exist" << std::endl;
- return 1;
- }
- std::cout << "(" << prefix << "," << key
- << ") size " << si_t(bl.length()) << std::endl;
-
- } else if (cmd == "set") {
- if (argc < 7) {
- usage(argv[0]);
- return 1;
- }
- string prefix(argv[3]);
- string key(argv[4]);
- string subcmd(argv[5]);
-
- bufferlist val;
- string errstr;
- if (subcmd == "ver") {
- version_t v = (version_t) strict_strtoll(argv[6], 10, &errstr);
- if (!errstr.empty()) {
- std::cerr << "error reading version: " << errstr << std::endl;
- return 1;
- }
- ::encode(v, val);
- } else if (subcmd == "in") {
- int ret = val.read_file(argv[6], &errstr);
- if (ret < 0 || !errstr.empty()) {
- std::cerr << "error reading file: " << errstr << std::endl;
- return 1;
- }
- } else {
- std::cerr << "unrecognized subcommand '" << subcmd << "'" << std::endl;
- usage(argv[0]);
- return 1;
- }
-
- bool ret = st.set(prefix, key, val);
- if (!ret) {
- std::cerr << "error setting ("
- << prefix << "," << key << ")" << std::endl;
- return 1;
- }
- } else if (cmd == "store-copy") {
- int num_keys_per_tx = 128; // magic number that just feels right.
- if (argc < 4) {
- usage(argv[0]);
- return 1;
- } else if (argc > 4) {
- string err;
- num_keys_per_tx = strict_strtol(argv[4], 10, &err);
- if (!err.empty()) {
- std::cerr << "invalid num_keys_per_tx: " << err << std::endl;
- return 1;
- }
- }
-
- int ret = st.copy_store_to(argv[3], num_keys_per_tx);
- if (ret < 0) {
- std::cerr << "error copying store to path '" << argv[3]
- << "': " << cpp_strerror(ret) << std::endl;
- return 1;
- }
-
- } else if (cmd == "store-crc") {
- uint32_t crc = st.traverse(string(), true, NULL);
- std::cout << "store at '" << path << "' crc " << crc << std::endl;
-
- } else {
- std::cerr << "Unrecognized command: " << cmd << std::endl;
- return 1;
- }
-
- return 0;
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 Inktank, Inc.
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License version 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#include <boost/scoped_ptr.hpp>
-#include <boost/lexical_cast.hpp>
-#include <boost/program_options/option.hpp>
-#include <boost/program_options/options_description.hpp>
-#include <boost/program_options/variables_map.hpp>
-#include <boost/program_options/cmdline.hpp>
-#include <boost/program_options/parsers.hpp>
-#include <iostream>
-#include <set>
-#include <sstream>
-#include <stdlib.h>
-#include <fstream>
-#include <string>
-#include <sstream>
-#include <map>
-#include <set>
-#include <boost/scoped_ptr.hpp>
-
-#include "global/global_init.h"
-#include "os/LevelDBStore.h"
-#include "mon/MonitorDBStore.h"
-#include "mon/Paxos.h"
-#include "common/Formatter.h"
-#include "include/stringify.h"
-
-namespace po = boost::program_options;
-using namespace std;
-
-class TraceIter {
- int fd;
- unsigned idx;
- MonitorDBStore::Transaction t;
-public:
- TraceIter(string fname) : fd(-1), idx(-1) {
- fd = ::open(fname.c_str(), O_RDONLY);
- }
- bool valid() {
- return fd != -1;
- }
- const MonitorDBStore::Transaction &cur() {
- assert(valid());
- return t;
- }
- unsigned num() { return idx; }
- void next() {
- ++idx;
- bufferlist bl;
- int r = bl.read_fd(fd, 6);
- if (r < 0) {
- std::cerr << "Got error: " << cpp_strerror(r) << " on read_fd"
- << std::endl;
- ::close(fd);
- fd = -1;
- return;
- } else if ((unsigned)r < 6) {
- std::cerr << "short read" << std::endl;
- ::close(fd);
- fd = -1;
- return;
- }
- bufferlist::iterator bliter = bl.begin();
- uint8_t ver, ver2;
- ::decode(ver, bliter);
- ::decode(ver2, bliter);
- uint32_t len;
- ::decode(len, bliter);
- r = bl.read_fd(fd, len);
- if (r < 0) {
- std::cerr << "Got error: " << cpp_strerror(r) << " on read_fd"
- << std::endl;
- ::close(fd);
- fd = -1;
- return;
- } else if ((unsigned)r < len) {
- std::cerr << "short read" << std::endl;
- ::close(fd);
- fd = -1;
- return;
- }
- bliter = bl.begin();
- t.decode(bliter);
- }
- void init() {
- next();
- }
- ~TraceIter() {
- if (fd != -1) {
- ::close(fd);
- fd = -1;
- }
- }
-};
-
-int main(int argc, char **argv) {
- po::options_description desc("Allowed options");
- int version = -1;
- string store_path, cmd, out_path, tfile;
- unsigned dstart = 0;
- unsigned dstop = ~0;
- unsigned num_replays = 1;
- unsigned tsize = 200;
- unsigned tvalsize = 1024;
- unsigned ntrans = 100;
- desc.add_options()
- ("help", "produce help message")
- ("mon-store-path", po::value<string>(&store_path),
- "path to mon directory, mandatory")
- ("out", po::value<string>(&out_path),
- "out path")
- ("version", po::value<int>(&version),
- "version requested")
- ("trace-file", po::value<string>(&tfile),
- "trace file")
- ("dump-start", po::value<unsigned>(&dstart),
- "transaction num to start dumping at")
- ("dump-end", po::value<unsigned>(&dstop),
- "transaction num to stop dumping at")
- ("num-replays", po::value<unsigned>(&num_replays),
- "number of times to replay")
- ("trans-size", po::value<unsigned>(&tsize),
- "keys to write in each transaction")
- ("trans-val-size", po::value<unsigned>(&tvalsize),
- "val to write in each key")
- ("num-trans", po::value<unsigned>(&ntrans),
- "number of transactions to run")
- ("command", po::value<string>(&cmd),
- "command")
- ;
- po::positional_options_description p;
- p.add("command", 1);
- p.add("version", 1);
-
- po::variables_map vm;
- po::parsed_options parsed =
- po::command_line_parser(argc, argv).options(desc).positional(p).run();
- po::store(
- parsed,
- vm);
- try {
- po::notify(vm);
- } catch (...) {
- cout << desc << std::endl;
- return 1;
- }
-
- vector<const char *> ceph_options, def_args;
- vector<string> ceph_option_strings = po::collect_unrecognized(
- parsed.options, po::include_positional);
- ceph_options.reserve(ceph_option_strings.size());
- for (vector<string>::iterator i = ceph_option_strings.begin();
- i != ceph_option_strings.end();
- ++i) {
- ceph_options.push_back(i->c_str());
- }
-
- global_init(
- &def_args, ceph_options, CEPH_ENTITY_TYPE_MON,
- CODE_ENVIRONMENT_UTILITY, 0);
- common_init_finish(g_ceph_context);
- g_ceph_context->_conf->apply_changes(NULL);
- g_conf = g_ceph_context->_conf;
-
- if (vm.count("help")) {
- std::cerr << desc << std::endl;
- return 1;
- }
-
- int fd;
- if (vm.count("out")) {
- if ((fd = open(out_path.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666)) < 0) {
- int _err = errno;
- if (_err != EISDIR) {
- std::cerr << "Couldn't open " << out_path << ": " << cpp_strerror(_err) << std::endl;
- return 1;
- }
- }
- } else {
- fd = STDOUT_FILENO;
- }
-
- if (fd < 0 && cmd != "store-copy") {
- std::cerr << "error: '" << out_path << "' is a directory!" << std::endl;
- return 1;
- }
-
- MonitorDBStore st(store_path);
- if (store_path.size()) {
- stringstream ss;
- int r = st.open(ss);
- if (r < 0) {
- std::cerr << ss.str() << std::endl;
- goto done;
- }
- }
- if (cmd == "dump-keys") {
- KeyValueDB::WholeSpaceIterator iter = st.get_iterator();
- while (iter->valid()) {
- pair<string,string> key(iter->raw_key());
- cout << key.first << " / " << key.second << std::endl;
- iter->next();
- }
- } else if (cmd == "compact") {
- st.compact();
- } else if (cmd == "getmonmap") {
- assert(fd >= 0);
- if (!store_path.size()) {
- std::cerr << "need mon store path" << std::endl;
- std::cerr << desc << std::endl;
- goto done;
- }
- version_t v;
- if (version <= 0) {
- v = st.get("monmap", "last_committed");
- } else {
- v = version;
- }
-
- bufferlist bl;
- /// XXX: this is not ok, osdmap and full should be abstracted somewhere
- int r = st.get("monmap", v, bl);
- if (r < 0) {
- std::cerr << "Error getting map: " << cpp_strerror(r) << std::endl;
- goto done;
- }
- bl.write_fd(fd);
- } else if (cmd == "getosdmap") {
- if (!store_path.size()) {
- std::cerr << "need mon store path" << std::endl;
- std::cerr << desc << std::endl;
- goto done;
- }
- version_t v;
- if (version == -1) {
- v = st.get("osdmap", "last_committed");
- } else {
- v = version;
- }
-
- bufferlist bl;
- /// XXX: this is not ok, osdmap and full should be abstracted somewhere
- int r = st.get("osdmap", st.combine_strings("full", v), bl);
- if (r < 0) {
- std::cerr << "Error getting map: " << cpp_strerror(r) << std::endl;
- goto done;
- }
- bl.write_fd(fd);
- } else if (cmd == "dump-paxos") {
- for (version_t v = dstart; v <= dstop; ++v) {
- bufferlist bl;
- st.get("paxos", v, bl);
- if (bl.length() == 0)
- break;
- cout << "\n--- " << v << " ---" << std::endl;
- MonitorDBStore::Transaction tx;
- Paxos::decode_append_transaction(tx, bl);
- JSONFormatter f(true);
- tx.dump(&f);
- f.flush(cout);
- }
- } else if (cmd == "dump-trace") {
- if (tfile.empty()) {
- std::cerr << "Need trace_file" << std::endl;
- std::cerr << desc << std::endl;
- goto done;
- }
- TraceIter iter(tfile.c_str());
- iter.init();
- while (true) {
- if (!iter.valid())
- break;
- if (iter.num() >= dstop) {
- break;
- }
- if (iter.num() >= dstart) {
- JSONFormatter f(true);
- iter.cur().dump(&f, false);
- f.flush(std::cout);
- std::cout << std::endl;
- }
- iter.next();
- }
- std::cerr << "Read up to transaction " << iter.num() << std::endl;
- } else if (cmd == "replay-trace") {
- if (!store_path.size()) {
- std::cerr << "need mon store path" << std::endl;
- std::cerr << desc << std::endl;
- goto done;
- }
- if (tfile.empty()) {
- std::cerr << "Need trace_file" << std::endl;
- std::cerr << desc << std::endl;
- goto done;
- }
- unsigned num = 0;
- for (unsigned i = 0; i < num_replays; ++i) {
- TraceIter iter(tfile.c_str());
- iter.init();
- while (true) {
- if (!iter.valid())
- break;
- std::cerr << "Replaying trans num " << num << std::endl;
- st.apply_transaction(iter.cur());
- iter.next();
- ++num;
- }
- std::cerr << "Read up to transaction " << iter.num() << std::endl;
- }
- } else if (cmd == "random-gen") {
- if (!store_path.size()) {
- std::cerr << "need mon store path" << std::endl;
- std::cerr << desc << std::endl;
- goto done;
- }
- unsigned num = 0;
- for (unsigned i = 0; i < ntrans; ++i) {
- std::cerr << "Applying trans " << i << std::endl;
- MonitorDBStore::Transaction t;
- string prefix;
- prefix.push_back((i%26)+'a');
- for (unsigned j = 0; j < tsize; ++j) {
- stringstream os;
- os << num;
- bufferlist bl;
- for (unsigned k = 0; k < tvalsize; ++k) bl.append(rand());
- t.put(prefix, os.str(), bl);
- ++num;
- }
- t.compact_prefix(prefix);
- st.apply_transaction(t);
- }
- } else if (cmd == "store-copy") {
- if (!store_path.size()) {
- std::cerr << "need mon store path to copy from" << std::endl;
- std::cerr << desc << std::endl;
- goto done;
- }
- if (!out_path.size()) {
- std::cerr << "need mon store path to copy to (--out <mon_data_dir>)"
- << std::endl;
- std::cerr << desc << std::endl;
- goto done;
- }
- if (fd > 0) {
- std::cerr << "supplied out path '" << out_path << "' is not a directory"
- << std::endl;
- goto done;
- }
-
- MonitorDBStore out_store(out_path);
- {
- stringstream ss;
- int r = out_store.create_and_open(ss);
- if (r < 0) {
- std::cerr << ss.str() << std::endl;
- goto done;
- }
- }
-
-
- KeyValueDB::WholeSpaceIterator it = st.get_iterator();
- uint64_t total_keys = 0;
- uint64_t total_size = 0;
- uint64_t total_tx = 0;
-
- do {
- uint64_t num_keys = 0;
-
- MonitorDBStore::Transaction tx;
-
- while (it->valid() && num_keys < 128) {
- pair<string,string> k = it->raw_key();
- bufferlist v = it->value();
- tx.put(k.first, k.second, v);
-
- num_keys ++;
- total_tx ++;
- total_size += v.length();
-
- it->next();
- }
-
- total_keys += num_keys;
-
- if (!tx.empty())
- out_store.apply_transaction(tx);
-
- std::cout << "copied " << total_keys << " keys so far ("
- << stringify(si_t(total_size)) << ")" << std::endl;
-
- } while (it->valid());
-
- std::cout << "summary: copied " << total_keys << " keys, using "
- << total_tx << " transactions, totalling "
- << stringify(si_t(total_size)) << std::endl;
- std::cout << "from '" << store_path << "' to '" << out_path << "'"
- << std::endl;
- } else {
- std::cerr << "Unrecognized command: " << cmd << std::endl;
- goto done;
- }
-
- done:
- if (vm.count("out") && fd > 0) {
- ::close(fd);
- }
- return 0;
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
-* Ceph - scalable distributed file system
-*
-* Copyright (C) 2012 Inktank, Inc.
-*
-* This is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License kkjversion 2.1, as published by the Free Software
-* Foundation. See file COPYING.
-*/
-#include <boost/scoped_ptr.hpp>
-#include <boost/lexical_cast.hpp>
-#include <boost/program_options/option.hpp>
-#include <boost/program_options/options_description.hpp>
-#include <boost/program_options/variables_map.hpp>
-#include <boost/program_options/cmdline.hpp>
-#include <boost/program_options/parsers.hpp>
-#include <iostream>
-#include <set>
-#include <sstream>
-#include <stdlib.h>
-#include <fstream>
-#include <string>
-#include <sstream>
-#include <map>
-#include <set>
-#include <boost/scoped_ptr.hpp>
-
-#include "global/global_init.h"
-#include "os/LevelDBStore.h"
-#include "mon/MonitorDBStore.h"
-#include "os/DBObjectMap.h"
-
-namespace po = boost::program_options;
-using namespace std;
-
-int main(int argc, char **argv) {
- po::options_description desc("Allowed options");
- string store_path, cmd, out_path;
- bool paranoid = false;
- desc.add_options()
- ("help", "produce help message")
- ("omap-path", po::value<string>(&store_path),
- "path to mon directory, mandatory (current/omap usually)")
- ("paranoid", po::value<bool>(¶noid),
- "use paranoid checking")
- ("command", po::value<string>(&cmd),
- "command")
- ;
- po::positional_options_description p;
- p.add("command", 1);
-
- po::variables_map vm;
- po::parsed_options parsed =
- po::command_line_parser(argc, argv).options(desc).positional(p).run();
- po::store(
- parsed,
- vm);
- try {
- po::notify(vm);
- } catch (...) {
- cout << desc << std::endl;
- return 1;
- }
-
- vector<const char *> ceph_options, def_args;
- vector<string> ceph_option_strings = po::collect_unrecognized(
- parsed.options, po::include_positional);
- ceph_options.reserve(ceph_option_strings.size());
- for (vector<string>::iterator i = ceph_option_strings.begin();
- i != ceph_option_strings.end();
- ++i) {
- ceph_options.push_back(i->c_str());
- }
-
- global_init(
- &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD,
- CODE_ENVIRONMENT_UTILITY, 0);
- common_init_finish(g_ceph_context);
- g_ceph_context->_conf->apply_changes(NULL);
- g_conf = g_ceph_context->_conf;
-
- if (vm.count("help")) {
- std::cerr << desc << std::endl;
- return 1;
- }
-
- LevelDBStore* store(new LevelDBStore(g_ceph_context, store_path));
- if (paranoid) {
- std::cerr << "Enabling paranoid checks" << std::endl;
- store->options.paranoid_checks = paranoid;
- }
- DBObjectMap omap(store);
- stringstream out;
- int r = store->open(out);
- if (r < 0) {
- std::cerr << "Store open got: " << cpp_strerror(r) << std::endl;
- std::cerr << "Output: " << out.str() << std::endl;
- goto done;
- }
- r = 0;
-
-
- if (cmd == "dump-raw-keys") {
- KeyValueDB::WholeSpaceIterator i = store->get_iterator();
- for (i->seek_to_first(); i->valid(); i->next()) {
- std::cout << i->raw_key() << std::endl;
- }
- } else if (cmd == "dump-raw-key-vals") {
- KeyValueDB::WholeSpaceIterator i = store->get_iterator();
- for (i->seek_to_first(); i->valid(); i->next()) {
- std::cout << i->raw_key() << std::endl;
- i->value().hexdump(std::cout);
- }
- } else if (cmd == "dump-objects") {
- vector<ghobject_t> objects;
- r = omap.list_objects(&objects);
- if (r < 0) {
- std::cerr << "list_objects got: " << cpp_strerror(r) << std::endl;
- goto done;
- }
- for (vector<ghobject_t>::iterator i = objects.begin();
- i != objects.end();
- ++i) {
- std::cout << *i << std::endl;
- }
- r = 0;
- } else if (cmd == "dump-objects-with-keys") {
- vector<ghobject_t> objects;
- r = omap.list_objects(&objects);
- if (r < 0) {
- std::cerr << "list_objects got: " << cpp_strerror(r) << std::endl;
- goto done;
- }
- for (vector<ghobject_t>::iterator i = objects.begin();
- i != objects.end();
- ++i) {
- std::cout << "Object: " << *i << std::endl;
- ObjectMap::ObjectMapIterator j = omap.get_iterator(i->hobj);
- for (j->seek_to_first(); j->valid(); j->next()) {
- std::cout << j->key() << std::endl;
- j->value().hexdump(std::cout);
- }
- }
- } else if (cmd == "check") {
- r = omap.check(std::cout);
- if (!r) {
- std::cerr << "check got: " << cpp_strerror(r) << std::endl;
- goto done;
- }
- std::cout << "check succeeded" << std::endl;
- } else {
- std::cerr << "Did not recognize command " << cmd << std::endl;
- goto done;
- }
-
- done:
- return r;
-}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/program_options/option.hpp>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <boost/program_options/cmdline.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <iostream>
+#include <set>
+#include <sstream>
+#include <stdlib.h>
+#include <fstream>
+
+#include "common/Formatter.h"
+
+#include "global/global_init.h"
+#include "os/ObjectStore.h"
+#include "os/FileStore.h"
+#include "common/perf_counters.h"
+#include "common/errno.h"
+#include "osd/PGLog.h"
+#include "osd/OSD.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+enum {
+ TYPE_NONE = 0,
+ TYPE_PG_BEGIN,
+ TYPE_PG_END,
+ TYPE_OBJECT_BEGIN,
+ TYPE_OBJECT_END,
+ TYPE_DATA,
+ TYPE_ATTRS,
+ TYPE_OMAP_HDR,
+ TYPE_OMAP,
+ TYPE_PG_METADATA,
+ END_OF_TYPES, //Keep at the end
+};
+
+//#define INTERNAL_TEST
+//#define INTERNAL_TEST2
+
+#ifdef INTERNAL_TEST
+CompatSet get_test_compat_set() {
+ CompatSet::FeatureSet ceph_osd_feature_compat;
+ CompatSet::FeatureSet ceph_osd_feature_ro_compat;
+ CompatSet::FeatureSet ceph_osd_feature_incompat;
+ ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_BASE);
+ ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_PGINFO);
+ ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_OLOC);
+ ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEC);
+ ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_CATEGORIES);
+ ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_HOBJECTPOOL);
+ ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_BIGINFO);
+ ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEVELDBINFO);
+ ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_LEVELDBLOG);
+#ifdef INTERNAL_TEST2
+ ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SNAPMAPPER);
+ ceph_osd_feature_incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS);
+#endif
+ return CompatSet(ceph_osd_feature_compat, ceph_osd_feature_ro_compat,
+ ceph_osd_feature_incompat);
+}
+#endif
+
+typedef uint8_t sectiontype_t;
+typedef uint32_t mymagic_t;
+typedef int64_t mysize_t;
+const ssize_t max_read = 1024 * 1024;
+const uint16_t shortmagic = 0xffce; //goes into stream as "ceff"
+//endmagic goes into stream as "ceff ffec"
+const mymagic_t endmagic = (0xecff << 16) | shortmagic;
+const int fd_none = INT_MIN;
+
+//The first FIXED_LENGTH bytes are a fixed
+//portion of the export output. This includes the overall
+//version number, and size of header and footer.
+//THIS STRUCTURE CAN ONLY BE APPENDED TO. If it needs to expand,
+//the version can be bumped and then anything
+//can be added to the export format.
+struct super_header {
+ static const uint32_t super_magic = (shortmagic << 16) | shortmagic;
+ static const uint32_t super_ver = 2;
+ static const uint32_t FIXED_LENGTH = 16;
+ uint32_t magic;
+ uint32_t version;
+ uint32_t header_size;
+ uint32_t footer_size;
+
+ super_header() : magic(0), version(0), header_size(0), footer_size(0) { }
+ int read_super();
+
+ void encode(bufferlist& bl) const {
+ ::encode(magic, bl);
+ ::encode(version, bl);
+ ::encode(header_size, bl);
+ ::encode(footer_size, bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ ::decode(magic, bl);
+ ::decode(version, bl);
+ ::decode(header_size, bl);
+ ::decode(footer_size, bl);
+ }
+};
+
+struct header {
+ sectiontype_t type;
+ mysize_t size;
+ header(sectiontype_t type, mysize_t size) :
+ type(type), size(size) { }
+ header(): type(0), size(0) { }
+
+ int get_header();
+
+ void encode(bufferlist& bl) const {
+ uint32_t debug_type = (type << 24) | (type << 16) | shortmagic;
+ ENCODE_START(1, 1, bl);
+ ::encode(debug_type, bl);
+ ::encode(size, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ uint32_t debug_type;
+ DECODE_START(1, bl);
+ ::decode(debug_type, bl);
+ type = debug_type >> 24;
+ ::decode(size, bl);
+ DECODE_FINISH(bl);
+ }
+};
+
+struct footer {
+ mymagic_t magic;
+ footer() : magic(endmagic) { }
+
+ int get_footer();
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(magic, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(magic, bl);
+ DECODE_FINISH(bl);
+ }
+};
+
+struct pg_begin {
+ pg_t pgid;
+ OSDSuperblock superblock;
+
+ pg_begin(pg_t pg, const OSDSuperblock& sb):
+ pgid(pg), superblock(sb) { }
+ pg_begin() { }
+
+ void encode(bufferlist& bl) const {
+ // New super_ver prevents decode from ver 1
+ ENCODE_START(2, 2, bl);
+ ::encode(pgid, bl);
+ ::encode(superblock, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(2, bl);
+ ::decode(pgid, bl);
+ if (struct_v > 1) {
+ ::decode(superblock, bl);
+ }
+ DECODE_FINISH(bl);
+ }
+};
+
+struct object_begin {
+ hobject_t hoid;
+ object_begin(const hobject_t &hoid): hoid(hoid) { }
+ object_begin() { }
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(hoid, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(hoid, bl);
+ DECODE_FINISH(bl);
+ }
+};
+
+struct data_section {
+ uint64_t offset;
+ uint64_t len;
+ bufferlist databl;
+ data_section(uint64_t offset, uint64_t len, bufferlist bl):
+ offset(offset), len(len), databl(bl) { }
+ data_section(): offset(0), len(0) { }
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(offset, bl);
+ ::encode(len, bl);
+ ::encode(databl, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(offset, bl);
+ ::decode(len, bl);
+ ::decode(databl, bl);
+ DECODE_FINISH(bl);
+ }
+};
+
+struct attr_section {
+ map<string,bufferptr> data;
+ attr_section(const map<string,bufferptr> &data) : data(data) { }
+ attr_section() { }
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(data, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(data, bl);
+ DECODE_FINISH(bl);
+ }
+};
+
+struct omap_hdr_section {
+ bufferlist hdr;
+ omap_hdr_section(bufferlist hdr) : hdr(hdr) { }
+ omap_hdr_section() { }
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(hdr, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(hdr, bl);
+ DECODE_FINISH(bl);
+ }
+};
+
+struct omap_section {
+ map<string, bufferlist> omap;
+ omap_section(const map<string, bufferlist> &omap) :
+ omap(omap) { }
+ omap_section() { }
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(omap, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(omap, bl);
+ DECODE_FINISH(bl);
+ }
+};
+
+struct metadata_section {
+ __u8 struct_ver;
+ epoch_t map_epoch;
+ pg_info_t info;
+ pg_log_t log;
+
+ metadata_section(__u8 struct_ver, epoch_t map_epoch, const pg_info_t &info,
+ const pg_log_t &log)
+ : struct_ver(struct_ver),
+ map_epoch(map_epoch),
+ info(info),
+ log(log) { }
+ metadata_section()
+ : struct_ver(0),
+ map_epoch(0) { }
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ ::encode(struct_ver, bl);
+ ::encode(map_epoch, bl);
+ ::encode(info, bl);
+ ::encode(log, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::iterator& bl) {
+ DECODE_START(1, bl);
+ ::decode(struct_ver, bl);
+ ::decode(map_epoch, bl);
+ ::decode(info, bl);
+ ::decode(log, bl);
+ DECODE_FINISH(bl);
+ }
+};
+
+hobject_t infos_oid = OSD::make_infos_oid();
+hobject_t biginfo_oid, log_oid;
+
+int file_fd = fd_none;
+bool debug = false;
+super_header sh;
+
+template <typename T>
+int write_section(sectiontype_t type, const T& obj, int fd) {
+ bufferlist blhdr, bl, blftr;
+ obj.encode(bl);
+ header hdr(type, bl.length());
+ hdr.encode(blhdr);
+ footer ft;
+ ft.encode(blftr);
+
+ int ret = blhdr.write_fd(fd);
+ if (ret) return ret;
+ ret = bl.write_fd(fd);
+ if (ret) return ret;
+ ret = blftr.write_fd(fd);
+ return ret;
+}
+
+int write_simple(sectiontype_t type, int fd)
+{
+ bufferlist hbl;
+
+ header hdr(type, 0);
+ hdr.encode(hbl);
+ return hbl.write_fd(fd);
+}
+
+static void invalid_path(string &path)
+{
+ cout << "Invalid path to osd store specified: " << path << "\n";
+ exit(1);
+}
+
+int get_log(ObjectStore *fs, coll_t coll, pg_t pgid, const pg_info_t &info,
+ PGLog::IndexedLog &log, pg_missing_t &missing)
+{
+ map<eversion_t, hobject_t> divergent_priors;
+ try {
+ ostringstream oss;
+ PGLog::read_log(fs, coll, log_oid, info, divergent_priors, log, missing, oss);
+ if (debug && oss.str().size())
+ cerr << oss.str() << std::endl;
+ }
+ catch (const buffer::error &e) {
+ cout << "read_log threw exception error " << e.what() << std::endl;
+ return 1;
+ }
+ return 0;
+}
+
+//Based on RemoveWQ::_process()
+void remove_coll(ObjectStore *store, const coll_t &coll)
+{
+ spg_t pg;
+ coll.is_pg_prefix(pg);
+ OSDriver driver(
+ store,
+ coll_t(),
+ OSD::make_snapmapper_oid());
+ SnapMapper mapper(&driver, 0, 0, 0, pg.shard);
+
+ vector<ghobject_t> objects;
+ ghobject_t next;
+ int r = 0;
+ int64_t num = 0;
+ ObjectStore::Transaction *t = new ObjectStore::Transaction;
+ cout << "remove_coll " << coll << std::endl;
+ while (!next.is_max()) {
+ r = store->collection_list_partial(coll, next, 200, 300, 0,
+ &objects, &next);
+ if (r < 0)
+ goto out;
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end();
+ ++i, ++num) {
+
+ assert(i->generation == ghobject_t::NO_GEN);
+ OSDriver::OSTransaction _t(driver.get_transaction(t));
+ cout << "remove " << *i << std::endl;
+ int r = mapper.remove_oid(i->hobj, &_t);
+ if (r != 0 && r != -ENOENT) {
+ assert(0);
+ }
+
+ t->remove(coll, *i);
+ if (num >= 30) {
+ store->apply_transaction(*t);
+ delete t;
+ t = new ObjectStore::Transaction;
+ num = 0;
+ }
+ }
+ }
+ t->remove_collection(coll);
+ store->apply_transaction(*t);
+out:
+ delete t;
+}
+
+//Based on part of OSD::load_pgs()
+int finish_remove_pgs(ObjectStore *store, uint64_t *next_removal_seq)
+{
+ vector<coll_t> ls;
+ int r = store->list_collections(ls);
+ if (r < 0) {
+ cout << "finish_remove_pgs: failed to list pgs: " << cpp_strerror(-r)
+ << std::endl;
+ return r;
+ }
+
+ for (vector<coll_t>::iterator it = ls.begin();
+ it != ls.end();
+ ++it) {
+ spg_t pgid;
+ snapid_t snap;
+
+ if (it->is_temp(pgid)) {
+ cout << "finish_remove_pgs " << *it << " clearing temp" << std::endl;
+ OSD::recursive_remove_collection(store, *it);
+ continue;
+ }
+
+ if (it->is_pg(pgid, snap)) {
+ continue;
+ }
+
+ uint64_t seq;
+ if (it->is_removal(&seq, &pgid)) {
+ if (seq >= *next_removal_seq)
+ *next_removal_seq = seq + 1;
+ cout << "finish_remove_pgs removing " << *it << ", seq is "
+ << seq << " pgid is " << pgid << std::endl;
+ remove_coll(store, *it);
+ continue;
+ }
+
+ //cout << "finish_remove_pgs ignoring unrecognized " << *it << std::endl;
+ }
+ return 0;
+}
+
+int initiate_new_remove_pg(ObjectStore *store, pg_t r_pgid,
+ uint64_t *next_removal_seq)
+{
+ ObjectStore::Transaction *rmt = new ObjectStore::Transaction;
+
+ if (store->collection_exists(coll_t(spg_t(r_pgid, ghobject_t::no_shard())))) {
+ coll_t to_remove = coll_t::make_removal_coll((*next_removal_seq)++,
+ spg_t(r_pgid, ghobject_t::no_shard()));
+ cout << "collection rename " << coll_t(spg_t(r_pgid, ghobject_t::no_shard()))
+ << " to " << to_remove
+ << std::endl;
+ rmt->collection_rename(coll_t(spg_t(r_pgid, ghobject_t::no_shard())), to_remove);
+ } else {
+ delete rmt;
+ return ENOENT;
+ }
+
+ cout << "remove " << coll_t::META_COLL << " " << log_oid.oid << std::endl;
+ rmt->remove(coll_t::META_COLL, log_oid);
+ cout << "remove " << coll_t::META_COLL << " " << biginfo_oid.oid << std::endl;
+ rmt->remove(coll_t::META_COLL, biginfo_oid);
+
+ store->apply_transaction(*rmt);
+
+ return 0;
+}
+
+int header::get_header()
+{
+ bufferlist ebl;
+ bufferlist::iterator ebliter = ebl.begin();
+ ssize_t bytes;
+
+ bytes = ebl.read_fd(file_fd, sh.header_size);
+ if ((size_t)bytes != sh.header_size) {
+ cout << "Unexpected EOF" << std::endl;
+ return EFAULT;
+ }
+
+ decode(ebliter);
+
+ return 0;
+}
+
+int footer::get_footer()
+{
+ bufferlist ebl;
+ bufferlist::iterator ebliter = ebl.begin();
+ ssize_t bytes;
+
+ bytes = ebl.read_fd(file_fd, sh.footer_size);
+ if ((size_t)bytes != sh.footer_size) {
+ cout << "Unexpected EOF" << std::endl;
+ return EFAULT;
+ }
+
+ decode(ebliter);
+
+ if (magic != endmagic) {
+ cout << "Bad footer magic" << std::endl;
+ return EFAULT;
+ }
+
+ return 0;
+}
+
+int write_info(ObjectStore::Transaction &t, epoch_t epoch, pg_info_t &info,
+ __u8 struct_ver)
+{
+ //Empty for this
+ interval_set<snapid_t> snap_collections; // obsolete
+ map<epoch_t,pg_interval_t> past_intervals;
+ coll_t coll(info.pgid);
+
+ int ret = PG::_write_info(t, epoch,
+ info, coll,
+ past_intervals,
+ snap_collections,
+ infos_oid,
+ struct_ver,
+ true, true);
+ if (ret < 0) ret = -ret;
+ if (ret) cout << "Failed to write info" << std::endl;
+ return ret;
+}
+
+void write_log(ObjectStore::Transaction &t, pg_log_t &log)
+{
+ map<eversion_t, hobject_t> divergent_priors;
+ PGLog::write_log(t, log, log_oid, divergent_priors);
+}
+
+int write_pg(ObjectStore::Transaction &t, epoch_t epoch, pg_info_t &info,
+ pg_log_t &log, __u8 struct_ver)
+{
+ int ret = write_info(t, epoch, info, struct_ver);
+ if (ret) return ret;
+ write_log(t, log);
+ return 0;
+}
+
+int export_file(ObjectStore *store, coll_t cid, hobject_t &obj)
+{
+ struct stat st;
+ mysize_t total;
+ ostringstream objname;
+ footer ft;
+
+ int ret = store->stat(cid, obj, &st);
+ if (ret < 0)
+ return ret;
+
+ objname << obj;
+ if (debug && file_fd != STDOUT_FILENO)
+ cout << "objname=" << objname.str() << std::endl;
+
+ total = st.st_size;
+ if (debug && file_fd != STDOUT_FILENO)
+ cout << "size=" << total << std::endl;
+
+ object_begin objb(obj);
+ ret = write_section(TYPE_OBJECT_BEGIN, objb, file_fd);
+ if (ret < 0)
+ return ret;
+
+ uint64_t offset = 0;
+ bufferlist rawdatabl, databl;
+ while(total > 0) {
+ rawdatabl.clear();
+ databl.clear();
+ mysize_t len = max_read;
+ if (len > total)
+ len = total;
+
+ ret = store->read(cid, obj, offset, len, rawdatabl);
+ if (ret < 0)
+ return ret;
+ if (ret == 0)
+ return -EINVAL;
+
+ data_section dblock(offset, len, rawdatabl);
+ total -= ret;
+ offset += ret;
+
+ if (debug && file_fd != STDOUT_FILENO)
+ cout << "data section offset=" << offset << " len=" << len << std::endl;
+
+ ret = write_section(TYPE_DATA, dblock, file_fd);
+ if (ret) return ret;
+ }
+
+ //Handle attrs for this object
+ map<string,bufferptr> aset;
+ ret = store->getattrs(cid, obj, aset, false);
+ if (ret) return ret;
+ attr_section as(aset);
+ ret = write_section(TYPE_ATTRS, as, file_fd);
+ if (ret)
+ return ret;
+
+ if (debug && file_fd != STDOUT_FILENO) {
+ cout << "attrs size " << aset.size() << std::endl;
+ }
+
+ //Handle omap information
+ databl.clear();
+ bufferlist hdrbuf;
+ map<string, bufferlist> out;
+ ret = store->omap_get(cid, obj, &hdrbuf, &out);
+ if (ret < 0)
+ return ret;
+
+ omap_hdr_section ohs(hdrbuf);
+ ret = write_section(TYPE_OMAP_HDR, ohs, file_fd);
+ if (ret)
+ return ret;
+
+ if (!out.empty()) {
+ omap_section oms(out);
+ ret = write_section(TYPE_OMAP, oms, file_fd);
+ if (ret)
+ return ret;
+
+ if (debug && file_fd != STDOUT_FILENO)
+ cout << "omap map size " << out.size() << std::endl;
+ }
+
+ ret = write_simple(TYPE_OBJECT_END, file_fd);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+int export_files(ObjectStore *store, coll_t coll)
+{
+ vector<ghobject_t> objects;
+ ghobject_t next;
+
+ while (!next.is_max()) {
+ int r = store->collection_list_partial(coll, next, 200, 300, 0,
+ &objects, &next);
+ if (r < 0)
+ return r;
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end();
+ ++i) {
+ assert(i->generation == ghobject_t::NO_GEN);
+ r = export_file(store, coll, i->hobj);
+ if (r < 0)
+ return r;
+ }
+ }
+ return 0;
+}
+
+//Write super_header with its fixed 16 byte length
+void write_super()
+{
+ bufferlist superbl;
+ super_header sh;
+ footer ft;
+
+ header hdr(TYPE_NONE, 0);
+ hdr.encode(superbl);
+
+ sh.magic = super_header::super_magic;
+ sh.version = super_header::super_ver;
+ sh.header_size = superbl.length();
+ superbl.clear();
+ ft.encode(superbl);
+ sh.footer_size = superbl.length();
+ superbl.clear();
+
+ sh.encode(superbl);
+ assert(super_header::FIXED_LENGTH == superbl.length());
+ superbl.write_fd(file_fd);
+}
+
+int do_export(ObjectStore *fs, coll_t coll, pg_t pgid, pg_info_t &info,
+ epoch_t map_epoch, __u8 struct_ver, const OSDSuperblock& superblock)
+{
+ PGLog::IndexedLog log;
+ pg_missing_t missing;
+
+ int ret = get_log(fs, coll, pgid, info, log, missing);
+ if (ret > 0)
+ return ret;
+
+ write_super();
+
+ pg_begin pgb(pgid, superblock);
+ ret = write_section(TYPE_PG_BEGIN, pgb, file_fd);
+ if (ret)
+ return ret;
+
+ export_files(fs, coll);
+
+ metadata_section ms(struct_ver, map_epoch, info, log);
+ ret = write_section(TYPE_PG_METADATA, ms, file_fd);
+ if (ret)
+ return ret;
+
+ ret = write_simple(TYPE_PG_END, file_fd);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+int super_header::read_super()
+{
+ bufferlist ebl;
+ bufferlist::iterator ebliter = ebl.begin();
+ ssize_t bytes;
+
+ bytes = ebl.read_fd(file_fd, super_header::FIXED_LENGTH);
+ if ((size_t)bytes != super_header::FIXED_LENGTH) {
+ cout << "Unexpected EOF" << std::endl;
+ return EFAULT;
+ }
+
+ decode(ebliter);
+
+ return 0;
+}
+
+int read_section(int fd, sectiontype_t *type, bufferlist *bl)
+{
+ header hdr;
+ ssize_t bytes;
+
+ int ret = hdr.get_header();
+ if (ret)
+ return ret;
+
+ *type = hdr.type;
+
+ bl->clear();
+ bytes = bl->read_fd(fd, hdr.size);
+ if (bytes != hdr.size) {
+ cout << "Unexpected EOF" << std::endl;
+ return EFAULT;
+ }
+
+ if (hdr.size > 0) {
+ footer ft;
+ ret = ft.get_footer();
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+int get_data(ObjectStore *store, coll_t coll, hobject_t hoid,
+ ObjectStore::Transaction *t, bufferlist &bl)
+{
+ bufferlist::iterator ebliter = bl.begin();
+ data_section ds;
+ ds.decode(ebliter);
+
+ if (debug)
+ cout << "\tdata: offset " << ds.offset << " len " << ds.len << std::endl;
+ t->write(coll, hoid, ds.offset, ds.len, ds.databl);
+ return 0;
+}
+
+int get_attrs(ObjectStore *store, coll_t coll, hobject_t hoid,
+ ObjectStore::Transaction *t, bufferlist &bl,
+ OSDriver &driver, SnapMapper &snap_mapper)
+{
+ bufferlist::iterator ebliter = bl.begin();
+ attr_section as;
+ as.decode(ebliter);
+
+ if (debug)
+ cout << "\tattrs: len " << as.data.size() << std::endl;
+ t->setattrs(coll, hoid, as.data);
+
+ if (hoid.snap < CEPH_MAXSNAP) {
+ map<string,bufferptr>::iterator mi = as.data.find(OI_ATTR);
+ if (mi != as.data.end()) {
+ bufferlist attr_bl;
+ attr_bl.push_back(mi->second);
+ object_info_t oi(attr_bl);
+
+ if (debug)
+ cout << "object_info " << oi << std::endl;
+
+ OSDriver::OSTransaction _t(driver.get_transaction(t));
+ set<snapid_t> oi_snaps(oi.snaps.begin(), oi.snaps.end());
+ snap_mapper.add_oid(hoid, oi_snaps, &_t);
+ }
+ }
+
+ return 0;
+}
+
+int get_omap_hdr(ObjectStore *store, coll_t coll, hobject_t hoid,
+ ObjectStore::Transaction *t, bufferlist &bl)
+{
+ bufferlist::iterator ebliter = bl.begin();
+ omap_hdr_section oh;
+ oh.decode(ebliter);
+
+ if (debug)
+ cout << "\tomap header: " << string(oh.hdr.c_str(), oh.hdr.length())
+ << std::endl;
+ t->omap_setheader(coll, hoid, oh.hdr);
+ return 0;
+}
+
+int get_omap(ObjectStore *store, coll_t coll, hobject_t hoid,
+ ObjectStore::Transaction *t, bufferlist &bl)
+{
+ bufferlist::iterator ebliter = bl.begin();
+ omap_section os;
+ os.decode(ebliter);
+
+ if (debug)
+ cout << "\tomap: size " << os.omap.size() << std::endl;
+ t->omap_setkeys(coll, hoid, os.omap);
+ return 0;
+}
+
+int get_object(ObjectStore *store, coll_t coll, bufferlist &bl)
+{
+ ObjectStore::Transaction tran;
+ ObjectStore::Transaction *t = &tran;
+ bufferlist::iterator ebliter = bl.begin();
+ object_begin ob;
+ ob.decode(ebliter);
+ OSDriver driver(
+ store,
+ coll_t(),
+ OSD::make_snapmapper_oid());
+ spg_t pg;
+ coll.is_pg_prefix(pg);
+ SnapMapper mapper(&driver, 0, 0, 0, pg.shard);
+
+ t->touch(coll, ob.hoid);
+
+ if (debug) {
+ ostringstream objname;
+ objname << ob.hoid.oid;
+ cout << "name " << objname.str() << " snap " << ob.hoid.snap << std::endl;
+ }
+
+ bufferlist ebl;
+ bool done = false;
+ while(!done) {
+ sectiontype_t type;
+ int ret = read_section(file_fd, &type, &ebl);
+ if (ret)
+ return ret;
+
+ //cout << "\tdo_object: Section type " << hex << type << dec << std::endl;
+ //cout << "\t\tsection size " << ebl.length() << std::endl;
+ if (type >= END_OF_TYPES) {
+ cout << "Skipping unknown object section type" << std::endl;
+ continue;
+ }
+ switch(type) {
+ case TYPE_DATA:
+ ret = get_data(store, coll, ob.hoid, t, ebl);
+ if (ret) return ret;
+ break;
+ case TYPE_ATTRS:
+ ret = get_attrs(store, coll, ob.hoid, t, ebl, driver, mapper);
+ if (ret) return ret;
+ break;
+ case TYPE_OMAP_HDR:
+ ret = get_omap_hdr(store, coll, ob.hoid, t, ebl);
+ if (ret) return ret;
+ break;
+ case TYPE_OMAP:
+ ret = get_omap(store, coll, ob.hoid, t, ebl);
+ if (ret) return ret;
+ break;
+ case TYPE_OBJECT_END:
+ done = true;
+ break;
+ default:
+ return EFAULT;
+ }
+ }
+ store->apply_transaction(*t);
+ return 0;
+}
+
+int get_pg_metadata(ObjectStore *store, coll_t coll, bufferlist &bl)
+{
+ ObjectStore::Transaction tran;
+ ObjectStore::Transaction *t = &tran;
+ bufferlist::iterator ebliter = bl.begin();
+ metadata_section ms;
+ ms.decode(ebliter);
+
+#if DIAGNOSTIC
+ Formatter *formatter = new JSONFormatter(true);
+ cout << "struct_v " << (int)ms.struct_ver << std::endl;
+ cout << "epoch " << ms.map_epoch << std::endl;
+ formatter->open_object_section("info");
+ ms.info.dump(formatter);
+ formatter->close_section();
+ formatter->flush(cout);
+ cout << std::endl;
+
+ formatter->open_object_section("log");
+ ms.log.dump(formatter);
+ formatter->close_section();
+ formatter->flush(cout);
+ cout << std::endl;
+#endif
+
+ coll_t newcoll(ms.info.pgid);
+ t->collection_rename(coll, newcoll);
+
+ int ret = write_pg(*t, ms.map_epoch, ms.info, ms.log, ms.struct_ver);
+ if (ret) return ret;
+
+ store->apply_transaction(*t);
+
+ return 0;
+}
+
+int do_import(ObjectStore *store, OSDSuperblock& sb)
+{
+ bufferlist ebl;
+ pg_info_t info;
+ PGLog::IndexedLog log;
+
+ uint64_t next_removal_seq = 0; //My local seq
+ finish_remove_pgs(store, &next_removal_seq);
+
+ int ret = sh.read_super();
+ if (ret)
+ return ret;
+
+ if (sh.magic != super_header::super_magic) {
+ cout << "Invalid magic number" << std::endl;
+ return EFAULT;
+ }
+
+ if (sh.version > super_header::super_ver) {
+ cout << "Can't handle export format version=" << sh.version << std::endl;
+ return EINVAL;
+ }
+
+ //First section must be TYPE_PG_BEGIN
+ sectiontype_t type;
+ ret = read_section(file_fd, &type, &ebl);
+ if (type != TYPE_PG_BEGIN) {
+ return EFAULT;
+ }
+
+ bufferlist::iterator ebliter = ebl.begin();
+ pg_begin pgb;
+ pgb.decode(ebliter);
+ pg_t pgid = pgb.pgid;
+
+ if (debug) {
+ cout << "Exported features: " << pgb.superblock.compat_features << std::endl;
+ }
+ if (sb.compat_features.compare(pgb.superblock.compat_features) == -1) {
+ cout << "Export has incompatible features set "
+ << pgb.superblock.compat_features << std::endl;
+ return 1;
+ }
+
+ log_oid = OSD::make_pg_log_oid(spg_t(pgid, ghobject_t::no_shard()));
+ biginfo_oid = OSD::make_pg_biginfo_oid(spg_t(pgid, ghobject_t::no_shard()));
+
+ //Check for PG already present.
+ coll_t coll(spg_t(pgid, ghobject_t::no_shard()));
+ if (store->collection_exists(coll)) {
+ cout << "pgid " << pgid << " already exists" << std::endl;
+ return 1;
+ }
+
+ //Switch to collection which will be removed automatically if
+ //this program is interupted.
+ coll_t rmcoll = coll_t::make_removal_coll(
+ next_removal_seq, spg_t(pgid, ghobject_t::no_shard()));
+ ObjectStore::Transaction *t = new ObjectStore::Transaction;
+ t->create_collection(rmcoll);
+ store->apply_transaction(*t);
+ delete t;
+
+ cout << "Importing pgid " << pgid << std::endl;
+
+ bool done = false;
+ bool found_metadata = false;
+ while(!done) {
+ ret = read_section(file_fd, &type, &ebl);
+ if (ret)
+ return ret;
+
+ //cout << "do_import: Section type " << hex << type << dec << std::endl;
+ if (type >= END_OF_TYPES) {
+ cout << "Skipping unknown section type" << std::endl;
+ continue;
+ }
+ switch(type) {
+ case TYPE_OBJECT_BEGIN:
+ ret = get_object(store, rmcoll, ebl);
+ if (ret) return ret;
+ break;
+ case TYPE_PG_METADATA:
+ ret = get_pg_metadata(store, rmcoll, ebl);
+ if (ret) return ret;
+ found_metadata = true;
+ break;
+ case TYPE_PG_END:
+ done = true;
+ break;
+ default:
+ return EFAULT;
+ }
+ }
+
+ if (!found_metadata) {
+ cout << "Missing metadata section" << std::endl;
+ return EFAULT;
+ }
+
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ string fspath, jpath, pgidstr, type, file;
+ Formatter *formatter = new JSONFormatter(true);
+
+ po::options_description desc("Allowed options");
+ desc.add_options()
+ ("help", "produce help message")
+ ("filestore-path", po::value<string>(&fspath),
+ "path to filestore directory, mandatory")
+ ("journal-path", po::value<string>(&jpath),
+ "path to journal, mandatory")
+ ("pgid", po::value<string>(&pgidstr),
+ "PG id, mandatory")
+ ("type", po::value<string>(&type),
+ "Type one of info, log, remove, export, or import, mandatory")
+ ("file", po::value<string>(&file),
+ "path of file to export or import")
+ ("debug", "Enable diagnostic output to stderr")
+ ;
+
+ po::variables_map vm;
+ po::parsed_options parsed =
+ po::command_line_parser(argc, argv).options(desc).allow_unregistered().run();
+ po::store( parsed, vm);
+ try {
+ po::notify(vm);
+ }
+ catch(...) {
+ cout << desc << std::endl;
+ exit(1);
+ }
+
+ if (vm.count("help")) {
+ cout << desc << std::endl;
+ return 1;
+ }
+
+ if (!vm.count("filestore-path")) {
+ cout << "Must provide filestore-path" << std::endl
+ << desc << std::endl;
+ return 1;
+ }
+ if (!vm.count("journal-path")) {
+ cout << "Must provide journal-path" << std::endl
+ << desc << std::endl;
+ return 1;
+ }
+ if (!vm.count("type")) {
+ cout << "Must provide type (info, log, remove, export, import)"
+ << std::endl << desc << std::endl;
+ return 1;
+ }
+ if (type != "import" && !vm.count("pgid")) {
+ cout << "Must provide pgid" << std::endl
+ << desc << std::endl;
+ return 1;
+ }
+
+ file_fd = fd_none;
+ if (type == "export") {
+ if (!vm.count("file")) {
+ file_fd = STDOUT_FILENO;
+ } else {
+ file_fd = open(file.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666);
+ }
+ } else if (type == "import") {
+ if (!vm.count("file")) {
+ file_fd = STDIN_FILENO;
+ } else {
+ file_fd = open(file.c_str(), O_RDONLY);
+ }
+ }
+
+ if (vm.count("file") && file_fd == fd_none) {
+ cout << "--file option only applies to import or export" << std::endl;
+ return 1;
+ }
+
+ if (file_fd != fd_none && file_fd < 0) {
+ perror("open");
+ return 1;
+ }
+
+ if ((fspath.length() == 0 || jpath.length() == 0) ||
+ (type != "info" && type != "log" && type != "remove" && type != "export"
+ && type != "import") ||
+ (type != "import" && pgidstr.length() == 0)) {
+ cerr << "Invalid params" << std::endl;
+ exit(1);
+ }
+
+ if (type == "import" && pgidstr.length()) {
+ cerr << "--pgid option invalid with import" << std::endl;
+ exit(1);
+ }
+
+ vector<const char *> ceph_options, def_args;
+ vector<string> ceph_option_strings = po::collect_unrecognized(
+ parsed.options, po::include_positional);
+ ceph_options.reserve(ceph_option_strings.size());
+ for (vector<string>::iterator i = ceph_option_strings.begin();
+ i != ceph_option_strings.end();
+ ++i) {
+ ceph_options.push_back(i->c_str());
+ }
+
+ //Suppress derr() output to stderr by default
+ if (!vm.count("debug")) {
+ close(STDERR_FILENO);
+ (void)open("/dev/null", O_WRONLY);
+ debug = false;
+ } else {
+ debug = true;
+ }
+
+ global_init(
+ &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD,
+ CODE_ENVIRONMENT_UTILITY, 0);
+ //CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+ common_init_finish(g_ceph_context);
+ g_ceph_context->_conf->apply_changes(NULL);
+ g_conf = g_ceph_context->_conf;
+
+ //Verify that fspath really is an osd store
+ struct stat st;
+ if (::stat(fspath.c_str(), &st) == -1) {
+ perror("fspath");
+ invalid_path(fspath);
+ }
+ if (!S_ISDIR(st.st_mode)) {
+ invalid_path(fspath);
+ }
+ string check = fspath + "/whoami";
+ if (::stat(check.c_str(), &st) == -1) {
+ perror("whoami");
+ invalid_path(fspath);
+ }
+ if (!S_ISREG(st.st_mode)) {
+ invalid_path(fspath);
+ }
+ check = fspath + "/current";
+ if (::stat(check.c_str(), &st) == -1) {
+ perror("current");
+ invalid_path(fspath);
+ }
+ if (!S_ISDIR(st.st_mode)) {
+ invalid_path(fspath);
+ }
+
+ pg_t pgid;
+ if (pgidstr.length() && !pgid.parse(pgidstr.c_str())) {
+ cout << "Invalid pgid '" << pgidstr << "' specified" << std::endl;
+ exit(1);
+ }
+
+ ObjectStore *fs = new FileStore(fspath, jpath);
+
+ int r = fs->mount();
+ if (r < 0) {
+ if (r == -EBUSY) {
+ cout << "OSD has the store locked" << std::endl;
+ } else {
+ cout << "Mount failed with '" << cpp_strerror(-r) << "'" << std::endl;
+ }
+ return 1;
+ }
+
+ bool fs_sharded_objects = fs->get_allow_sharded_objects();
+
+ int ret = 0;
+ vector<coll_t> ls;
+ vector<coll_t>::iterator it;
+ CompatSet supported;
+
+#ifdef INTERNAL_TEST
+ supported = get_test_compat_set();
+#else
+ supported = OSD::get_osd_compat_set();
+#endif
+
+ bufferlist bl;
+ OSDSuperblock superblock;
+ bufferlist::iterator p;
+ ret = fs->read(coll_t::META_COLL, OSD_SUPERBLOCK_POBJECT, 0, 0, bl);
+ if (ret < 0) {
+ cout << "Failure to read OSD superblock error= " << r << std::endl;
+ goto out;
+ }
+
+ p = bl.begin();
+ ::decode(superblock, p);
+
+#ifdef INTERNAL_TEST2
+ fs->set_allow_sharded_objects();
+ assert(fs->get_allow_sharded_objects());
+ fs_sharded_objects = true;
+ superblock.compat_features.incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS);
+#endif
+
+ if (debug && file_fd != STDOUT_FILENO) {
+ cout << "Supported features: " << supported << std::endl;
+ cout << "On-disk features: " << superblock.compat_features << std::endl;
+ }
+ if (supported.compare(superblock.compat_features) == -1) {
+ cout << "On-disk OSD incompatible features set "
+ << superblock.compat_features << std::endl;
+ ret = EINVAL;
+ goto out;
+ }
+
+ // If there was a crash as an OSD was transitioning to sharded objects
+ // and hadn't completed a set_allow_sharded_objects().
+ // This utility does not want to attempt to finish that transition.
+ if (superblock.compat_features.incompat.contains(CEPH_OSD_FEATURE_INCOMPAT_SHARDS) != fs_sharded_objects) {
+ // An OSD should never have call set_allow_sharded_objects() before
+ // updating its own OSD features.
+ if (fs_sharded_objects)
+ cout << "FileStore sharded but OSD not set, Corruption?" << std::endl;
+ else
+ cout << "Found incomplete transition to sharded objects" << std::endl;
+ ret = EINVAL;
+ goto out;
+ }
+
+ if (type == "import") {
+
+ try {
+ ret = do_import(fs, superblock);
+ }
+ catch (const buffer::error &e) {
+ cout << "do_import threw exception error " << e.what() << std::endl;
+ ret = EFAULT;
+ }
+ if (ret == EFAULT) {
+ cout << "Corrupt input for import" << std::endl;
+ }
+ goto out;
+ }
+
+ log_oid = OSD::make_pg_log_oid(spg_t(pgid, ghobject_t::no_shard()));
+ biginfo_oid = OSD::make_pg_biginfo_oid(spg_t(pgid, ghobject_t::no_shard()));
+
+ if (type == "remove") {
+ uint64_t next_removal_seq = 0; //My local seq
+ finish_remove_pgs(fs, &next_removal_seq);
+ int r = initiate_new_remove_pg(fs, pgid, &next_removal_seq);
+ if (r) {
+ cout << "PG '" << pgid << "' not found" << std::endl;
+ ret = 1;
+ goto out;
+ }
+ finish_remove_pgs(fs, &next_removal_seq);
+ cout << "Remove successful" << std::endl;
+ goto out;
+ }
+
+ r = fs->list_collections(ls);
+ if (r < 0) {
+ cout << "failed to list pgs: " << cpp_strerror(-r) << std::endl;
+ exit(1);
+ }
+
+ for (it = ls.begin(); it != ls.end(); ++it) {
+ snapid_t snap;
+ spg_t tmppgid;
+
+ if (!it->is_pg(tmppgid, snap)) {
+ continue;
+ }
+
+ if (tmppgid.pgid != pgid) {
+ continue;
+ }
+ if (snap != CEPH_NOSNAP && debug) {
+ cerr << "skipping snapped dir " << *it
+ << " (pg " << pgid << " snap " << snap << ")" << std::endl;
+ continue;
+ }
+
+ //Found!
+ break;
+ }
+
+ epoch_t map_epoch;
+ if (it != ls.end()) {
+
+ coll_t coll = *it;
+
+ bufferlist bl;
+ map_epoch = PG::peek_map_epoch(fs, coll, infos_oid, &bl);
+ if (debug)
+ cerr << "map_epoch " << map_epoch << std::endl;
+
+ pg_info_t info(spg_t(pgid, ghobject_t::no_shard()));
+ map<epoch_t,pg_interval_t> past_intervals;
+ hobject_t biginfo_oid = OSD::make_pg_biginfo_oid(
+ spg_t(pgid, ghobject_t::no_shard()));
+ interval_set<snapid_t> snap_collections;
+
+ __u8 struct_ver;
+ r = PG::read_info(fs, coll, bl, info, past_intervals, biginfo_oid,
+ infos_oid, snap_collections, struct_ver);
+ if (r < 0) {
+ cout << "read_info error " << cpp_strerror(-r) << std::endl;
+ ret = 1;
+ goto out;
+ }
+ if (debug)
+ cerr << "struct_v " << (int)struct_ver << std::endl;
+
+ if (type == "export") {
+ ret = do_export(fs, coll, pgid, info, map_epoch, struct_ver, superblock);
+ } else if (type == "info") {
+ formatter->open_object_section("info");
+ info.dump(formatter);
+ formatter->close_section();
+ formatter->flush(cout);
+ cout << std::endl;
+ } else if (type == "log") {
+ PGLog::IndexedLog log;
+ pg_missing_t missing;
+ ret = get_log(fs, coll, pgid, info, log, missing);
+ if (ret > 0)
+ goto out;
+
+ formatter->open_object_section("log");
+ log.dump(formatter);
+ formatter->close_section();
+ formatter->flush(cout);
+ cout << std::endl;
+ formatter->open_object_section("missing");
+ missing.dump(formatter);
+ formatter->close_section();
+ formatter->flush(cout);
+ cout << std::endl;
+ }
+ } else {
+ cout << "PG '" << pgid << "' not found" << std::endl;
+ ret = 1;
+ }
+
+out:
+ if (fs->umount() < 0) {
+ cout << "umount failed" << std::endl;
+ return 1;
+ }
+
+ return (ret != 0);
+}
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/program_options/option.hpp>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <boost/program_options/cmdline.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <iostream>
+#include <set>
+#include <sstream>
+#include <stdlib.h>
+#include <fstream>
+
+#include "common/Formatter.h"
+
+#include "global/global_init.h"
+#include "os/ObjectStore.h"
+#include "os/FileStore.h"
+#include "common/perf_counters.h"
+#include "common/errno.h"
+#include "osd/PGLog.h"
+#include "osd/osd_types.h"
+#include "osd/OSD.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+static void invalid_path(string &path)
+{
+ cout << "Invalid path to osd store specified: " << path << "\n";
+ exit(1);
+}
+
+int main(int argc, char **argv)
+{
+ string fspath, jpath, pgidstr;
+ bool list_lost_objects = false;
+ bool fix_lost_objects = false;
+ unsigned LIST_AT_A_TIME = 100;
+ unsigned scanned = 0;
+
+ po::options_description desc("Allowed options");
+ desc.add_options()
+ ("help", "produce help message")
+ ("filestore-path", po::value<string>(&fspath),
+ "path to filestore directory, mandatory")
+ ("journal-path", po::value<string>(&jpath),
+ "path to journal, mandatory")
+ ("pgid", po::value<string>(&pgidstr),
+ "PG id")
+ ("list-lost-objects", po::value<bool>(
+ &list_lost_objects)->default_value(false),
+ "list lost objects")
+ ("fix-lost-objects", po::value<bool>(
+ &fix_lost_objects)->default_value(false),
+ "fix lost objects")
+ ;
+
+ po::variables_map vm;
+ po::parsed_options parsed =
+ po::command_line_parser(argc, argv).options(desc).
+ allow_unregistered().run();
+ po::store( parsed, vm);
+ try {
+ po::notify(vm);
+ }
+ catch(...) {
+ cout << desc << std::endl;
+ exit(1);
+ }
+
+ if (vm.count("help")) {
+ cout << desc << std::endl;
+ return 1;
+ }
+
+ if (!vm.count("filestore-path")) {
+ cerr << "Must provide filestore-path" << std::endl
+ << desc << std::endl;
+ return 1;
+ }
+ if (!vm.count("journal-path")) {
+ cerr << "Must provide journal-path" << std::endl
+ << desc << std::endl;
+ return 1;
+ }
+
+ if ((fspath.length() == 0 || jpath.length() == 0)) {
+ cerr << "Invalid params" << desc << std::endl;
+ exit(1);
+ }
+
+ vector<const char *> ceph_options, def_args;
+ vector<string> ceph_option_strings = po::collect_unrecognized(
+ parsed.options, po::include_positional);
+ ceph_options.reserve(ceph_option_strings.size());
+ for (vector<string>::iterator i = ceph_option_strings.begin();
+ i != ceph_option_strings.end();
+ ++i) {
+ ceph_options.push_back(i->c_str());
+ }
+
+ global_init(
+ &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD,
+ CODE_ENVIRONMENT_UTILITY, 0);
+ //CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+ common_init_finish(g_ceph_context);
+ g_ceph_context->_conf->apply_changes(NULL);
+ g_conf = g_ceph_context->_conf;
+
+ //Verify that fspath really is an osd store
+ struct stat st;
+ if (::stat(fspath.c_str(), &st) == -1) {
+ perror("fspath");
+ invalid_path(fspath);
+ }
+ if (!S_ISDIR(st.st_mode)) {
+ invalid_path(fspath);
+ }
+ string check = fspath + "/whoami";
+ if (::stat(check.c_str(), &st) == -1) {
+ perror("whoami");
+ invalid_path(fspath);
+ }
+ if (!S_ISREG(st.st_mode)) {
+ invalid_path(fspath);
+ }
+ check = fspath + "/current";
+ if (::stat(check.c_str(), &st) == -1) {
+ perror("current");
+ invalid_path(fspath);
+ }
+ if (!S_ISDIR(st.st_mode)) {
+ invalid_path(fspath);
+ }
+
+ ObjectStore *fs = new FileStore(fspath, jpath);
+
+ int r = fs->mount();
+ if (r < 0) {
+ if (r == -EBUSY) {
+ cout << "OSD has the store locked" << std::endl;
+ } else {
+ cout << "Mount failed with '" << cpp_strerror(-r) << "'" << std::endl;
+ }
+ return 1;
+ }
+
+ vector<coll_t> colls_to_check;
+ if (pgidstr.length()) {
+ spg_t pgid;
+ if (!pgid.parse(pgidstr.c_str())) {
+ cout << "Invalid pgid '" << pgidstr << "' specified" << std::endl;
+ exit(1);
+ }
+ colls_to_check.push_back(coll_t(pgid));
+ } else {
+ vector<coll_t> candidates;
+ r = fs->list_collections(candidates);
+ if (r < 0) {
+ cerr << "Error listing collections: " << cpp_strerror(r) << std::endl;
+ goto UMOUNT;
+ }
+ for (vector<coll_t>::iterator i = candidates.begin();
+ i != candidates.end();
+ ++i) {
+ spg_t pgid;
+ snapid_t snap;
+ if (i->is_pg(pgid, snap)) {
+ colls_to_check.push_back(*i);
+ }
+ }
+ }
+
+ cerr << colls_to_check.size() << " pgs to scan" << std::endl;
+ for (vector<coll_t>::iterator i = colls_to_check.begin();
+ i != colls_to_check.end();
+ ++i, ++scanned) {
+ cerr << "Scanning " << *i << ", " << scanned << "/"
+ << colls_to_check.size() << " completed" << std::endl;
+ ghobject_t next;
+ while (!next.is_max()) {
+ vector<ghobject_t> list;
+ r = fs->collection_list_partial(
+ *i,
+ next,
+ LIST_AT_A_TIME,
+ LIST_AT_A_TIME,
+ CEPH_NOSNAP,
+ &list,
+ &next);
+ if (r < 0) {
+ cerr << "Error listing collection: " << *i << ", "
+ << cpp_strerror(r) << std::endl;
+ goto UMOUNT;
+ }
+ for (vector<ghobject_t>::iterator obj = list.begin();
+ obj != list.end();
+ ++obj) {
+ bufferlist attr;
+ r = fs->getattr(*i, *obj, OI_ATTR, attr);
+ if (r < 0) {
+ cerr << "Error getting attr on : " << make_pair(*i, *obj) << ", "
+ << cpp_strerror(r) << std::endl;
+ goto UMOUNT;
+ }
+ object_info_t oi;
+ bufferlist::iterator bp = attr.begin();
+ try {
+ ::decode(oi, bp);
+ } catch (...) {
+ r = -EINVAL;
+ cerr << "Error getting attr on : " << make_pair(*i, *obj) << ", "
+ << cpp_strerror(r) << std::endl;
+ goto UMOUNT;
+ }
+ if (oi.is_lost()) {
+ if (list_lost_objects) {
+ cout << *i << "/" << *obj << " is lost" << std::endl;
+ }
+ if (fix_lost_objects) {
+ cerr << *i << "/" << *obj << " is lost, fixing" << std::endl;
+ oi.clear_flag(object_info_t::FLAG_LOST);
+ bufferlist bl2;
+ ::encode(oi, bl2);
+ ObjectStore::Transaction t;
+ t.setattr(*i, *obj, OI_ATTR, bl2);
+ r = fs->apply_transaction(t);
+ if (r < 0) {
+ cerr << "Error getting fixing attr on : " << make_pair(*i, *obj)
+ << ", "
+ << cpp_strerror(r) << std::endl;
+ goto UMOUNT;
+ }
+ }
+ }
+ }
+ }
+ }
+ cerr << "Completed" << std::endl;
+
+ UMOUNT:
+ fs->sync_and_flush();
+ fs->umount();
+ return r;
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 Inktank, Inc.
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include <iostream>
+#include <string>
+#include <sstream>
+#include <map>
+#include <set>
+#include <boost/scoped_ptr.hpp>
+
+#include "os/LevelDBStore.h"
+
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "common/errno.h"
+#include "common/safe_io.h"
+#include "common/config.h"
+#include "common/strtol.h"
+#include "include/stringify.h"
+
+using namespace std;
+
+class StoreTool
+{
+ boost::scoped_ptr<KeyValueDB> db;
+ string store_path;
+
+ public:
+ StoreTool(const string &path) : store_path(path) {
+ LevelDBStore *db_ptr = new LevelDBStore(g_ceph_context, store_path);
+ assert(!db_ptr->open(std::cerr));
+ db.reset(db_ptr);
+ }
+
+ uint32_t traverse(const string &prefix,
+ const bool do_crc,
+ ostream *out) {
+ KeyValueDB::WholeSpaceIterator iter = db->get_iterator();
+
+ if (prefix.empty())
+ iter->seek_to_first();
+ else
+ iter->seek_to_first(prefix);
+
+ uint32_t crc = -1;
+
+ while (iter->valid()) {
+ pair<string,string> rk = iter->raw_key();
+ if (!prefix.empty() && (rk.first != prefix))
+ break;
+
+ if (out)
+ *out << rk.first << ":" << rk.second;
+ if (do_crc) {
+ bufferlist bl;
+ bl.append(rk.first);
+ bl.append(rk.second);
+ bl.append(iter->value());
+
+ crc = bl.crc32c(crc);
+ if (out) {
+ *out << " (" << bl.crc32c(0) << ")";
+ }
+ }
+ if (out)
+ *out << std::endl;
+ iter->next();
+ }
+
+ return crc;
+ }
+
+ void list(const string &prefix, const bool do_crc) {
+ traverse(prefix, do_crc, &std::cout);
+ }
+
+ bool exists(const string &prefix) {
+ assert(!prefix.empty());
+ KeyValueDB::WholeSpaceIterator iter = db->get_iterator();
+ iter->seek_to_first(prefix);
+ return (iter->valid() && (iter->raw_key().first == prefix));
+ }
+
+ bool exists(const string &prefix, const string &key) {
+ assert(!prefix.empty());
+
+ if (key.empty()) {
+ return exists(prefix);
+ }
+
+ bool exists = false;
+ get(prefix, key, exists);
+ return exists;
+ }
+
+ bufferlist get(const string &prefix, const string &key, bool &exists) {
+ assert(!prefix.empty() && !key.empty());
+
+ map<string,bufferlist> result;
+ std::set<std::string> keys;
+ keys.insert(key);
+ db->get(prefix, keys, &result);
+
+ if (result.count(key) > 0) {
+ exists = true;
+ return result[key];
+ }
+ exists = false;
+ return bufferlist();
+ }
+
+ uint64_t get_size() {
+ map<string,uint64_t> extras;
+ uint64_t s = db->get_estimated_size(extras);
+ for (map<string,uint64_t>::iterator p = extras.begin();
+ p != extras.end(); ++p) {
+ std::cout << p->first << " - " << p->second << std::endl;
+ }
+ std::cout << "total: " << s << std::endl;
+ return s;
+ }
+
+ bool set(const string &prefix, const string &key, bufferlist &val) {
+ assert(!prefix.empty());
+ assert(!key.empty());
+ assert(val.length() > 0);
+
+ KeyValueDB::Transaction tx = db->get_transaction();
+ tx->set(prefix, key, val);
+ int ret = db->submit_transaction_sync(tx);
+
+ return (ret == 0);
+ }
+
+ int copy_store_to(const string &other_path, const int num_keys_per_tx) {
+
+ if (num_keys_per_tx <= 0) {
+ std::cerr << "must specify a number of keys/tx > 0" << std::endl;
+ return -EINVAL;
+ }
+
+ // open or create a leveldb store at @p other_path
+ LevelDBStore other(g_ceph_context, other_path);
+ int err = other.create_and_open(std::cerr);
+ if (err < 0)
+ return err;
+
+ KeyValueDB::WholeSpaceIterator it = db->get_iterator();
+ it->seek_to_first();
+ uint64_t total_keys = 0;
+ uint64_t total_size = 0;
+ uint64_t total_txs = 0;
+
+ utime_t started_at = ceph_clock_now(g_ceph_context);
+
+ do {
+ int num_keys = 0;
+
+ KeyValueDB::Transaction tx = other.get_transaction();
+
+
+ while (it->valid() && num_keys < num_keys_per_tx) {
+ pair<string,string> k = it->raw_key();
+ bufferlist v = it->value();
+ tx->set(k.first, k.second, v);
+
+ num_keys ++;
+ total_size += v.length();
+
+ it->next();
+ }
+
+ total_txs ++;
+ total_keys += num_keys;
+
+ if (num_keys > 0)
+ other.submit_transaction_sync(tx);
+
+ utime_t cur_duration = ceph_clock_now(g_ceph_context) - started_at;
+ std::cout << "ts = " << cur_duration << "s, copied " << total_keys
+ << " keys so far (" << stringify(si_t(total_size)) << ")"
+ << std::endl;
+
+ } while (it->valid());
+
+ utime_t time_taken = ceph_clock_now(g_ceph_context) - started_at;
+
+ std::cout << "summary:" << std::endl;
+ std::cout << " copied " << total_keys << " keys" << std::endl;
+ std::cout << " used " << total_txs << " transactions" << std::endl;
+ std::cout << " total size " << stringify(si_t(total_size)) << std::endl;
+ std::cout << " from '" << store_path << "' to '" << other_path << "'"
+ << std::endl;
+ std::cout << " duration " << time_taken << " seconds" << std::endl;
+
+ return 0;
+ }
+};
+
+void usage(const char *pname)
+{
+ std::cerr << "Usage: " << pname << " <store path> command [args...]\n"
+ << "\n"
+ << "Commands:\n"
+ << " list [prefix]\n"
+ << " list-crc [prefix]\n"
+ << " exists <prefix> [key]\n"
+ << " get <prefix> <key> [out <file>]\n"
+ << " crc <prefix> <key>\n"
+ << " get-size [<prefix> <key>]\n"
+ << " set <prefix> <key> [ver <N>|in <file>]\n"
+ << " store-copy <path> [num-keys-per-tx]\n"
+ << " store-crc <path>\n"
+ << std::endl;
+}
+
+int main(int argc, const char *argv[])
+{
+ vector<const char*> args;
+ argv_to_vec(argc, argv, args);
+ env_to_vec(args);
+
+ global_init(
+ NULL, args,
+ CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+
+
+ if (args.size() < 2) {
+ usage(argv[0]);
+ return 1;
+ }
+
+ string path(args[0]);
+ string cmd(args[1]);
+
+ StoreTool st(path);
+
+ if (cmd == "list" || cmd == "list-crc") {
+ string prefix;
+ if (argc > 3)
+ prefix = argv[3];
+
+ bool do_crc = (cmd == "list-crc");
+
+ st.list(prefix, do_crc);
+
+ } else if (cmd == "exists") {
+ string key;
+ if (argc < 4) {
+ usage(argv[0]);
+ return 1;
+ }
+ string prefix(argv[3]);
+ if (argc > 4)
+ key = argv[4];
+
+ bool ret = st.exists(prefix, key);
+ std::cout << "(" << prefix << ", " << key << ") "
+ << (ret ? "exists" : "does not exist")
+ << std::endl;
+ return (ret ? 0 : 1);
+
+ } else if (cmd == "get") {
+ if (argc < 5) {
+ usage(argv[0]);
+ return 1;
+ }
+ string prefix(argv[3]);
+ string key(argv[4]);
+
+ bool exists = false;
+ bufferlist bl = st.get(prefix, key, exists);
+ std::cout << "(" << prefix << ", " << key << ")";
+ if (!exists) {
+ std::cout << " does not exist" << std::endl;
+ return 1;
+ }
+ std::cout << std::endl;
+
+ if (argc >= 6) {
+ string subcmd(argv[5]);
+ string out(argv[6]);
+
+ if (subcmd != "out") {
+ std::cerr << "unrecognized subcmd '" << subcmd << "'"
+ << std::endl;
+ return 1;
+ }
+
+ if (out.empty()) {
+ std::cerr << "unspecified out file" << std::endl;
+ return 1;
+ }
+
+ int err = bl.write_file(argv[6], 0644);
+ if (err < 0) {
+ std::cerr << "error writing value to '" << out << "': "
+ << cpp_strerror(err) << std::endl;
+ return 1;
+ }
+ } else {
+ ostringstream os;
+ bl.hexdump(os);
+ std::cout << os.str() << std::endl;
+ }
+
+ } else if (cmd == "crc") {
+ if (argc < 5) {
+ usage(argv[0]);
+ return 1;
+ }
+ string prefix(argv[3]);
+ string key(argv[4]);
+
+ bool exists = false;
+ bufferlist bl = st.get(prefix, key, exists);
+ std::cout << "(" << prefix << ", " << key << ") ";
+ if (!exists) {
+ std::cout << " does not exist" << std::endl;
+ return 1;
+ }
+ std::cout << " crc " << bl.crc32c(0) << std::endl;
+
+ } else if (cmd == "get-size") {
+ std::cout << "estimated store size: " << st.get_size() << std::endl;
+
+ if (argc < 4)
+ return 0;
+
+ if (argc < 5) {
+ usage(argv[0]);
+ return 1;
+ }
+ string prefix(argv[3]);
+ string key(argv[4]);
+
+ bool exists = false;
+ bufferlist bl = st.get(prefix, key, exists);
+ if (!exists) {
+ std::cerr << "(" << prefix << "," << key
+ << ") does not exist" << std::endl;
+ return 1;
+ }
+ std::cout << "(" << prefix << "," << key
+ << ") size " << si_t(bl.length()) << std::endl;
+
+ } else if (cmd == "set") {
+ if (argc < 7) {
+ usage(argv[0]);
+ return 1;
+ }
+ string prefix(argv[3]);
+ string key(argv[4]);
+ string subcmd(argv[5]);
+
+ bufferlist val;
+ string errstr;
+ if (subcmd == "ver") {
+ version_t v = (version_t) strict_strtoll(argv[6], 10, &errstr);
+ if (!errstr.empty()) {
+ std::cerr << "error reading version: " << errstr << std::endl;
+ return 1;
+ }
+ ::encode(v, val);
+ } else if (subcmd == "in") {
+ int ret = val.read_file(argv[6], &errstr);
+ if (ret < 0 || !errstr.empty()) {
+ std::cerr << "error reading file: " << errstr << std::endl;
+ return 1;
+ }
+ } else {
+ std::cerr << "unrecognized subcommand '" << subcmd << "'" << std::endl;
+ usage(argv[0]);
+ return 1;
+ }
+
+ bool ret = st.set(prefix, key, val);
+ if (!ret) {
+ std::cerr << "error setting ("
+ << prefix << "," << key << ")" << std::endl;
+ return 1;
+ }
+ } else if (cmd == "store-copy") {
+ int num_keys_per_tx = 128; // magic number that just feels right.
+ if (argc < 4) {
+ usage(argv[0]);
+ return 1;
+ } else if (argc > 4) {
+ string err;
+ num_keys_per_tx = strict_strtol(argv[4], 10, &err);
+ if (!err.empty()) {
+ std::cerr << "invalid num_keys_per_tx: " << err << std::endl;
+ return 1;
+ }
+ }
+
+ int ret = st.copy_store_to(argv[3], num_keys_per_tx);
+ if (ret < 0) {
+ std::cerr << "error copying store to path '" << argv[3]
+ << "': " << cpp_strerror(ret) << std::endl;
+ return 1;
+ }
+
+ } else if (cmd == "store-crc") {
+ uint32_t crc = st.traverse(string(), true, NULL);
+ std::cout << "store at '" << path << "' crc " << crc << std::endl;
+
+ } else {
+ std::cerr << "Unrecognized command: " << cmd << std::endl;
+ return 1;
+ }
+
+ return 0;
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 Inktank, Inc.
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License version 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/program_options/option.hpp>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <boost/program_options/cmdline.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <iostream>
+#include <set>
+#include <sstream>
+#include <stdlib.h>
+#include <fstream>
+#include <string>
+#include <sstream>
+#include <map>
+#include <set>
+#include <boost/scoped_ptr.hpp>
+
+#include "global/global_init.h"
+#include "os/LevelDBStore.h"
+#include "mon/MonitorDBStore.h"
+#include "mon/Paxos.h"
+#include "common/Formatter.h"
+#include "include/stringify.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+class TraceIter {
+ int fd;
+ unsigned idx;
+ MonitorDBStore::Transaction t;
+public:
+ TraceIter(string fname) : fd(-1), idx(-1) {
+ fd = ::open(fname.c_str(), O_RDONLY);
+ }
+ bool valid() {
+ return fd != -1;
+ }
+ const MonitorDBStore::Transaction &cur() {
+ assert(valid());
+ return t;
+ }
+ unsigned num() { return idx; }
+ void next() {
+ ++idx;
+ bufferlist bl;
+ int r = bl.read_fd(fd, 6);
+ if (r < 0) {
+ std::cerr << "Got error: " << cpp_strerror(r) << " on read_fd"
+ << std::endl;
+ ::close(fd);
+ fd = -1;
+ return;
+ } else if ((unsigned)r < 6) {
+ std::cerr << "short read" << std::endl;
+ ::close(fd);
+ fd = -1;
+ return;
+ }
+ bufferlist::iterator bliter = bl.begin();
+ uint8_t ver, ver2;
+ ::decode(ver, bliter);
+ ::decode(ver2, bliter);
+ uint32_t len;
+ ::decode(len, bliter);
+ r = bl.read_fd(fd, len);
+ if (r < 0) {
+ std::cerr << "Got error: " << cpp_strerror(r) << " on read_fd"
+ << std::endl;
+ ::close(fd);
+ fd = -1;
+ return;
+ } else if ((unsigned)r < len) {
+ std::cerr << "short read" << std::endl;
+ ::close(fd);
+ fd = -1;
+ return;
+ }
+ bliter = bl.begin();
+ t.decode(bliter);
+ }
+ void init() {
+ next();
+ }
+ ~TraceIter() {
+ if (fd != -1) {
+ ::close(fd);
+ fd = -1;
+ }
+ }
+};
+
+int main(int argc, char **argv) {
+ po::options_description desc("Allowed options");
+ int version = -1;
+ string store_path, cmd, out_path, tfile;
+ unsigned dstart = 0;
+ unsigned dstop = ~0;
+ unsigned num_replays = 1;
+ unsigned tsize = 200;
+ unsigned tvalsize = 1024;
+ unsigned ntrans = 100;
+ desc.add_options()
+ ("help", "produce help message")
+ ("mon-store-path", po::value<string>(&store_path),
+ "path to mon directory, mandatory")
+ ("out", po::value<string>(&out_path),
+ "out path")
+ ("version", po::value<int>(&version),
+ "version requested")
+ ("trace-file", po::value<string>(&tfile),
+ "trace file")
+ ("dump-start", po::value<unsigned>(&dstart),
+ "transaction num to start dumping at")
+ ("dump-end", po::value<unsigned>(&dstop),
+ "transaction num to stop dumping at")
+ ("num-replays", po::value<unsigned>(&num_replays),
+ "number of times to replay")
+ ("trans-size", po::value<unsigned>(&tsize),
+ "keys to write in each transaction")
+ ("trans-val-size", po::value<unsigned>(&tvalsize),
+ "val to write in each key")
+ ("num-trans", po::value<unsigned>(&ntrans),
+ "number of transactions to run")
+ ("command", po::value<string>(&cmd),
+ "command")
+ ;
+ po::positional_options_description p;
+ p.add("command", 1);
+ p.add("version", 1);
+
+ po::variables_map vm;
+ po::parsed_options parsed =
+ po::command_line_parser(argc, argv).options(desc).positional(p).run();
+ po::store(
+ parsed,
+ vm);
+ try {
+ po::notify(vm);
+ } catch (...) {
+ cout << desc << std::endl;
+ return 1;
+ }
+
+ vector<const char *> ceph_options, def_args;
+ vector<string> ceph_option_strings = po::collect_unrecognized(
+ parsed.options, po::include_positional);
+ ceph_options.reserve(ceph_option_strings.size());
+ for (vector<string>::iterator i = ceph_option_strings.begin();
+ i != ceph_option_strings.end();
+ ++i) {
+ ceph_options.push_back(i->c_str());
+ }
+
+ global_init(
+ &def_args, ceph_options, CEPH_ENTITY_TYPE_MON,
+ CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+ g_ceph_context->_conf->apply_changes(NULL);
+ g_conf = g_ceph_context->_conf;
+
+ if (vm.count("help")) {
+ std::cerr << desc << std::endl;
+ return 1;
+ }
+
+ int fd;
+ if (vm.count("out")) {
+ if ((fd = open(out_path.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666)) < 0) {
+ int _err = errno;
+ if (_err != EISDIR) {
+ std::cerr << "Couldn't open " << out_path << ": " << cpp_strerror(_err) << std::endl;
+ return 1;
+ }
+ }
+ } else {
+ fd = STDOUT_FILENO;
+ }
+
+ if (fd < 0 && cmd != "store-copy") {
+ std::cerr << "error: '" << out_path << "' is a directory!" << std::endl;
+ return 1;
+ }
+
+ MonitorDBStore st(store_path);
+ if (store_path.size()) {
+ stringstream ss;
+ int r = st.open(ss);
+ if (r < 0) {
+ std::cerr << ss.str() << std::endl;
+ goto done;
+ }
+ }
+ if (cmd == "dump-keys") {
+ KeyValueDB::WholeSpaceIterator iter = st.get_iterator();
+ while (iter->valid()) {
+ pair<string,string> key(iter->raw_key());
+ cout << key.first << " / " << key.second << std::endl;
+ iter->next();
+ }
+ } else if (cmd == "compact") {
+ st.compact();
+ } else if (cmd == "getmonmap") {
+ assert(fd >= 0);
+ if (!store_path.size()) {
+ std::cerr << "need mon store path" << std::endl;
+ std::cerr << desc << std::endl;
+ goto done;
+ }
+ version_t v;
+ if (version <= 0) {
+ v = st.get("monmap", "last_committed");
+ } else {
+ v = version;
+ }
+
+ bufferlist bl;
+ /// XXX: this is not ok, osdmap and full should be abstracted somewhere
+ int r = st.get("monmap", v, bl);
+ if (r < 0) {
+ std::cerr << "Error getting map: " << cpp_strerror(r) << std::endl;
+ goto done;
+ }
+ bl.write_fd(fd);
+ } else if (cmd == "getosdmap") {
+ if (!store_path.size()) {
+ std::cerr << "need mon store path" << std::endl;
+ std::cerr << desc << std::endl;
+ goto done;
+ }
+ version_t v;
+ if (version == -1) {
+ v = st.get("osdmap", "last_committed");
+ } else {
+ v = version;
+ }
+
+ bufferlist bl;
+ /// XXX: this is not ok, osdmap and full should be abstracted somewhere
+ int r = st.get("osdmap", st.combine_strings("full", v), bl);
+ if (r < 0) {
+ std::cerr << "Error getting map: " << cpp_strerror(r) << std::endl;
+ goto done;
+ }
+ bl.write_fd(fd);
+ } else if (cmd == "dump-paxos") {
+ for (version_t v = dstart; v <= dstop; ++v) {
+ bufferlist bl;
+ st.get("paxos", v, bl);
+ if (bl.length() == 0)
+ break;
+ cout << "\n--- " << v << " ---" << std::endl;
+ MonitorDBStore::Transaction tx;
+ Paxos::decode_append_transaction(tx, bl);
+ JSONFormatter f(true);
+ tx.dump(&f);
+ f.flush(cout);
+ }
+ } else if (cmd == "dump-trace") {
+ if (tfile.empty()) {
+ std::cerr << "Need trace_file" << std::endl;
+ std::cerr << desc << std::endl;
+ goto done;
+ }
+ TraceIter iter(tfile.c_str());
+ iter.init();
+ while (true) {
+ if (!iter.valid())
+ break;
+ if (iter.num() >= dstop) {
+ break;
+ }
+ if (iter.num() >= dstart) {
+ JSONFormatter f(true);
+ iter.cur().dump(&f, false);
+ f.flush(std::cout);
+ std::cout << std::endl;
+ }
+ iter.next();
+ }
+ std::cerr << "Read up to transaction " << iter.num() << std::endl;
+ } else if (cmd == "replay-trace") {
+ if (!store_path.size()) {
+ std::cerr << "need mon store path" << std::endl;
+ std::cerr << desc << std::endl;
+ goto done;
+ }
+ if (tfile.empty()) {
+ std::cerr << "Need trace_file" << std::endl;
+ std::cerr << desc << std::endl;
+ goto done;
+ }
+ unsigned num = 0;
+ for (unsigned i = 0; i < num_replays; ++i) {
+ TraceIter iter(tfile.c_str());
+ iter.init();
+ while (true) {
+ if (!iter.valid())
+ break;
+ std::cerr << "Replaying trans num " << num << std::endl;
+ st.apply_transaction(iter.cur());
+ iter.next();
+ ++num;
+ }
+ std::cerr << "Read up to transaction " << iter.num() << std::endl;
+ }
+ } else if (cmd == "random-gen") {
+ if (!store_path.size()) {
+ std::cerr << "need mon store path" << std::endl;
+ std::cerr << desc << std::endl;
+ goto done;
+ }
+ unsigned num = 0;
+ for (unsigned i = 0; i < ntrans; ++i) {
+ std::cerr << "Applying trans " << i << std::endl;
+ MonitorDBStore::Transaction t;
+ string prefix;
+ prefix.push_back((i%26)+'a');
+ for (unsigned j = 0; j < tsize; ++j) {
+ stringstream os;
+ os << num;
+ bufferlist bl;
+ for (unsigned k = 0; k < tvalsize; ++k) bl.append(rand());
+ t.put(prefix, os.str(), bl);
+ ++num;
+ }
+ t.compact_prefix(prefix);
+ st.apply_transaction(t);
+ }
+ } else if (cmd == "store-copy") {
+ if (!store_path.size()) {
+ std::cerr << "need mon store path to copy from" << std::endl;
+ std::cerr << desc << std::endl;
+ goto done;
+ }
+ if (!out_path.size()) {
+ std::cerr << "need mon store path to copy to (--out <mon_data_dir>)"
+ << std::endl;
+ std::cerr << desc << std::endl;
+ goto done;
+ }
+ if (fd > 0) {
+ std::cerr << "supplied out path '" << out_path << "' is not a directory"
+ << std::endl;
+ goto done;
+ }
+
+ MonitorDBStore out_store(out_path);
+ {
+ stringstream ss;
+ int r = out_store.create_and_open(ss);
+ if (r < 0) {
+ std::cerr << ss.str() << std::endl;
+ goto done;
+ }
+ }
+
+
+ KeyValueDB::WholeSpaceIterator it = st.get_iterator();
+ uint64_t total_keys = 0;
+ uint64_t total_size = 0;
+ uint64_t total_tx = 0;
+
+ do {
+ uint64_t num_keys = 0;
+
+ MonitorDBStore::Transaction tx;
+
+ while (it->valid() && num_keys < 128) {
+ pair<string,string> k = it->raw_key();
+ bufferlist v = it->value();
+ tx.put(k.first, k.second, v);
+
+ num_keys ++;
+ total_tx ++;
+ total_size += v.length();
+
+ it->next();
+ }
+
+ total_keys += num_keys;
+
+ if (!tx.empty())
+ out_store.apply_transaction(tx);
+
+ std::cout << "copied " << total_keys << " keys so far ("
+ << stringify(si_t(total_size)) << ")" << std::endl;
+
+ } while (it->valid());
+
+ std::cout << "summary: copied " << total_keys << " keys, using "
+ << total_tx << " transactions, totalling "
+ << stringify(si_t(total_size)) << std::endl;
+ std::cout << "from '" << store_path << "' to '" << out_path << "'"
+ << std::endl;
+ } else {
+ std::cerr << "Unrecognized command: " << cmd << std::endl;
+ goto done;
+ }
+
+ done:
+ if (vm.count("out") && fd > 0) {
+ ::close(fd);
+ }
+ return 0;
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+* Ceph - scalable distributed file system
+*
+* Copyright (C) 2012 Inktank, Inc.
+*
+* This is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License kkjversion 2.1, as published by the Free Software
+* Foundation. See file COPYING.
+*/
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/program_options/option.hpp>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <boost/program_options/cmdline.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <iostream>
+#include <set>
+#include <sstream>
+#include <stdlib.h>
+#include <fstream>
+#include <string>
+#include <sstream>
+#include <map>
+#include <set>
+#include <boost/scoped_ptr.hpp>
+
+#include "global/global_init.h"
+#include "os/LevelDBStore.h"
+#include "mon/MonitorDBStore.h"
+#include "os/DBObjectMap.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+int main(int argc, char **argv) {
+ po::options_description desc("Allowed options");
+ string store_path, cmd, out_path;
+ bool paranoid = false;
+ desc.add_options()
+ ("help", "produce help message")
+ ("omap-path", po::value<string>(&store_path),
+ "path to mon directory, mandatory (current/omap usually)")
+ ("paranoid", po::value<bool>(¶noid),
+ "use paranoid checking")
+ ("command", po::value<string>(&cmd),
+ "command")
+ ;
+ po::positional_options_description p;
+ p.add("command", 1);
+
+ po::variables_map vm;
+ po::parsed_options parsed =
+ po::command_line_parser(argc, argv).options(desc).positional(p).run();
+ po::store(
+ parsed,
+ vm);
+ try {
+ po::notify(vm);
+ } catch (...) {
+ cout << desc << std::endl;
+ return 1;
+ }
+
+ vector<const char *> ceph_options, def_args;
+ vector<string> ceph_option_strings = po::collect_unrecognized(
+ parsed.options, po::include_positional);
+ ceph_options.reserve(ceph_option_strings.size());
+ for (vector<string>::iterator i = ceph_option_strings.begin();
+ i != ceph_option_strings.end();
+ ++i) {
+ ceph_options.push_back(i->c_str());
+ }
+
+ global_init(
+ &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD,
+ CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+ g_ceph_context->_conf->apply_changes(NULL);
+ g_conf = g_ceph_context->_conf;
+
+ if (vm.count("help")) {
+ std::cerr << desc << std::endl;
+ return 1;
+ }
+
+ LevelDBStore* store(new LevelDBStore(g_ceph_context, store_path));
+ if (paranoid) {
+ std::cerr << "Enabling paranoid checks" << std::endl;
+ store->options.paranoid_checks = paranoid;
+ }
+ DBObjectMap omap(store);
+ stringstream out;
+ int r = store->open(out);
+ if (r < 0) {
+ std::cerr << "Store open got: " << cpp_strerror(r) << std::endl;
+ std::cerr << "Output: " << out.str() << std::endl;
+ goto done;
+ }
+ r = 0;
+
+
+ if (cmd == "dump-raw-keys") {
+ KeyValueDB::WholeSpaceIterator i = store->get_iterator();
+ for (i->seek_to_first(); i->valid(); i->next()) {
+ std::cout << i->raw_key() << std::endl;
+ }
+ } else if (cmd == "dump-raw-key-vals") {
+ KeyValueDB::WholeSpaceIterator i = store->get_iterator();
+ for (i->seek_to_first(); i->valid(); i->next()) {
+ std::cout << i->raw_key() << std::endl;
+ i->value().hexdump(std::cout);
+ }
+ } else if (cmd == "dump-objects") {
+ vector<ghobject_t> objects;
+ r = omap.list_objects(&objects);
+ if (r < 0) {
+ std::cerr << "list_objects got: " << cpp_strerror(r) << std::endl;
+ goto done;
+ }
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end();
+ ++i) {
+ std::cout << *i << std::endl;
+ }
+ r = 0;
+ } else if (cmd == "dump-objects-with-keys") {
+ vector<ghobject_t> objects;
+ r = omap.list_objects(&objects);
+ if (r < 0) {
+ std::cerr << "list_objects got: " << cpp_strerror(r) << std::endl;
+ goto done;
+ }
+ for (vector<ghobject_t>::iterator i = objects.begin();
+ i != objects.end();
+ ++i) {
+ std::cout << "Object: " << *i << std::endl;
+ ObjectMap::ObjectMapIterator j = omap.get_iterator(i->hobj);
+ for (j->seek_to_first(); j->valid(); j->next()) {
+ std::cout << j->key() << std::endl;
+ j->value().hexdump(std::cout);
+ }
+ }
+ } else if (cmd == "check") {
+ r = omap.check(std::cout);
+ if (!r) {
+ std::cerr << "check got: " << cpp_strerror(r) << std::endl;
+ goto done;
+ }
+ std::cout << "check succeeded" << std::endl;
+ } else {
+ std::cerr << "Did not recognize command " << cmd << std::endl;
+ goto done;
+ }
+
+ done:
+ return r;
+}
projects / ceph.git / commitdiff