pid = os.getpid()
TESTDIR = "/tmp/test.{pid}".format(pid=pid)
DATADIR = "/tmp/data.{pid}".format(pid=pid)
- CFSD_PREFIX = "./ceph_filestore_dump --filestore-path dev/{osd} --journal-path dev/{osd}.journal "
+ CFSD_PREFIX = "./ceph_objectstore_tool --data-path dev/{osd} --journal-path dev/{osd}.journal "
DATALINECOUNT = 10000
PROFNAME = "testecprofile"
print "Test invalid parameters"
# On export can't use stdout to a terminal
- cmd = (CFSD_PREFIX + "--type export --pgid {pg}").format(osd=ONEOSD, pg=ONEPG)
+ cmd = (CFSD_PREFIX + "--op export --pgid {pg}").format(osd=ONEOSD, pg=ONEPG)
ERRORS += test_failure(cmd, "stdout is a tty and no --file option specified")
OTHERFILE = "/tmp/foo.{pid}".format(pid=pid)
foofd.close()
# On import can't specify a PG
- cmd = (CFSD_PREFIX + "--type import --pgid {pg} --file {FOO}").format(osd=ONEOSD, pg=ONEPG, FOO=OTHERFILE)
+ cmd = (CFSD_PREFIX + "--op import --pgid {pg} --file {FOO}").format(osd=ONEOSD, pg=ONEPG, FOO=OTHERFILE)
ERRORS += test_failure(cmd, "--pgid option invalid with import")
os.unlink(OTHERFILE)
- cmd = (CFSD_PREFIX + "--type import --file {FOO}").format(osd=ONEOSD, FOO=OTHERFILE)
+ cmd = (CFSD_PREFIX + "--op import --file {FOO}").format(osd=ONEOSD, FOO=OTHERFILE)
ERRORS += test_failure(cmd, "open: No such file or directory")
# On import can't use stdin from a terminal
- cmd = (CFSD_PREFIX + "--type import --pgid {pg}").format(osd=ONEOSD, pg=ONEPG)
+ cmd = (CFSD_PREFIX + "--op import --pgid {pg}").format(osd=ONEOSD, pg=ONEPG)
ERRORS += test_failure(cmd, "stdin is a tty and no --file option specified")
- # Test --type list and generate json for all objects
- print "Test --type list by generating json for all objects"
+ # Specify a bad --type
+ cmd = (CFSD_PREFIX + "--type foobar --op list --pgid {pg}").format(osd=ONEOSD, pg=ONEPG)
+ ERRORS += test_failure(cmd, "Must provide --type (filestore, memstore, keyvaluestore-dev)")
+
+ # Don't specify a data-path
+ cmd = "./ceph_objectstore_tool --journal-path dev/{osd}.journal --type memstore --op list --pgid {pg}".format(osd=ONEOSD, pg=ONEPG)
+ ERRORS += test_failure(cmd, "Must provide --data-path")
+
+ # Don't specify a journal-path for filestore
+ cmd = "./ceph_objectstore_tool --type filestore --data-path dev/{osd} --op list --pgid {pg}".format(osd=ONEOSD, pg=ONEPG)
+ ERRORS += test_failure(cmd, "Must provide --journal-path")
+
+ # Test --op list and generate json for all objects
+ print "Test --op list by generating json for all objects"
TMPFILE = r"/tmp/tmp.{pid}".format(pid=pid)
ALLPGS = OBJREPPGS + OBJECPGS
for pg in ALLPGS:
OSDS = get_osds(pg, OSDDIR)
for osd in OSDS:
- cmd = (CFSD_PREFIX + "--type list --pgid {pg}").format(osd=osd, pg=pg)
+ cmd = (CFSD_PREFIX + "--op list --pgid {pg}").format(osd=osd, pg=pg)
tmpfd = open(TMPFILE, "a")
logging.debug(cmd)
ret = call(cmd, shell=True, stdout=tmpfd)
if ret != 0:
- logging.error("Bad exit status {ret} from --type list request".format(ret=ret))
+ logging.error("Bad exit status {ret} from --op list request".format(ret=ret))
ERRORS += 1
tmpfd.close()
print "Test pg info"
for pg in ALLREPPGS + ALLECPGS:
for osd in get_osds(pg, OSDDIR):
- cmd = (CFSD_PREFIX + "--type info --pgid {pg} | grep '\"pgid\": \"{pg}\"'").format(osd=osd, pg=pg)
+ cmd = (CFSD_PREFIX + "--op info --pgid {pg} | grep '\"pgid\": \"{pg}\"'").format(osd=osd, pg=pg)
logging.debug(cmd)
ret = call(cmd, shell=True, stdout=nullfd)
if ret != 0:
for pg in ALLREPPGS + ALLECPGS:
for osd in get_osds(pg, OSDDIR):
tmpfd = open(TMPFILE, "w")
- cmd = (CFSD_PREFIX + "--type log --pgid {pg}").format(osd=osd, pg=pg)
+ cmd = (CFSD_PREFIX + "--op log --pgid {pg}").format(osd=osd, pg=pg)
logging.debug(cmd)
ret = call(cmd, shell=True, stdout=tmpfd)
if ret != 0:
for osd in get_osds(pg, OSDDIR):
mydir = os.path.join(TESTDIR, osd)
fname = os.path.join(mydir, pg)
- cmd = (CFSD_PREFIX + "--type export --pgid {pg} --file {file}").format(osd=osd, pg=pg, file=fname)
+ cmd = (CFSD_PREFIX + "--op export --pgid {pg} --file {file}").format(osd=osd, pg=pg, file=fname)
logging.debug(cmd)
ret = call(cmd, shell=True, stdout=nullfd, stderr=nullfd)
if ret != 0:
RM_ERRORS = 0
for pg in ALLREPPGS + ALLECPGS:
for osd in get_osds(pg, OSDDIR):
- cmd = (CFSD_PREFIX + "--type remove --pgid {pg}").format(pg=pg, osd=osd)
+ cmd = (CFSD_PREFIX + "--op remove --pgid {pg}").format(pg=pg, osd=osd)
logging.debug(cmd)
ret = call(cmd, shell=True, stdout=nullfd)
if ret != 0:
dir = os.path.join(TESTDIR, osd)
for pg in [f for f in os.listdir(dir) if os.path.isfile(os.path.join(dir, f))]:
file = os.path.join(dir, pg)
- cmd = (CFSD_PREFIX + "--type import --file {file}").format(osd=osd, file=file)
+ cmd = (CFSD_PREFIX + "--op import --file {file}").format(osd=osd, file=file)
logging.debug(cmd)
ret = call(cmd, shell=True, stdout=nullfd)
if ret != 0:
return 0;
}
-static void invalid_path(string &path)
+static void invalid_filestore_path(string &path)
{
- cerr << "Invalid path to osd store specified: " << path << "\n";
+ cerr << "Invalid filestore path specified: " << path << "\n";
exit(1);
}
cerr << std::endl;
cerr << "Positional syntax:" << std::endl;
cerr << std::endl;
- cerr << "(requires --filestore-path, --journal-path and --pgid to be specified)" << std::endl;
+ cerr << "(requires --data, --journal (for filestore type) and --pgid to be specified)" << std::endl;
cerr << "(optional [file] argument will read stdin or write stdout if not specified or if '-' specified)" << std::endl;
- cerr << "ceph-filestore-dump ... <object> (get|set)-bytes [file]" << std::endl;
- cerr << "ceph-filestore-dump ... <object> (set-(attr|omap) <key> [file]" << std::endl;
- cerr << "ceph-filestore-dump ... <object> (set-omaphdr) [file]" << std::endl;
- cerr << "ceph-filestore-dump ... <object> (get|rm)-(attr|omap) <key>" << std::endl;
- cerr << "ceph-filestore-dump ... <object> (get-omaphdr)" << std::endl;
- cerr << "ceph-filestore-dump ... <object> list-attrs" << std::endl;
- cerr << "ceph-filestore-dump ... <object> list-omap" << std::endl;
- cerr << "ceph-filestore-dump ... <object> remove" << std::endl;
+ cerr << "ceph_objectstore_tool ... <object> (get|set)-bytes [file]" << std::endl;
+ cerr << "ceph_objectstore_tool ... <object> (set-(attr|omap) <key> [file]" << std::endl;
+ cerr << "ceph_objectstore_tool ... <object> (set-omaphdr) [file]" << std::endl;
+ cerr << "ceph_objectstore_tool ... <object> (get|rm)-(attr|omap) <key>" << std::endl;
+ cerr << "ceph_objectstore_tool ... <object> (get-omaphdr)" << std::endl;
+ cerr << "ceph_objectstore_tool ... <object> list-attrs" << std::endl;
+ cerr << "ceph_objectstore_tool ... <object> list-omap" << std::endl;
+ cerr << "ceph_objectstore_tool ... <object> remove" << std::endl;
cerr << std::endl;
exit(1);
}
int main(int argc, char **argv)
{
- string fspath, jpath, pgidstr, type, file, object, objcmd, arg1, arg2;
+ string dpath, jpath, pgidstr, op, file, object, objcmd, arg1, arg2, type;
ghobject_t ghobj;
po::options_description desc("Allowed options");
desc.add_options()
("help", "produce help message")
- ("filestore-path", po::value<string>(&fspath),
- "path to filestore directory, mandatory")
+ ("type", po::value<string>(&type),
+ "Arg is one of [filestore (default), memstore, keyvaluestore-dev]")
+ ("data-path", po::value<string>(&dpath),
+ "path to object store, mandatory")
("journal-path", po::value<string>(&jpath),
- "path to journal, mandatory")
+ "path to journal, mandatory for filestore type")
("pgid", po::value<string>(&pgidstr),
- "PG id, mandatory except for import")
- ("type", po::value<string>(&type),
- "Arg is one of [info, log, remove, export, import, list]")
+ "PG id, mandatory except for import, list-lost, fix-lost")
+ ("op", po::value<string>(&op),
+ "Arg is one of [info, log, remove, export, import, list, list-lost, fix-lost]")
("file", po::value<string>(&file),
"path of file to export or import")
("debug", "Enable diagnostic output to stderr")
usage(desc);
}
- if (!vm.count("filestore-path")) {
- cerr << "Must provide --filestore-path" << std::endl;
+ if (!vm.count("data-path")) {
+ cerr << "Must provide --data-path" << std::endl;
usage(desc);
}
- if (!vm.count("journal-path")) {
+ if (!vm.count("type")) {
+ type = "filestore";
+ }
+ if (type == "filestore" && !vm.count("journal-path")) {
cerr << "Must provide --journal-path" << std::endl;
usage(desc);
}
cerr << "Invalid syntax, missing command" << std::endl;
usage(desc);
}
- if (!vm.count("type") && !(vm.count("object") && vm.count("objcmd"))) {
- cerr << "Must provide --type or object command..."
- << std::endl;
+ if (!vm.count("op") && !(vm.count("object") && vm.count("objcmd"))) {
+ cerr << "Must provide --op or object command..." << std::endl;
usage(desc);
}
- if (vm.count("type") && vm.count("object")) {
- cerr << "Can't specify both --type and object command syntax" << std::endl;
+ if (vm.count("op") && vm.count("object")) {
+ cerr << "Can't specify both --op and object command syntax" << std::endl;
usage(desc);
}
- if (type != "import" && !vm.count("pgid")) {
+ if (op != "import" && op != "list-lost" && op != "fix-lost"
+ && !vm.count("pgid")) {
cerr << "Must provide pgid" << std::endl;
usage(desc);
}
outistty = isatty(STDOUT_FILENO);
file_fd = fd_none;
- if (type == "export") {
+ if (op == "export") {
if (!vm.count("file")) {
if (outistty) {
cerr << "stdout is a tty and no --file option specified" << std::endl;
} else {
file_fd = open(file.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666);
}
- } else if (type == "import") {
+ } else if (op == "import") {
if (!vm.count("file")) {
if (isatty(STDIN_FILENO)) {
cerr << "stdin is a tty and no --file option specified" << std::endl;
return 1;
}
- if ((fspath.length() == 0 || jpath.length() == 0) ||
- (type != "import" && pgidstr.length() == 0)) {
+ if (dpath.length() == 0) {
cerr << "Invalid params" << std::endl;
return 1;
}
- if (type == "import" && pgidstr.length()) {
+ if (op == "import" && pgidstr.length()) {
cerr << "--pgid option invalid with import" << std::endl;
return 1;
}
}
g_conf->apply_changes(NULL);
- //Verify that fspath really is an osd store
+ //Verify that data-path really exists
struct stat st;
- if (::stat(fspath.c_str(), &st) == -1) {
- perror("fspath");
- invalid_path(fspath);
- }
- if (!S_ISDIR(st.st_mode)) {
- invalid_path(fspath);
- }
- string check = fspath + "/whoami";
- if (::stat(check.c_str(), &st) == -1) {
- perror("whoami");
- invalid_path(fspath);
- }
- if (!S_ISREG(st.st_mode)) {
- invalid_path(fspath);
- }
- check = fspath + "/current";
- if (::stat(check.c_str(), &st) == -1) {
- perror("current");
- invalid_path(fspath);
- }
- if (!S_ISDIR(st.st_mode)) {
- invalid_path(fspath);
+ if (::stat(dpath.c_str(), &st) == -1) {
+ perror("data-path");
+ exit(1);
+ }
+ //Verify data data-path really is a filestore
+ if (type == "filestore") {
+ if (!S_ISDIR(st.st_mode)) {
+ invalid_filestore_path(dpath);
+ }
+ string check = dpath + "/whoami";
+ if (::stat(check.c_str(), &st) == -1) {
+ perror("whoami");
+ invalid_filestore_path(dpath);
+ }
+ if (!S_ISREG(st.st_mode)) {
+ invalid_filestore_path(dpath);
+ }
+ check = dpath + "/current";
+ if (::stat(check.c_str(), &st) == -1) {
+ perror("current");
+ invalid_filestore_path(dpath);
+ }
+ if (!S_ISDIR(st.st_mode)) {
+ invalid_filestore_path(dpath);
+ }
}
spg_t pgid;
return 1;
}
- ObjectStore *fs = ObjectStore::create(NULL, "filestore", fspath, jpath, flags);
+ ObjectStore *fs = ObjectStore::create(NULL, type, dpath, jpath, flags);
+ if (fs == NULL) {
+ cerr << "Must provide --type (filestore, memstore, keyvaluestore-dev)" << std::endl;
+ exit(1);
+ }
int r = fs->mount();
if (r < 0) {
goto out;
}
- if (type == "import") {
+ if (op == "import") {
try {
ret = do_import(fs, superblock);
log_oid = OSD::make_pg_log_oid(pgid);
biginfo_oid = OSD::make_pg_biginfo_oid(pgid);
- if (type == "remove") {
+ if (op == "remove") {
uint64_t next_removal_seq = 0; //My local seq
finish_remove_pgs(fs, &next_removal_seq);
int r = initiate_new_remove_pg(fs, pgid, &next_removal_seq);
goto out;
}
+ if (op == "list-lost" || op == "fix-lost") {
+ unsigned LIST_AT_A_TIME = 100;
+ unsigned scanned = 0;
+ int r;
+ vector<coll_t> colls_to_check;
+ if (pgidstr.length()) {
+ colls_to_check.push_back(coll_t(pgid));
+ } else {
+ vector<coll_t> candidates;
+ r = fs->list_collections(candidates);
+ if (r < 0) {
+ cerr << "Error listing collections: " << cpp_strerror(r) << std::endl;
+ goto UMOUNT;
+ }
+ for (vector<coll_t>::iterator i = candidates.begin();
+ i != candidates.end();
+ ++i) {
+ spg_t pgid;
+ snapid_t snap;
+ if (i->is_pg(pgid, snap)) {
+ colls_to_check.push_back(*i);
+ }
+ }
+ }
+
+ cerr << colls_to_check.size() << " pgs to scan" << std::endl;
+ for (vector<coll_t>::iterator i = colls_to_check.begin();
+ i != colls_to_check.end();
+ ++i, ++scanned) {
+ cerr << "Scanning " << *i << ", " << scanned << "/"
+ << colls_to_check.size() << " completed" << std::endl;
+ ghobject_t next;
+ while (!next.is_max()) {
+ vector<ghobject_t> list;
+ r = fs->collection_list_partial(
+ *i,
+ next,
+ LIST_AT_A_TIME,
+ LIST_AT_A_TIME,
+ CEPH_NOSNAP,
+ &list,
+ &next);
+ if (r < 0) {
+ cerr << "Error listing collection: " << *i << ", "
+ << cpp_strerror(r) << std::endl;
+ goto UMOUNT;
+ }
+ for (vector<ghobject_t>::iterator obj = list.begin();
+ obj != list.end();
+ ++obj) {
+ bufferlist attr;
+ r = fs->getattr(*i, *obj, OI_ATTR, attr);
+ if (r < 0) {
+ cerr << "Error getting attr on : " << make_pair(*i, *obj) << ", "
+ << cpp_strerror(r) << std::endl;
+ goto UMOUNT;
+ }
+ object_info_t oi;
+ bufferlist::iterator bp = attr.begin();
+ try {
+ ::decode(oi, bp);
+ } catch (...) {
+ r = -EINVAL;
+ cerr << "Error getting attr on : " << make_pair(*i, *obj) << ", "
+ << cpp_strerror(r) << std::endl;
+ goto UMOUNT;
+ }
+ if (oi.is_lost()) {
+ if (op == "list-lost") {
+ cout << *i << "/" << *obj << " is lost" << std::endl;
+ }
+ if (op == "fix-lost") {
+ cerr << *i << "/" << *obj << " is lost, fixing" << std::endl;
+ oi.clear_flag(object_info_t::FLAG_LOST);
+ bufferlist bl2;
+ ::encode(oi, bl2);
+ ObjectStore::Transaction t;
+ t.setattr(*i, *obj, OI_ATTR, bl2);
+ r = fs->apply_transaction(t);
+ if (r < 0) {
+ cerr << "Error getting fixing attr on : " << make_pair(*i, *obj)
+ << ", "
+ << cpp_strerror(r) << std::endl;
+ goto UMOUNT;
+ }
+ }
+ }
+ }
+ }
+ }
+ cerr << "Completed" << std::endl;
+
+ UMOUNT:
+ fs->sync_and_flush();
+ ret = r;
+ goto out;
+ }
+
r = fs->list_collections(ls);
if (r < 0) {
cerr << "failed to list pgs: " << cpp_strerror(-r) << std::endl;
usage(desc);
}
- if (type == "list") {
+ if (op == "list") {
Formatter *formatter = new JSONFormatter(false);
r = do_list(fs, coll, formatter);
if (r) {
if (debug)
cerr << "struct_v " << (int)struct_ver << std::endl;
- if (type == "export") {
+ if (op == "export") {
ret = do_export(fs, coll, pgid, info, map_epoch, struct_ver, superblock);
if (ret == 0 && file_fd != STDOUT_FILENO)
cout << "Export successful" << std::endl;
- } else if (type == "info") {
+ } else if (op == "info") {
formatter->open_object_section("info");
info.dump(formatter);
formatter->close_section();
formatter->flush(cout);
cout << std::endl;
- } else if (type == "log") {
+ } else if (op == "log") {
PGLog::IndexedLog log;
pg_missing_t missing;
ret = get_log(fs, coll, pgid, info, log, missing);
formatter->flush(cout);
cout << std::endl;
} else {
- cerr << "Must provide --type (info, log, remove, export, import, list)"
+ cerr << "Must provide --op (info, log, remove, export, import, list, list-lost, fix-lost)"
<< std::endl;
usage(desc);
}
projects / ceph.git / commitdiff