From 647723ecd283273657bdb8b35dbb904e55c37add Mon Sep 17 00:00:00 2001 From: David Zafman Date: Thu, 4 Jun 2015 10:37:42 -0700 Subject: [PATCH] tools, test: Add ceph-objectstore-tool to operate on the meta collection Signed-off-by: David Zafman (cherry picked from commit 31fa89f24997c3e9b4f8795ea28938ab65493d10) Conflicts: src/tools/ceph_objectstore_tool.cc is_pg() still needs snap argument --- src/test/ceph_objectstore_tool.py | 54 +++++++++++++-- src/tools/ceph_objectstore_tool.cc | 108 +++++++++++++++++++---------- 2 files changed, 119 insertions(+), 43 deletions(-) diff --git a/src/test/ceph_objectstore_tool.py b/src/test/ceph_objectstore_tool.py index 61b0f57b79dcf..93a44388a6582 100755 --- a/src/test/ceph_objectstore_tool.py +++ b/src/test/ceph_objectstore_tool.py @@ -907,7 +907,7 @@ def main(argv): # Specify a bad --op command cmd = (CFSD_PREFIX + "--op oops").format(osd=ONEOSD) - ERRORS += test_failure(cmd, "Must provide --op (info, log, remove, export, import, list, fix-lost, list-pgs, rm-past-intervals, set-allow-sharded-objects, dump-journal, dump-super, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete)") + ERRORS += test_failure(cmd, "Must provide --op (info, log, remove, export, import, list, fix-lost, list-pgs, rm-past-intervals, set-allow-sharded-objects, dump-journal, dump-super, meta-list, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete)") # Provide just the object param not a command cmd = (CFSD_PREFIX + "object").format(osd=ONEOSD) @@ -965,7 +965,7 @@ def main(argv): tmpfd.close() lines = get_lines(TMPFILE) JSONOBJ = sorted(set(lines)) - (pgid, jsondict) = json.loads(JSONOBJ[0])[0] + (pgid, coll, jsondict) = json.loads(JSONOBJ[0])[0] # retrieve all objects in a given PG tmpfd = open(OTHERFILE, "a") @@ -978,9 +978,9 @@ def main(argv): tmpfd.close() lines = get_lines(OTHERFILE) JSONOBJ = sorted(set(lines)) - (other_pgid, other_jsondict) = json.loads(JSONOBJ[0])[0] + (other_pgid, other_coll, other_jsondict) = json.loads(JSONOBJ[0])[0] - if pgid != other_pgid or jsondict != other_jsondict: + if pgid != other_pgid or jsondict != other_jsondict or coll != other_coll: logging.error("the first line of --op list is different " "from the first line of --op list --pgid {pg}".format(pg=pgid)) ERRORS += 1 @@ -996,9 +996,9 @@ def main(argv): tmpfd.close() lines = get_lines(OTHERFILE) JSONOBJ = sorted(set(lines)) - (other_pgid, other_jsondict) in json.loads(JSONOBJ[0])[0] + (other_pgid, other_coll, other_jsondict) in json.loads(JSONOBJ[0])[0] - if pgid != other_pgid or jsondict != other_jsondict: + if pgid != other_pgid or jsondict != other_jsondict or coll != other_coll: logging.error("the first line of --op list is different " "from the first line of --op list --pgid {pg} {object}".format(pg=pgid, object=jsondict['oid'])) ERRORS += 1 @@ -1415,6 +1415,48 @@ def main(argv): logging.error("Not all keys found, remaining keys:") print values + print "Test --op meta-list" + tmpfd = open(TMPFILE, "w") + cmd = (CFSD_PREFIX + "--op meta-list").format(osd=ONEOSD) + logging.debug(cmd) + ret = call(cmd, shell=True, stdout=tmpfd) + if ret != 0: + logging.error("Bad exit status {ret} from --op meta-list request".format(ret=ret)) + ERRORS += 1 + + print "Test get-bytes on meta" + tmpfd.close() + lines = get_lines(TMPFILE) + JSONOBJ = sorted(set(lines)) + for JSON in JSONOBJ: + (pgid, jsondict) = json.loads(JSON) + if pgid != "meta": + logging.error("pgid incorrect for --op meta-list {pgid}".format(pgid=pgid)) + ERRORS += 1 + if jsondict['namespace'] != "": + logging.error("namespace non null --op meta-list {ns}".format(ns=jsondict['namespace'])) + ERRORS += 1 + logging.info(JSON) + try: + os.unlink(GETNAME) + except: + pass + cmd = (CFSD_PREFIX + "'{json}' get-bytes {fname}").format(osd=ONEOSD, json=JSON, fname=GETNAME) + logging.debug(cmd) + ret = call(cmd, shell=True) + if ret != 0: + logging.error("Bad exit status {ret}".format(ret=ret)) + ERRORS += 1 + + try: + os.unlink(GETNAME) + except: + pass + try: + os.unlink(TESTNAME) + except: + pass + print "Test pg info" for pg in ALLREPPGS + ALLECPGS: for osd in get_osds(pg, OSDDIR): diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc index a6d885a57062a..6553a0f490663 100644 --- a/src/tools/ceph_objectstore_tool.cc +++ b/src/tools/ceph_objectstore_tool.cc @@ -409,22 +409,24 @@ int _action_on_all_objects_in_pg(ObjectStore *store, coll_t coll, action_on_obje ++obj) { if (obj->is_pgmeta()) continue; - bufferlist attr; - r = store->getattr(coll, *obj, OI_ATTR, attr); - if (r < 0) { - cerr << "Error getting attr on : " << make_pair(coll, *obj) << ", " - << cpp_strerror(r) << std::endl; - continue; - } object_info_t oi; - bufferlist::iterator bp = attr.begin(); - try { - ::decode(oi, bp); - } catch (...) { - r = -EINVAL; - cerr << "Error getting attr on : " << make_pair(coll, *obj) << ", " - << cpp_strerror(r) << std::endl; - continue; + if (coll != META_COLL) { + bufferlist attr; + r = store->getattr(coll, *obj, OI_ATTR, attr); + if (r < 0) { + cerr << "Error getting attr on : " << make_pair(coll, *obj) << ", " + << cpp_strerror(r) << std::endl; + continue; + } + bufferlist::iterator bp = attr.begin(); + try { + ::decode(oi, bp); + } catch (...) { + r = -EINVAL; + cerr << "Error getting attr on : " << make_pair(coll, *obj) << ", " + << cpp_strerror(r) << std::endl; + continue; + } } r = action.call(store, coll, *obj, oi); if (r < 0) @@ -545,12 +547,13 @@ struct pgid_object_list { i != _objects.end(); ++i) { f->open_array_section("pgid_object"); - string pgid = i->first.c_str(); - std::size_t pos = pgid.find("_"); - if (pos == string::npos) - f->dump_string("pgid", pgid); - else - f->dump_string("pgid", pgid.substr(0, pos)); + snapid_t snap; + spg_t pgid; + bool is_pg = i->first.is_pg(pgid, snap); + if (is_pg) + f->dump_string("pgid", stringify(pgid)); + if (!is_pg || !human_readable) + f->dump_string("coll", i->first.to_str()); f->open_object_section("ghobject"); i->second.dump(f); f->close_section(); @@ -2298,6 +2301,18 @@ int do_list(ObjectStore *store, string pgidstr, string object, return 0; } +int do_meta(ObjectStore *store, string object, Formatter *formatter, bool debug, bool human_readable) +{ + int r; + lookup_ghobject lookup(object); + r = action_on_all_objects_in_exact_pg(store, META_COLL, lookup, debug); + if (r) + return r; + lookup.dump(formatter, human_readable); + formatter->flush(cout); + return 0; +} + int do_remove_object(ObjectStore *store, coll_t coll, ghobject_t &ghobj) { spg_t pg; @@ -3069,7 +3084,7 @@ int main(int argc, char **argv) ("pgid", po::value(&pgidstr), "PG id, mandatory for info, log, remove, export, rm-past-intervals, mark-complete") ("op", po::value(&op), - "Arg is one of [info, log, remove, export, import, list, fix-lost, list-pgs, rm-past-intervals, set-allow-sharded-objects, dump-journal, dump-super, " + "Arg is one of [info, log, remove, export, import, list, fix-lost, list-pgs, rm-past-intervals, set-allow-sharded-objects, dump-journal, dump-super, meta-list, " "get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete]") ("epoch", po::value(&epoch), "epoch# for get-osdmap and get-inc-osdmap, the current epoch in use if not specified") @@ -3274,7 +3289,7 @@ int main(int argc, char **argv) // Special list handling. Treating pretty_format as human readable, // with one object per line and not an enclosing array. human_readable = ends_with(format, "-pretty"); - if (op == "list" && human_readable) { + if ((op == "list" || op == "meta-list") && human_readable) { // Remove -pretty from end of format which we know is there format = format.substr(0, format.size() - strlen("-pretty")); } @@ -3452,19 +3467,23 @@ int main(int argc, char **argv) throw std::runtime_error(ss.str()); } string object_pgidstr = i->get_str(); - spg_t object_pgid; - object_pgid.parse(object_pgidstr.c_str()); - if (pgidstr.length() > 0) { - if (object_pgid != pgid) { - ss << "object '" << object - << "' has a pgid different from the --pgid=" - << pgidstr << " option"; - throw std::runtime_error(ss.str()); + if (object_pgidstr != "meta") { + spg_t object_pgid; + object_pgid.parse(object_pgidstr.c_str()); + if (pgidstr.length() > 0) { + if (object_pgid != pgid) { + ss << "object '" << object + << "' has a pgid different from the --pgid=" + << pgidstr << " option"; + throw std::runtime_error(ss.str()); + } + } else { + pgidstr = object_pgidstr; + pgid = object_pgid; } - } else { - pgidstr = object_pgidstr; - pgid = object_pgid; - } + } else { + pgidstr = object_pgidstr; + } ++i; v = *i; } @@ -3474,7 +3493,7 @@ int main(int argc, char **argv) ss << "Decode object JSON error: " << e.what(); throw std::runtime_error(ss.str()); } - if ((uint64_t)pgid.pgid.m_pool != (uint64_t)ghobj.hobj.pool) { + if (pgidstr != "meta" && (uint64_t)pgid.pgid.m_pool != (uint64_t)ghobj.hobj.pool) { cerr << "Object pool and pgid pool don't match" << std::endl; ret = 1; goto out; @@ -3715,6 +3734,14 @@ int main(int argc, char **argv) goto out; } + if (op == "meta-list") { + ret = do_meta(fs, object, formatter, debug, human_readable); + if (ret < 0) { + cerr << "do_meta failed: " << cpp_strerror(ret) << std::endl; + } + goto out; + } + ret = fs->list_collections(ls); if (ret < 0) { cerr << "failed to list pgs: " << cpp_strerror(ret) << std::endl; @@ -3729,6 +3756,13 @@ int main(int argc, char **argv) snapid_t snap; spg_t tmppgid; + if (pgidstr == "meta") { + if (it->to_str() == "meta") + break; + else + continue; + } + if (!it->is_pg(tmppgid, snap)) { continue; } @@ -3763,7 +3797,7 @@ int main(int argc, char **argv) // If not an object command nor any of the ops handled below, then output this usage // before complaining about a bad pgid if (!vm.count("objcmd") && op != "export" && op != "info" && op != "log" && op != "rm-past-intervals" && op != "mark-complete") { - cerr << "Must provide --op (info, log, remove, export, import, list, fix-lost, list-pgs, rm-past-intervals, set-allow-sharded-objects, dump-journal, dump-super, " + cerr << "Must provide --op (info, log, remove, export, import, list, fix-lost, list-pgs, rm-past-intervals, set-allow-sharded-objects, dump-journal, dump-super, meta-list, " "get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete)" << std::endl; usage(desc); -- 2.39.5