tools, test: Add ceph-objectstore-tool to operate on the meta collection

Signed-off-by: David Zafman <dzafman@redhat.com>
(cherry picked from commit 31fa89f24997c3e9b4f8795ea28938ab65493d10)

Conflicts:
	src/tools/ceph_objectstore_tool.cc
is_pg() still needs snap argument

diff --git a/src/test/ceph_objectstore_tool.py b/src/test/ceph_objectstore_tool.py
index 61b0f57b79dcf5683355d2edd6925499e09aec79..93a44388a6582c22d06316b0fc98735104f49dc8 100755 (executable)
--- a/src/test/ceph_objectstore_tool.py
+++ b/src/test/ceph_objectstore_tool.py
@@ -907,7 +907,7 @@ def main(argv):
 
     # Specify a bad --op command
     cmd = (CFSD_PREFIX + "--op oops").format(osd=ONEOSD)
-    ERRORS += test_failure(cmd, "Must provide --op (info, log, remove, export, import, list, fix-lost, list-pgs, rm-past-intervals, set-allow-sharded-objects, dump-journal, dump-super, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete)")
+    ERRORS += test_failure(cmd, "Must provide --op (info, log, remove, export, import, list, fix-lost, list-pgs, rm-past-intervals, set-allow-sharded-objects, dump-journal, dump-super, meta-list, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete)")
 
     # Provide just the object param not a command
     cmd = (CFSD_PREFIX + "object").format(osd=ONEOSD)
@@ -965,7 +965,7 @@ def main(argv):
     tmpfd.close()
     lines = get_lines(TMPFILE)
     JSONOBJ = sorted(set(lines))
-    (pgid, jsondict) = json.loads(JSONOBJ[0])[0]
+    (pgid, coll, jsondict) = json.loads(JSONOBJ[0])[0]
 
     # retrieve all objects in a given PG
     tmpfd = open(OTHERFILE, "a")
@@ -978,9 +978,9 @@ def main(argv):
     tmpfd.close()
     lines = get_lines(OTHERFILE)
     JSONOBJ = sorted(set(lines))
-    (other_pgid, other_jsondict) = json.loads(JSONOBJ[0])[0]
+    (other_pgid, other_coll, other_jsondict) = json.loads(JSONOBJ[0])[0]
 
-    if pgid != other_pgid or jsondict != other_jsondict:
+    if pgid != other_pgid or jsondict != other_jsondict or coll != other_coll:
         logging.error("the first line of --op list is different "
                       "from the first line of --op list --pgid {pg}".format(pg=pgid))
         ERRORS += 1
@@ -996,9 +996,9 @@ def main(argv):
     tmpfd.close()
     lines = get_lines(OTHERFILE)
     JSONOBJ = sorted(set(lines))
-    (other_pgid, other_jsondict) in json.loads(JSONOBJ[0])[0]
+    (other_pgid, other_coll, other_jsondict) in json.loads(JSONOBJ[0])[0]
 
-    if pgid != other_pgid or jsondict != other_jsondict:
+    if pgid != other_pgid or jsondict != other_jsondict or coll != other_coll:
         logging.error("the first line of --op list is different "
                       "from the first line of --op list --pgid {pg} {object}".format(pg=pgid, object=jsondict['oid']))
         ERRORS += 1
@@ -1415,6 +1415,48 @@ def main(argv):
                         logging.error("Not all keys found, remaining keys:")
                         print values
 
+    print "Test --op meta-list"
+    tmpfd = open(TMPFILE, "w")
+    cmd = (CFSD_PREFIX + "--op meta-list").format(osd=ONEOSD)
+    logging.debug(cmd)
+    ret = call(cmd, shell=True, stdout=tmpfd)
+    if ret != 0:
+        logging.error("Bad exit status {ret} from --op meta-list request".format(ret=ret))
+        ERRORS += 1
+
+    print "Test get-bytes on meta"
+    tmpfd.close()
+    lines = get_lines(TMPFILE)
+    JSONOBJ = sorted(set(lines))
+    for JSON in JSONOBJ:
+        (pgid, jsondict) = json.loads(JSON)
+        if pgid != "meta":
+            logging.error("pgid incorrect for --op meta-list {pgid}".format(pgid=pgid))
+            ERRORS += 1
+        if jsondict['namespace'] != "":
+            logging.error("namespace non null --op meta-list {ns}".format(ns=jsondict['namespace']))
+            ERRORS += 1
+        logging.info(JSON)
+        try:
+            os.unlink(GETNAME)
+        except:
+            pass
+        cmd = (CFSD_PREFIX + "'{json}' get-bytes {fname}").format(osd=ONEOSD, json=JSON, fname=GETNAME)
+        logging.debug(cmd)
+        ret = call(cmd, shell=True)
+        if ret != 0:
+            logging.error("Bad exit status {ret}".format(ret=ret))
+            ERRORS += 1
+
+    try:
+        os.unlink(GETNAME)
+    except:
+        pass
+    try:
+        os.unlink(TESTNAME)
+    except:
+        pass
+
     print "Test pg info"
     for pg in ALLREPPGS + ALLECPGS:
         for osd in get_osds(pg, OSDDIR):

diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc
index a6d885a57062a794ecb3b5887f68b6e91691e3e0..6553a0f4906636fe5e797a621cae95f23e5743c0 100644 (file)
--- a/src/tools/ceph_objectstore_tool.cc
+++ b/src/tools/ceph_objectstore_tool.cc
@@ -409,22 +409,24 @@ int _action_on_all_objects_in_pg(ObjectStore *store, coll_t coll, action_on_obje
         ++obj) {
       if (obj->is_pgmeta())
        continue;
-      bufferlist attr;
-      r = store->getattr(coll, *obj, OI_ATTR, attr);
-      if (r < 0) {
-       cerr << "Error getting attr on : " << make_pair(coll, *obj) << ", "
-            << cpp_strerror(r) << std::endl;
-       continue;
-      }
       object_info_t oi;
-      bufferlist::iterator bp = attr.begin();
-      try {
-       ::decode(oi, bp);
-      } catch (...) {
-       r = -EINVAL;
-       cerr << "Error getting attr on : " << make_pair(coll, *obj) << ", "
-            << cpp_strerror(r) << std::endl;
-       continue;
+      if (coll != META_COLL) {
+        bufferlist attr;
+        r = store->getattr(coll, *obj, OI_ATTR, attr);
+        if (r < 0) {
+         cerr << "Error getting attr on : " << make_pair(coll, *obj) << ", "
+              << cpp_strerror(r) << std::endl;
+         continue;
+        }
+        bufferlist::iterator bp = attr.begin();
+        try {
+         ::decode(oi, bp);
+        } catch (...) {
+         r = -EINVAL;
+         cerr << "Error getting attr on : " << make_pair(coll, *obj) << ", "
+              << cpp_strerror(r) << std::endl;
+         continue;
+        }
       }
       r = action.call(store, coll, *obj, oi);
       if (r < 0)
@@ -545,12 +547,13 @@ struct pgid_object_list {
         i != _objects.end();
         ++i) {
       f->open_array_section("pgid_object");
-      string pgid = i->first.c_str();
-      std::size_t pos = pgid.find("_");
-      if (pos == string::npos)
-        f->dump_string("pgid", pgid);
-      else
-        f->dump_string("pgid", pgid.substr(0, pos));
+      snapid_t snap;
+      spg_t pgid;
+      bool is_pg = i->first.is_pg(pgid, snap);
+      if (is_pg)
+        f->dump_string("pgid", stringify(pgid));
+      if (!is_pg || !human_readable)
+        f->dump_string("coll", i->first.to_str());
       f->open_object_section("ghobject");
       i->second.dump(f);
       f->close_section();
@@ -2298,6 +2301,18 @@ int do_list(ObjectStore *store, string pgidstr, string object,
   return 0;
 }
 
+int do_meta(ObjectStore *store, string object, Formatter *formatter, bool debug, bool human_readable)
+{
+  int r;
+  lookup_ghobject lookup(object);
+  r = action_on_all_objects_in_exact_pg(store, META_COLL, lookup, debug);
+  if (r)
+    return r;
+  lookup.dump(formatter, human_readable);
+  formatter->flush(cout);
+  return 0;
+}
+
 int do_remove_object(ObjectStore *store, coll_t coll, ghobject_t &ghobj)
 {
   spg_t pg;
@@ -3069,7 +3084,7 @@ int main(int argc, char **argv)
     ("pgid", po::value<string>(&pgidstr),
      "PG id, mandatory for info, log, remove, export, rm-past-intervals, mark-complete")
     ("op", po::value<string>(&op),
-     "Arg is one of [info, log, remove, export, import, list, fix-lost, list-pgs, rm-past-intervals, set-allow-sharded-objects, dump-journal, dump-super, "
+     "Arg is one of [info, log, remove, export, import, list, fix-lost, list-pgs, rm-past-intervals, set-allow-sharded-objects, dump-journal, dump-super, meta-list, "
         "get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete]")
     ("epoch", po::value<unsigned>(&epoch),
      "epoch# for get-osdmap and get-inc-osdmap, the current epoch in use if not specified")
@@ -3274,7 +3289,7 @@ int main(int argc, char **argv)
   // Special list handling.  Treating pretty_format as human readable,
   // with one object per line and not an enclosing array.
   human_readable = ends_with(format, "-pretty");
-  if (op == "list" && human_readable) {
+  if ((op == "list" || op == "meta-list") && human_readable) {
     // Remove -pretty from end of format which we know is there
     format = format.substr(0, format.size() - strlen("-pretty"));
   }
@@ -3452,19 +3467,23 @@ int main(int argc, char **argv)
            throw std::runtime_error(ss.str());
          }
          string object_pgidstr = i->get_str();
-         spg_t object_pgid;
-         object_pgid.parse(object_pgidstr.c_str());
-         if (pgidstr.length() > 0) {
-           if (object_pgid != pgid) {
-             ss << "object '" << object
-                << "' has a pgid different from the --pgid="
-                << pgidstr << " option";
-             throw std::runtime_error(ss.str());
+          if (object_pgidstr != "meta") {
+           spg_t object_pgid;
+           object_pgid.parse(object_pgidstr.c_str());
+           if (pgidstr.length() > 0) {
+             if (object_pgid != pgid) {
+               ss << "object '" << object
+                  << "' has a pgid different from the --pgid="
+                  << pgidstr << " option";
+               throw std::runtime_error(ss.str());
+             }
+           } else {
+             pgidstr = object_pgidstr;
+             pgid = object_pgid;
            }
-         } else {
-           pgidstr = object_pgidstr;
-           pgid = object_pgid;
-         }
+          } else {
+            pgidstr = object_pgidstr;
+          }
          ++i;
          v = *i;
        }
@@ -3474,7 +3493,7 @@ int main(int argc, char **argv)
          ss << "Decode object JSON error: " << e.what();
          throw std::runtime_error(ss.str());
        }
-        if ((uint64_t)pgid.pgid.m_pool != (uint64_t)ghobj.hobj.pool) {
+        if (pgidstr != "meta" && (uint64_t)pgid.pgid.m_pool != (uint64_t)ghobj.hobj.pool) {
           cerr << "Object pool and pgid pool don't match" << std::endl;
           ret = 1;
           goto out;
@@ -3715,6 +3734,14 @@ int main(int argc, char **argv)
     goto out;
   }
 
+  if (op == "meta-list") {
+    ret = do_meta(fs, object, formatter, debug, human_readable);
+    if (ret < 0) {
+      cerr << "do_meta failed: " << cpp_strerror(ret) << std::endl;
+    }
+    goto out;
+  }
+
   ret = fs->list_collections(ls);
   if (ret < 0) {
     cerr << "failed to list pgs: " << cpp_strerror(ret) << std::endl;
@@ -3729,6 +3756,13 @@ int main(int argc, char **argv)
     snapid_t snap;
     spg_t tmppgid;
 
+    if (pgidstr == "meta") {
+      if (it->to_str() == "meta")
+        break;
+      else
+        continue;
+    }
+
     if (!it->is_pg(tmppgid, snap)) {
       continue;
     }
@@ -3763,7 +3797,7 @@ int main(int argc, char **argv)
   // If not an object command nor any of the ops handled below, then output this usage
   // before complaining about a bad pgid
   if (!vm.count("objcmd") && op != "export" && op != "info" && op != "log" && op != "rm-past-intervals" && op != "mark-complete") {
-    cerr << "Must provide --op (info, log, remove, export, import, list, fix-lost, list-pgs, rm-past-intervals, set-allow-sharded-objects, dump-journal, dump-super, "
+    cerr << "Must provide --op (info, log, remove, export, import, list, fix-lost, list-pgs, rm-past-intervals, set-allow-sharded-objects, dump-journal, dump-super, meta-list, "
       "get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete)"
         << std::endl;
     usage(desc);