tools: Fix ceph_filestore_dump to fully operate on EC pools

Add spg_t and ghobject_t handling

Fixes: #7698
Signed-off-by: David Zafman <david.zafman@inktank.com>

diff --git a/src/tools/ceph_filestore_dump.cc b/src/tools/ceph_filestore_dump.cc
index fbb68982ef841b7075b2040bc0f16e7ae6d2d0c5..87a8387390c54839724bb366ae891d8e8e4f432a 100644 (file)
--- a/src/tools/ceph_filestore_dump.cc
+++ b/src/tools/ceph_filestore_dump.cc
@@ -164,43 +164,64 @@ struct footer {
 };
 
 struct pg_begin {
-  pg_t pgid;
+  spg_t pgid;
   OSDSuperblock superblock;
 
-  pg_begin(pg_t pg, const OSDSuperblock& sb):
+  pg_begin(spg_t pg, const OSDSuperblock& sb):
     pgid(pg), superblock(sb) { }
   pg_begin() { }
 
   void encode(bufferlist& bl) const {
-    // New super_ver prevents decode from ver 1
-    ENCODE_START(2, 2, bl);
-    ::encode(pgid, bl);
+    // If superblock doesn't include CEPH_FS_FEATURE_INCOMPAT_SHARDS then
+    // shard will be NO_SHARD for a replicated pool.  This means
+    // that we allow the decode by struct_v 2.
+    ENCODE_START(3, 2, bl);
+    ::encode(pgid.pgid, bl);
     ::encode(superblock, bl);
+    ::encode(pgid.shard, bl);
     ENCODE_FINISH(bl);
   }
+  // NOTE: New super_ver prevents decode from ver 1
   void decode(bufferlist::iterator& bl) {
-    DECODE_START(2, bl);
-    ::decode(pgid, bl);
+    DECODE_START(3, bl);
+    ::decode(pgid.pgid, bl);
     if (struct_v > 1) {
       ::decode(superblock, bl);
     }
+    if (struct_v > 2) {
+      ::decode(pgid.shard, bl);
+    } else {
+      pgid.shard = ghobject_t::NO_SHARD;
+    }
     DECODE_FINISH(bl);
   }
 };
 
 struct object_begin {
-  hobject_t hoid;
-  object_begin(const hobject_t &hoid): hoid(hoid) { }
+  ghobject_t hoid;
+  object_begin(const ghobject_t &hoid): hoid(hoid) { }
   object_begin() { }
 
+  // If superblock doesn't include CEPH_FS_FEATURE_INCOMPAT_SHARDS then
+  // generation will be NO_GEN, shard_id will be NO_SHARD for a replicated
+  // pool.  This means we will allow the decode by struct_v 1.
   void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
-    ::encode(hoid, bl);
+    ENCODE_START(2, 1, bl);
+    ::encode(hoid.hobj, bl);
+    ::encode(hoid.generation, bl);
+    ::encode(hoid.shard_id, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::iterator& bl) {
-    DECODE_START(1, bl);
-    ::decode(hoid, bl);
+    DECODE_START(2, bl);
+    ::decode(hoid.hobj, bl);
+    if (struct_v > 1) {
+      ::decode(hoid.generation, bl);
+      ::decode(hoid.shard_id, bl);
+    } else {
+      hoid.generation = ghobject_t::NO_GEN;
+      hoid.shard_id = ghobject_t::NO_SHARD;
+    }
     DECODE_FINISH(bl);
   }
 };
@@ -354,7 +375,7 @@ static void invalid_path(string &path)
   exit(1);
 }
 
-int get_log(ObjectStore *fs, coll_t coll, pg_t pgid, const pg_info_t &info,
+int get_log(ObjectStore *fs, coll_t coll, spg_t pgid, const pg_info_t &info,
    PGLog::IndexedLog &log, pg_missing_t &missing)
 { 
   map<eversion_t, hobject_t> divergent_priors;
@@ -397,7 +418,6 @@ void remove_coll(ObjectStore *store, const coll_t &coll)
         i != objects.end();
         ++i, ++num) {
 
-      assert(i->generation == ghobject_t::NO_GEN);
       OSDriver::OSTransaction _t(driver.get_transaction(t));
       cout << "remove " << *i << std::endl;
       int r = mapper.remove_oid(i->hobj, &_t);
@@ -462,18 +482,17 @@ int finish_remove_pgs(ObjectStore *store, uint64_t *next_removal_seq)
   return 0;
 }
 
-int initiate_new_remove_pg(ObjectStore *store, pg_t r_pgid,
+int initiate_new_remove_pg(ObjectStore *store, spg_t r_pgid,
     uint64_t *next_removal_seq)
 {
   ObjectStore::Transaction *rmt = new ObjectStore::Transaction;
 
-  if (store->collection_exists(coll_t(spg_t(r_pgid, ghobject_t::no_shard())))) {
-      coll_t to_remove = coll_t::make_removal_coll((*next_removal_seq)++,
-        spg_t(r_pgid, ghobject_t::no_shard()));
-      cout << "collection rename " << coll_t(spg_t(r_pgid, ghobject_t::no_shard()))
+  if (store->collection_exists(coll_t(r_pgid))) {
+      coll_t to_remove = coll_t::make_removal_coll((*next_removal_seq)++, r_pgid);
+      cout << "collection rename " << coll_t(r_pgid)
           << " to " << to_remove
         << std::endl;
-      rmt->collection_rename(coll_t(spg_t(r_pgid, ghobject_t::no_shard())), to_remove);
+      rmt->collection_rename(coll_t(r_pgid), to_remove);
   } else {
     delete rmt;
     return ENOENT;
@@ -563,7 +582,7 @@ int write_pg(ObjectStore::Transaction &t, epoch_t epoch, pg_info_t &info,
   return 0;
 }
 
-int export_file(ObjectStore *store, coll_t cid, hobject_t &obj)
+int export_file(ObjectStore *store, coll_t cid, ghobject_t &obj)
 {
   struct stat st;
   mysize_t total;
@@ -667,8 +686,7 @@ int export_files(ObjectStore *store, coll_t coll)
     for (vector<ghobject_t>::iterator i = objects.begin();
         i != objects.end();
         ++i) {
-      assert(i->generation == ghobject_t::NO_GEN);
-      r = export_file(store, coll, i->hobj);
+      r = export_file(store, coll, *i);
       if (r < 0)
         return r;
     }
@@ -699,7 +717,7 @@ void write_super()
   superbl.write_fd(file_fd);
 }
 
-int do_export(ObjectStore *fs, coll_t coll, pg_t pgid, pg_info_t &info,
+int do_export(ObjectStore *fs, coll_t coll, spg_t pgid, pg_info_t &info,
     epoch_t map_epoch, __u8 struct_ver, const OSDSuperblock& superblock)
 {
   PGLog::IndexedLog log;
@@ -783,7 +801,7 @@ int read_section(int fd, sectiontype_t *type, bufferlist *bl)
   return 0;
 }
 
-int get_data(ObjectStore *store, coll_t coll, hobject_t hoid,
+int get_data(ObjectStore *store, coll_t coll, ghobject_t hoid,
     ObjectStore::Transaction *t, bufferlist &bl)
 {
   bufferlist::iterator ebliter = bl.begin();
@@ -796,7 +814,7 @@ int get_data(ObjectStore *store, coll_t coll, hobject_t hoid,
   return 0;
 }
 
-int get_attrs(ObjectStore *store, coll_t coll, hobject_t hoid,
+int get_attrs(ObjectStore *store, coll_t coll, ghobject_t hoid,
     ObjectStore::Transaction *t, bufferlist &bl,
     OSDriver &driver, SnapMapper &snap_mapper)
 {
@@ -808,7 +826,7 @@ int get_attrs(ObjectStore *store, coll_t coll, hobject_t hoid,
     cout << "\tattrs: len " << as.data.size() << std::endl;
   t->setattrs(coll, hoid, as.data);
 
-  if (hoid.snap < CEPH_MAXSNAP) {
+  if (hoid.hobj.snap < CEPH_MAXSNAP && hoid.generation == ghobject_t::NO_GEN) {
     map<string,bufferptr>::iterator mi = as.data.find(OI_ATTR);
     if (mi != as.data.end()) {
       bufferlist attr_bl;
@@ -820,14 +838,14 @@ int get_attrs(ObjectStore *store, coll_t coll, hobject_t hoid,
   
       OSDriver::OSTransaction _t(driver.get_transaction(t));
       set<snapid_t> oi_snaps(oi.snaps.begin(), oi.snaps.end());
-      snap_mapper.add_oid(hoid, oi_snaps, &_t);
+      snap_mapper.add_oid(hoid.hobj, oi_snaps, &_t);
     }
   }
 
   return 0;
 }
 
-int get_omap_hdr(ObjectStore *store, coll_t coll, hobject_t hoid,
+int get_omap_hdr(ObjectStore *store, coll_t coll, ghobject_t hoid,
     ObjectStore::Transaction *t, bufferlist &bl)
 {
   bufferlist::iterator ebliter = bl.begin();
@@ -841,7 +859,7 @@ int get_omap_hdr(ObjectStore *store, coll_t coll, hobject_t hoid,
   return 0;
 }
 
-int get_omap(ObjectStore *store, coll_t coll, hobject_t hoid,
+int get_omap(ObjectStore *store, coll_t coll, ghobject_t hoid,
     ObjectStore::Transaction *t, bufferlist &bl)
 {
   bufferlist::iterator ebliter = bl.begin();
@@ -986,7 +1004,7 @@ int do_import(ObjectStore *store, OSDSuperblock& sb)
   bufferlist::iterator ebliter = ebl.begin();
   pg_begin pgb;
   pgb.decode(ebliter);
-  pg_t pgid = pgb.pgid;
+  spg_t pgid = pgb.pgid;
 
   if (debug) {
     cout << "Exported features: " << pgb.superblock.compat_features << std::endl;
@@ -997,11 +1015,11 @@ int do_import(ObjectStore *store, OSDSuperblock& sb)
     return 1;
   }
 
-  log_oid = OSD::make_pg_log_oid(spg_t(pgid, ghobject_t::no_shard()));
-  biginfo_oid = OSD::make_pg_biginfo_oid(spg_t(pgid, ghobject_t::no_shard()));
+  log_oid = OSD::make_pg_log_oid(pgid);
+  biginfo_oid = OSD::make_pg_biginfo_oid(pgid);
 
   //Check for PG already present.
-  coll_t coll(spg_t(pgid, ghobject_t::no_shard()));
+  coll_t coll(pgid);
   if (store->collection_exists(coll)) {
     cout << "pgid " << pgid << " already exists" << std::endl;
     return 1;
@@ -1009,8 +1027,7 @@ int do_import(ObjectStore *store, OSDSuperblock& sb)
 
   //Switch to collection which will be removed automatically if
   //this program is interupted.
-  coll_t rmcoll = coll_t::make_removal_coll(
-    next_removal_seq, spg_t(pgid, ghobject_t::no_shard()));
+  coll_t rmcoll = coll_t::make_removal_coll(next_removal_seq, pgid);
   ObjectStore::Transaction *t = new ObjectStore::Transaction;
   t->create_collection(rmcoll);
   store->apply_transaction(*t);
@@ -1206,7 +1223,7 @@ int main(int argc, char **argv)
     invalid_path(fspath);
   }
 
-  pg_t pgid;
+  spg_t pgid;
   if (pgidstr.length() && !pgid.parse(pgidstr.c_str())) {
     cout << "Invalid pgid '" << pgidstr << "' specified" << std::endl;
     exit(1);
@@ -1298,8 +1315,8 @@ int main(int argc, char **argv)
     goto out;
   }
 
-  log_oid = OSD::make_pg_log_oid(spg_t(pgid, ghobject_t::no_shard()));
-  biginfo_oid = OSD::make_pg_biginfo_oid(spg_t(pgid, ghobject_t::no_shard()));
+  log_oid = OSD::make_pg_log_oid(pgid);
+  biginfo_oid = OSD::make_pg_biginfo_oid(pgid);
 
   if (type == "remove") {
     uint64_t next_removal_seq = 0;     //My local seq
@@ -1329,7 +1346,7 @@ int main(int argc, char **argv)
       continue;
     }
 
-    if (tmppgid.pgid != pgid) {
+    if (tmppgid != pgid) {
       continue;
     }
     if (snap != CEPH_NOSNAP && debug) {
@@ -1352,10 +1369,9 @@ int main(int argc, char **argv)
     if (debug)
       cerr << "map_epoch " << map_epoch << std::endl;
 
-    pg_info_t info(spg_t(pgid, ghobject_t::no_shard()));
+    pg_info_t info(pgid);
     map<epoch_t,pg_interval_t> past_intervals;
-    hobject_t biginfo_oid = OSD::make_pg_biginfo_oid(
-      spg_t(pgid, ghobject_t::no_shard()));
+    hobject_t biginfo_oid = OSD::make_pg_biginfo_oid(pgid);
     interval_set<snapid_t> snap_collections;
   
     __u8 struct_ver;