mds: Backtrace for create,rename,mkdir,setlayout

Design info:
http://www.spinics.net/lists/ceph-devel/msg11872.html

Adds a backtrace to the data pool for supporting lookup-by-ino,
storing the backtrace on the first object in the data pool
or the metadata pool for a directory, as the 'parent' xattr
on the object (named by inode) in that pool.  For create, rename,
mkdir, and setlayout operations, the backtrace is
queued (with the current log segment) after the journal is
committed and the safe reply is returned to the client, but the
the backtrace operation itself isn't started until the log segment is
expired.

For journal replay, we queue the backtrace so that it gets
written out on journal expire.  Inodes get added to the EMetaBlob
in the fullbits list, so we queue backtraces while iterating through
the fullbits during replay.

Using setlayout or setxattr('ceph.file.layout.pool'),
the data pool for a file can be changed after it is created
but before anything is written to the file.  A forwarding backtrace
is written to the old pool on a setlayout, to ensure we can always find
the latest backtrace.  We store a list of old pools with the backtrace
for cleaning up all forwarding pointers of an inode.

Signed-off-by: Sam Lang <sam.lang@inktank.com>
Reviewed-by: Greg Farnum <greg@inktank.com>

diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index ae75840978ffe027c8cd4dd38ebe7ab8d2460164..21f5e166ac032dd932e571c8c7e1d4d784b7c5af 100644 (file)
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -1015,12 +1015,12 @@ void CInode::build_backtrace(int64_t location, inode_backtrace_t* bt)
     in = diri;
     pdn = in->get_parent_dn();
   }
-  vector<ceph_file_layout>::iterator i = inode.old_layouts.begin();
-  while(i != inode.old_layouts.end()) {
+  vector<int64_t>::iterator i = inode.old_pools.begin();
+  while(i != inode.old_pools.end()) {
     // don't add our own pool id to old_pools to avoid looping (e.g. setlayout 0, 1, 0)
-    if (i->fl_pg_pool == location)
+    if (*i == location)
       continue;
-    bt->old_pools.insert(i->fl_pg_pool);
+    bt->old_pools.insert(*i);
     i++;
   }
 }

diff --git a/src/mds/LogSegment.h b/src/mds/LogSegment.h
index bcdd54b44321ca16f7b8553b96e264d6fff0abc1..8cf58a183062abf33a1be3a7b1e01ebc492ea135 100644 (file)
--- a/src/mds/LogSegment.h
+++ b/src/mds/LogSegment.h
@@ -104,6 +104,6 @@ class LogSegment {
   void store_backtrace_update(MDS *mds, BacktraceInfo *info, Context *fin);
   void _stored_backtrace(BacktraceInfo *info, Context *fin);
   unsigned encode_parent_mutation(ObjectOperation& m, BacktraceInfo *info);
-    };
+};
 
 #endif

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index fddcfc68f8eca9b7f46ef8aafab12ddfdea8fffc..de256975651d7c51893bb12341406f7b8ac5ff18 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -8419,6 +8419,91 @@ public:
   }
 };
 
+class C_MDC_PurgeForwardingPointers : public Context {
+  MDCache *cache;
+  CDentry *dn;
+  Context *fin;
+public:
+  inode_backtrace_t backtrace;
+  C_MDC_PurgeForwardingPointers(MDCache *c, CDentry *d, Context *f) :
+    cache(c), dn(d), fin(f) {}
+  void finish(int r) {
+    cache->_purge_forwarding_pointers(&backtrace, dn, r, fin);
+  }
+};
+
+class C_MDC_PurgeStray : public Context {
+  MDCache *cache;
+  CDentry *dn;
+public:
+  C_MDC_PurgeStray(MDCache *c, CDentry *d) :
+    cache(c), dn(d) {}
+  void finish(int r) {
+    cache->_purge_stray(dn, r);
+  }
+};
+
+void MDCache::_purge_forwarding_pointers(inode_backtrace_t *backtrace, CDentry *d, int r, Context *fin)
+{
+  assert(r == 0 || r == -ENOENT);
+  // setup gathering context
+  C_GatherBuilder gather_bld(g_ceph_context);
+
+  // remove all the objects with forwarding pointer backtraces (aka sentinels)
+  for (set<int64_t>::const_iterator i = backtrace->old_pools.begin();
+       i != backtrace->old_pools.end();
+       i++) {
+    SnapContext snapc;
+    object_t oid = CInode::get_object_name(backtrace->ino, frag_t(), "");
+    object_locator_t oloc(*i);
+
+    mds->objecter->remove(oid, oloc, snapc, ceph_clock_now(g_ceph_context), 0,
+                         NULL, gather_bld.new_sub());
+  }
+
+  if (gather_bld.has_subs()) {
+    gather_bld.set_finisher(fin);
+    gather_bld.activate();
+  } else {
+    fin->finish(r);
+  }
+}
+
+void MDCache::_purge_stray(CDentry *dn, int r)
+{
+  // purge the strays
+  CDentry::linkage_t *dnl = dn->get_projected_linkage();
+  CInode *in = dnl->get_inode();
+  dout(10) << "_purge_stray " << *dn << " " << *in << dendl;
+
+  SnapRealm *realm = in->find_snaprealm();
+  SnapContext nullsnap;
+  const SnapContext *snapc;
+  if (realm) {
+    dout(10) << " realm " << *realm << dendl;
+    snapc = &realm->get_snap_context();
+  } else {
+    dout(10) << " NO realm, using null context" << dendl;
+    snapc = &nullsnap;
+    assert(in->last == CEPH_NOSNAP);
+  }
+
+  uint64_t period = (uint64_t)in->inode.layout.fl_object_size * (uint64_t)in->inode.layout.fl_stripe_count;
+  uint64_t cur_max_size = in->inode.get_max_size();
+  uint64_t to = MAX(in->inode.size, cur_max_size);
+  if (to && period) {
+    uint64_t num = (to + period - 1) / period;
+    dout(10) << "purge_stray 0~" << to << " objects 0~" << num << " snapc " << snapc << " on " << *in << dendl;
+    mds->filer->purge_range(in->inode.ino, &in->inode.layout, *snapc,
+                           0, num, ceph_clock_now(g_ceph_context), 0,
+                          new C_MDC_PurgeStrayPurged(this, dn));
+
+  } else {
+    dout(10) << "purge_stray 0 objects snapc " << snapc << " on " << *in << dendl;
+    _purge_stray_purged(dn);
+  }
+}
+
 void MDCache::purge_stray(CDentry *dn)
 {
   CDentry::linkage_t *dnl = dn->get_projected_linkage();
@@ -8441,35 +8526,20 @@ void MDCache::purge_stray(CDentry *dn)
   // that is implicit in the dentry's presence and non-use in the stray
   // dir.  on recovery, we'll need to re-eval all strays anyway.
   
-  SnapRealm *realm = in->find_snaprealm();
-  SnapContext nullsnap;
-  const SnapContext *snapc;
-  if (realm) {
-    dout(10) << " realm " << *realm << dendl;
-    snapc = &realm->get_snap_context();
-  } else {
-    dout(10) << " NO realm, using null context" << dendl;
-    snapc = &nullsnap;
-    assert(in->last == CEPH_NOSNAP);
-  }
-
   if (in->is_dir()) {
     dout(10) << "purge_stray dir ... implement me!" << dendl;  // FIXME XXX
-    _purge_stray_purged(dn);
+    // remove the backtrace
+    SnapContext snapc;
+    object_t oid = CInode::get_object_name(in->ino(), frag_t(), "");
+    object_locator_t oloc(mds->mdsmap->get_metadata_pool());
+
+    mds->objecter->removexattr(oid, oloc, "parent", snapc, ceph_clock_now(g_ceph_context), 0,
+                               NULL, new C_MDC_PurgeStrayPurged(this, dn));
   } else if (in->is_file()) {
-    uint64_t period = (uint64_t)in->inode.layout.fl_object_size * (uint64_t)in->inode.layout.fl_stripe_count;
-    uint64_t cur_max_size = in->inode.get_max_size();
-    uint64_t to = MAX(in->inode.size, cur_max_size);
-    if (to && period) {
-      uint64_t num = (to + period - 1) / period;
-      dout(10) << "purge_stray 0~" << to << " objects 0~" << num << " snapc " << snapc << " on " << *in << dendl;
-      mds->filer->purge_range(in->inode.ino, &in->inode.layout, *snapc,
-                             0, num, ceph_clock_now(g_ceph_context), 0,
-                             new C_MDC_PurgeStrayPurged(this, dn));
-    } else {
-      dout(10) << "purge_stray 0 objects snapc " << snapc << " on " << *in << dendl;
-      _purge_stray_purged(dn);
-    }
+    // get the backtrace before blowing away the object
+    C_MDC_PurgeStray *strayfin = new C_MDC_PurgeStray(this, dn);
+    C_MDC_PurgeForwardingPointers *fpfin = new C_MDC_PurgeForwardingPointers(this, dn, strayfin);
+    in->fetch_backtrace(&fpfin->backtrace, fpfin);
   } else {
     // not a dir or file; purged!
     _purge_stray_purged(dn);

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index f07ea74be67ac9d75347c9964f9e921daa6502b6..7ac129a225f700eb0f9e742384db1fe114819127 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -785,10 +785,14 @@ public:
       eval_stray(dn);
   }
 protected:
+  void _purge_forwarding_pointers(inode_backtrace_t *backtrace, CDentry *dn, int r, Context *fin);
+  void _purge_stray(CDentry *dn, int r);
   void purge_stray(CDentry *dn);
   void _purge_stray_purged(CDentry *dn, int r=0);
   void _purge_stray_logged(CDentry *dn, version_t pdv, LogSegment *ls);
   void _purge_stray_logged_truncate(CDentry *dn, LogSegment *ls);
+  friend class C_MDC_PurgeForwardingPointers;
+  friend class C_MDC_PurgeStray;
   friend class C_MDC_PurgeStrayLogged;
   friend class C_MDC_PurgeStrayLoggedTruncate;
   friend class C_MDC_PurgeStrayPurged;

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 888ea12344f02605010f753bfe97efda50f5fe57..4880f9cae3050bb7820bdcee4161e4955408711b 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -2619,8 +2619,6 @@ void Server::handle_client_open(MDRequest *mdr)
   reply_request(mdr, 0, cur, dn);
 }
 
-
-
 class C_MDS_openc_finish : public Context {
   MDS *mds;
   MDRequest *mdr;
@@ -2650,6 +2648,9 @@ public:
     MClientReply *reply = new MClientReply(mdr->client_request, 0);
     reply->set_extra_bl(mdr->reply_extra_bl);
     mds->server->reply_request(mdr, reply);
+
+    mdr->ls->queue_backtrace_update(newi, newi->inode.layout.fl_pg_pool);
+    assert(g_conf->mds_kill_openc_at != 1);
   }
 };
 
@@ -2762,7 +2763,7 @@ void Server::handle_client_openc(MDRequest *mdr)
   CInode *in = prepare_new_inode(mdr, dn->get_dir(), inodeno_t(req->head.ino),
                                 req->head.args.open.mode | S_IFREG, &layout);
   assert(in);
-  
+
   // it's a file.
   dn->push_projected_linkage(in);
 
@@ -3050,6 +3051,8 @@ public:
   void finish(int r) {
     assert(r == 0);
 
+    int64_t old_pool = in->inode.layout.fl_pg_pool;
+
     // apply
     in->pop_and_dirty_projected_inode(mdr->ls);
     mdr->apply();
@@ -3066,6 +3069,16 @@ public:
 
     if (changed_ranges)
       mds->locker->share_inode_max_size(in);
+
+    // if pool changed, queue a new backtrace and set forward pointer on old
+    if (old_pool != in->inode.layout.fl_pg_pool) {
+      mdr->ls->remove_pending_backtraces(in->ino(), in->inode.layout.fl_pg_pool);
+      mdr->ls->queue_backtrace_update(in, in->inode.layout.fl_pg_pool);
+
+      // set forwarding pointer on old backtrace
+      mdr->ls->remove_pending_backtraces(in->ino(), old_pool);
+      mdr->ls->queue_backtrace_update(in, old_pool, in->inode.layout.fl_pg_pool);
+    }
   }
 };
 
@@ -3396,6 +3409,8 @@ void Server::handle_client_setlayout(MDRequest *mdr)
 
   // validate layout
   ceph_file_layout layout = cur->get_projected_inode()->layout;
+  // save existing layout for later
+  int64_t old_pool = layout.fl_pg_pool;
 
   if (req->head.args.setlayout.layout.fl_object_size > 0)
     layout.fl_object_size = req->head.args.setlayout.layout.fl_object_size;
@@ -3434,6 +3449,8 @@ void Server::handle_client_setlayout(MDRequest *mdr)
   // project update
   inode_t *pi = cur->project_inode();
   pi->layout = layout;
+  // add the old pool to the inode
+  pi->add_old_pool(old_pool);
   pi->version = cur->pre_dirty();
   pi->ctime = ceph_clock_now(g_ceph_context);
   
@@ -3442,6 +3459,8 @@ void Server::handle_client_setlayout(MDRequest *mdr)
   EUpdate *le = new EUpdate(mdlog, "setlayout");
   mdlog->start_entry(le);
   le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+  // add the old pool to the metablob to indicate the pool changed with this event
+  le->metablob.add_old_pool(old_pool);
   mdcache->predirty_journal_parents(mdr, &le->metablob, cur, 0, PREDIRTY_PRIMARY, false);
   mdcache->journal_dirty_inode(mdr, &le->metablob, cur);
   
@@ -3624,6 +3643,7 @@ void Server::handle_set_vxattr(MDRequest *mdr, CInode *cur,
       name.find("ceph.dir.layout") == 0) {
     inode_t *pi;
     string rest;
+    int64_t old_pool = -1;
     if (name.find("ceph.dir.layout") == 0) {
       if (!cur->is_dir()) {
        reply_request(mdr, -EINVAL);
@@ -3691,6 +3711,8 @@ void Server::handle_set_vxattr(MDRequest *mdr, CInode *cur,
        return;
 
       pi = cur->project_inode();
+      old_pool = pi->layout.fl_pg_pool;
+      pi->add_old_pool(old_pool);
       pi->layout = layout;
       pi->ctime = ceph_clock_now(g_ceph_context);
     }
@@ -3702,6 +3724,10 @@ void Server::handle_set_vxattr(MDRequest *mdr, CInode *cur,
     EUpdate *le = new EUpdate(mdlog, "set vxattr layout");
     mdlog->start_entry(le);
     le->metablob.add_client_req(req->get_reqid(), req->get_oldest_client_tid());
+    if (cur->is_file()) {
+      assert(old_pool != -1);
+      le->metablob.add_old_pool(old_pool);
+    }
     mdcache->predirty_journal_parents(mdr, &le->metablob, cur, 0, PREDIRTY_PRIMARY, false);
     mdcache->journal_dirty_inode(mdr, &le->metablob, cur);
 
@@ -3955,6 +3981,15 @@ public:
     // hit pop
     mds->balancer->hit_inode(mdr->now, newi, META_POP_IWR);
 
+    // store the backtrace on the 'parent' xattr
+    if (newi->inode.is_dir()) {
+      // if its a dir, put it in the metadata pool
+      mdr->ls->queue_backtrace_update(newi, mds->mdsmap->get_metadata_pool());
+    } else {
+      // if its a file, put it in the data pool for that file
+      mdr->ls->queue_backtrace_update(newi, newi->inode.layout.fl_pg_pool);
+    }
+
     // reply
     MClientReply *reply = new MClientReply(mdr->client_request, 0);
     reply->set_result(0);
@@ -5869,7 +5904,20 @@ void Server::_rename_finish(MDRequest *mdr, CDentry *srcdn, CDentry *destdn, CDe
     mds->balancer->hit_inode(mdr->now, destdnl->get_inode(), META_POP_IWR);
 
   // did we import srci?  if so, explicitly ack that import that, before we unlock and reply.
-  
+
+  // backtrace
+  if (destdnl->inode->is_dir()) {
+    // replace previous backtrace on this inode with myself
+    mdr->ls->remove_pending_backtraces(destdnl->inode->ino(), mds->mdsmap->get_metadata_pool());
+    // queue an updated backtrace
+    mdr->ls->queue_backtrace_update(destdnl->inode, mds->mdsmap->get_metadata_pool());
+
+  } else {
+    // remove all pending backtraces going to the same pool
+    mdr->ls->remove_pending_backtraces(destdnl->inode->ino(), destdnl->inode->inode.layout.fl_pg_pool);
+    // queue an updated backtrace
+    mdr->ls->queue_backtrace_update(destdnl->inode, destdnl->inode->inode.layout.fl_pg_pool);
+  }
 
   // reply
   MClientReply *reply = new MClientReply(mdr->client_request, 0);
@@ -6242,6 +6290,9 @@ void Server::_rename_prepare(MDRequest *mdr,
     mdcache->project_subtree_rename(oldin, destdn->get_dir(), straydn->get_dir());
   if (srci->is_dir())
     mdcache->project_subtree_rename(srci, srcdn->get_dir(), destdn->get_dir());
+
+  // always update the backtrace
+  metablob->update_backtrace();
 }
 
 

diff --git a/src/mds/events/EMetaBlob.h b/src/mds/events/EMetaBlob.h
index b098f84a08e5dfdd2d946fd6c57151c86da09d19..4ff320edd2c330445390559b09de3a07f0b28d8e 100644 (file)
--- a/src/mds/events/EMetaBlob.h
+++ b/src/mds/events/EMetaBlob.h
@@ -318,6 +318,9 @@ private:
   // idempotent op(s)
   list<pair<metareqid_t,uint64_t> > client_reqs;
 
+  int64_t old_pool;
+  bool update_bt;
+
  public:
   void encode(bufferlist& bl) const;
   void decode(bufferlist::iterator& bl);
@@ -520,6 +523,13 @@ private:
   
   void add_dir_context(CDir *dir, int mode = TO_AUTH_SUBTREE_ROOT);
  
+  void add_old_pool(int64_t pool) {
+    old_pool = pool;
+  }
+  void update_backtrace() {
+    update_bt = true;
+  }
+
   void print(ostream& out) const {
     out << "[metablob";
     if (!lump_order.empty()) 

diff --git a/src/mds/journal.cc b/src/mds/journal.cc
index 8ba70a23a98b8d5de7d976a677c32bd3b40c2c81..cdf11a609352422102213e92d7617c2935b6d755 100644 (file)
--- a/src/mds/journal.cc
+++ b/src/mds/journal.cc
@@ -276,7 +276,7 @@ BacktraceInfo::BacktraceInfo(
   if (pool == -1) pool = location;
 
   bt.pool = pool;
-  i->build_backtrace(l, bt);
+  i->build_backtrace(l, &bt);
   ls->update_backtraces.push_back(&item_logseg);
 }
 
@@ -363,6 +363,8 @@ void LogSegment::_stored_backtrace(BacktraceInfo *info, Context *fin)
 EMetaBlob::EMetaBlob(MDLog *mdlog) : opened_ino(0), renamed_dirino(0),
                                     inotablev(0), sessionmapv(0),
                                     allocated_ino(0),
+                                    old_pool(-1),
+                                    update_bt(false),
                                     last_subtree_map(mdlog ? mdlog->get_last_segment_offset() : 0),
                                     my_offset(mdlog ? mdlog->get_write_pos() : 0) //, _segment(0)
 { }
@@ -813,7 +815,7 @@ void EMetaBlob::dirlump::generate_test_instances(list<dirlump*>& ls)
  */
 void EMetaBlob::encode(bufferlist& bl) const
 {
-  ENCODE_START(5, 5, bl);
+  ENCODE_START(6, 5, bl);
   ::encode(lump_order, bl);
   ::encode(lump_map, bl);
   ::encode(roots, bl);
@@ -831,11 +833,13 @@ void EMetaBlob::encode(bufferlist& bl) const
   ::encode(client_reqs, bl);
   ::encode(renamed_dirino, bl);
   ::encode(renamed_dir_frags, bl);
+  ::encode(old_pool, bl);
+  ::encode(update_bt, bl);
   ENCODE_FINISH(bl);
 }
 void EMetaBlob::decode(bufferlist::iterator &bl)
 {
-  DECODE_START_LEGACY_COMPAT_LEN(5, 5, 5, bl);
+  DECODE_START_LEGACY_COMPAT_LEN(6, 5, 5, bl);
   ::decode(lump_order, bl);
   ::decode(lump_map, bl);
   if (struct_v >= 4) {
@@ -873,6 +877,10 @@ void EMetaBlob::decode(bufferlist::iterator &bl)
     ::decode(renamed_dirino, bl);
     ::decode(renamed_dir_frags, bl);
   }
+  if (struct_v >= 6) {
+    ::decode(old_pool, bl);
+    ::decode(update_bt, bl);
+  }
   DECODE_FINISH(bl);
 }
 
@@ -982,6 +990,7 @@ void EMetaBlob::replay(MDS *mds, LogSegment *logseg, MDSlaveUpdate *slaveup)
     if (!in)
       in = new CInode(mds->mdcache, true);
     (*p)->update_inode(mds, in);
+
     if (isnew)
       mds->mdcache->add_inode(in);
     if ((*p)->dirty) in->_mark_dirty(logseg);
@@ -1149,6 +1158,35 @@ void EMetaBlob::replay(MDS *mds, LogSegment *logseg, MDSlaveUpdate *slaveup)
        assert(in->first == p->dnfirst ||
               (in->is_multiversion() && in->first > p->dnfirst));
       }
+
+      // store backtrace for allocated inos (create, mkdir, symlink, mknod)
+      if (allocated_ino || used_preallocated_ino) {
+       if (in->inode.is_dir()) {
+         logseg->queue_backtrace_update(in, mds->mdsmap->get_metadata_pool());
+       } else {
+         logseg->queue_backtrace_update(in, in->inode.layout.fl_pg_pool);
+       }
+      }
+      // handle change of pool with backtrace update
+      if (old_pool != -1 && old_pool != in->inode.layout.fl_pg_pool) {
+       // update backtrace on new data pool
+       logseg->queue_backtrace_update(in, in->inode.layout.fl_pg_pool);
+
+       // set forwarding pointer on old backtrace
+       logseg->queue_backtrace_update(in, old_pool, in->inode.layout.fl_pg_pool);
+      }
+      // handle backtrace update if specified (used by rename)
+      if (update_bt) {
+       if (in->is_dir()) {
+         // replace previous backtrace on this inode with myself
+         logseg->remove_pending_backtraces(in->ino(), mds->mdsmap->get_metadata_pool());
+         logseg->queue_backtrace_update(in, mds->mdsmap->get_metadata_pool());
+       } else {
+         // remove all pending backtraces going to the same pool
+         logseg->remove_pending_backtraces(in->ino(), in->inode.layout.fl_pg_pool);
+         logseg->queue_backtrace_update(in, in->inode.layout.fl_pg_pool);
+       }
+      }
     }
 
     // remote dentries

diff --git a/src/mds/mdstypes.cc b/src/mds/mdstypes.cc
index 51103bd6d188eb7735146d5492c64ccf39d6fd5c..ad4a71acba56ae0a76e40908324b81dff538c022 100644 (file)
--- a/src/mds/mdstypes.cc
+++ b/src/mds/mdstypes.cc
@@ -318,9 +318,9 @@ void inode_t::dump(Formatter *f) const
   f->close_section();
 
   f->open_array_section("old_pools");
-  vector<ceph_file_layout>::const_iterator i = old_pools.begin();
+  vector<int64_t>::const_iterator i = old_pools.begin();
   while(i != old_pools.end()) {
-    ::dump(*i, f);
+    f->dump_int("pool", *i);
   }
   f->close_section();