mds: purge preallocated inos/files when client disconnected

When a client is kicked out of the cluster, purge any data written to
preallocated inos.  This should be the first object in the file sequence.

diff --git a/src/TODO b/src/TODO
index b58407c00c56b55469f2b406ceebf53e43bc195a..69d62292f953926a85112906c2e8328b81ed73b6 100644 (file)
--- a/src/TODO
+++ b/src/TODO
@@ -14,6 +14,7 @@ v0.9
 /- librados
   - async io
   - list_objects
+  - perl swig wrapper
 /- object classes
 - optionally separate osd interfaces (ips) for clients and osds (replication, peering, etc.)
 
@@ -100,14 +101,8 @@ userspace client
 - fix readdir vs fragment race by keeping a separate frag pos, and ignoring dentries below it
 
 mds
-- fix client op replay
-  - ordering
-  - should to set caps/lock states so that ops can proceed without revoking from client
-  - need to make sure replayed ops finish before new ops proceed.  globally across the cluster.  ick.
 - on replay, but dirty scatter replicas on lists so that they get flushed?  or does rejoin handle that?
 - linkage vs cdentry replicas and remote rename....
-- make recovery work with early replies
-  - purge each session's unused preallocated inodes
 - hard link backpointers
   - anchor source dir
   - build snaprealm for any hardlinked file

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 18eefea2b9b8a2d0fb0f7f31f5557f24d8967935..22768113ec2b70ef9513e1ea7579026769bd886d 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -4363,7 +4363,15 @@ void MDCache::_recovered(CInode *in, int r)
   do_file_recover();
 }
 
-
+void MDCache::purge_prealloc_ino(inodeno_t ino, Context *fin)
+{
+  object_t oid(ino, 0);
+  dout(10) << "purge_prealloc_ino " << ino << " oid " << oid << dendl;
+  ceph_object_layout ol = mds->osdmap->make_object_layout(oid,
+                                                         mds->mdsmap->get_metadata_pg_pool());
+  SnapContext snapc;
+  mds->objecter->remove(oid, ol, snapc, g_clock.now(), 0, 0, fin);
+}  
 
 
 

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 32ef9bc644dcf33d178e90ffdc64d6d9c6e7c2e0..444b7505e34f0d9d6dbed8f8ca11a5eb0962ccc7 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -752,6 +752,9 @@ public:
   void do_file_recover();
   void _recovered(CInode *in, int r);
 
+  void purge_prealloc_ino(inodeno_t ino, Context *fin);
+
+
 
  public:
 

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 908a397f34f0b93ea710dfe5a02dd1c04b8a6392..a894243b1021fa7f3dc3669a2bdbfbc84ec12cf0 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -337,6 +337,15 @@ void Server::terminate_sessions()
 }
 
 
+struct C_MDS_session_purged : public Context {
+  Server *server;
+  Session *session;
+  C_MDS_session_purged(Server *s, Session *ss) : server(s), session(ss) {}
+  void finish(int r) {
+    server->_finish_session_purge(session);
+  }
+};
+
 void Server::find_idle_sessions()
 {
   dout(10) << "find_idle_sessions" << dendl;
@@ -379,13 +388,31 @@ void Server::find_idle_sessions()
     }
 
     dout(10) << "autoclosing stale session " << session->inst << " last " << session->last_cap_renew << dendl;
-    mds->sessionmap.set_state(session, Session::STATE_STALE_CLOSING);
-    version_t pv = ++mds->sessionmap.projected;
-    mdlog->submit_entry(new ESession(session->inst, false, pv),
-                       new C_MDS_session_finish(mds, session, false, pv));
-    mdlog->flush();
+    if (session->prealloc_inos.empty()) {
+      _finish_session_purge(session);
+    } else {
+      mds->sessionmap.set_state(session, Session::STATE_STALE_PURGING);
+      C_Gather *fin = new C_Gather(new C_MDS_session_purged(this, session));
+      for (map<inodeno_t,inodeno_t>::iterator p = session->prealloc_inos.m.begin();
+          p != session->prealloc_inos.m.end();
+          p++) {
+       inodeno_t last = p->first + p->second;
+       for (inodeno_t i = p->first; i < last; i = i + 1)
+         mds->mdcache->purge_prealloc_ino(i, fin->new_sub());
+      }
+    }
   }
+}
 
+void Server::_finish_session_purge(Session *session)
+{
+  dout(10) << "_finish_session_purge " << session->inst << dendl;
+  assert(session->is_stale_purging());
+  mds->sessionmap.set_state(session, Session::STATE_STALE_CLOSING);
+  version_t pv = ++mds->sessionmap.projected;
+  mdlog->submit_entry(new ESession(session->inst, false, pv),
+                     new C_MDS_session_finish(mds, session, false, pv));
+  mdlog->flush();
 }
 
 
@@ -534,7 +561,6 @@ void Server::reconnect_tick()
   }
 }
 
-
 void Server::recall_client_state(float ratio)
 {
   int max_caps_per_client = g_conf.mds_cache_size * .8;

diff --git a/src/mds/Server.h b/src/mds/Server.h
index db9dcb5d90ba9a743fc474f9e3a321ee7a446c8a..9f258f6ed679bf7001ecbce05be237457e783e73 100644 (file)
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -69,6 +69,7 @@ public:
 
   void handle_client_session(class MClientSession *m);
   void _session_logged(Session *session, bool open, version_t pv, interval_set<inodeno_t>& inos,version_t piv);
+  void _finish_session_purge(Session *);
   version_t prepare_force_open_sessions(map<__u32,entity_inst_t> &cm);
   void finish_force_open_sessions(map<__u32,entity_inst_t> &cm);
   void terminate_sessions();
@@ -78,7 +79,7 @@ public:
   void process_reconnect_cap(CInode *in, int from, ceph_mds_cap_reconnect& capinfo);
   void reconnect_gather_finish();
   void reconnect_tick();
-  
+
   void recall_client_state(float ratio);
 
   // -- requests --

diff --git a/src/mds/SessionMap.h b/src/mds/SessionMap.h
index ff7ebafc73e536f5f766becafc3f0986d3b12db2..918eada1a67b737d63c558c9a2dafb2acaee5365 100644 (file)
--- a/src/mds/SessionMap.h
+++ b/src/mds/SessionMap.h
@@ -43,7 +43,8 @@ public:
   static const int STATE_OPEN = 2;
   static const int STATE_CLOSING = 3;   // journaling close
   static const int STATE_STALE = 4;
-  static const int STATE_STALE_CLOSING = 5;
+  static const int STATE_STALE_PURGING = 5;
+  static const int STATE_STALE_CLOSING = 6;
   //static const int STATE_RECONNECTING = 5; // ?
 
 private:
@@ -84,6 +85,7 @@ public:
   bool is_open() { return state == STATE_OPEN; }
   bool is_closing() { return state == STATE_CLOSING; }
   bool is_stale() { return state == STATE_STALE; }
+  bool is_stale_purging() { return state == STATE_STALE_PURGING; }
   bool is_stale_closing() { return state == STATE_STALE_CLOSING; }
 
   // -- caps --