mds mon: propagate deleted snaps to osdmap

diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index 541a4d0ebfc29bba5b60625f3d03def3d4c05999..910aa72453e0cde749786ac0e1b20351d6efca96 100644 (file)
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -399,6 +399,8 @@ void MDS::tick()
 
   if (is_active()) {
     balancer->tick();
+    if (snapserver)
+      snapserver->check_osd_map(false);
   }
 }
 
@@ -1151,6 +1153,8 @@ void MDS::_dispatch(Message *m)
       break;
     case CEPH_MSG_OSD_MAP:
       objecter->handle_osd_map((MOSDMap*)m);
+      if (snapserver)
+       snapserver->check_osd_map(true);
       break;
       
       // MDS

diff --git a/src/mds/SnapServer.cc b/src/mds/SnapServer.cc
index 868560a1df9a23baafd3a5acf277d0ead44bb214..1f3283f9525f5e4ee075ffe665e2c04f1b4600f0 100644 (file)
--- a/src/mds/SnapServer.cc
+++ b/src/mds/SnapServer.cc
@@ -14,9 +14,13 @@
 
 #include "SnapServer.h"
 #include "MDS.h"
+#include "osd/OSDMap.h"
 
 #include "include/types.h"
 #include "messages/MMDSTableRequest.h"
+#include "messages/MRemoveSnaps.h"
+
+#include "msg/Messenger.h"
 
 #include "config.h"
 
@@ -32,32 +36,7 @@ void SnapServer::reset_state()
 {
   last_snap = 0;
   snaps.clear();
-  pending_removal.clear();
-}
-
-snapid_t SnapServer::create(inodeno_t base, const string& name, utime_t stamp, version_t *psnapv)
-{
-  assert(is_active());
-  
-  snapid_t sn = ++last_snap;
-  snaps[sn].snapid = sn;
-  snaps[sn].ino = base;
-  snaps[sn].name = name;
-  snaps[sn].stamp = stamp;
-  *psnapv = ++version;
-
-  dout(10) << "create(" << base << "," << name << ") = " << sn << dendl;
-
-  return sn;
-}
-
-void SnapServer::remove(snapid_t sn) 
-{
-  assert(is_active());
-
-  snaps.erase(sn);
-  pending_removal.insert(sn);
-  version++;
+  pending_purge.clear();
 }
 
 
@@ -66,10 +45,10 @@ void SnapServer::remove(snapid_t sn)
 void SnapServer::_prepare(bufferlist &bl, __u64 reqid, int bymds)
 {
   bufferlist::iterator p = bl.begin();
-  __u32 what;
-  ::decode(what, p);
+  __u32 op;
+  ::decode(op, p);
 
-  switch (what) {
+  switch (op) {
   case TABLE_OP_CREATE:
     {
       version++;
@@ -81,18 +60,23 @@ void SnapServer::_prepare(bufferlist &bl, __u64 reqid, int bymds)
        ::decode(info.stamp, p);
        info.snapid = version;
        pending_create[version] = info;
+       dout(10) << "prepare v" << version << " create " << info << dendl;
       } else {
        pending_noop.insert(version);
+       dout(10) << "prepare v" << version << " noop" << dendl;
       }
     }
     break;
 
   case TABLE_OP_DESTROY:
     {
+      inodeno_t ino;
       snapid_t snapid;
+      ::decode(ino, p);    // not used, currently.
       ::decode(snapid, p);
       version++;
       pending_destroy[version] = snapid;
+      dout(10) << "prepare v" << version << " destroy " << snapid << dendl;
     }
     break;
     
@@ -118,8 +102,10 @@ void SnapServer::_commit(version_t tid)
   } 
 
   else if (pending_destroy.count(tid)) {
-    dout(7) << "commit " << tid << " destroy " << pending_destroy[tid] << dendl;
-    snaps.erase(pending_destroy[tid]);
+    snapid_t sn = pending_destroy[tid];
+    dout(7) << "commit " << tid << " destroy " << sn << dendl;
+    snaps.erase(sn);
+    pending_purge.insert(sn);
     pending_destroy.erase(tid);
   }
   else if (pending_noop.count(tid)) {
@@ -189,3 +175,39 @@ void SnapServer::handle_query(MMDSTableRequest *req)
 }
 
 
+
+void SnapServer::check_osd_map(bool force)
+{
+  if (!force && version == last_checked_osdmap) {
+    dout(10) << "check_osd_map - version unchanged" << dendl;
+    return;
+  }
+  dout(10) << "check_osd_map pending_purge=" << pending_purge << dendl;
+
+  vector<snapid_t> purge;
+  bool purged = false;
+
+  set<snapid_t>::iterator p = pending_purge.begin();
+  while (p != pending_purge.end()) {
+    if (mds->osdmap->is_removed_snap(*p)) {
+      dout(10) << " osdmap marks " << *p << " as removed" << dendl;
+      pending_purge.erase(p++);
+      purged = true;
+    } else {
+      purge.push_back(*p);
+      p++;
+    }
+  }
+
+  if (purged)
+    version++;
+
+  if (!purge.empty()) {
+    dout(10) << "requesting removal of " << purge << dendl;
+    MRemoveSnaps *m = new MRemoveSnaps(purge);
+    int mon = mds->monmap->pick_mon();
+    mds->messenger->send_message(m, mds->monmap->get_inst(mon));
+  }
+
+  last_checked_osdmap = version;
+}

diff --git a/src/mds/SnapServer.h b/src/mds/SnapServer.h
index 4b50c59f2678dea7b4993d06beaab54b930e3cd8..a0371f6363254b02784b5399ed76377b0f1caf89 100644 (file)
--- a/src/mds/SnapServer.h
+++ b/src/mds/SnapServer.h
@@ -26,25 +26,23 @@ public:
 protected:
   snapid_t last_snap;
   map<snapid_t, SnapInfo> snaps;
-  set<snapid_t> pending_removal;
+  set<snapid_t> pending_purge;
   
   map<version_t, SnapInfo> pending_create;
   map<version_t, snapid_t> pending_destroy;
   set<version_t>           pending_noop;
 
+  version_t last_checked_osdmap;
+
 public:
   SnapServer(MDS *m) : MDSTableServer(m, TABLE_SNAP) { }
-  
-  // alloc or reclaim ids
-  snapid_t create(inodeno_t base, const string& name, utime_t stamp, version_t *snapv);
-  void remove(snapid_t sn);
-  
+    
   void init_inode();
   void reset_state();
   void encode_state(bufferlist& bl) {
     ::encode(last_snap, bl);
     ::encode(snaps, bl);
-    ::encode(pending_removal, bl);
+    ::encode(pending_purge, bl);
     ::encode(pending_create, bl);
     ::encode(pending_destroy, bl);
     ::encode(pending_noop, bl);
@@ -53,7 +51,7 @@ public:
   void decode_state(bufferlist::iterator& bl) {
     ::decode(last_snap, bl);
     ::decode(snaps, bl);
-    ::decode(pending_removal, bl);
+    ::decode(pending_purge, bl);
     ::decode(pending_create, bl);
     ::decode(pending_destroy, bl);
     ::decode(pending_noop, bl);
@@ -66,6 +64,8 @@ public:
   void _commit(version_t tid);
   void _rollback(version_t tid);
   void handle_query(MMDSTableRequest *m);
+
+  void check_osd_map(bool force);
 };
 
 #endif

diff --git a/src/messages/MRemoveSnaps.h b/src/messages/MRemoveSnaps.h
new file mode 100644 (file)
index 0000000..a63b07e
--- /dev/null
+++ b/src/messages/MRemoveSnaps.h
@@ -0,0 +1,46 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef __MREMOVESNAPS_H
+#define __MREMOVESNAPS_H
+
+#include "msg/Message.h"
+
+struct MRemoveSnaps : public Message {
+  vector<snapid_t> snaps;
+  
+  MRemoveSnaps() : 
+    Message(MSG_REMOVE_SNAPS) { }
+  MRemoveSnaps(vector<snapid_t>& s) : 
+    Message(MSG_REMOVE_SNAPS) {
+    snaps.swap(s);
+  }
+  
+  const char *get_type_name() { return "remove_snaps"; }
+  void print(ostream& out) {
+    out << "remove_snaps(" << snaps << ")";
+  }
+
+  void encode_payload() {
+    ::encode(snaps, payload);
+  }
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(snaps, p);
+    assert(p.end());
+  }
+
+};
+
+#endif

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 6dc5acb6d993d4e4b50e07c7f8c9e6efae7c8429..7142258595309755f65b790721709a279bd5bd42 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -291,6 +291,7 @@ void Monitor::dispatch(Message *m)
     case MSG_OSD_IN:
     case MSG_OSD_OUT:
     case MSG_OSD_ALIVE:
+    case MSG_REMOVE_SNAPS:
       osdmon->dispatch(m);
       break;
 

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 1ac624e220a4672b095a659d87e0e40f9958b713..a9ade5969ac892f85e66acf80cef4bfcdc03d1af 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -27,6 +27,7 @@
 #include "messages/MOSDBoot.h"
 #include "messages/MOSDAlive.h"
 #include "messages/MMonCommand.h"
+#include "messages/MRemoveSnaps.h"
 
 #include "common/Timer.h"
 
@@ -280,6 +281,9 @@ bool OSDMonitor::preprocess_query(Message *m)
   case MSG_OSD_OUT:
     return preprocess_out((MOSDOut*)m);
     */
+
+  case MSG_REMOVE_SNAPS:
+    return preprocess_remove_snaps((MRemoveSnaps*)m);
     
   default:
     assert(0);
@@ -309,6 +313,9 @@ bool OSDMonitor::prepare_update(Message *m)
     return prepare_out((MOSDOut*)m);
     */
 
+  case MSG_REMOVE_SNAPS:
+    return prepare_remove_snaps((MRemoveSnaps*)m);
+
   default:
     assert(0);
     delete m;
@@ -577,6 +584,53 @@ void OSDMonitor::_alive(MOSDAlive *m)
 }
 
 
+// ---
+
+bool OSDMonitor::preprocess_remove_snaps(MRemoveSnaps *m)
+{
+  dout(7) << "preprocess_remove_snaps " << *m << dendl;
+  
+  for (vector<snapid_t>::iterator p = m->snaps.begin(); 
+       p != m->snaps.end();
+       p++) {
+    if (*p > osdmap.max_snap ||
+       !osdmap.removed_snaps.contains(*p))
+      return false;
+  }
+  delete m;
+  return true;
+}
+
+bool OSDMonitor::prepare_remove_snaps(MRemoveSnaps *m)
+{
+  dout(7) << "prepare_remove_snaps " << *m << dendl;
+
+  snapid_t max;
+  for (vector<snapid_t>::iterator p = m->snaps.begin(); 
+       p != m->snaps.end();
+       p++) {
+    if (*p > max)
+      max = *p;
+
+    if (!osdmap.removed_snaps.contains(*p)) {
+      dout(10) << " adding " << *p << " to removed_snaps" << dendl;
+      pending_inc.removed_snaps.insert(*p);
+    }
+  }
+
+  if (max > osdmap.max_snap && 
+      (__s64)max > pending_inc.new_max_snap) {
+    dout(10) << " new_max_snap " << max << dendl;
+    pending_inc.new_max_snap = max;
+  } else {
+    dout(10) << " max_snap " << osdmap.max_snap << " still >= " << max << dendl;
+  }
+
+  delete m;
+  return true;
+}
+
+
 // ---------------
 // map helpers
 

diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index faf7370b28a61a7fc21ab4753be9b47ec127cb98..261c363ea0a4dfef6b744f50c7f3b9fed7813a7f 100644 (file)
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -119,6 +119,9 @@ private:
   bool preprocess_out(class MOSDOut *m);
   bool prepare_out(class MOSDOut *m);
 
+  bool preprocess_remove_snaps(class MRemoveSnaps *m);
+  bool prepare_remove_snaps(class MRemoveSnaps *m);
+
  public:
   OSDMonitor(Monitor *mn, Paxos *p) : 
     PaxosService(mn, p) { }

diff --git a/src/msg/Message.cc b/src/msg/Message.cc
index 7c91740ec9a2fad5c9542460c60ec6615a3fd36d..549ea63455d236fa8da5f56315df259a847d58ba 100644 (file)
--- a/src/msg/Message.cc
+++ b/src/msg/Message.cc
@@ -46,6 +46,8 @@ using namespace std;
 #include "messages/MOSDPGInfo.h"
 #include "messages/MOSDPGCreate.h"
 
+#include "messages/MRemoveSnaps.h"
+
 #include "messages/MMonMap.h"
 #include "messages/MMonGetMap.h"
 
@@ -224,6 +226,10 @@ decode_message(ceph_msg_header& env, bufferlist& front, bufferlist& data)
     m = new MOSDPGCreate;
     break;
 
+  case MSG_REMOVE_SNAPS:
+    m = new MRemoveSnaps;
+    break;
+
 
     // clients
   case CEPH_MSG_CLIENT_MOUNT:

diff --git a/src/msg/Message.h b/src/msg/Message.h
index 333bb4a621712c527b5e85ae61ae81b219a05806..69de7cb2fcc1931b14a6d856df7c02dbf44df7a7 100644 (file)
--- a/src/msg/Message.h
+++ b/src/msg/Message.h
@@ -48,7 +48,7 @@
 #define MSG_PGSTATSACK 87
 
 #define MSG_OSD_PG_CREATE      88
-
+#define MSG_REMOVE_SNAPS 89
 
 
 

diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index 46d4ba14fdd38347c353ede12b3924f0e2d89409..1977b696c3574fdbfaf1dfce31e17a1d2b28271c 100644 (file)
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -95,7 +95,7 @@ public:
     map<pg_t,uint32_t> new_pg_swap_primary;
     list<pg_t> old_pg_swap_primary;
 
-    int64_t new_max_snap;
+    snapid_t new_max_snap;
     interval_set<snapid_t> removed_snaps;
     
     void encode(bufferlist& bl) {
@@ -245,6 +245,13 @@ private:
     return last_pg_change;
   }
 
+  snapid_t get_max_snap() { return max_snap; }
+  bool is_removed_snap(snapid_t sn) { 
+    if (sn > max_snap)
+      return false;
+    return removed_snaps.contains(sn); 
+  }
+
   /***** cluster state *****/
   /* osds */
   int get_max_osd() const { return max_osd; }
@@ -348,14 +355,6 @@ private:
     return -1;
   }
 
-  // snaps
-  
-  // returns true if a snap has been deleted.
-  bool snap_removed(snapid_t s) {
-    if (s > max_snap)
-      return false;
-    return removed_snaps.contains(s);
-  }
 
   void apply_incremental(Incremental &inc) {
     if (inc.epoch == 1)

diff --git a/src/vstartnew.sh b/src/vstartnew.sh
index 4faa481ea9446fc0d1a5ea634446edc4d1448e24..44905bddb6923c75f776eb59a0154976088b2706 100755 (executable)
--- a/src/vstartnew.sh
+++ b/src/vstartnew.sh
@@ -38,7 +38,7 @@ for osd in 0 #1 #2 3 #4 5 6 7 8 9 10 11 12 13 14 15
 do
  $CEPH_BIN/cosd --mkfs_for_osd $osd dev/osd$osd  # initialize empty object store
 # echo valgrind --leak-check=full --show-reachable=yes $CEPH_BIN/cosd dev/osd$osd --debug_ms 1 --debug_osd 20 --debug_filestore 10 --debug_ebofs 20 #1>out/o$osd #& #--debug_osd 40
- $CEPH_BIN/cosd -m $IP:12345 dev/osd$osd -d --debug_ms 1 --debug_osd 20 # --debug_filestore 10
+ $CEPH_BIN/cosd -m $IP:12345 dev/osd$osd -d --debug_ms 1 --debug_osd 20 --debug_filestore 20
 done
 
 # mds