objecter: pause writes when FULL flag is set

Also, subscribe to all osdmap updates while FULL flag is set, so that we
discover when it is unset.

Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index 41d893a72e4298fcc42f96e2b127f443e4723aeb..8b7d82adab541ffe8aad39982dfc59e493ffe8ed 100644 (file)
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -98,6 +98,8 @@ MDS::MDS(const char *n, Messenger *m, MonClient *mc) :
   osdmap = new OSDMap;
 
   objecter = new Objecter(messenger, monc, osdmap, mds_lock);
+  objecter->unset_honor_osdmap_full();
+
   filer = new Filer(objecter);
 
   mdcache = new MDCache(this);

diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index f86699a97836b5c0207f945cf8b952e83401186f..215213bf6913aa0566eed1de9e4814efeecab069 100644 (file)
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -122,10 +122,8 @@ void Objecter::handle_osd_map(MOSDMap *m)
 
        bool was_pauserd = osdmap->test_flag(CEPH_OSDMAP_PAUSERD);
        bool was_pausewr = osdmap->test_flag(CEPH_OSDMAP_PAUSEWR);
-
-       if (was_pauserd || was_pausewr)
-         maybe_request_map();
-    
+       bool was_full = osdmap->test_flag(CEPH_OSDMAP_FULL);
+  
        if (m->incremental_maps.count(e)) {
          dout(3) << "handle_osd_map decoding incremental epoch " << e << dendl;
          OSDMap::Incremental inc(m->incremental_maps[e]);
@@ -144,17 +142,20 @@ void Objecter::handle_osd_map(MOSDMap *m)
        }
        else {
          dout(3) << "handle_osd_map requesting missing epoch " << osdmap->get_epoch()+1 << dendl;
-         monc->sub_want("osdmap", osdmap->get_epoch() + 1, CEPH_SUBSCRIBE_ONETIME);
-         monc->renew_subs();
+         maybe_request_map();
          break;
        }
+
+       if (was_pauserd || was_pausewr || was_full)
+         maybe_request_map();
        
        // scan pgs for changes
        scan_pgs(changed_pgs);
 
        // kick paused
        if ((was_pauserd && !osdmap->test_flag(CEPH_OSDMAP_PAUSERD)) ||
-           (was_pausewr && !osdmap->test_flag(CEPH_OSDMAP_PAUSEWR))) {
+           (was_pausewr && !osdmap->test_flag(CEPH_OSDMAP_PAUSEWR)) ||
+           (was_full && !osdmap->test_flag(CEPH_OSDMAP_FULL))) {
          for (hash_map<tid_t,Op*>::iterator p = op_osd.begin();
               p != op_osd.end();
               p++) {
@@ -213,8 +214,14 @@ void Objecter::handle_osd_map(MOSDMap *m)
 
 void Objecter::maybe_request_map()
 {
-  dout(10) << "maybe_request_map subscribing (onetime) to next osd map" << dendl;
-  if (monc->sub_want("osdmap", osdmap->get_epoch() ? osdmap->get_epoch()+1 : 0, CEPH_SUBSCRIBE_ONETIME))
+  int flag = 0;
+  if (osdmap->test_flag(CEPH_OSDMAP_FULL)) {
+    dout(10) << "maybe_request_map subscribing (continuous) to next osd map (FULL flag is set)" << dendl;
+  } else {
+    dout(10) << "maybe_request_map subscribing (onetime) to next osd map" << dendl;
+    flag = CEPH_SUBSCRIBE_ONETIME;
+  }
+  if (monc->sub_want("osdmap", osdmap->get_epoch() ? osdmap->get_epoch()+1 : 0, flag))
     monc->renew_subs();
 }
 
@@ -435,6 +442,11 @@ tid_t Objecter::op_submit(Op *op)
     dout(10) << " paused read " << op << " tid " << last_tid << dendl;
     op->paused = true;
     maybe_request_map();
+ } else if ((op->flags & CEPH_OSD_FLAG_WRITE) &&
+           osdmap->test_flag(CEPH_OSDMAP_FULL)) {
+    dout(10) << " FULL, paused modify " << op << " tid " << last_tid << dendl;
+    op->paused = true;
+    maybe_request_map();
   } else if (pg.primary() >= 0) {
     int flags = op->flags;
     if (op->oncommit)

diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index eda4f7282ea3d66fdd994ee19e1c427e3cb16776..279fb8fc64276567982f59e59ee91f10c939efa8 100644 (file)
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -186,6 +186,7 @@ class Objecter {
   int num_unacked;
   int num_uncommitted;
   bool keep_balanced_budget;
+  bool honor_osdmap_full;
 
   void maybe_request_map();
 
@@ -436,7 +437,7 @@ public:
     messenger(m), monc(mc), osdmap(om),
     last_tid(0), client_inc(-1),
     num_unacked(0), num_uncommitted(0),
-    keep_balanced_budget(false),
+    keep_balanced_budget(false), honor_osdmap_full(true),
     last_seen_osdmap_version(0),
     last_seen_pgmap_version(0),
     client_lock(l), timer(l),
@@ -456,6 +457,10 @@ public:
    */
   void set_balanced_budget() { keep_balanced_budget = true; }
   void unset_balanced_budget() { keep_balanced_budget = false; }
+
+  void set_honor_osdmap_full() { honor_osdmap_full = true; }
+  void unset_honor_osdmap_full() { honor_osdmap_full = false; }
+
   // messages
  public:
   void dispatch(Message *m);