From 0e84f49bf6b77a41a0734b99f05dcc798dba3918 Mon Sep 17 00:00:00 2001
From: Greg Farnum <gregf@hq.newdream.net>
Date: Thu, 22 Oct 2009 17:15:20 -0700
Subject: [PATCH] mon: Move fullness checks into OSDMonitor where they belong

---
 src/mon/OSDMonitor.cc | 48 +++++++++++++++++++++++++++++++++++++++----
 src/mon/OSDMonitor.h  | 21 +++++++++++++++----
 src/mon/PGMonitor.cc  | 14 +------------
 3 files changed, 62 insertions(+), 21 deletions(-)

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 2ba4307c590b4..2b0b803309f37 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -271,7 +271,9 @@ bool OSDMonitor::should_propose(double& delay)
   if (pending_inc.fullmap.length())
     return true;
 
-  if (pending_inc.new_flags & CEPH_OSDMAP_FULL) { //get that info out there!
+  if ((pending_inc.new_flags ^ osdmap.flags)
+      & CEPH_OSDMAP_FULL) { //get that info out there!
+    dout(1) << "should_propose returning true, 0.0 because full status changed! " << dendl;
     delay = 0.0;
     return true;
   }
@@ -754,6 +756,43 @@ void OSDMonitor::check_sub(Subscription *sub)
   }
 }
 
+void OSDMonitor::handle_osd_stat(osd_stat_t& stat, int from)
+{
+  float ratio = ((float)stat.kb_used) / (float) stat.kb;
+  if ( ratio > CEPH_OSD_FULL_RATIO ) {
+    dout(5) << "osd" << from << " has ratio " << ratio 
+	    << "which is beyond CEPH_OSD_FULL_RATIO "
+	    << CEPH_OSD_FULL_RATIO << dendl;
+    add_flag(CEPH_OSDMAP_FULL);
+    full_osds.insert(from);
+  }
+  else if ( ratio > CEPH_OSD_NEARFULL_RATIO ) {
+    dout(10) << "osd" << from << " has ratio " << ratio 
+	     << "which is beyond CEPH_OSD_NEARFULL_RATIO "
+	     << CEPH_OSD_NEARFULL_RATIO << dendl;
+    add_flag(CEPH_OSDMAP_NEARFULL);
+    nearfull_osds.insert(from);
+    if (full_osds.erase(from)) { //it actually erased an element
+      if (full_osds.empty()) {
+	dout(1) << "full_osds is empty, removing full flag!" << dendl;
+	remove_flag(CEPH_OSDMAP_FULL);
+      }
+    }
+  }
+  else {//it's not full or near-full
+    if (full_osds.erase(from)) {
+      dout(1) << "erased osd" << from << "from full_osds" << dendl;
+      if (full_osds.empty()) {
+	dout(1) << "full_osds is empty, removing full flag!" << dendl;
+	remove_flag(CEPH_OSDMAP_FULL);
+      }
+    }
+    if (nearfull_osds.erase(from)) {
+      if (nearfull_osds.empty())
+	remove_flag(CEPH_OSDMAP_NEARFULL);
+    }
+  }
+}
 
 // TICK
 
@@ -807,10 +846,11 @@ void OSDMonitor::tick()
     }
   }
 
-  //if map is set to full, get that info out there!
-  if (pending_inc.new_flags & CEPH_OSDMAP_FULL)
+  //if map full setting has changed, get that info out there!
+  if ((pending_inc.new_flags ^ osdmap.flags) & CEPH_OSDMAP_FULL) {
+    dout(1) << "New setting for CEPH_OSDMAP_FULL -- doing propose" << dendl;
     do_propose = true;
-
+  }
   // ---------------
 #define SWAP_PRIMARIES_AT_START 0
 #define SWAP_TIME 1
diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index 35dc36c60d58f..af79182a694ba 100644
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -48,7 +48,8 @@ private:
   map<int,utime_t>    down_pending_out;  // osd down -> out
 
   map<int,double> osd_weight;
-
+  set<int> nearfull_osds;
+  set<int> full_osds;
   // svc
 public:  
   void create_initial(bufferlist& bl);
@@ -163,10 +164,22 @@ private:
   void check_subs();
   void check_sub(Subscription *sub);
 
+  void handle_osd_stat(osd_stat_t& stat, int from);
+
   void add_flag(int flag) {
-    if (pending_inc.new_flags < 0)
-      pending_inc.new_flags = osdmap.flags;
-    pending_inc.new_flags |= flag;
+    if (!(osdmap.flags & flag)) {
+      if (pending_inc.new_flags < 0)
+	pending_inc.new_flags = osdmap.flags;
+      pending_inc.new_flags |= flag;
+    }
+  }
+
+  void remove_flag(int flag) {
+    if(osdmap.flags & flag) {
+      if (pending_inc.new_flags < 0)
+	pending_inc.new_flags = osdmap.flags;
+      pending_inc.new_flags &= ~flag;
+    }
   }
 };
 
diff --git a/src/mon/PGMonitor.cc b/src/mon/PGMonitor.cc
index f6ad0f69a031c..de5de1c4a770c 100644
--- a/src/mon/PGMonitor.cc
+++ b/src/mon/PGMonitor.cc
@@ -326,19 +326,7 @@ bool PGMonitor::prepare_pg_stats(MPGStats *stats)
     dout(10) << " got osd " << from << " " << stats->osd_stat << " (first report)" << dendl;
 
   //Check how full the OSD is; set OSDMap to full/near-full if needed
-  float ratio = ((float)stats->osd_stat.kb_used) / (float) stats->osd_stat.kb;
-  if ( ratio > CEPH_OSD_FULL_RATIO ) {
-    dout(5) << "osd" << from << " has ratio " << ratio 
-	    << "which is beyond CEPH_OSD_FULL_RATIO "
-	    << CEPH_OSD_FULL_RATIO << dendl;
-    mon->osdmon()->add_flag(CEPH_OSDMAP_FULL);
-  } else if ( ratio > CEPH_OSD_NEARFULL_RATIO ) {
-    dout(10) << "osd" << from << " has ratio " << ratio 
-	    << "which is beyond CEPH_OSD_NEARFULL_RATIO "
-	    << CEPH_OSD_NEARFULL_RATIO << dendl;
-    mon->osdmon()->add_flag(CEPH_OSDMAP_NEARFULL);
-  }
-
+  mon->osdmon()->handle_osd_stat(stats->osd_stat, from);
   // apply to live map too (screw consistency)
   /*
     actually, no, don't.  that screws up our "latest" stash.  and can
-- 
2.39.5