From 4379d8a42e999c05b07e002d14ee7184739faee2 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@newdream.net>
Date: Tue, 17 Jun 2008 15:54:03 -0700
Subject: [PATCH] osd: fix up osd_auto_weight

---
 src/crush/CrushWrapper.h | 15 +++++++++++++++
 src/mon/OSDMonitor.cc    | 31 ++++++++++++++++++-------------
 src/osd/OSD.cc           |  3 +++
 src/osd/OSDMap.cc        |  6 +++---
 src/osd/OSDMap.h         | 13 +++++++------
 5 files changed, 46 insertions(+), 22 deletions(-)

diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index 2e6741beef00d..2a055f06ef103 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -339,6 +339,21 @@ public:
       out[i] = rawout[i];
   }
 
+
+  void adjust_osd_weights(map<int,double> &weights) {
+    float max = 0;
+    for (map<int,double>::iterator p = weights.begin(); p != weights.end(); p++)
+      if (p->second > max)
+	max = p->second;
+    
+    for (map<int,double>::iterator p = weights.begin(); p != weights.end(); p++) {
+      unsigned w = 0x10000 - (p->second / max * 0x10000);
+      set_offload(p->first, w);
+    }
+  }
+  
+
+
   int read_from_file(const char *fn) {
     bufferlist bl;
     int r = bl.read_file(fn);
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 642e77bca4511..855c78750b96a 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -318,18 +318,22 @@ bool OSDMonitor::prepare_update(Message *m)
 bool OSDMonitor::should_propose(double& delay)
 {
   dout(10) << "should_propose" << dendl;
-  if (osdmap.epoch == 1) {
-    if (pending_inc.new_up.size() == (unsigned)osdmap.get_max_osd()) {
-      delay = 0.0;
-      if (g_conf.osd_auto_weight) {
-	CrushWrapper crush;
-	OSDMap::build_simple_crush_map(crush, osdmap.get_max_osd(), osd_weight);
-	crush.encode(pending_inc.crush);
-      }
-      return true;
-    } else 
-      return false;
+
+  // adjust osd weights?
+  if (osd_weight.size() == (unsigned)osdmap.get_max_osd()) {
+    dout(0) << " adjusting crush osd weights based on " << osd_weight << dendl;
+    bufferlist bl;
+    osdmap.crush.encode(bl);
+    CrushWrapper crush;
+    bufferlist::iterator p = bl.begin();
+    crush.decode(p);
+    crush.adjust_osd_weights(osd_weight);
+    crush.encode(pending_inc.crush);
+    delay = 0.0;
+    osd_weight.clear();
+    return true;
   }
+
   return PaxosService::should_propose(delay);
 }
 
@@ -518,8 +522,9 @@ bool OSDMonitor::prepare_boot(MOSDBoot *m)
     
     // mark in?
     pending_inc.new_offload[from] = CEPH_OSD_IN;
-    
-    osd_weight[from] = m->sb.weight;
+
+    if (m->sb.weight)
+      osd_weight[from] = m->sb.weight;
 
     // wait
     paxos->wait_for_commit(new C_Booted(this, m));
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 02f8e8405e41c..1016e417329e9 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -141,6 +141,8 @@ ObjectStore *OSD::create_object_store(const char *dev)
 int OSD::mkfs(const char *dev, ceph_fsid fsid, int whoami)
 {
   ObjectStore *store = create_object_store(dev);
+  if (!store)
+    return -ENOENT;
   int err = store->mkfs();    
   if (err < 0) return err;
   err = store->mount();
@@ -172,6 +174,7 @@ int OSD::mkfs(const char *dev, ceph_fsid fsid, int whoami)
     bufferptr bp(1048576);
     bp.zero();
     bl.push_back(bp);
+    cout << "testing disk bandwidth..." << std::endl;
     utime_t start = g_clock.now();
     for (int i=0; i<1000; i++) 
       store->write(0, pobject_t(0, 0, object_t(999,i)), 0, bl.length(), bl, 0);
diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index e08f9199775a5..89248f85e1273 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -32,8 +32,7 @@ void OSDMap::build_simple(epoch_t e, ceph_fsid &fsid,
   pg_num = pgp_num = num_osd << pg_bits;
 
   // crush map
-  map<int,double> weights;
-  build_simple_crush_map(crush, num_osd, weights);
+  build_simple_crush_map(crush, num_osd);
 
   for (int i=0; i<num_osd; i++) {
     set_state(i, CEPH_OSD_EXISTS|CEPH_OSD_CLEAN);
@@ -52,7 +51,7 @@ void OSDMap::build_simple(epoch_t e, ceph_fsid &fsid,
   }
 }
 
-void OSDMap::build_simple_crush_map(CrushWrapper& crush, int num_osd, map<int,double>& weights)
+void OSDMap::build_simple_crush_map(CrushWrapper& crush, int num_osd)
 {
   // new
   crush.create();
@@ -165,3 +164,4 @@ void OSDMap::build_simple_crush_map(CrushWrapper& crush, int num_osd, map<int,do
 
   dout(20) << "crush max_devices " << crush.crush->max_devices << dendl;
 }
+
diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index 4e2ec8fc6f0bd..71a6b1e45976f 100644
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -357,11 +357,6 @@ private:
     if (inc.fullmap.length())
       return;
 
-    if (inc.crush.length()) {
-      bufferlist::iterator blp = inc.crush.begin();
-      crush.decode(blp);
-    }
-
     // nope, incremental.
     if (inc.new_flags >= 0)
       flags = inc.new_flags;
@@ -420,6 +415,12 @@ private:
 	 i != inc.old_pg_swap_primary.end();
 	 i++)
       pg_swap_primary.erase(*i);
+
+    // do new crush map last (after up/down stuff)
+    if (inc.crush.length()) {
+      bufferlist::iterator blp = inc.crush.begin();
+      crush.decode(blp);
+    }
   }
 
   // serialize, unserialize
@@ -715,7 +716,7 @@ private:
    */
   void build_simple(epoch_t e, ceph_fsid &fsid,
 		    int num_osd, int pg_bits, int mds_local_osd);
-  static void build_simple_crush_map(CrushWrapper& crush, int num_osd, map<int,double>& weights);
+  static void build_simple_crush_map(CrushWrapper& crush, int num_osd);
 
 };
 
-- 
2.39.5