osd: replay cleanup, send_alive fix

diff --git a/src/config.cc b/src/config.cc
index 089769bf3f847fc1c4b45b46b778b02ef6443819..84181a607b5fa69955825673d0d08d358d3c9112 100644 (file)
--- a/src/config.cc
+++ b/src/config.cc
@@ -384,7 +384,7 @@ md_config_t g_conf = {
   osd_heartbeat_interval: 1,
   osd_heartbeat_grace: 30,
   osd_mon_report_interval:  5,  // pg stats, failures, up_thru, boot.
-  osd_replay_window: 5,
+  osd_replay_window: 45,
   osd_max_pull: 2,
   osd_pad_pg_log: false,
 

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index ba2e5dc9f0eee47ff7234c323640ca1df81ecc31..e8ef8b8fbe681ef5ffe0b7a0f6708ab80896c46e 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1004,12 +1004,20 @@ void OSD::send_boot()
 
 void OSD::queue_want_up_thru(epoch_t want)
 {
+  epoch_t cur = osdmap->get_up_thru(whoami);
   if (want > up_thru_wanted) {
+    dout(10) << "queue_want_up_thru now " << want << " (was " << up_thru_wanted << ")" 
+            << ", currently " << cur
+            << dendl;
     up_thru_wanted = want;
 
     // expedite, a bit.  WARNING this will somewhat delay other mon queries.
     last_mon_report = g_clock.now();
     send_alive();
+  } else {
+    dout(10) << "queue_want_up_thru want " << want << " <= queued " << up_thru_wanted 
+            << ", currently " << cur
+            << dendl;
   }
 }
 
@@ -1018,7 +1026,8 @@ void OSD::send_alive()
   if (!osdmap->exists(whoami))
     return;
   epoch_t up_thru = osdmap->get_up_thru(whoami);
-  if (up_thru_wanted < up_thru) {
+  dout(10) << "send_alive up_thru currently " << up_thru << " want " << up_thru_wanted << dendl;
+  if (up_thru_wanted > up_thru) {
     up_thru_pending = up_thru_wanted;
     int mon = monmap->pick_mon();
     dout(10) << "send_alive to mon" << mon << " (want " << up_thru_wanted << ")" << dendl;
@@ -1520,7 +1529,7 @@ void OSD::handle_osd_map(MOSDMap *m)
        osdmap->get_addr(whoami) == messenger->get_myaddr()) {
       // yay!
       activate_map(t);
-    
+
       // process waiters
       take_waiters(waiting_for_osdmap);
     }
@@ -1726,20 +1735,19 @@ void OSD::activate_map(ObjectStore::Transaction& t)
   for (hash_map<pg_t,PG*>::iterator it = pg_map.begin();
        it != pg_map.end();
        it++) {
-    //pg_t pgid = it->first;
     PG *pg = it->second;
     pg->lock();
     if (pg->is_active()) {
       // update started counter
       pg->info.history.last_epoch_started = osdmap->get_epoch();
-    } 
+    }
     else if (pg->get_role() == 0 && !pg->is_active()) {
       // i am (inactive) primary
       pg->build_prior();
       pg->peer(t, query_map, &info_map);
     }
     else if (pg->is_stray() &&
-             pg->get_primary() >= 0) {
+            pg->get_primary() >= 0) {
       // i am residual|replica
       notify_list[pg->get_primary()].push_back(pg->info);
     }
@@ -1748,6 +1756,8 @@ void OSD::activate_map(ObjectStore::Transaction& t)
     pg->unlock();
   }  
 
+  last_active_epoch = osdmap->get_epoch();
+
   do_notifies(notify_list);  // notify? (residual|replica)
   do_queries(query_map);
   do_infos(info_map);

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 5111f7806aa1bc7868c8fba8ab4af672e43229c1..346b49f3cb313533f6294d794cb3f56c45b62507 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -79,7 +79,8 @@ public:
 private:
   /** superblock **/
   OSDSuperblock superblock;
-  epoch_t  boot_epoch;      
+  epoch_t boot_epoch;      
+  epoch_t last_active_epoch;
 
   void write_superblock();
   void write_superblock(ObjectStore::Transaction& t);

diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index 35e18984695b2c2647f998a3001d89c6e71c9e27..f28f5ac392bb73664e73526667d1d7645e9b9b99 100644 (file)
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -273,6 +273,10 @@ private:
     return n;
   }
 
+  int get_state(int o) {
+    assert(o < max_osd);
+    return osd_state[o];
+  }
   void set_state(int o, unsigned s) {
     assert(o < max_osd);
     osd_state[o] = s;
@@ -328,19 +332,6 @@ private:
     return -1;
   }
 
-  void mark_down(int o, bool clean) { 
-    osd_state[o] &= ~CEPH_OSD_UP;
-  }
-  void mark_up(int o) { 
-    osd_state[o] |= CEPH_OSD_UP;
-  }
-  void mark_out(int o) { 
-    set_offload(o, CEPH_OSD_OUT);
-  }
-  void mark_in(int o) { 
-    set_offload(o, CEPH_OSD_IN);
-  }
-
   void apply_incremental(Incremental &inc) {
     if (inc.epoch == 1)
       fsid = inc.fsid;

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index fc2d331e7d13888a590d10b0ffe2ac54495614f2..95d3c225b5a91a77c492057a01b2499ac856c6a4 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -614,9 +614,12 @@ void PG::build_prior()
     int num_still_up_or_clean = 0;
     for (unsigned i=0; i<acting.size(); i++) {
       if (osd->osdmap->is_up(acting[i])) {  // is up now
-       num_still_up_or_clean++;
        if (acting[i] != osd->whoami)       // and is not me
          prior_set.insert(acting[i]);
+
+       // has it been up this whole time?
+       if (osd->osdmap->get_up_from(acting[i]) <= first_epoch)
+         num_still_up_or_clean++;
       } else {
        dout(10) << "build_prior  prior osd" << acting[i] << " is down, must notify mon" << dendl;
        must_notify_mon = true;

diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index 0043269dd83609858364c45881ccceb774a9ea3b..2a94f9a03e0643062131f20c4bb459a8734a202e 100644 (file)
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -918,11 +918,14 @@ void Objecter::handle_osd_modify_reply(MOSDOpReply *m)
     num_unacked--;
     dout(15) << "handle_osd_modify_reply ack" << dendl;
     
+    /*
+      osd uses v to reorder during replay, but doesn't preserve it
     if (wr->tid_version.count(tid) &&
        wr->tid_version[tid].version != m->get_version().version) {
       dout(-10) << "handle_osd_modify_reply WARNING: replay of tid " << tid 
                << " did not achieve previous ordering" << dendl;
     }
+    */
     wr->tid_version[tid] = m->get_version();
     
     if (wr->waitfor_ack.empty()) {
@@ -942,8 +945,11 @@ void Objecter::handle_osd_modify_reply(MOSDOpReply *m)
   }
   if (m->is_safe()) {
     // safe
+    /*
+      osd uses v to reorder during replay, but doesn't preserve it
     assert(wr->tid_version.count(tid) == 0 ||
            m->get_version() == wr->tid_version[tid]);
+    */
 
     wr->waitfor_commit.erase(tid);
     num_uncommitted--;

diff --git a/src/psim.cc b/src/psim.cc
index dd0e78a34a789d8dd0210071667bb402481a6b94..30d12585c72723e15d0b7cb2acce86c9924c99f5 100644 (file)
--- a/src/psim.cc
+++ b/src/psim.cc
@@ -20,8 +20,8 @@ int main()
   int n = osdmap.get_max_osd();
   int count[n];
   for (int i=0; i<n; i++) {
-    osdmap.mark_up(i);
-    osdmap.mark_in(i);
+    osdmap.set_state(i, osdmap.get_state(i) | CEPH_OSD_UP);
+    osdmap.set_offload(i, CEPH_OSD_IN);
     count[i] = 0;
   }
 

diff --git a/src/vstartnew.sh b/src/vstartnew.sh
index 626ec6eff4486a36a62f113f6849774316631b2e..dbd119b56519492924f566d107a6061105d079d7 100755 (executable)
--- a/src/vstartnew.sh
+++ b/src/vstartnew.sh
@@ -39,7 +39,8 @@ $CEPH_BIN/cmonctl osd setmap -i .ceph_osdmap
 for osd in 0 1 2 3 #4 5 6 7
 do
  $CEPH_BIN/cosd --mkfs_for_osd $osd dev/osd$osd  # initialize empty object store
- $CEPH_BIN/cosd $ARGS dev/osd$osd --debug_ms 1 --debug_osd 20 --debug_fakestore 10 #--debug_osd 40
+ #valgrind --tool=massif $CEPH_BIN/cosd dev/osd$osd --debug_ms 1 --debug_osd 20 --debug_fakestore 10 1>out/o$osd & #--debug_osd 40
+ $CEPH_BIN/cosd dev/osd$osd -d --debug_ms 1 --debug_osd 20 --debug_fakestore 10
 done
 
 # mds