From 95ebd79774dd4c59bd4d52e3e77a053c6c7fc48c Mon Sep 17 00:00:00 2001 From: sageweil Date: Wed, 24 Jan 2007 03:14:37 +0000 Subject: [PATCH] more journaling work. initial bits of MDS states and revised startup process. git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@1030 29311d96-e01e-0410-9327-a35deaab8ce9 --- branches/sage/cephmds2/TODO | 4 +- branches/sage/cephmds2/fakesyn.cc | 12 ++ branches/sage/cephmds2/mds/CDir.cc | 16 ++- branches/sage/cephmds2/mds/CDir.h | 3 +- branches/sage/cephmds2/mds/LogEvent.h | 5 +- branches/sage/cephmds2/mds/MDBalancer.cc | 2 +- branches/sage/cephmds2/mds/MDCache.cc | 44 ++++--- branches/sage/cephmds2/mds/MDCache.h | 1 + branches/sage/cephmds2/mds/MDLog.cc | 12 +- branches/sage/cephmds2/mds/MDS.cc | 26 +++-- branches/sage/cephmds2/mds/MDS.h | 21 ++-- branches/sage/cephmds2/mds/MDSMap.h | 57 ++++++--- branches/sage/cephmds2/mds/Migrator.cc | 15 ++- branches/sage/cephmds2/mds/Migrator.h | 8 +- branches/sage/cephmds2/mds/Server.cc | 14 +-- branches/sage/cephmds2/mds/events/EAlloc.h | 41 +------ .../sage/cephmds2/mds/events/EExportFinish.h | 13 ++- .../sage/cephmds2/mds/events/EExportStart.h | 8 +- .../sage/cephmds2/mds/events/EImportFinish.h | 13 ++- .../sage/cephmds2/mds/events/EImportMap.h | 20 +++- branches/sage/cephmds2/mds/events/EMetaBlob.h | 94 +++++++++++---- branches/sage/cephmds2/mds/journal.cc | 109 +++++++++++++++--- branches/sage/cephmds2/mon/MDSMonitor.cc | 15 ++- branches/sage/cephmds2/mon/OSDMonitor.cc | 4 +- 24 files changed, 377 insertions(+), 180 deletions(-) diff --git a/branches/sage/cephmds2/TODO b/branches/sage/cephmds2/TODO index c168cb8547245..0700bfeb0cb83 100644 --- a/branches/sage/cephmds2/TODO +++ b/branches/sage/cephmds2/TODO @@ -1,7 +1,9 @@ blech: -- importmap cleanup on log flush/shutdown - metablob shoudl mark explicitly dirtied items, and return 'expired' if they have higher versions (and are thus described by a newer journal entry) +- mds failure detection, failure decision by monitor. + - keepalive beacons! + mds - journal+recovery diff --git a/branches/sage/cephmds2/fakesyn.cc b/branches/sage/cephmds2/fakesyn.cc index 312ad8b345ef8..0968d79440e8e 100644 --- a/branches/sage/cephmds2/fakesyn.cc +++ b/branches/sage/cephmds2/fakesyn.cc @@ -42,6 +42,14 @@ public: } }; +class C_Die : public Context { +public: + void finish(int) { + cerr << "die" << endl; + exit(1); + } +}; + int main(int argc, char **argv) { @@ -68,6 +76,10 @@ int main(int argc, char **argv) assert(nargs.empty()); + if (g_conf.kill_after) + g_timer.add_event_after(g_conf.kill_after, new C_Die); + + g_clock.tare(); MonMap *monmap = new MonMap(g_conf.num_mon); diff --git a/branches/sage/cephmds2/mds/CDir.cc b/branches/sage/cephmds2/mds/CDir.cc index 436a56710f340..dfabfbc7c5ebf 100644 --- a/branches/sage/cephmds2/mds/CDir.cc +++ b/branches/sage/cephmds2/mds/CDir.cc @@ -487,20 +487,24 @@ version_t CDir::pre_dirty() return projected_version; } -void CDir::mark_dirty(version_t pv) +void CDir::_mark_dirty() { - ++version; - assert(pv == version); - if (!state_test(CDIR_STATE_DIRTY)) { state_set(CDIR_STATE_DIRTY); - dout(10) << "mark_dirty (was clean) " << *this << " new version " << version << endl; + dout(10) << "mark_dirty (was clean) " << *this << " version " << version << endl; get(PIN_DIRTY); } else { - dout(10) << "mark_dirty (already dirty) " << *this << " new version " << version << endl; + dout(10) << "mark_dirty (already dirty) " << *this << " version " << version << endl; } } +void CDir::mark_dirty(version_t pv) +{ + ++version; + assert(pv == version); + _mark_dirty(); +} + void CDir::mark_clean() { dout(10) << "mark_clean " << *this << " version " << version << endl; diff --git a/branches/sage/cephmds2/mds/CDir.h b/branches/sage/cephmds2/mds/CDir.h index e84a8cca4ca8c..aa9114f764bc1 100644 --- a/branches/sage/cephmds2/mds/CDir.h +++ b/branches/sage/cephmds2/mds/CDir.h @@ -358,7 +358,7 @@ class CDir : public MDSCacheObject { // -- dirtyness -- version_t get_version() { return version; } - void set_version(version_t v) { version = v; } + void set_version(version_t v) { projected_version = version = v; } version_t get_projected_version() { return projected_version; } version_t get_committing_version() { return committing_version; } @@ -368,6 +368,7 @@ class CDir : public MDSCacheObject { void set_last_committed_version(version_t v) { last_committed_version = v; } version_t pre_dirty(); + void _mark_dirty(); void mark_dirty(version_t pv); void mark_clean(); void mark_complete() { state_set(CDIR_STATE_COMPLETE); } diff --git a/branches/sage/cephmds2/mds/LogEvent.h b/branches/sage/cephmds2/mds/LogEvent.h index f0120c9102596..6895ed54074d4 100644 --- a/branches/sage/cephmds2/mds/LogEvent.h +++ b/branches/sage/cephmds2/mds/LogEvent.h @@ -50,14 +50,15 @@ class MDS; class LogEvent { private: int _type; - off_t _end_off; + off_t _start_off,_end_off; friend class MDLog; public: - LogEvent(int t) : _type(t), _end_off(0) { } + LogEvent(int t) : _type(t), _start_off(0), _end_off(0) { } virtual ~LogEvent() { } int get_type() { return _type; } + off_t get_start_off() { return _start_off; } off_t get_end_off() { return _end_off; } // encoding diff --git a/branches/sage/cephmds2/mds/MDBalancer.cc b/branches/sage/cephmds2/mds/MDBalancer.cc index 97d649b73557c..85072a629c89d 100644 --- a/branches/sage/cephmds2/mds/MDBalancer.cc +++ b/branches/sage/cephmds2/mds/MDBalancer.cc @@ -73,7 +73,7 @@ void MDBalancer::tick() (num_bal_times || (g_conf.mds_bal_max_until >= 0 && elapsed.sec() > g_conf.mds_bal_max_until)) && - !mds->is_stopping() && !mds->is_stopped() && + mds->is_active() && now.sec() - last_heartbeat.sec() >= g_conf.mds_bal_interval) { last_heartbeat = now; send_heartbeat(); diff --git a/branches/sage/cephmds2/mds/MDCache.cc b/branches/sage/cephmds2/mds/MDCache.cc index c5f6f44c9d237..e9a64b2904a32 100644 --- a/branches/sage/cephmds2/mds/MDCache.cc +++ b/branches/sage/cephmds2/mds/MDCache.cc @@ -213,6 +213,7 @@ void MDCache::log_import_map(Context *onsync) CDir *im = *p; le->imports.insert(im->ino()); le->metablob.add_dir_context(im, true); + le->metablob.add_dir(im, false); if (nested_exports.count(im)) { for (set::iterator q = nested_exports[im].begin(); @@ -222,6 +223,7 @@ void MDCache::log_import_map(Context *onsync) le->nested_exports[im->ino()].insert(ex->ino()); le->exports.insert(ex->ino()); le->metablob.add_dir_context(ex); + le->metablob.add_dir(ex, false); } } } @@ -502,7 +504,7 @@ bool MDCache::shutdown_pass() { dout(7) << "shutdown_pass" << endl; //assert(mds->is_shutting_down()); - if (mds->is_stopped()) { + if (mds->is_down()) { dout(7) << " already shut down" << endl; show_cache(); show_imports(); @@ -669,6 +671,27 @@ bool MDCache::shutdown_pass() +CInode *MDCache::create_root_inode() +{ + CInode *root = new CInode(this); + memset(&root->inode, 0, sizeof(inode_t)); + root->inode.ino = 1; + root->inode.hash_seed = 0; // not hashed! + + // make it up (FIXME) + root->inode.mode = 0755 | INODE_MODE_DIR; + root->inode.size = 0; + root->inode.ctime = 0; + root->inode.mtime = g_clock.gettime(); + + root->inode.nlink = 1; + root->inode.layout = g_OSD_MDDirLayout; + + set_root( root ); + add_inode( root ); + + return root; +} int MDCache::open_root(Context *c) @@ -678,22 +701,7 @@ int MDCache::open_root(Context *c) // open root inode if (whoami == 0) { // i am root inode - CInode *root = new CInode(this); - memset(&root->inode, 0, sizeof(inode_t)); - root->inode.ino = 1; - root->inode.hash_seed = 0; // not hashed! - - // make it up (FIXME) - root->inode.mode = 0755 | INODE_MODE_DIR; - root->inode.size = 0; - root->inode.ctime = 0; - root->inode.mtime = g_clock.gettime(); - - root->inode.nlink = 1; - root->inode.layout = g_OSD_MDDirLayout; - - set_root( root ); - add_inode( root ); + CInode *root = create_root_inode(); // root directory too assert(root->dir == NULL); @@ -2236,7 +2244,7 @@ int MDCache::send_dir_updates(CDir *dir, bool bcast) set who; if (bcast) { - who = mds->get_mds_map()->get_mds(); + who = mds->get_mds_map()->get_mds_set(); } else { for (map::iterator p = dir->replicas_begin(); p != dir->replicas_end(); diff --git a/branches/sage/cephmds2/mds/MDCache.h b/branches/sage/cephmds2/mds/MDCache.h index 7ff277337f72b..fd3c41c7d09c0 100644 --- a/branches/sage/cephmds2/mds/MDCache.h +++ b/branches/sage/cephmds2/mds/MDCache.h @@ -212,6 +212,7 @@ class MDCache { public: + CInode *create_root_inode(); int open_root(Context *c); int path_traverse(filepath& path, vector& trace, bool follow_trailing_sym, Message *req, Context *ondelay, diff --git a/branches/sage/cephmds2/mds/MDLog.cc b/branches/sage/cephmds2/mds/MDLog.cc index 4b8bd393b3e2b..7708689b3f2d7 100644 --- a/branches/sage/cephmds2/mds/MDLog.cc +++ b/branches/sage/cephmds2/mds/MDLog.cc @@ -120,6 +120,8 @@ void MDLog::submit_entry( LogEvent *le, Context *c ) { if (g_conf.mds_log) { + dout(5) << "submit_entry " << journaler->get_write_pos() << " : " << *le << endl; + // encode it, with event type bufferlist bl; bl.append((char*)&le->_type, sizeof(le->_type)); @@ -128,8 +130,6 @@ void MDLog::submit_entry( LogEvent *le, // journal it. journaler->append_entry(bl); - dout(5) << "submit_entry " << journaler->get_write_pos() << " : " << *le << endl; - assert(!capped); delete le; @@ -224,7 +224,7 @@ void MDLog::_did_read() void MDLog::_trimmed(LogEvent *le) { - dout(7) << "trimmed : " << le->get_end_off() << " : " << *le << endl; + dout(7) << "trimmed : " << le->get_start_off() << " : " << *le << endl; assert(le->has_expired(mds)); if (trimming.begin()->first == le->_end_off) { @@ -267,23 +267,25 @@ void MDLog::trim(Context *c) } bufferlist bl; + off_t so = journaler->get_read_pos(); if (journaler->try_read_entry(bl)) { // decode logevent LogEvent *le = LogEvent::decode(bl); + le->_start_off = so; le->_end_off = journaler->get_read_pos(); num_events--; // we just read an event. if (le->has_expired(mds)) { // obsolete - dout(7) << "trim obsolete : " << le->get_end_off() << " : " << *le << endl; + dout(7) << "trim obsolete : " << le->get_start_off() << " : " << *le << endl; delete le; logger->inc("obs"); } else { assert ((int)trimming.size() < g_conf.mds_log_max_trimming); // trim! - dout(7) << "trim expiring : " << le->get_end_off() << " : " << *le << endl; + dout(7) << "trim expiring : " << le->get_start_off() << " : " << *le << endl; trimming[le->_end_off] = le; le->expire(mds, new C_MDL_Trimmed(this, le)); logger->inc("expire"); diff --git a/branches/sage/cephmds2/mds/MDS.cc b/branches/sage/cephmds2/mds/MDS.cc index c0648b6259b45..fec7267cd077e 100644 --- a/branches/sage/cephmds2/mds/MDS.cc +++ b/branches/sage/cephmds2/mds/MDS.cc @@ -107,7 +107,7 @@ MDS::MDS(int whoami, Messenger *m, MonMap *mm) { req_rate = 0; - state = STATE_BOOTING; + state = MDSMap::STATE_DOWN; // booting // log string name; @@ -381,10 +381,11 @@ public: void MDS::boot_recover(int step) { if (is_booting()) - state = STATE_RECOVERING; + state = MDSMap::STATE_STARTING; switch (step) { case 0: + /* no, EImportMap takes care of all this. if (whoami == 0) { dout(2) << "boot_recover " << step << ": creating root inode" << endl; mdcache->open_root(0); @@ -393,7 +394,8 @@ void MDS::boot_recover(int step) } else { // FIXME assert(0); - } + }*/ + step = 1; case 1: dout(2) << "boot_recover " << step << ": opening idalloc" << endl; @@ -438,7 +440,7 @@ void MDS::boot_recover(int step) void MDS::mark_active() { dout(3) << "mark_active" << endl; - state = STATE_ACTIVE; + state = MDSMap::STATE_ACTIVE; finish_contexts(waitfor_active); // kick waiters } @@ -451,12 +453,14 @@ int MDS::shutdown_start() dout(1) << "shutdown_start" << endl; derr(0) << "mds shutdown start" << endl; - for (set::iterator p = mdsmap->get_mds().begin(); - p != mdsmap->get_mds().end(); + for (set::iterator p = mdsmap->get_mds_set().begin(); + p != mdsmap->get_mds_set().end(); p++) { - dout(1) << "sending MShutdownStart to mds" << *p << endl; - send_message_mds(new MGenericMessage(MSG_MDS_SHUTDOWNSTART), - *p, MDS_PORT_MAIN); + if (mdsmap->is_starting(*p) || mdsmap->is_active(*p)) { + dout(1) << "sending MShutdownStart to mds" << *p << endl; + send_message_mds(new MGenericMessage(MSG_MDS_SHUTDOWNSTART), + *p, MDS_PORT_MAIN); + } } if (idalloc) idalloc->shutdown(); @@ -471,7 +475,7 @@ void MDS::handle_shutdown_start(Message *m) dout(1) << " handle_shutdown_start" << endl; // set flag - state = STATE_STOPPING; + state = MDSMap::STATE_STOPPING; mdcache->shutdown_start(); @@ -494,7 +498,7 @@ int MDS::shutdown_final() { dout(1) << "shutdown" << endl; - state = STATE_STOPPED; + state = MDSMap::STATE_DOWN; // shut down cache mdcache->shutdown(); diff --git a/branches/sage/cephmds2/mds/MDS.h b/branches/sage/cephmds2/mds/MDS.h index 27ef0eb847874..dc2d7213440a6 100644 --- a/branches/sage/cephmds2/mds/MDS.h +++ b/branches/sage/cephmds2/mds/MDS.h @@ -34,6 +34,7 @@ using namespace __gnu_cxx; #include "common/Mutex.h" #include "mon/MonMap.h" +#include "MDSMap.h" #include "ClientMap.h" @@ -67,7 +68,6 @@ using namespace __gnu_cxx; class filepath; -class MDSMap; class OSDMap; class Objecter; class Filer; @@ -133,24 +133,19 @@ class MDS : public Dispatcher { protected: // -- MDS state -- - static const int STATE_BOOTING = 1; // fetching mds and osd maps - static const int STATE_MKFS = 2; // creating a file system - static const int STATE_RECOVERING = 3; // recovering mds log - static const int STATE_ACTIVE = 4; // up and active! - static const int STATE_STOPPING = 5; - static const int STATE_STOPPED = 6; - int state; list waitfor_active; public: void queue_waitfor_active(Context *c) { waitfor_active.push_back(c); } - bool is_booting() { return state == STATE_BOOTING; } - bool is_recovering() { return state == STATE_RECOVERING; } - bool is_active() { return state == STATE_ACTIVE; } - bool is_stopping() { return state == STATE_STOPPING; } - bool is_stopped() { return state == STATE_STOPPED; } + bool is_down() { return state == MDSMap::STATE_DOWN; } + bool is_booting() { return state == MDSMap::STATE_DOWN; } + bool is_creating() { return state == MDSMap::STATE_CREATING; } + bool is_standby() { return state == MDSMap::STATE_STANDBY; } + bool is_starting() { return state == MDSMap::STATE_STARTING; } + bool is_active() { return state == MDSMap::STATE_ACTIVE; } + bool is_stopping() { return state == MDSMap::STATE_STOPPING; } void mark_active(); diff --git a/branches/sage/cephmds2/mds/MDSMap.h b/branches/sage/cephmds2/mds/MDSMap.h index 6117e6943d3c7..a02d67eda9f77 100644 --- a/branches/sage/cephmds2/mds/MDSMap.h +++ b/branches/sage/cephmds2/mds/MDSMap.h @@ -26,20 +26,31 @@ using namespace std; class MDSMap { + public: + static const int STATE_OUT = 0; // offline, no metadata. need not exist in map at all. + static const int STATE_DOWN = 1; // offline, holds (er, held) metadata; needs to be recovered. + + static const int STATE_STANDBY = 2; // online, but inactive; waiting for someone to fail. + static const int STATE_CREATING = 3; // online, creating MDS instance (initializing journal, etc.) + static const int STATE_STARTING = 4; // online, scanning journal, recoverying any shared state + static const int STATE_ACTIVE = 5; // online, active + static const int STATE_STOPPING = 6; // online, exporting metadata (-> standby or out) + protected: epoch_t epoch; utime_t ctime; - int anchortable; + int anchortable; // which MDS has anchortable (fixme someday) + int root; // which MDS has root directory - set all_mds; - set down_mds; - map mds_inst; + set mds_set; // set of MDSs + map mds_state; // MDS state + map mds_inst; // online (non-out, non-down) instances friend class MDSMonitor; public: - MDSMap() : epoch(0), anchortable(0) {} + MDSMap() : epoch(0), anchortable(0), root(0) {} epoch_t get_epoch() const { return epoch; } void inc_epoch() { epoch++; } @@ -47,16 +58,20 @@ class MDSMap { const utime_t& get_ctime() const { return ctime; } int get_anchortable() const { return anchortable; } + int get_root() const { return root; } - int get_num_mds() const { return all_mds.size(); } - int get_num_up_mds() const { return all_mds.size() - down_mds.size(); } + int get_num_mds() const { return mds_set.size(); } - const set& get_mds() const { return all_mds; } - const set& get_down_mds() const { return down_mds; } - - bool is_down(int m) const { return down_mds.count(m); } - bool is_up(int m) const { return !is_down(m); } + const set& get_mds_set() const { return mds_set; } + bool is_out(int m) { return mds_state.count(m) == 0 && mds_state[m] == STATE_OUT; } + bool is_down(int m) { return mds_state.count(m) && mds_state[m] == STATE_DOWN; } + bool is_standby(int m) { return mds_state.count(m) && mds_state[m] == STATE_STANDBY; } + bool is_creating(int m) { return mds_state.count(m) && mds_state[m] == STATE_CREATING; } + bool is_starting(int m) { return mds_state.count(m) && mds_state[m] == STATE_STARTING; } + bool is_active(int m) { return mds_state.count(m) && mds_state[m] == STATE_ACTIVE; } + bool is_stopping(int m) { return mds_state.count(m) && mds_state[m] == STATE_STOPPING; } + const entity_inst_t& get_inst(int m) { assert(mds_inst.count(m)); return mds_inst[m]; @@ -69,14 +84,22 @@ class MDSMap { return false; } + void remove_mds(int m) { + mds_set.erase(m); + mds_inst.erase(m); + mds_state.erase(m); + } + + // serialize, unserialize void encode(bufferlist& blist) { blist.append((char*)&epoch, sizeof(epoch)); blist.append((char*)&ctime, sizeof(ctime)); blist.append((char*)&anchortable, sizeof(anchortable)); + blist.append((char*)&root, sizeof(root)); - _encode(all_mds, blist); - _encode(down_mds, blist); + _encode(mds_set, blist); + _encode(mds_state, blist); _encode(mds_inst, blist); } @@ -88,9 +111,11 @@ class MDSMap { off += sizeof(ctime); blist.copy(off, sizeof(anchortable), (char*)&anchortable); off += sizeof(anchortable); + blist.copy(off, sizeof(root), (char*)&root); + off += sizeof(root); - _decode(all_mds, blist, off); - _decode(down_mds, blist, off); + _decode(mds_set, blist, off); + _decode(mds_state, blist, off); _decode(mds_inst, blist, off); } diff --git a/branches/sage/cephmds2/mds/Migrator.cc b/branches/sage/cephmds2/mds/Migrator.cc index 6a60121eb7b08..96adbe8c6fb37 100644 --- a/branches/sage/cephmds2/mds/Migrator.cc +++ b/branches/sage/cephmds2/mds/Migrator.cc @@ -267,6 +267,8 @@ void Migrator::export_dir(CDir *dir, // ok, let's go. + exporting.insert(dir); + // send ExportDirDiscover (ask target) mds->send_message_mds(new MExportDirDiscover(dir->inode), dest, MDS_PORT_MIGRATOR); dir->auth_pin(); // pin dir, to hang up our freeze (unpin on prep ack) @@ -764,7 +766,7 @@ void Migrator::handle_export_dir_notify_ack(MExportDirNotifyAck *m) << ", last one!" << endl; // log export completion, then finish (unfreeze, trigger finish context, etc.) - mds->mdlog->submit_entry(new EExportFinish(dir), + mds->mdlog->submit_entry(new EExportFinish(dir, true), new C_MDS_ExportFinishLogged(this, dir)); } else { @@ -784,6 +786,9 @@ void Migrator::export_dir_finish(CDir *dir) // send finish/commit to new auth mds->send_message_mds(new MExportDirFinish(dir->ino()), dir->authority(), MDS_PORT_MIGRATOR); + // remove from exporting list + exporting.erase(dir); + // unfreeze dout(7) << "export_dir_finish " << *dir << ", unfreezing" << endl; dir->unfreeze_tree(); @@ -1323,7 +1328,7 @@ void Migrator::handle_export_dir_finish(MExportDirFinish *m) assert(dir->is_auth()); // log - mds->mdlog->submit_entry(new EImportFinish(dir), + mds->mdlog->submit_entry(new EImportFinish(dir, true), new C_MDS_ImportDirLoggedFinish(this,dir)); delete m; } @@ -1479,7 +1484,7 @@ int Migrator::import_dir_block(bufferlist& bl, dir->remove_replica(mds->get_nodeid()); // add to journal entry - le->metablob.add_dir(dir); + le->metablob.add_dir(dir, true); // Hmm: false would be okay in some cases if (dir->is_hashed()) { @@ -1554,7 +1559,7 @@ int Migrator::import_dir_block(bufferlist& bl, } // add dentry to journal entry - le->metablob.add_dentry(dn); + le->metablob.add_dentry(dn, true); // Hmm: might we do dn->is_dirty() here instead? } return num_imported; @@ -3186,7 +3191,7 @@ void Migrator::handle_unhash_dir(MUnhashDir *m) } // init gather set - hash_gather[dir] = mds->get_mds_map()->get_mds(); + hash_gather[dir] = mds->get_mds_map()->get_mds_set(); // fixme hash_gather[dir].erase(mds->get_nodeid()); // send unhash message diff --git a/branches/sage/cephmds2/mds/Migrator.h b/branches/sage/cephmds2/mds/Migrator.h index 613e7fd1ed32d..09925c667b7fa 100644 --- a/branches/sage/cephmds2/mds/Migrator.h +++ b/branches/sage/cephmds2/mds/Migrator.h @@ -62,6 +62,7 @@ private: MDCache *cache; // export fun + set exporting; map > export_notify_ack_waiting; // nodes i am waiting to get export_notify_ack's from map > export_proxy_inos; map > export_proxy_dirinos; @@ -87,8 +88,11 @@ public: void dispatch(Message*); //bool is_importing(); - bool is_exporting() { - return !export_notify_ack_waiting.empty() || !export_finish_waiters.empty(); + bool is_exporting(CDir *dir = 0) { + if (dir) + return exporting.count(dir); + else + return !exporting.empty(); } // -- import/export -- diff --git a/branches/sage/cephmds2/mds/Server.cc b/branches/sage/cephmds2/mds/Server.cc index 12676012a83aa..68fd1187aa89a 100644 --- a/branches/sage/cephmds2/mds/Server.cc +++ b/branches/sage/cephmds2/mds/Server.cc @@ -617,7 +617,7 @@ void Server::handle_client_utime(MClientRequest *req, // log + wait EUpdate *le = new EUpdate("utime"); le->metablob.add_dir_context(cur->get_parent_dir()); - inode_t *pi = le->metablob.add_dentry(cur->parent); + inode_t *pi = le->metablob.add_dentry(cur->parent, true); pi->mtime = mtime; pi->atime = mtime; pi->version = pdv; @@ -680,7 +680,7 @@ void Server::handle_client_chmod(MClientRequest *req, // log + wait EUpdate *le = new EUpdate("chmod"); le->metablob.add_dir_context(cur->get_parent_dir()); - inode_t *pi = le->metablob.add_dentry(cur->parent); + inode_t *pi = le->metablob.add_dentry(cur->parent, true); pi->mode = mode; pi->version = pdv; @@ -738,7 +738,7 @@ void Server::handle_client_chown(MClientRequest *req, // log + wait EUpdate *le = new EUpdate("chown"); le->metablob.add_dir_context(cur->get_parent_dir()); - inode_t *pi = le->metablob.add_dentry(cur->parent); + inode_t *pi = le->metablob.add_dentry(cur->parent, true); if (uid >= 0) pi->uid = uid; if (gid >= 0) pi->gid = gid; pi->version = pdv; @@ -1094,7 +1094,7 @@ void Server::handle_client_mknod(MClientRequest *req, CInode *diri) C_MDS_mknod_finish *fin = new C_MDS_mknod_finish(mds, req, dn, newi); EUpdate *le = new EUpdate("mknod"); le->metablob.add_dir_context(diri->dir); - inode_t *pi = le->metablob.add_dentry(dn, newi); + inode_t *pi = le->metablob.add_dentry(dn, true, newi); pi->version = dn->get_projected_version(); // log + wait @@ -1256,9 +1256,9 @@ void Server::handle_client_mkdir(MClientRequest *req, CInode *diri) C_MDS_mknod_finish *fin = new C_MDS_mknod_finish(mds, req, dn, newi); EUpdate *le = new EUpdate("mkdir"); le->metablob.add_dir_context(diri->dir); - inode_t *pi = le->metablob.add_dentry(dn, newi); + inode_t *pi = le->metablob.add_dentry(dn, true, newi); pi->version = dn->get_projected_version(); - le->metablob.add_dir(newi->dir); + le->metablob.add_dir(newi->dir, true); // log + wait mdlog->submit_entry(le); @@ -1304,7 +1304,7 @@ void Server::handle_client_symlink(MClientRequest *req, CInode *diri) C_MDS_mknod_finish *fin = new C_MDS_mknod_finish(mds, req, dn, newi); EUpdate *le = new EUpdate("symlink"); le->metablob.add_dir_context(diri->dir); - inode_t *pi = le->metablob.add_dentry(dn, newi); + inode_t *pi = le->metablob.add_dentry(dn, true, newi); pi->version = dn->get_projected_version(); // log + wait diff --git a/branches/sage/cephmds2/mds/events/EAlloc.h b/branches/sage/cephmds2/mds/events/EAlloc.h index c6f82c5a977cb..9360db4ab49bb 100644 --- a/branches/sage/cephmds2/mds/events/EAlloc.h +++ b/branches/sage/cephmds2/mds/events/EAlloc.h @@ -61,46 +61,15 @@ class EAlloc : public LogEvent { void print(ostream& out) { if (what == EALLOC_EV_ALLOC) - out << "alloc " << hex << id << dec << " tablev " << table_version; + out << "EAlloc alloc " << hex << id << dec << " tablev " << table_version; else - out << "dealloc " << hex << id << dec << " tablev " << table_version; + out << "EAlloc dealloc " << hex << id << dec << " tablev " << table_version; } - // live journal - bool has_expired(MDS *mds) { - if (mds->idalloc->get_committed_version() < table_version) - return false; // still dirty - else - return true; // already flushed - } - - void expire(MDS *mds, Context *c) { - mds->idalloc->save(c, table_version); - } - - - // recovery - void replay(MDS *mds) { - if (mds->idalloc->get_version() >= table_version) { - cout << " event " << table_version << " <= table " << mds->idalloc->get_version() << endl; - } else { - cout << " event " << table_version << " - 1 == table " << mds->idalloc->get_version() << endl; - assert(table_version-1 == mds->idalloc->get_version()); - - if (what == EALLOC_EV_ALLOC) { - idno_t nid = mds->idalloc->alloc_id(true); - assert(nid == id); // this should match. - } - else if (what == EALLOC_EV_FREE) { - mds->idalloc->reclaim_id(id, true); - } - else - assert(0); - - assert(table_version == mds->idalloc->get_version()); - } - } + bool has_expired(MDS *mds); + void expire(MDS *mds, Context *c); + void replay(MDS *mds); }; diff --git a/branches/sage/cephmds2/mds/events/EExportFinish.h b/branches/sage/cephmds2/mds/events/EExportFinish.h index b5f7a2811c198..114d580b6a499 100644 --- a/branches/sage/cephmds2/mds/events/EExportFinish.h +++ b/branches/sage/cephmds2/mds/events/EExportFinish.h @@ -23,22 +23,31 @@ class EExportFinish : public LogEvent { protected: inodeno_t dirino; // exported dir + bool success; public: - EExportFinish(CDir *dir) : LogEvent(EVENT_EXPORTFINISH), - dirino(dir->ino()) { } + EExportFinish(CDir *dir, bool s) : LogEvent(EVENT_EXPORTFINISH), + dirino(dir->ino()), + success(s) { } EExportFinish() : LogEvent(EVENT_EXPORTFINISH) { } void print(ostream& out) { out << "export_finish " << dirino; + if (success) + out << " success"; + else + out << " failure"; } virtual void encode_payload(bufferlist& bl) { bl.append((char*)&dirino, sizeof(dirino)); + bl.append((char*)&success, sizeof(success)); } void decode_payload(bufferlist& bl, int& off) { bl.copy(off, sizeof(dirino), (char*)&dirino); off += sizeof(dirino); + bl.copy(off, sizeof(success), (char*)&success); + off += sizeof(success); } bool has_expired(MDS *mds); diff --git a/branches/sage/cephmds2/mds/events/EExportStart.h b/branches/sage/cephmds2/mds/events/EExportStart.h index 68a8826bdeaf1..d3e9c91dd64fe 100644 --- a/branches/sage/cephmds2/mds/events/EExportStart.h +++ b/branches/sage/cephmds2/mds/events/EExportStart.h @@ -25,26 +25,30 @@ class EExportStart : public LogEvent { protected: EMetaBlob metablob; // exported dir + inodeno_t dirino; int dest; // dest mds public: EExportStart(CDir *dir, int d) : LogEvent(EVENT_EXPORTSTART), + dirino(dir->ino()), dest(d) { metablob.add_dir_context(dir); } EExportStart() : LogEvent(EVENT_EXPORTSTART) { } void print(ostream& out) { - out << "export_start "; - out << " -> " << dest; + out << "export_start " << dirino << " -> " << dest; } virtual void encode_payload(bufferlist& bl) { metablob._encode(bl); + bl.append((char*)&dirino, sizeof(dirino)); bl.append((char*)&dest, sizeof(dest)); } void decode_payload(bufferlist& bl, int& off) { metablob._decode(bl, off); + bl.copy(off, sizeof(dirino), (char*)&dirino); + off += sizeof(dirino); bl.copy(off, sizeof(dest), (char*)&dest); off += sizeof(dest); } diff --git a/branches/sage/cephmds2/mds/events/EImportFinish.h b/branches/sage/cephmds2/mds/events/EImportFinish.h index 250230ddd412e..14a9ab6403af6 100644 --- a/branches/sage/cephmds2/mds/events/EImportFinish.h +++ b/branches/sage/cephmds2/mds/events/EImportFinish.h @@ -23,22 +23,31 @@ class EImportFinish : public LogEvent { protected: inodeno_t dirino; // imported dir + bool success; public: - EImportFinish(CDir *dir) : LogEvent(EVENT_IMPORTFINISH), - dirino(dir->ino()) { } + EImportFinish(CDir *dir, bool s) : LogEvent(EVENT_IMPORTFINISH), + dirino(dir->ino()), + success(s) { } EImportFinish() : LogEvent(EVENT_IMPORTFINISH) { } void print(ostream& out) { out << "import_finish " << dirino; + if (success) + out << " success"; + else + out << " failed"; } virtual void encode_payload(bufferlist& bl) { bl.append((char*)&dirino, sizeof(dirino)); + bl.append((char*)&success, sizeof(success)); } void decode_payload(bufferlist& bl, int& off) { bl.copy(off, sizeof(dirino), (char*)&dirino); off += sizeof(dirino); + bl.copy(off, sizeof(success), (char*)&success); + off += sizeof(success); } bool has_expired(MDS *mds); diff --git a/branches/sage/cephmds2/mds/events/EImportMap.h b/branches/sage/cephmds2/mds/events/EImportMap.h index 66ca390a582e0..41e1fec300237 100644 --- a/branches/sage/cephmds2/mds/events/EImportMap.h +++ b/branches/sage/cephmds2/mds/events/EImportMap.h @@ -22,7 +22,7 @@ public: EMetaBlob metablob; set imports; set exports; - set hashdirs; + //set hashdirs; map > nested_exports; EImportMap() : LogEvent(EVENT_IMPORTMAP) { } @@ -34,9 +34,27 @@ public: void encode_payload(bufferlist& bl) { metablob._encode(bl); + ::_encode(imports, bl); + ::_encode(exports, bl); + for (set::iterator p = imports.begin(); + p != imports.end(); + ++p) { + ::_encode(nested_exports[*p], bl); + if (nested_exports[*p].empty()) + nested_exports.erase(*p); + } } void decode_payload(bufferlist& bl, int& off) { metablob._decode(bl, off); + ::_decode(imports, bl, off); + ::_decode(exports, bl, off); + for (set::iterator p = imports.begin(); + p != imports.end(); + ++p) { + ::_decode(nested_exports[*p], bl, off); + if (nested_exports[*p].empty()) + nested_exports.erase(*p); + } } bool has_expired(MDS *mds); diff --git a/branches/sage/cephmds2/mds/events/EMetaBlob.h b/branches/sage/cephmds2/mds/events/EMetaBlob.h index 0525e0e68e470..e97fccd5a91a1 100644 --- a/branches/sage/cephmds2/mds/events/EMetaBlob.h +++ b/branches/sage/cephmds2/mds/events/EMetaBlob.h @@ -49,21 +49,29 @@ class EMetaBlob { version_t dnv; inode_t inode; // if it's not string symlink; - fullbit(const string& d, inode_t& i) : dn(d), inode(i) { } - fullbit(const string& d, inode_t& i, string& sym) : dn(d), inode(i), symlink(sym) { } + bool dirty; + + fullbit(const string& d, version_t v, inode_t& i, bool dr) : dn(d), dnv(v), inode(i), dirty(dr) { } + fullbit(const string& d, version_t v, inode_t& i, string& sym, bool dr) : dn(d), dnv(v), inode(i), symlink(sym), dirty(dr) { } fullbit(bufferlist& bl, int& off) { _decode(bl, off); } void _encode(bufferlist& bl) { ::_encode(dn, bl); + bl.append((char*)&dnv, sizeof(dnv)); bl.append((char*)&inode, sizeof(inode)); if (inode.is_symlink()) ::_encode(symlink, bl); + bl.append((char*)&dirty, sizeof(dirty)); } void _decode(bufferlist& bl, int& off) { ::_decode(dn, bl, off); + bl.copy(off, sizeof(dnv), (char*)&dnv); + off += sizeof(dnv); bl.copy(off, sizeof(inode), (char*)&inode); off += sizeof(inode); if (inode.is_symlink()) ::_decode(symlink, bl, off); + bl.copy(off, sizeof(dirty), (char*)&dirty); + off += sizeof(dirty); } }; @@ -73,13 +81,15 @@ class EMetaBlob { string dn; version_t dnv; inodeno_t ino; + bool dirty; - remotebit(const string& d, version_t v, inodeno_t i) : dn(d), dnv(v), ino(i) { } + remotebit(const string& d, version_t v, inodeno_t i, bool dr) : dn(d), dnv(v), ino(i), dirty(dr) { } remotebit(bufferlist& bl, int& off) { _decode(bl, off); } void _encode(bufferlist& bl) { ::_encode(dn, bl); bl.append((char*)&dnv, sizeof(dnv)); bl.append((char*)&ino, sizeof(ino)); + bl.append((char*)&dirty, sizeof(dirty)); } void _decode(bufferlist& bl, int& off) { ::_decode(dn, bl, off); @@ -87,6 +97,8 @@ class EMetaBlob { off += sizeof(dnv); bl.copy(off, sizeof(ino), (char*)&ino); off += sizeof(ino); + bl.copy(off, sizeof(dirty), (char*)&dirty); + off += sizeof(dirty); } }; @@ -96,16 +108,20 @@ class EMetaBlob { struct nullbit { string dn; version_t dnv; - nullbit(const string& d, version_t v) : dn(d), dnv(v) { } + bool dirty; + nullbit(const string& d, version_t v, bool dr) : dn(d), dnv(v), dirty(dr) { } nullbit(bufferlist& bl, int& off) { _decode(bl, off); } void _encode(bufferlist& bl) { ::_encode(dn, bl); bl.append((char*)&dnv, sizeof(dnv)); + bl.append((char*)&dirty, sizeof(dirty)); } void _decode(bufferlist& bl, int& off) { ::_decode(dn, bl, off); bl.copy(off, sizeof(dnv), (char*)&dnv); off += sizeof(dnv); + bl.copy(off, sizeof(dirty), (char*)&dirty); + off += sizeof(dirty); } }; @@ -115,6 +131,7 @@ class EMetaBlob { struct dirlump { static const int STATE_IMPORT = (1<<0); static const int STATE_COMPLETE = (1<<1); + static const int STATE_DIRTY = (1<<2); // dirty due to THIS journal item, that is! dirslice_t dirslice; version_t dirv; @@ -131,6 +148,8 @@ class EMetaBlob { void mark_import() { state |= STATE_IMPORT; } bool is_complete() { return state & STATE_COMPLETE; } void mark_complete() { state |= STATE_COMPLETE; } + bool is_dirty() { return state & STATE_DIRTY; } + void mark_dirty() { state |= STATE_DIRTY; } void _encode_bits() { for (list::iterator p = dfull.begin(); p != dfull.end(); ++p) @@ -185,30 +204,69 @@ class EMetaBlob { public: // remote pointer to to-be-journaled inode iff it's a normal (non-remote) dentry - inode_t *add_dentry(CDentry *dn, CInode *in=0) { + inode_t *add_dentry(CDentry *dn, bool dirty, CInode *in=0) { CDir *dir = dn->get_dir(); if (!in) in = dn->get_inode(); // add the dir - dirlump& lump = add_dir(dir); + dirlump& lump = add_dir(dir, false); // add the dirbit if (dn->is_remote()) { - lump.dremote.push_back(remotebit(dn->get_name(), dn->get_projected_version(), dn->get_remote_ino())); lump.nremote++; + if (dirty) + lump.dremote.push_front(remotebit(dn->get_name(), + dn->get_projected_version(), + dn->get_remote_ino(), + dirty)); + else + lump.dremote.push_back(remotebit(dn->get_name(), + dn->get_projected_version(), + dn->get_remote_ino(), + dirty)); } else if (!in) { - lump.dnull.push_back(nullbit(dn->get_name(), dn->get_projected_version())); lump.nnull++; + if (dirty) + lump.dnull.push_front(nullbit(dn->get_name(), + dn->get_projected_version(), + dirty)); + else + lump.dnull.push_back(nullbit(dn->get_name(), + dn->get_projected_version(), + dirty)); } else { - lump.dfull.push_back(fullbit(dn->get_name(), in->inode, in->symlink)); lump.nfull++; - return &lump.dfull.back().inode; + if (dirty) { + lump.dfull.push_front(fullbit(dn->get_name(), + dn->get_projected_version(), + in->inode, in->symlink, + dirty)); + return &lump.dfull.front().inode; + } else { + lump.dfull.push_back(fullbit(dn->get_name(), + dn->get_projected_version(), + in->inode, in->symlink, + dirty)); + return &lump.dfull.back().inode; + } } return 0; } + dirlump& add_dir(CDir *dir, bool dirty) { + if (lump_map.count(dir->ino()) == 0) { + lump_order.push_back(dir->ino()); + lump_map[dir->ino()].dirv = dir->get_projected_version(); + } + dirlump& l = lump_map[dir->ino()]; + if (dir->is_complete()) l.mark_complete(); + if (dir->is_import()) l.mark_import(); + if (dirty) l.mark_dirty(); + return l; + } + void add_dir_context(CDir *dir, bool toroot=false) { // already have this dir? (we must always add in order) if (lump_map.count(dir->ino())) @@ -223,20 +281,12 @@ class EMetaBlob { CDentry *parent = diri->get_parent_dn(); add_dir_context(parent->get_dir(), toroot); - add_dentry(parent); + add_dentry(parent, false); } - dirlump& add_dir(CDir *dir) { - if (lump_map.count(dir->ino()) == 0) { - lump_order.push_back(dir->ino()); - lump_map[dir->ino()].dirv = dir->get_projected_version(); - } - dirlump& l = lump_map[dir->ino()]; - if (dir->is_complete()) l.mark_complete(); - if (dir->is_import()) l.mark_import(); - return l; - } - + + // encoding + void _encode(bufferlist& bl) { int n = lump_map.size(); bl.append((char*)&n, sizeof(n)); diff --git a/branches/sage/cephmds2/mds/journal.cc b/branches/sage/cephmds2/mds/journal.cc index 061858093409b..eab1a07e201db 100644 --- a/branches/sage/cephmds2/mds/journal.cc +++ b/branches/sage/cephmds2/mds/journal.cc @@ -12,6 +12,7 @@ */ #include "events/EMetaBlob.h" +#include "events/EAlloc.h" #include "events/EUpdate.h" #include "events/EImportMap.h" @@ -150,6 +151,8 @@ void EMetaBlob::expire(MDS *mds, Context *c) void EMetaBlob::replay(MDS *mds) { + dout(10) << "EMetaBlob.replay " << lump_map.size() << " dirlumps" << endl; + // walk through my dirs (in order!) for (list::iterator lp = lump_order.begin(); lp != lump_order.end(); @@ -161,23 +164,20 @@ void EMetaBlob::replay(MDS *mds) CInode *diri = mds->mdcache->get_inode(*lp); CDir *dir; if (!diri) { - diri = new CInode(mds->mdcache); - diri->inode.ino = *lp; - diri->inode.mode = INODE_MODE_DIR; - mds->mdcache->add_inode(diri); - - dir = diri->get_or_open_dir(mds->mdcache); - - if (dir->ino() == 1) - mds->mdcache->set_root(diri); - - dout(10) << "EMetaBlob.replay added dir " << *dir << endl; + assert(*lp == 1); + diri = mds->mdcache->create_root_inode(); + dout(10) << "EMetaBlob.replay created root " << *diri << endl; + } + if (diri->dir) { + dir = diri->dir; + dout(20) << "EMetaBlob.replay had dir " << *dir << endl; } else { dir = diri->get_or_open_dir(mds->mdcache); - dout(20) << "EMetaBlob.replay had dir " << *dir << endl; + dout(10) << "EMetaBlob.replay added dir " << *dir << endl; } - dir->set_version( lump.dirv-1 ); - dir->mark_dirty( lump.dirv ); + dir->set_version( lump.dirv ); + if (lump.is_dirty()) + dir->_mark_dirty(); if (lump.is_complete()) dir->mark_complete(); @@ -252,6 +252,53 @@ void EMetaBlob::replay(MDS *mds) +// ----------------------- +// EAlloc + +bool EAlloc::has_expired(MDS *mds) +{ + version_t cv = mds->idalloc->get_committed_version(); + if (cv < table_version) { + dout(10) << "EAlloc.has_expired v " << table_version << " > " << cv + << ", still dirty" << endl; + return false; // still dirty + } else { + dout(10) << "EAlloc.has_expired v " << table_version << " <= " << cv + << ", already flushed" << endl; + return true; // already flushed + } +} + +void EAlloc::expire(MDS *mds, Context *c) +{ + dout(10) << "EAlloc.expire saving idalloc table" << endl; + mds->idalloc->save(c, table_version); +} + +void EAlloc::replay(MDS *mds) +{ + if (mds->idalloc->get_version() >= table_version) { + dout(10) << "EAlloc.replay event " << table_version + << " <= table " << mds->idalloc->get_version() << endl; + } else { + dout(10) << " EAlloc.replay event " << table_version + << " - 1 == table " << mds->idalloc->get_version() << endl; + assert(table_version-1 == mds->idalloc->get_version()); + + if (what == EALLOC_EV_ALLOC) { + idno_t nid = mds->idalloc->alloc_id(true); + assert(nid == id); // this should match. + } + else if (what == EALLOC_EV_FREE) { + mds->idalloc->reclaim_id(id, true); + } + else + assert(0); + + assert(table_version == mds->idalloc->get_version()); + } +} + // ----------------------- // EUpdate @@ -316,7 +363,7 @@ void EImportMap::expire(MDS *mds, Context *c) void EImportMap::replay(MDS *mds) { dout(10) << "EImportMap.replay -- reconstructing import/export spanning tree" << endl; - assert(imports.empty()); + assert(mds->mdcache->imports.empty()); // first, stick the spanning tree in my cache metablob.replay(mds); @@ -438,19 +485,44 @@ void EPurgeFinish::replay(MDS *mds) - +// ----------------------- +// EExportStart bool EExportStart::has_expired(MDS *mds) { - return true; + CInode *diri = mds->mdcache->get_inode(dirino); + if (!diri) return true; + CDir *dir = diri->dir; + if (!dir) return true; + if (!mds->mdcache->migrator->is_exporting(dir)) + return true; + dout(10) << "EExportStart.has_expired still exporting " << *dir << endl; + return false; } + void EExportStart::expire(MDS *mds, Context *c) { + CInode *diri = mds->mdcache->get_inode(dirino); + assert(diri); + CDir *dir = diri->dir; + assert(dir); + assert(mds->mdcache->migrator->is_exporting(dir)); + + dout(10) << "EExportStart.expire waiting for export of " << *dir << endl; + mds->mdcache->migrator->add_export_finish_waiter(dir, c); } + void EExportStart::replay(MDS *mds) { + dout(10) << "EExportStart.replay " << dirino << " -> " << dest << endl; + + metablob.replay(mds); + + } +// ----------------------- +// EExportFinish bool EExportFinish::has_expired(MDS *mds) { @@ -464,6 +536,9 @@ void EExportFinish::replay(MDS *mds) } +// ----------------------- +// EImportStart + bool EImportStart::has_expired(MDS *mds) { return metablob.has_expired(mds); diff --git a/branches/sage/cephmds2/mon/MDSMonitor.cc b/branches/sage/cephmds2/mon/MDSMonitor.cc index e2e2553670fe7..eea1257a37116 100644 --- a/branches/sage/cephmds2/mon/MDSMonitor.cc +++ b/branches/sage/cephmds2/mon/MDSMonitor.cc @@ -36,8 +36,8 @@ void MDSMonitor::create_initial() mdsmap.epoch = 0; // until everyone boots mdsmap.ctime = g_clock.now(); for (int i=0; iget_source_inst(); - mdsmap.down_mds.erase(from); + mdsmap.mds_state[from] = MDSMap::STATE_STARTING; if ((int)mdsmap.mds_inst.size() == mdsmap.get_num_mds()) { mdsmap.inc_epoch(); @@ -103,11 +103,10 @@ void MDSMonitor::handle_mds_shutdown(Message *m) assert(m->get_source().is_mds()); int from = m->get_source().num(); - mdsmap.mds_inst.erase(from); - mdsmap.all_mds.erase(from); + mdsmap.remove_mds(from); dout(7) << "mds_shutdown from " << m->get_source() - << ", still have " << mdsmap.all_mds + << ", still have " << mdsmap.mds_set << endl; // tell someone? @@ -132,8 +131,8 @@ void MDSMonitor::bcast_latest_mds() dout(10) << "bcast_latest_mds " << mdsmap.get_epoch() << endl; // tell mds - for (set::iterator p = mdsmap.get_mds().begin(); - p != mdsmap.get_mds().end(); + for (set::iterator p = mdsmap.get_mds_set().begin(); + p != mdsmap.get_mds_set().end(); p++) { if (mdsmap.is_down(*p)) continue; send_full(MSG_ADDR_MDS(*p), mdsmap.get_inst(*p)); diff --git a/branches/sage/cephmds2/mon/OSDMonitor.cc b/branches/sage/cephmds2/mon/OSDMonitor.cc index 7fafbff48b2f1..f81b397fa2ded 100644 --- a/branches/sage/cephmds2/mon/OSDMonitor.cc +++ b/branches/sage/cephmds2/mon/OSDMonitor.cc @@ -567,8 +567,8 @@ void OSDMonitor::bcast_latest_mds() dout(1) << "bcast_latest_mds epoch " << e << endl; // tell mds - for (set::iterator i = mon->mdsmon->mdsmap.get_mds().begin(); - i != mon->mdsmon->mdsmap.get_mds().end(); + for (set::iterator i = mon->mdsmon->mdsmap.get_mds_set().begin(); + i != mon->mdsmon->mdsmap.get_mds_set().end(); i++) { if (mon->mdsmon->mdsmap.is_down(*i)) continue; send_incremental(osdmap.get_epoch()-1, MSG_ADDR_MDS(*i), mon->mdsmon->mdsmap.get_inst(*i)); -- 2.39.5