From 93528d8d8313e90dae78a31846eb49aea27f33db Mon Sep 17 00:00:00 2001 From: sageweil Date: Fri, 23 Feb 2007 23:27:19 +0000 Subject: [PATCH] monitorstore fixes git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@1124 29311d96-e01e-0410-9327-a35deaab8ce9 --- branches/riccardo/monitor2/client/Client.cc | 2 +- branches/riccardo/monitor2/mon/MDSMonitor.cc | 87 +++++++++++++------ branches/riccardo/monitor2/mon/MDSMonitor.h | 10 ++- branches/riccardo/monitor2/mon/Monitor.cc | 4 +- .../riccardo/monitor2/mon/MonitorStore.cc | 12 +-- branches/riccardo/monitor2/mon/MonitorStore.h | 35 ++++---- branches/riccardo/monitor2/mon/OSDMonitor.cc | 38 ++++++-- branches/riccardo/monitor2/mon/OSDMonitor.h | 4 +- 8 files changed, 129 insertions(+), 63 deletions(-) diff --git a/branches/riccardo/monitor2/client/Client.cc b/branches/riccardo/monitor2/client/Client.cc index 928096ace4667..19153ae30803e 100644 --- a/branches/riccardo/monitor2/client/Client.cc +++ b/branches/riccardo/monitor2/client/Client.cc @@ -2619,7 +2619,7 @@ int Client::lazyio_synchronize(int fd, off_t offset, size_t count) void Client::ms_handle_failure(Message *m, const entity_inst_t& inst) { - msg_addr_t dest = inst.name; + entity_name_t dest = inst.name; if (dest.is_mon()) { // resend to a different monitor. diff --git a/branches/riccardo/monitor2/mon/MDSMonitor.cc b/branches/riccardo/monitor2/mon/MDSMonitor.cc index 3ea7ad235ada0..68bc61fbfe6bd 100644 --- a/branches/riccardo/monitor2/mon/MDSMonitor.cc +++ b/branches/riccardo/monitor2/mon/MDSMonitor.cc @@ -14,6 +14,7 @@ #include "MDSMonitor.h" #include "Monitor.h" +#include "MonitorStore.h" #include "messages/MMDSMap.h" #include "messages/MMDSGetMap.h" @@ -30,14 +31,6 @@ /********* MDS map **************/ -void MDSMonitor::create_initial() -{ - mdsmap.epoch = 0; // until everyone boots - mdsmap.ctime = g_clock.now(); - - print_map(); -} - void MDSMonitor::dispatch(Message *m) { switch (m->get_type()) { @@ -55,6 +48,50 @@ void MDSMonitor::dispatch(Message *m) } } + + +void MDSMonitor::election_finished() +{ + if (mon->is_leader()) { + + // FIXME be smarter later. + + if (g_conf.mkfs) { + create_initial(); + save_map(); + } else { + load_map(); + } + } +} + + +void MDSMonitor::create_initial() +{ + mdsmap.epoch = 0; // until everyone boots + mdsmap.ctime = g_clock.now(); + + mdsmap.encode(encoded_map); + + print_map(); +} + +void MDSMonitor::load_map() +{ + int r = mon->store->get_bl_ss(encoded_map, "mdsmap", "current"); + assert(r > 0); + mdsmap.decode(encoded_map); + dout(7) << "load_map epoch " << mdsmap.get_epoch() << endl; +} + +void MDSMonitor::save_map() +{ + dout(7) << "save_map epoch " << mdsmap.get_epoch() << endl; + + int r = mon->store->put_bl_ss(encoded_map, "mdsmap", "current"); + assert(r>=0); +} + void MDSMonitor::print_map() { dout(7) << "print_map epoch " << mdsmap.get_epoch() << endl; @@ -72,6 +109,21 @@ void MDSMonitor::print_map() } +void MDSMonitor::issue_map() +{ + mdsmap.inc_epoch(); + mdsmap.encode(encoded_map); + + dout(7) << "issue_map epoch " << mdsmap.get_epoch() << endl; + + save_map(); + print_map(); + + // bcast map + bcast_latest_mds(); + send_current(); +} + void MDSMonitor::handle_mds_beacon(MMDSBeacon *m) { @@ -189,15 +241,7 @@ void MDSMonitor::handle_mds_beacon(MMDSBeacon *m) else mdsmap.mds_state_seq.erase(from); - // inc map version - mdsmap.inc_epoch(); - mdsmap.encode(maps[mdsmap.get_epoch()]); - - print_map(); - - // bcast map - bcast_latest_mds(); - send_current(); + issue_map(); } delete m; @@ -319,14 +363,7 @@ void MDSMonitor::tick() } if (changed) { - mdsmap.inc_epoch(); - mdsmap.encode(maps[mdsmap.get_epoch()]); - - print_map(); - - // bcast map - bcast_latest_mds(); - send_current(); + issue_map(); } } } diff --git a/branches/riccardo/monitor2/mon/MDSMonitor.h b/branches/riccardo/monitor2/mon/MDSMonitor.h index 31c3e2a3d242b..c3bc3d165883c 100644 --- a/branches/riccardo/monitor2/mon/MDSMonitor.h +++ b/branches/riccardo/monitor2/mon/MDSMonitor.h @@ -35,7 +35,7 @@ class MDSMonitor : public Dispatcher { MDSMap mdsmap; private: - map maps; + bufferlist encoded_map; //map inc_maps; //MDSMap::Incremental pending_inc; @@ -54,6 +54,10 @@ class MDSMonitor : public Dispatcher { void send_full(entity_inst_t dest); void bcast_latest_mds(); + void issue_map(); + + void save_map(); + void load_map(); void print_map(); //void accept_pending(); // accept pending, new map. @@ -68,12 +72,14 @@ class MDSMonitor : public Dispatcher { public: MDSMonitor(Monitor *mn, Messenger *m, Mutex& l) : mon(mn), messenger(m), lock(l) { - create_initial(); } void dispatch(Message *m); void tick(); // check state, take actions + void election_starting(); + void election_finished(); + void send_latest(entity_inst_t dest); }; diff --git a/branches/riccardo/monitor2/mon/Monitor.cc b/branches/riccardo/monitor2/mon/Monitor.cc index 5fc16256aa270..93fc043d8c42c 100644 --- a/branches/riccardo/monitor2/mon/Monitor.cc +++ b/branches/riccardo/monitor2/mon/Monitor.cc @@ -55,6 +55,8 @@ void Monitor::init() if (g_conf.mkfs) store->mkfs(); + else + store->mount(); // create osdmon = new OSDMonitor(this, messenger, lock); @@ -144,7 +146,7 @@ void Monitor::win_election(set& active) // init osdmon->election_finished(); - //mdsmon->election_finished(); + mdsmon->election_finished(); // init paxos test_paxos.leader_start(); diff --git a/branches/riccardo/monitor2/mon/MonitorStore.cc b/branches/riccardo/monitor2/mon/MonitorStore.cc index a56dcb229e4f0..52351bc6cf0d6 100644 --- a/branches/riccardo/monitor2/mon/MonitorStore.cc +++ b/branches/riccardo/monitor2/mon/MonitorStore.cc @@ -25,9 +25,9 @@ #include -void MonitorStore::init() +void MonitorStore::mount() { - dout(1) << "init" << endl; + dout(1) << "mount" << endl; // verify dir exists DIR *d = ::opendir(dir.c_str()); if (!d) { @@ -43,7 +43,7 @@ void MonitorStore::mkfs() dout(1) << "mkfs" << endl; char cmd[200]; - sprintf(cmd, "/bin/rm -r %s ; mkdir %s", dir.c_str(), dir.c_str()); + sprintf(cmd, "test -d %s && /bin/rm -r %s ; mkdir -p %s", dir.c_str(), dir.c_str(), dir.c_str()); dout(1) << cmd << endl; system(cmd); } @@ -106,7 +106,7 @@ void MonitorStore::put_int(version_t val, const char *a, const char *b) // ---------------------------------------- // buffers -bool MonitorStore::exists_bl(const char *a, const char *b) +bool MonitorStore::exists_bl_ss(const char *a, const char *b) { char fn[200]; if (b) { @@ -123,7 +123,7 @@ bool MonitorStore::exists_bl(const char *a, const char *b) } -int MonitorStore::get_bl(bufferlist& bl, const char *a, const char *b) +int MonitorStore::get_bl_ss(bufferlist& bl, const char *a, const char *b) { char fn[200]; if (b) { @@ -162,7 +162,7 @@ int MonitorStore::get_bl(bufferlist& bl, const char *a, const char *b) return len; } -int MonitorStore::put_bl(bufferlist& bl, const char *a, const char *b) +int MonitorStore::put_bl_ss(bufferlist& bl, const char *a, const char *b) { char fn[200]; sprintf(fn, "%s/%s", dir.c_str(), a); diff --git a/branches/riccardo/monitor2/mon/MonitorStore.h b/branches/riccardo/monitor2/mon/MonitorStore.h index 0ae1c96a84ea0..f1d5f67ab3473 100644 --- a/branches/riccardo/monitor2/mon/MonitorStore.h +++ b/branches/riccardo/monitor2/mon/MonitorStore.h @@ -22,39 +22,38 @@ class MonitorStore { string dir; - void init(); - public: MonitorStore(char *d) : dir(d) { - init(); } ~MonitorStore() { } void mkfs(); // wipe + void mount(); // ints (stored as ascii) version_t get_int(const char *a, const char *b=0); void put_int(version_t v, const char *a, const char *b=0); // buffers - bool exists_bl(const char *a, const char *b=0); - int get_bl(bufferlist& bl, const char *a, const char *b); - int put_bl(bufferlist& bl, const char *a, const char *b); - bool exists_bl(const char *a, unsigned b) { - char bs[16]; - sprintf(bs, "%0u", b); - return exists_bl(a, bs); + // ss and sn varieties. + bool exists_bl_ss(const char *a, const char *b=0); + int get_bl_ss(bufferlist& bl, const char *a, const char *b); + int put_bl_ss(bufferlist& bl, const char *a, const char *b); + bool exists_bl_sn(const char *a, version_t b) { + char bs[20]; + sprintf(bs, "%llu", b); + return exists_bl_ss(a, bs); } - int get_bl(bufferlist& bl, const char *a, version_t b) { - char bs[16]; - sprintf(bs, "%0llu", b); - return get_bl(bl, a, bs); + int get_bl_sn(bufferlist& bl, const char *a, version_t b) { + char bs[20]; + sprintf(bs, "%llu", b); + return get_bl_ss(bl, a, bs); } - int put_bl(bufferlist& bl, const char *a, version_t b) { - char bs[16]; - sprintf(bs, "%0llu", b); - return put_bl(bl, a, bs); + int put_bl_sn(bufferlist& bl, const char *a, version_t b) { + char bs[20]; + sprintf(bs, "%llu", b); + return put_bl_ss(bl, a, bs); } /* diff --git a/branches/riccardo/monitor2/mon/OSDMonitor.cc b/branches/riccardo/monitor2/mon/OSDMonitor.cc index 9c51d40d6a8a0..ac4d49427d59c 100644 --- a/branches/riccardo/monitor2/mon/OSDMonitor.cc +++ b/branches/riccardo/monitor2/mon/OSDMonitor.cc @@ -91,6 +91,7 @@ void OSDMonitor::fake_reorg() +/* void OSDMonitor::init() { // start with blank map @@ -106,6 +107,7 @@ void OSDMonitor::init() pending_inc.epoch = osdmap.get_epoch()+1; } } +*/ @@ -224,18 +226,18 @@ void OSDMonitor::create_initial() bool OSDMonitor::get_map_bl(epoch_t epoch, bufferlist& bl) { - if (!mon->store->exists_bl("osdmap", epoch)) + if (!mon->store->exists_bl_sn("osdmap", epoch)) return false; - int r = mon->store->get_bl(bl, "osdmap", epoch); + int r = mon->store->get_bl_sn(bl, "osdmap", epoch); assert(r > 0); return true; } bool OSDMonitor::get_inc_map_bl(epoch_t epoch, bufferlist& bl) { - if (!mon->store->exists_bl("osdincmap", epoch)) + if (!mon->store->exists_bl_sn("osdincmap", epoch)) return false; - int r = mon->store->get_bl(bl, "osdincmap", epoch); + int r = mon->store->get_bl_sn(bl, "osdincmap", epoch); assert(r > 0); return true; } @@ -246,7 +248,7 @@ void OSDMonitor::save_map() bufferlist bl; osdmap.encode(bl); - mon->store->put_bl(bl, "osdmap", osdmap.get_epoch()); + mon->store->put_bl_sn(bl, "osdmap", osdmap.get_epoch()); mon->store->put_int(osdmap.get_epoch(), "osd_epoch"); } @@ -258,8 +260,8 @@ void OSDMonitor::save_inc_map(OSDMap::Incremental &inc) bufferlist incbl; inc.encode(incbl); - mon->store->put_bl(bl, "osdmap", osdmap.get_epoch()); - mon->store->put_bl(incbl, "osdincmap", osdmap.get_epoch()); + mon->store->put_bl_sn(bl, "osdmap", osdmap.get_epoch()); + mon->store->put_bl_sn(incbl, "osdincmap", osdmap.get_epoch()); mon->store->put_int(osdmap.get_epoch(), "osd_epoch"); } @@ -614,6 +616,26 @@ void OSDMonitor::election_finished() { dout(10) << "election_finished" << endl; + if (mon->is_leader()) { + if (g_conf.mkfs) { + create_initial(); + save_map(); + } else { + // + epoch_t epoch = mon->store->get_int("osd_epoch"); + dout(10) << " last epoch was " << epoch << endl; + bufferlist bl, blinc; + int r = mon->store->get_bl_sn(bl, "osdmap", epoch); + assert(r>0); + osdmap.decode(bl); + + // pending_inc + pending_inc.epoch = epoch+1; + } + + } + + /* state = STATE_INIT; // map? @@ -636,7 +658,7 @@ void OSDMonitor::election_finished() //messenger->send_message(new MMonOSDMapInfo(osdmap.epoch, osdmap.mon_epoch), // mon->monmap->get_inst(mon->leader)); } - + */ } diff --git a/branches/riccardo/monitor2/mon/OSDMonitor.h b/branches/riccardo/monitor2/mon/OSDMonitor.h index d2e6b2e284118..150a9de6de9f7 100644 --- a/branches/riccardo/monitor2/mon/OSDMonitor.h +++ b/branches/riccardo/monitor2/mon/OSDMonitor.h @@ -60,7 +60,7 @@ private: int state; utime_t lease_expire; // when lease expires - void init(); + //void init(); // maps void accept_pending(); // accept pending, new map. @@ -89,7 +89,7 @@ private: OSDMonitor(Monitor *mn, Messenger *m, Mutex& l) : mon(mn), messenger(m), lock(l), state(STATE_SYNC) { - init(); + //init(); } void dispatch(Message *m); -- 2.39.5