From fe5639319140aa0a30734cf6a2f6b2746b2f9f8f Mon Sep 17 00:00:00 2001 From: Greg Farnum Date: Mon, 26 Oct 2009 15:18:33 -0700 Subject: [PATCH] mon: Move full OSD test logic around --- src/mon/OSDMonitor.cc | 62 +++++++++---------------------------------- src/mon/OSDMonitor.h | 4 --- src/mon/PGMap.h | 30 ++++++++++++++++++--- src/mon/PGMonitor.cc | 2 -- 4 files changed, 40 insertions(+), 58 deletions(-) diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 9839b1ada4782..fd72d2507b859 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -271,13 +271,6 @@ bool OSDMonitor::should_propose(double& delay) if (pending_inc.fullmap.length()) return true; - if ((pending_inc.new_flags ^ osdmap.flags) - & CEPH_OSDMAP_FULL) { //get that info out there! - dout(1) << "should_propose returning true, 0.0 because full status changed! " << dendl; - delay = 0.0; - return true; - } - // adjust osd weights? if (osd_weight.size() == (unsigned)osdmap.get_max_osd()) { dout(0) << " adjusting osd weights based on " << osd_weight << dendl; @@ -756,44 +749,6 @@ void OSDMonitor::check_sub(Subscription *sub) } } -void OSDMonitor::handle_osd_stat(osd_stat_t& stat, int from) -{ - float ratio = ((float)stat.kb_used) / (float) stat.kb; - if ( ratio > CEPH_OSD_FULL_RATIO ) { - dout(5) << "osd" << from << " has ratio " << ratio - << "which is beyond CEPH_OSD_FULL_RATIO " - << CEPH_OSD_FULL_RATIO << dendl; - add_flag(CEPH_OSDMAP_FULL); - full_osds.insert(from); - } - else if ( ratio > CEPH_OSD_NEARFULL_RATIO ) { - dout(10) << "osd" << from << " has ratio " << ratio - << "which is beyond CEPH_OSD_NEARFULL_RATIO " - << CEPH_OSD_NEARFULL_RATIO << dendl; - add_flag(CEPH_OSDMAP_NEARFULL); - nearfull_osds.insert(from); - if (full_osds.erase(from)) { //it actually erased an element - if (full_osds.empty()) { - dout(1) << "full_osds is empty, removing full flag!" << dendl; - remove_flag(CEPH_OSDMAP_FULL); - } - } - } - else {//it's not full or near-full - if (full_osds.erase(from)) { - dout(1) << "erased osd" << from << "from full_osds" << dendl; - if (full_osds.empty()) { - dout(1) << "full_osds is empty, removing full flag!" << dendl; - remove_flag(CEPH_OSDMAP_FULL); - } - } - if (nearfull_osds.erase(from)) { - if (nearfull_osds.empty()) - remove_flag(CEPH_OSDMAP_NEARFULL); - } - } -} - // TICK @@ -847,10 +802,19 @@ void OSDMonitor::tick() } //if map full setting has changed, get that info out there! - if (pending_inc.new_flags != -1 && - (pending_inc.new_flags ^ osdmap.flags) & CEPH_OSDMAP_FULL) { - dout(1) << "New setting for CEPH_OSDMAP_FULL -- doing propose" << dendl; - do_propose = true; + if (mon->pgmon()->paxos->is_readable()) { + if (!mon->pgmon()->pg_map.full_osds.empty()) { + dout(5) << "There are full osds, setting full flag" << dendl; + add_flag(CEPH_OSDMAP_FULL); + } else { + dout(10) << "No full osds, removing full flag (if it's set)" << dendl; + remove_flag(CEPH_OSDMAP_FULL); + } + if (pending_inc.new_flags != -1 && + (pending_inc.new_flags ^ osdmap.flags) & CEPH_OSDMAP_FULL) { + dout(1) << "New setting for CEPH_OSDMAP_FULL -- doing propose" << dendl; + do_propose = true; + } } // --------------- #define SWAP_PRIMARIES_AT_START 0 diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h index af79182a694ba..ca82a225c9f34 100644 --- a/src/mon/OSDMonitor.h +++ b/src/mon/OSDMonitor.h @@ -48,8 +48,6 @@ private: map down_pending_out; // osd down -> out map osd_weight; - set nearfull_osds; - set full_osds; // svc public: void create_initial(bufferlist& bl); @@ -164,8 +162,6 @@ private: void check_subs(); void check_sub(Subscription *sub); - void handle_osd_stat(osd_stat_t& stat, int from); - void add_flag(int flag) { if (!(osdmap.flags & flag)) { if (pending_inc.new_flags < 0) diff --git a/src/mon/PGMap.h b/src/mon/PGMap.h index ca1ce3aa63a4c..0d16c5e01b1b9 100644 --- a/src/mon/PGMap.h +++ b/src/mon/PGMap.h @@ -33,6 +33,8 @@ public: hash_map pg_stat; set pg_set; hash_map osd_stat; + set full_osds; + set nearfull_osds; class Incremental { public: @@ -83,7 +85,23 @@ public: stat_osd_sub(osd_stat[p->first]); osd_stat[p->first] = p->second; stat_osd_add(p->second); + //update the full/nearful_osd sets + int from = p->first; + float ratio = ((float)p->second.kb_used) / (float) p->second.kb; + if ( ratio > full_ratio ) { + full_osds.insert(from); + //sets don't double-insert so this might be a (very expensive) null-op + } + else if ( ratio > nearfull_ratio ) { + nearfull_osds.insert(from); + full_osds.erase(from); + } + else {//it's not full or near-full + full_osds.erase(from); + nearfull_osds.erase(from); + } } + for (set::iterator p = inc.osd_stat_rm.begin(); p != inc.osd_stat_rm.end(); p++) @@ -104,6 +122,9 @@ public: pool_stat_t pg_sum; osd_stat_t osd_sum; + float full_ratio; + float nearfull_ratio; + set creating_pgs; // lru: front = new additions, back = recently pinged void stat_zero() { @@ -142,8 +163,10 @@ public: PGMap() : version(0), last_osdmap_epoch(0), last_pg_scan(0), - num_pg(0), - num_osd(0) {} + num_pg(0), + num_osd(0), + full_ratio(CEPH_OSD_FULL_RATIO), + nearfull_ratio(CEPH_OSD_NEARFULL_RATIO) {} void encode(bufferlist &bl) { ::encode(version, bl); @@ -176,6 +199,8 @@ public: ss << "version " << version << std::endl; ss << "last_osdmap_epoch " << last_osdmap_epoch << std::endl; ss << "last_pg_scan " << last_pg_scan << std::endl; + ss << "full_ratio " << full_ratio << std::endl; + ss << "nearfull_ratio " << nearfull_ratio << std::endl; ss << "pg_stat\tobjects\tmip\tdegr\tkb\tbytes\tlog\tdisklog\tstate\tv\treported\tup\tacting\tlast_scrub" << std::endl; for (set::iterator p = pg_set.begin(); p != pg_set.end(); @@ -220,7 +245,6 @@ public: << "\t" << pg_sum.log_size << "\t" << pg_sum.ondisk_log_size << std::endl; - ss << "osdstat\tkbused\tkbavail\tkb\thb in\thb out" << std::endl; for (hash_map::iterator p = osd_stat.begin(); p != osd_stat.end(); diff --git a/src/mon/PGMonitor.cc b/src/mon/PGMonitor.cc index 416d1fd80501a..2da9b67592fde 100644 --- a/src/mon/PGMonitor.cc +++ b/src/mon/PGMonitor.cc @@ -325,8 +325,6 @@ bool PGMonitor::prepare_pg_stats(MPGStats *stats) else dout(10) << " got osd " << from << " " << stats->osd_stat << " (first report)" << dendl; - //Check how full the OSD is; set OSDMap to full/near-full if needed - mon->osdmon()->handle_osd_stat(stats->osd_stat, from); // apply to live map too (screw consistency) /* actually, no, don't. that screws up our "latest" stash. and can -- 2.39.5