From: Yan, Zheng Date: Thu, 12 Apr 2018 08:06:10 +0000 (+0800) Subject: mds: automaticly allow multi-active MDS after removing all old snapshots X-Git-Tag: v13.1.0~2^2~13 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=49f4b432b2f587a6b0cd1ea23af9b9348b6aab2d;p=ceph.git mds: automaticly allow multi-active MDS after removing all old snapshots Signed-off-by: "Yan, Zheng" --- diff --git a/src/mds/MDSMap.h b/src/mds/MDSMap.h index 1c30686310e..b1b52e672f4 100644 --- a/src/mds/MDSMap.h +++ b/src/mds/MDSMap.h @@ -283,6 +283,12 @@ public: bool allows_snaps() const { return test_flag(CEPH_MDSMAP_ALLOW_SNAPS); } bool was_snaps_ever_allowed() const { return ever_allowed_features & CEPH_MDSMAP_ALLOW_SNAPS; } + void set_multimds_snaps_allowed() { + set_flag(CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS); + ever_allowed_features |= CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS; + explicitly_allowed_features |= CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS; + } + void clear_multimds_snaps_allowed() { clear_flag(CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS); } bool allows_multimds_snaps() const { return test_flag(CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS); } epoch_t get_epoch() const { return epoch; } diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index ca60ea60285..b26c9063e31 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -240,6 +240,47 @@ void MDSRank::hit_export_target(utime_t now, mds_rank_t rank, double amount) em.first->second.hit(now, amount); } +class C_MDS_MonCommand : public MDSInternalContext { + std::string cmd; +public: + std::string outs; + C_MDS_MonCommand(MDSRank *m, std::string_view c) + : MDSInternalContext(m), cmd(c) {} + void finish(int r) override { + mds->_mon_command_finish(r, cmd, outs); + } +}; + +void MDSRank::_mon_command_finish(int r, std::string_view cmd, std::string_view outs) +{ + if (r < 0) { + dout(0) << __func__ << ": mon command " << cmd << " failed with errno " << r + << " (" << outs << ")" << dendl; + } else { + dout(1) << __func__ << ": mon command " << cmd << " succeed" << dendl; + } +} + +void MDSRank::set_mdsmap_multimds_snaps_allowed() +{ + static bool already_sent = false; + if (already_sent) + return; + + stringstream ss; + ss << "{\"prefix\":\"fs set\", \"fs_name\":\"" << mdsmap->get_fs_name() << "\", "; + ss << "\"var\":\"allow_multimds_snaps\", \"val\":\"true\", "; + ss << "\"confirm\":\"--yes-i-am-really-a-mds\"}"; + std::vector cmd = {ss.str()}; + + dout(0) << __func__ << ": sending mon command: " << cmd[0] << dendl; + + C_MDS_MonCommand *fin = new C_MDS_MonCommand(this, cmd[0]); + monc->start_mon_command(cmd, {}, nullptr, &fin->outs, new C_IO_Wrapper(this, fin)); + + already_sent = true; +} + void MDSRankDispatcher::tick() { heartbeat_reset(); @@ -285,8 +326,16 @@ void MDSRankDispatcher::tick() balancer->tick(); mdcache->find_stale_fragment_freeze(); mdcache->migrator->find_stale_export_freeze(); - if (snapserver) + + if (mdsmap->get_tableserver() == whoami) { snapserver->check_osd_map(false); + // Filesystem was created by pre-mimic mds. Allow multi-active mds after + // all old snapshots are deleted. + if (!mdsmap->allows_multimds_snaps() && + snapserver->can_allow_multimds_snaps()) { + set_mdsmap_multimds_snaps_allowed(); + } + } } if (is_active() || is_stopping()) { @@ -2755,7 +2804,8 @@ void MDSRank::check_ops_in_flight() void MDSRankDispatcher::handle_osd_map() { - if (is_active() && snapserver) { + if (is_active() && + mdsmap->get_tableserver() == whoami) { snapserver->check_osd_map(true); } diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h index ef77333dff6..983a6c3c7d6 100644 --- a/src/mds/MDSRank.h +++ b/src/mds/MDSRank.h @@ -217,6 +217,7 @@ class MDSRank { bool is_any_replay() const { return (is_replay() || is_standby_replay()); } bool is_stopped() const { return mdsmap->is_stopped(whoami); } bool is_cluster_degraded() const { return cluster_degraded; } + bool allows_multimds_snaps() const { return mdsmap->allows_multimds_snaps(); } void handle_write_error(int err); @@ -523,6 +524,9 @@ class MDSRank { /* Update MDSMap export_targets for this rank. Called on ::tick(). */ void update_targets(utime_t now); + friend class C_MDS_MonCommand; + void _mon_command_finish(int r, std::string_view cmd, std::string_view outs); + void set_mdsmap_multimds_snaps_allowed(); private: mono_time starttime = mono_clock::zero(); }; diff --git a/src/mds/SnapRealm.cc b/src/mds/SnapRealm.cc index 935880a2449..4a64415ac6f 100644 --- a/src/mds/SnapRealm.cc +++ b/src/mds/SnapRealm.cc @@ -154,6 +154,13 @@ bool SnapRealm::_open_parents(MDSInternalContextBase *finish, snapid_t first, sn if (!srnode.past_parent_snaps.empty()) assert(mdcache->mds->snapclient->get_cached_version() > 0); + if (!srnode.past_parents.empty() && + mdcache->mds->allows_multimds_snaps()) { + dout(10) << " skip non-empty past_parents since multimds_snaps is allowed" << dendl; + open = true; + return true; + } + // and my past parents too! assert(srnode.past_parents.size() >= num_open_past_parents); if (srnode.past_parents.size() > num_open_past_parents) { @@ -205,6 +212,13 @@ bool SnapRealm::have_past_parents_open(snapid_t first, snapid_t last) const if (!srnode.past_parent_snaps.empty()) assert(mdcache->mds->snapclient->get_cached_version() > 0); + if (!srnode.past_parents.empty() && + mdcache->mds->allows_multimds_snaps()) { + dout(10) << " skip non-empty past_parents since multimds_snaps is allowed" << dendl; + open = true; + return true; + } + for (auto p = srnode.past_parents.lower_bound(first); p != srnode.past_parents.end(); ++p) { diff --git a/src/mds/SnapServer.cc b/src/mds/SnapServer.cc index 7e97ab8b0ef..b7ae4519a16 100644 --- a/src/mds/SnapServer.cc +++ b/src/mds/SnapServer.cc @@ -60,6 +60,7 @@ void SnapServer::reset_state() } last_created = last_snap; last_destroyed = last_snap; + snaprealm_v2_since = last_snap + 1; version++; } diff --git a/src/mds/SnapServer.h b/src/mds/SnapServer.h index 94f7a5bfa11..3475099d350 100644 --- a/src/mds/SnapServer.h +++ b/src/mds/SnapServer.h @@ -26,6 +26,7 @@ protected: MonClient *mon_client = nullptr; snapid_t last_snap; snapid_t last_created, last_destroyed; + snapid_t snaprealm_v2_since; map snaps; map > need_to_purge; @@ -36,7 +37,7 @@ protected: version_t last_checked_osdmap; void encode_server_state(bufferlist& bl) const override { - ENCODE_START(4, 3, bl); + ENCODE_START(5, 3, bl); encode(last_snap, bl); encode(snaps, bl); encode(need_to_purge, bl); @@ -45,10 +46,11 @@ protected: encode(pending_noop, bl); encode(last_created, bl); encode(last_destroyed, bl); + encode(snaprealm_v2_since, bl); ENCODE_FINISH(bl); } void decode_server_state(bufferlist::iterator& bl) override { - DECODE_START_LEGACY_COMPAT_LEN(4, 3, 3, bl); + DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, bl); decode(last_snap, bl); decode(snaps, bl); decode(need_to_purge, bl); @@ -69,6 +71,11 @@ protected: last_created = last_snap; last_destroyed = last_snap; } + if (struct_v >= 5) + decode(snaprealm_v2_since, bl); + else + snaprealm_v2_since = last_snap + 1; + DECODE_FINISH(bl); } @@ -90,6 +97,10 @@ public: void check_osd_map(bool force); + bool can_allow_multimds_snaps() const { + return snaps.empty() || snaps.begin()->first >= snaprealm_v2_since; + } + void encode(bufferlist& bl) const { encode_server_state(bl); } diff --git a/src/mon/FSCommands.cc b/src/mon/FSCommands.cc index 25792b97610..054dd8beb88 100644 --- a/src/mon/FSCommands.cc +++ b/src/mon/FSCommands.cc @@ -380,6 +380,37 @@ public: << " parameter to control the number of active MDSs" << " allowed. This command is DEPRECATED and will be" << " REMOVED from future releases."; + } else if (var == "allow_multimds_snaps") { + bool enable = false; + int r = parse_bool(val, &enable, ss); + if (r != 0) { + return r; + } + + string confirm; + if (!cmd_getval(g_ceph_context, cmdmap, "confirm", confirm) || + confirm != "--yes-i-am-really-a-mds") { + ss << "Warning! This command is for MDS only. Do not run it manually"; + return -EPERM; + } + + if (enable) { + ss << "enabled multimds with snapshot"; + fsmap.modify_filesystem( + fs->fscid, + [](std::shared_ptr fs) + { + fs->mds_map.set_multimds_snaps_allowed(); + }); + } else { + ss << "disabled multimds with snapshot"; + fsmap.modify_filesystem( + fs->fscid, + [](std::shared_ptr fs) + { + fs->mds_map.clear_multimds_snaps_allowed(); + }); + } } else if (var == "allow_dirfrags") { ss << "Directory fragmentation is now permanently enabled." << " This command is DEPRECATED and will be REMOVED from future releases."; diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index 9e0079e3371..0c7d18f0a7d 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -315,8 +315,8 @@ COMMAND_WITH_FLAG("mds set_max_mds " \ "name=maxmds,type=CephInt,range=0", \ "set max MDS index", "mds", "rw", "cli,rest", FLAG(OBSOLETE)) COMMAND_WITH_FLAG("mds set " \ - "name=var,type=CephChoices,strings=max_mds|max_file_size" - "|allow_new_snaps|inline_data|allow_multimds|allow_dirfrags " \ + "name=var,type=CephChoices,strings=max_mds|max_file_size|inline_data|" + "allow_new_snaps|allow_multimds|allow_multimds_snaps|allow_dirfrags " \ "name=val,type=CephString " \ "name=confirm,type=CephString,req=false", \ "set mds parameter to ", "mds", "rw", "cli,rest", FLAG(OBSOLETE))