From 14b0db908f652032c358e419ffa90f5676698d0e Mon Sep 17 00:00:00 2001 From: Varsha Rao Date: Thu, 24 Oct 2019 18:18:35 +0530 Subject: [PATCH] mds: Reorganize class members in MDSRank header Fixes: https://tracker.ceph.com/issues/42465 Signed-off-by: Varsha Rao --- src/mds/MDSRank.cc | 21 +-- src/mds/MDSRank.h | 383 ++++++++++++++++++++++----------------------- 2 files changed, 191 insertions(+), 213 deletions(-) diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index 379b392ed55b0..f214416555b87 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -487,20 +487,13 @@ MDSRank::MDSRank( Context *respawn_hook_, Context *suicide_hook_, boost::asio::io_context& ioctx) : - whoami(whoami_), incarnation(0), - mds_lock(mds_lock_), cct(msgr->cct), clog(clog_), timer(timer_), - mdsmap(mdsmap_), + cct(msgr->cct), mds_lock(mds_lock_), clog(clog_), + timer(timer_), mdsmap(mdsmap_), objecter(new Objecter(g_ceph_context, msgr, monc_, ctxpool, 0, 0)), - server(NULL), mdcache(NULL), locker(NULL), mdlog(NULL), - balancer(NULL), scrubstack(NULL), - damage_table(whoami_), - inotable(NULL), snapserver(NULL), snapclient(NULL), - sessionmap(this), logger(NULL), mlogger(NULL), + damage_table(whoami_), sessionmap(this), op_tracker(g_ceph_context, g_conf()->mds_enable_op_tracker, g_conf()->osd_num_op_tracker_shard), - last_state(MDSMap::STATE_BOOT), - state(MDSMap::STATE_BOOT), - cluster_degraded(false), stopping(false), + progress_thread(this), whoami(whoami_), purge_queue(g_ceph_context, whoami_, mdsmap_->get_metadata_pool(), objecter, new LambdaContext([this](int r) { @@ -509,14 +502,10 @@ MDSRank::MDSRank( } ) ), - progress_thread(this), dispatch_depth(0), - hb(NULL), last_tid(0), osd_epoch_barrier(0), beacon(beacon_), - mds_slow_req_count(0), - last_client_mdsmap_bcast(0), + beacon(beacon_), messenger(msgr), monc(monc_), mgrc(mgrc), respawn_hook(respawn_hook_), suicide_hook(suicide_hook_), - standby_replaying(false), starttime(mono_clock::now()) { hb = g_ceph_context->get_heartbeat_map()->add_worker("MDSRank", pthread_self()); diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h index a2dc8928ea6aa..fac6b08e47ee2 100644 --- a/src/mds/MDSRank.h +++ b/src/mds/MDSRank.h @@ -132,30 +132,32 @@ class C_ExecAndReply; * to the other subsystems, and message-sending calls. */ class MDSRank { - protected: - const mds_rank_t whoami; - - // Incarnation as seen in MDSMap at the point where a rank is - // assigned. - int incarnation; - public: - friend class C_Flush_Journal; friend class C_Drop_Cache; - friend class C_CacheDropExecAndReply; friend class C_ScrubExecAndReply; friend class C_ScrubControlExecAndReply; + class CephContext *cct; + + MDSRank( + mds_rank_t whoami_, + ceph::mutex &mds_lock_, + LogChannelRef &clog_, + SafeTimer &timer_, + Beacon &beacon_, + std::unique_ptr & mdsmap_, + Messenger *msgr, + MonClient *monc_, + MgrClient *mgrc, + Context *respawn_hook_, + Context *suicide_hook_, + boost::asio::io_context& ictx); + mds_rank_t get_nodeid() const { return whoami; } int64_t get_metadata_pool(); - // Reference to global MDS::mds_lock, so that users of MDSRank don't - // carry around references to the outer MDS, and we can substitute - // a separate lock here in future potentially. - ceph::mutex &mds_lock; - mono_time get_starttime() const { return starttime; } @@ -164,58 +166,16 @@ class MDSRank { return chrono::duration(now-starttime); } - class CephContext *cct; - bool is_daemon_stopping() const; - // Reference to global cluster log client, just to avoid initialising - // a separate one here. - LogChannelRef &clog; - - // Reference to global timer utility, because MDSRank and MDSDaemon - // currently both use the same mds_lock, so it makes sense for them - // to share a timer. - SafeTimer &timer; - - std::unique_ptr &mdsmap; /* MDSDaemon::mdsmap */ - - ceph::async::io_context_pool ctxpool; - Objecter *objecter; - - // sub systems - Server *server; - MDCache *mdcache; - Locker *locker; - MDLog *mdlog; - MDBalancer *balancer; - ScrubStack *scrubstack; - DamageTable damage_table; - - - InoTable *inotable; - - SnapServer *snapserver; - SnapClient *snapclient; - MDSTableClient *get_table_client(int t); MDSTableServer *get_table_server(int t); - SessionMap sessionmap; Session *get_session(client_t client) { return sessionmap.get_session(entity_name_t::CLIENT(client.v)); } Session *get_session(const cref_t &m); - PerfCounters *logger, *mlogger; - OpTracker op_tracker; - - // The last different state I held before current - MDSMap::DaemonState last_state; - // The state assigned to me by the MDSMap - MDSMap::DaemonState state; - - bool cluster_degraded; - MDSMap::DaemonState get_state() const { return state; } MDSMap::DaemonState get_want_state() const { return beacon.get_want_state(); } @@ -242,76 +202,7 @@ class MDSRank { void handle_write_error(int err); void update_mlogger(); - protected: - // Flag to indicate we entered shutdown: anyone seeing this to be true - // after taking mds_lock must drop out. - bool stopping; - - // PurgeQueue is only used by StrayManager, but it is owned by MDSRank - // because its init/shutdown happens at the top level. - PurgeQueue purge_queue; - - class ProgressThread : public Thread { - MDSRank *mds; - ceph::condition_variable cond; - public: - explicit ProgressThread(MDSRank *mds_) : mds(mds_) {} - void * entry() override; - void shutdown(); - void signal() {cond.notify_all();} - } progress_thread; - - list> waiting_for_nolaggy; - MDSContext::que finished_queue; - // Dispatch, retry, queues - int dispatch_depth; - void inc_dispatch_depth() { ++dispatch_depth; } - void dec_dispatch_depth() { --dispatch_depth; } - void retry_dispatch(const cref_t &m); - bool handle_deferrable_message(const cref_t &m); - void _advance_queues(); - bool _dispatch(const cref_t &m, bool new_msg); - ceph::heartbeat_handle_d *hb; // Heartbeat for threads using mds_lock - - bool is_stale_message(const cref_t &m) const; - - map peer_mdsmap_epoch; - - ceph_tid_t last_tid; // for mds-initiated requests (e.g. stray rename) - - MDSContext::vec waiting_for_active, waiting_for_replay, waiting_for_rejoin, - waiting_for_reconnect, waiting_for_resolve; - MDSContext::vec waiting_for_any_client_connection; - MDSContext::que replay_queue; - bool replaying_requests_done = false; - - map waiting_for_active_peer; - map waiting_for_mdsmap; - - epoch_t osd_epoch_barrier; - - // Const reference to the beacon so that we can behave differently - // when it's laggy. - Beacon &beacon; - - /** - * Emit clog warnings for any ops reported as warnings by optracker - */ - void check_ops_in_flight(); - - int mds_slow_req_count; - - /** - * Share MDSMap with clients - */ - void bcast_mds_map(); // to mounted clients - epoch_t last_client_mdsmap_bcast; - - map export_targets; /* targets this MDS is exporting to or wants/tries to */ - - void create_logger(); - public: void queue_waiter(MDSContext *c) { finished_queue.push_back(c); progress_thread.signal(); @@ -333,25 +224,6 @@ class MDSRank { progress_thread.signal(); } - MDSRank( - mds_rank_t whoami_, - ceph::mutex &mds_lock_, - LogChannelRef &clog_, - SafeTimer &timer_, - Beacon &beacon_, - std::unique_ptr & mdsmap_, - Messenger *msgr, - MonClient *monc_, - MgrClient *mgrc, - Context *respawn_hook_, - Context *suicide_hook_, - boost::asio::io_context& ictx); - - protected: - ~MDSRank(); - - public: - // Daemon lifetime functions: these guys break the abstraction // and call up into the parent MDSDaemon instance. It's kind // of unavoidable: if we want any depth into our calls @@ -446,8 +318,6 @@ class MDSRank { ceph_tid_t issue_tid() { return ++last_tid; } - Finisher *finisher; - MDSMap *get_mds_map() { return mdsmap.get(); } uint64_t get_num_requests() const { return logger->get(l_mds_request); } @@ -470,7 +340,106 @@ class MDSRank { void mark_base_recursively_scrubbed(inodeno_t ino); + // Reference to global MDS::mds_lock, so that users of MDSRank don't + // carry around references to the outer MDS, and we can substitute + // a separate lock here in future potentially. + ceph::mutex &mds_lock; + + // Reference to global cluster log client, just to avoid initialising + // a separate one here. + LogChannelRef &clog; + + // Reference to global timer utility, because MDSRank and MDSDaemon + // currently both use the same mds_lock, so it makes sense for them + // to share a timer. + SafeTimer &timer; + + std::unique_ptr &mdsmap; /* MDSDaemon::mdsmap */ + + ceph::async::io_context_pool ctxpool; + Objecter *objecter; + + // sub systems + Server *server = nullptr; + MDCache *mdcache = nullptr; + Locker *locker = nullptr; + MDLog *mdlog = nullptr; + MDBalancer *balancer = nullptr; + ScrubStack *scrubstack = nullptr; + DamageTable damage_table; + + InoTable *inotable = nullptr; + + SnapServer *snapserver = nullptr; + SnapClient *snapclient = nullptr; + + SessionMap sessionmap; + + PerfCounters *logger = nullptr, *mlogger = nullptr; + OpTracker op_tracker; + + // The last different state I held before current + MDSMap::DaemonState last_state = MDSMap::STATE_BOOT; + // The state assigned to me by the MDSMap + MDSMap::DaemonState state = MDSMap::STATE_BOOT; + + bool cluster_degraded = false; + + Finisher *finisher; protected: + typedef enum { + // The MDSMap is available, configure default layouts and structures + MDS_BOOT_INITIAL = 0, + // We are ready to open some inodes + MDS_BOOT_OPEN_ROOT, + // We are ready to do a replay if needed + MDS_BOOT_PREPARE_LOG, + // Replay is complete + MDS_BOOT_REPLAY_DONE + } BootStep; + + class ProgressThread : public Thread { + public: + explicit ProgressThread(MDSRank *mds_) : mds(mds_) {} + void * entry() override; + void shutdown(); + void signal() {cond.notify_all();} + private: + MDSRank *mds; + ceph::condition_variable cond; + } progress_thread; + + class C_MDS_StandbyReplayRestart; + class C_MDS_StandbyReplayRestartFinish; + // Friended to access retry_dispatch + friend class C_MDS_RetryMessage; + friend class C_MDS_BootStart; + friend class C_MDS_InternalBootStart; + friend class C_MDS_MonCommand; + + const mds_rank_t whoami; + + ~MDSRank(); + + void inc_dispatch_depth() { ++dispatch_depth; } + void dec_dispatch_depth() { --dispatch_depth; } + void retry_dispatch(const cref_t &m); + bool handle_deferrable_message(const cref_t &m); + void _advance_queues(); + bool _dispatch(const cref_t &m, bool new_msg); + bool is_stale_message(const cref_t &m) const; + + /** + * Emit clog warnings for any ops reported as warnings by optracker + */ + void check_ops_in_flight(); + + /** + * Share MDSMap with clients + */ + void bcast_mds_map(); // to mounted clients + void create_logger(); + void dump_clientreplay_status(Formatter *f) const; void command_scrub_start(Formatter *f, std::string_view path, std::string_view tag, @@ -507,37 +476,12 @@ class MDSRank { void command_dump_inode(Formatter *f, const cmdmap_t &cmdmap, std::ostream &ss); void command_cache_drop(uint64_t timeout, Formatter *f, Context *on_finish); - protected: - Messenger *messenger; - MonClient *monc; - MgrClient *mgrc; - - Context *respawn_hook; - Context *suicide_hook; - - // Friended to access retry_dispatch - friend class C_MDS_RetryMessage; - // FIXME the state machine logic should be separable from the dispatch // logic that calls it. // >>> void calc_recovery_set(); void request_state(MDSMap::DaemonState s); - bool standby_replaying; // true if current replay pass is in standby-replay mode - - typedef enum { - // The MDSMap is available, configure default layouts and structures - MDS_BOOT_INITIAL = 0, - // We are ready to open some inodes - MDS_BOOT_OPEN_ROOT, - // We are ready to do a replay if needed - MDS_BOOT_PREPARE_LOG, - // Replay is complete - MDS_BOOT_REPLAY_DONE - } BootStep; - friend class C_MDS_BootStart; - friend class C_MDS_InternalBootStart; void boot_create(); // i am new mds. void boot_start(BootStep step=MDS_BOOT_INITIAL, int r=0); // starting|replay @@ -547,8 +491,6 @@ class MDSRank { void replay_done(); void standby_replay_restart(); void _standby_replay_restart_finish(int r, uint64_t old_read_pos); - class C_MDS_StandbyReplayRestart; - class C_MDS_StandbyReplayRestartFinish; void reopen_log(); @@ -567,22 +509,71 @@ class MDSRank { void stopping_done(); void validate_sessions(); - // <<< - - // >>> + void handle_mds_recovery(mds_rank_t who); void handle_mds_failure(mds_rank_t who); - // <<< /* Update MDSMap export_targets for this rank. Called on ::tick(). */ void update_targets(); - friend class C_MDS_MonCommand; void _mon_command_finish(int r, std::string_view cmd, std::string_view outs); void set_mdsmap_multimds_snaps_allowed(); -private: - mono_time starttime = mono_clock::zero(); + Context *create_async_exec_context(C_ExecAndReply *ctx); + + // Incarnation as seen in MDSMap at the point where a rank is + // assigned. + int incarnation = 0; + + // Flag to indicate we entered shutdown: anyone seeing this to be true + // after taking mds_lock must drop out. + bool stopping = false; + + // PurgeQueue is only used by StrayManager, but it is owned by MDSRank + // because its init/shutdown happens at the top level. + PurgeQueue purge_queue; + + list> waiting_for_nolaggy; + MDSContext::que finished_queue; + // Dispatch, retry, queues + int dispatch_depth = 0; + + ceph::heartbeat_handle_d *hb = nullptr; // Heartbeat for threads using mds_lock + + map peer_mdsmap_epoch; + + ceph_tid_t last_tid = 0; // for mds-initiated requests (e.g. stray rename) + + MDSContext::vec waiting_for_active, waiting_for_replay, waiting_for_rejoin, + waiting_for_reconnect, waiting_for_resolve; + MDSContext::vec waiting_for_any_client_connection; + MDSContext::que replay_queue; + bool replaying_requests_done = false; + + map waiting_for_active_peer; + map waiting_for_mdsmap; + + epoch_t osd_epoch_barrier = 0; + + // Const reference to the beacon so that we can behave differently + // when it's laggy. + Beacon &beacon; + + int mds_slow_req_count = 0; + + epoch_t last_client_mdsmap_bcast = 0; + + map export_targets; /* targets this MDS is exporting to or wants/tries to */ + + Messenger *messenger; + MonClient *monc; + MgrClient *mgrc; + + Context *respawn_hook; + Context *suicide_hook; + + bool standby_replaying = false; // true if current replay pass is in standby-replay mode +private: // "task" string that gets displayed in ceph status inline static const std::string SCRUB_STATUS_KEY = "scrub status"; @@ -590,8 +581,7 @@ private: void schedule_update_timer_task(); void send_task_status(); -protected: - Context *create_async_exec_context(C_ExecAndReply *ctx); + mono_time starttime = mono_clock::zero(); }; /* This expects to be given a reference which it is responsible for. @@ -616,7 +606,6 @@ public: MDSContext *build() { return new C_MDS_RetryMessage(mds, m); } - private: MDSRank *mds; cref_t m; @@ -630,6 +619,20 @@ private: class MDSRankDispatcher : public MDSRank, public md_config_obs_t { public: + MDSRankDispatcher( + mds_rank_t whoami_, + ceph::mutex &mds_lock_, + LogChannelRef &clog_, + SafeTimer &timer_, + Beacon &beacon_, + std::unique_ptr &mdsmap_, + Messenger *msgr, + MonClient *monc_, + MgrClient *mgrc, + Context *respawn_hook_, + Context *suicide_hook_, + boost::asio::io_context& ictx); + void init(); void tick(); void shutdown(); @@ -656,20 +659,6 @@ public: // Call into me from MDS::ms_dispatch bool ms_dispatch(const cref_t &m); - - MDSRankDispatcher( - mds_rank_t whoami_, - ceph::mutex &mds_lock_, - LogChannelRef &clog_, - SafeTimer &timer_, - Beacon &beacon_, - std::unique_ptr &mdsmap_, - Messenger *msgr, - MonClient *monc_, - MgrClient *mgrc, - Context *respawn_hook_, - Context *suicide_hook_, - boost::asio::io_context& ictx); }; // This utility for MDS and MDSRank dispatchers. -- 2.39.5