cephfs-mirror: restart failed/blocklisted replayer instances

This was buggy right from the start. Start maintaining per replayer
blocklisted or failed timestamp and use that to check if a replayer
restart is required.

Signed-off-by: Venky Shankar <vshankar@redhat.com>
(cherry picked from commit f2fb84ddd25ac66ca1e79339801913b35b597e83)

diff --git a/src/tools/cephfs_mirror/FSMirror.h b/src/tools/cephfs_mirror/FSMirror.h
index d2c9c07782adc2007b806f0cab2c8c098f41efe8..a9c1fab1025d9f42ee29bba6289ba1f7df744f7c 100644 (file)
--- a/src/tools/cephfs_mirror/FSMirror.h
+++ b/src/tools/cephfs_mirror/FSMirror.h
@@ -52,11 +52,35 @@ public:
            m_mirror_watcher->is_failed();
   }
 
+  utime_t get_failed_ts() {
+    std::scoped_lock locker(m_lock);
+    if (m_instance_watcher) {
+      return m_instance_watcher->get_failed_ts();
+    }
+    if (m_mirror_watcher) {
+      return m_mirror_watcher->get_failed_ts();
+    }
+
+    return utime_t();
+  }
+
   bool is_blocklisted() {
     std::scoped_lock locker(m_lock);
     return is_blocklisted(locker);
   }
 
+  utime_t get_blocklisted_ts() {
+    std::scoped_lock locker(m_lock);
+    if (m_instance_watcher) {
+      return m_instance_watcher->get_blocklisted_ts();
+    }
+    if (m_mirror_watcher) {
+      return m_mirror_watcher->get_blocklisted_ts();
+    }
+
+    return utime_t();
+  }
+
   Peers get_peers() {
     std::scoped_lock locker(m_lock);
     return m_all_peers;

diff --git a/src/tools/cephfs_mirror/InstanceWatcher.cc b/src/tools/cephfs_mirror/InstanceWatcher.cc
index aeb474274fb4a755d06d1be93842359383f58639..b6a51a141aad19cd3b072b487765e12058d3e7e4 100644 (file)
--- a/src/tools/cephfs_mirror/InstanceWatcher.cc
+++ b/src/tools/cephfs_mirror/InstanceWatcher.cc
@@ -116,12 +116,15 @@ void InstanceWatcher::handle_rewatch_complete(int r) {
     dout(0) << ": client blocklisted" <<dendl;
     std::scoped_lock locker(m_lock);
     m_blocklisted = true;
+    m_blocklisted_ts = ceph_clock_now();
   } else if (r == -ENOENT) {
     derr << ": mirroring object deleted" << dendl;
     m_failed = true;
+    m_failed_ts = ceph_clock_now();
   } else if (r < 0) {
     derr << ": rewatch error: " << cpp_strerror(r) << dendl;
     m_failed = true;
+    m_failed_ts = ceph_clock_now();
   }
 }
 

diff --git a/src/tools/cephfs_mirror/InstanceWatcher.h b/src/tools/cephfs_mirror/InstanceWatcher.h
index dfe0cd05990e553dbc7122b93849d28af47efc5d..a0740009605e4ae62c9f67cd3bd7c213808b31fc 100644 (file)
--- a/src/tools/cephfs_mirror/InstanceWatcher.h
+++ b/src/tools/cephfs_mirror/InstanceWatcher.h
@@ -49,11 +49,21 @@ public:
     return m_blocklisted;
   }
 
+  utime_t get_blocklisted_ts() {
+    std::scoped_lock locker(m_lock);
+    return m_blocklisted_ts;
+  }
+
   bool is_failed() {
     std::scoped_lock locker(m_lock);
     return m_failed;
   }
 
+  utime_t get_failed_ts() {
+    std::scoped_lock locker(m_lock);
+    return m_failed_ts;
+  }
+
 private:
   librados::IoCtx &m_ioctx;
   Listener &m_listener;
@@ -66,6 +76,9 @@ private:
   bool m_blocklisted = false;
   bool m_failed = false;
 
+  utime_t m_blocklisted_ts;
+  utime_t m_failed_ts;
+
   void create_instance();
   void handle_create_instance(int r);
 

diff --git a/src/tools/cephfs_mirror/Mirror.cc b/src/tools/cephfs_mirror/Mirror.cc
index 2b108f9f9fd43b865da070f9f5f5e270c47178db..edf903b921fb3a8d9fb2fcccb54d78effa385bcd 100644 (file)
--- a/src/tools/cephfs_mirror/Mirror.cc
+++ b/src/tools/cephfs_mirror/Mirror.cc
@@ -196,8 +196,6 @@ Mirror::Mirror(CephContext *cct, const std::vector<const char*> &args,
     m_monc(monc),
     m_msgr(msgr),
     m_listener(this),
-    m_last_blocklist_check(ceph_clock_now()),
-    m_last_failure_check(ceph_clock_now()),
     m_local(new librados::Rados()) {
   auto thread_pool = &(cct->lookup_or_create_singleton_object<ThreadPoolSingleton>(
                          "cephfs::mirror::thread_pool", false, cct));
@@ -498,51 +496,35 @@ void Mirror::update_fs_mirrors() {
   auto now = ceph_clock_now();
   double blocklist_interval = g_ceph_context->_conf.get_val<std::chrono::seconds>
     ("cephfs_mirror_restart_mirror_on_blocklist_interval").count();
-  bool check_blocklist = blocklist_interval > 0 && ((now - m_last_blocklist_check) >= blocklist_interval);
-
   double failed_interval = g_ceph_context->_conf.get_val<std::chrono::seconds>
     ("cephfs_mirror_restart_mirror_on_failure_interval").count();
-  bool check_failure = failed_interval > 0 && ((now - m_last_failure_check) >= failed_interval);
 
   {
     std::scoped_lock locker(m_lock);
     for (auto &[filesystem, mirror_action] : m_mirror_actions) {
-      auto failed = mirror_action.fs_mirror && mirror_action.fs_mirror->is_failed();
-      auto blocklisted = mirror_action.fs_mirror && mirror_action.fs_mirror->is_blocklisted();
-
-      if (check_failure && !mirror_action.action_in_progress &&
-         !_is_restarting(filesystem) && failed) {
-        // about to restart failed mirror instance -- nothing
-        // should interfere
-        dout(5) << ": filesystem=" << filesystem << " failed mirroring -- restarting" << dendl;
-       _set_restarting(filesystem);
-        auto peers = mirror_action.fs_mirror->get_peers();
-        auto ctx =  new C_RestartMirroring(this, filesystem, mirror_action.pool_id, peers);
-        ctx->complete(0);
-      } else if (check_blocklist && !mirror_action.action_in_progress &&
-                !_is_restarting(filesystem) && blocklisted) {
-        // about to restart blocklisted mirror instance -- nothing
-        // should interfere
-       _set_restarting(filesystem);
-        dout(5) << ": filesystem=" << filesystem << " is blocklisted -- restarting" << dendl;
-        auto peers = mirror_action.fs_mirror->get_peers();
-        auto ctx = new C_RestartMirroring(this, filesystem, mirror_action.pool_id, peers);
-        ctx->complete(0);
+      auto failed_restart = mirror_action.fs_mirror && mirror_action.fs_mirror->is_failed() &&
+       (failed_interval > 0 && (mirror_action.fs_mirror->get_failed_ts() - now) > failed_interval);
+      auto blocklisted_restart = mirror_action.fs_mirror && mirror_action.fs_mirror->is_blocklisted() &&
+       (blocklist_interval > 0 && (mirror_action.fs_mirror->get_blocklisted_ts() - now) > blocklist_interval);
+
+      if (!mirror_action.action_in_progress && !_is_restarting(filesystem)) {
+       if (failed_restart || blocklisted_restart) {
+         dout(5) << ": filesystem=" << filesystem << " failed mirroring (failed: "
+                 << failed_restart << ", blocklisted: " << blocklisted_restart << dendl;
+         _set_restarting(filesystem);
+         auto peers = mirror_action.fs_mirror->get_peers();
+         auto ctx =  new C_RestartMirroring(this, filesystem, mirror_action.pool_id, peers);
+         ctx->complete(0);
+       }
       }
-      if (!failed && !blocklisted && !mirror_action.action_ctxs.empty()
+
+      if (!failed_restart && !blocklisted_restart && !mirror_action.action_ctxs.empty()
           && !mirror_action.action_in_progress) {
         auto ctx = std::move(mirror_action.action_ctxs.front());
         mirror_action.action_ctxs.pop_front();
         ctx->complete(0);
       }
     }
-
-    if (check_blocklist) {
-      m_last_blocklist_check = now;
-    }
-    if (check_failure) {
-      m_last_failure_check = now;
-    }
   }
 
   schedule_mirror_update_task();

diff --git a/src/tools/cephfs_mirror/Mirror.h b/src/tools/cephfs_mirror/Mirror.h
index 74fe5f4d8d31b179d0a6b85f1f589ef738a798d3..2081b5b53050f0e08f098aa1cf6f18a2b4f9f191 100644 (file)
--- a/src/tools/cephfs_mirror/Mirror.h
+++ b/src/tools/cephfs_mirror/Mirror.h
@@ -101,9 +101,6 @@ private:
   std::unique_ptr<ClusterWatcher> m_cluster_watcher;
   std::map<Filesystem, MirrorAction> m_mirror_actions;
 
-  utime_t m_last_blocklist_check;
-  utime_t m_last_failure_check;
-
   RadosRef m_local;
   std::unique_ptr<ServiceDaemon> m_service_daemon;
 

diff --git a/src/tools/cephfs_mirror/MirrorWatcher.cc b/src/tools/cephfs_mirror/MirrorWatcher.cc
index 26b88d077e59809b8565eea6d29fe5f5b3906574..b3770d103ea3ef00371ee92de5b89a09d5802796 100644 (file)
--- a/src/tools/cephfs_mirror/MirrorWatcher.cc
+++ b/src/tools/cephfs_mirror/MirrorWatcher.cc
@@ -93,12 +93,15 @@ void MirrorWatcher::handle_rewatch_complete(int r) {
     dout(0) << ": client blocklisted" <<dendl;
     std::scoped_lock locker(m_lock);
     m_blocklisted = true;
+    m_blocklisted_ts = ceph_clock_now();
   } else if (r == -ENOENT) {
     derr << ": mirroring object deleted" << dendl;
     m_failed = true;
+    m_failed_ts = ceph_clock_now();
   } else if (r < 0) {
     derr << ": rewatch error: " << cpp_strerror(r) << dendl;
     m_failed = true;
+    m_failed_ts = ceph_clock_now();
   }
 }
 

diff --git a/src/tools/cephfs_mirror/MirrorWatcher.h b/src/tools/cephfs_mirror/MirrorWatcher.h
index c4d4f452294d8459d4c53ad7b6ed97fb8f03e732..54e185b95b04eed14b3f0dbc2dd36a09b1c98859 100644 (file)
--- a/src/tools/cephfs_mirror/MirrorWatcher.h
+++ b/src/tools/cephfs_mirror/MirrorWatcher.h
@@ -47,11 +47,21 @@ public:
     return m_blocklisted;
   }
 
+  utime_t get_blocklisted_ts() {
+    std::scoped_lock locker(m_lock);
+    return m_blocklisted_ts;
+  }
+
   bool is_failed() {
     std::scoped_lock locker(m_lock);
     return m_failed;
   }
 
+  utime_t get_failed_ts() {
+    std::scoped_lock locker(m_lock);
+    return m_failed_ts;
+  }
+
 private:
   librados::IoCtx &m_ioctx;
   FSMirror *m_fs_mirror;
@@ -66,6 +76,9 @@ private:
   bool m_blocklisted = false;
   bool m_failed = false;
 
+  utime_t m_blocklisted_ts;
+  utime_t m_failed_ts;
+
   void register_watcher();
   void handle_register_watcher(int r);