cephtool: retry 'ceph tell <who> ...' command if connection fails

It was easy to reproduce a hang with 'ceph osd tell osd.0 foo' and
messenger failure injection.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/tools/common.cc b/src/tools/common.cc
index 78ab62b5081d85c200227071c2fdd55b9d0b7a24..4c671c7cd83eaa54ee9e61f1ed30df248a1e5ee7 100644 (file)
--- a/src/tools/common.cc
+++ b/src/tools/common.cc
@@ -160,7 +160,8 @@ static void send_command(CephToolCtx *ctx)
       if (!ctx->concise)
        *ctx->log << ceph_clock_now(g_ceph_context) << " " << pending_target << " <- " << pending_cmd << std::endl;
 
-      messenger->send_message(m, osdmap->get_inst(n));
+      command_con = messenger->get_connection(osdmap->get_inst(n));
+      messenger->send_message(m, command_con);
     }
     return;
   }
@@ -553,6 +554,19 @@ void Admin::ms_handle_connect(Connection *con) {
   }
 }
 
+bool Admin::ms_handle_reset(Connection *con)
+{
+  Mutex::Locker l(ctx->lock);
+  if (con == command_con) {
+    command_con->put();
+    command_con = NULL;
+    if (pending_cmd.size())
+      send_command(ctx);
+    return true;
+  }
+  return false;
+}
+
 bool Admin::ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, 
                              bool force_new)
 {

diff --git a/src/tools/common.h b/src/tools/common.h
index a0bad8a30a369094430a5aa10532ce5467ad738e..93a7d4a12709574ec094197598422ff63cb7be4c 100644 (file)
--- a/src/tools/common.h
+++ b/src/tools/common.h
@@ -63,7 +63,7 @@ public:
 
   bool ms_dispatch(Message *m);
   void ms_handle_connect(Connection *con);
-  bool ms_handle_reset(Connection *con) { return false; }
+  bool ms_handle_reset(Connection *con);
   void ms_handle_remote_reset(Connection *con) {}
 
   bool ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer, bool force_new);