osd: do not send ENXIO on misdirected op by default

In practice this tends to get bubbled up the stack as an error on
the caller, and they usually do not handle it properly.  For example,
with librbd, this turns into EIO and break the VM.

Instead, this will manifest as a hung op on the client.  That is
also not ideal, but given that the root cause here is generally a
bug, it's not clear what else would be better.

We already log an error in the cluster log, so teuthology runs will
continue to fail.

Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit 923e7f5ce5ed437af15e178299a61029ff48e4a2)

# Conflicts:
#	PendingReleaseNotes

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 72f3a2f9acd83726e27d1c4c8e9da4b5c76c9ccc..efe5bea8b51171e818472bb949767652ed0eb22e 100644 (file)
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -7,3 +7,11 @@
    in old version would operate on different priority ranges
    than new ones. Once upgraded, cluster will operate on
    consistent values.
+
+* In previous versions, if a client sent an op to the wrong OSD, the OSD
+  would reply with ENXIO.  The rationale here is that the client or OSD is
+  clearly buggy and we want to surface the error as clearly as possible.
+  We now only send the ENXIO reply if the osd_enxio_on_misdirected_op option
+  is enabled (it's off by default).  This means that a VM using librbd that
+  previously would have gotten an EIO and gone read-only will now see a
+  blocked/hung IO instead.

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 8edeed6dd02a835d24124dd8319493f8bc31edc6..673da99a8acf0846acd0be9255f11dcb4cd0dfb0 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -841,6 +841,7 @@ OPTION(osd_debug_reject_backfill_probability, OPT_DOUBLE, 0)
 OPTION(osd_debug_inject_copyfrom_error, OPT_BOOL, false)  // inject failure during copyfrom completion
 OPTION(osd_debug_randomize_hobject_sort_order, OPT_BOOL, false)
 OPTION(osd_debug_misdirected_ops, OPT_BOOL, false)
+OPTION(osd_enxio_on_misdirected_op, OPT_BOOL, false)
 OPTION(osd_debug_verify_cached_snaps, OPT_BOOL, false)
 OPTION(osd_enable_op_tracker, OPT_BOOL, true) // enable/disable OSD op tracking
 OPTION(osd_num_op_tracker_shard, OPT_U32, 32) // The number of shards for holding the ops

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 3b62ef3755b80c74bb6ef8c8ca82a6622424ea68..727e244da31027114d5a18fbb4dbbe6314fe65be 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1432,11 +1432,14 @@ void OSDService::handle_misdirected_op(PG *pg, OpRequestRef op)
 
   dout(7) << *pg << " misdirected op in " << m->get_map_epoch() << dendl;
   clog->warn() << m->get_source_inst() << " misdirected " << m->get_reqid()
-             << " pg " << m->get_pg()
-             << " to osd." << whoami
-             << " not " << pg->acting
-             << " in e" << m->get_map_epoch() << "/" << osdmap->get_epoch() << "\n";
-  reply_op_error(op, -ENXIO);
+              << " pg " << m->get_pg()
+              << " to osd." << whoami
+              << " not " << pg->acting
+              << " in e" << m->get_map_epoch() << "/" << osdmap->get_epoch()
+              << "\n";
+  if (g_conf->osd_enxio_on_misdirected_op) {
+    reply_op_error(op, -ENXIO);
+  }
 }
 
 
@@ -8787,14 +8790,16 @@ void OSD::handle_op(OpRequestRef& op, OSDMapRef& osdmap)
   if (!send_map->osd_is_valid_op_target(pgid.pgid, whoami)) {
     dout(7) << "we are invalid target" << dendl;
     clog->warn() << m->get_source_inst() << " misdirected " << m->get_reqid()
-                     << " pg " << m->get_pg()
-                     << " to osd." << whoami
-                     << " in e" << osdmap->get_epoch()
-                     << ", client e" << m->get_map_epoch()
-                     << " pg " << pgid
-                     << " features " << m->get_connection()->get_features()
-                     << "\n";
-    service.reply_op_error(op, -ENXIO);
+                << " pg " << m->get_pg()
+                << " to osd." << whoami
+                << " in e" << osdmap->get_epoch()
+                << ", client e" << m->get_map_epoch()
+                << " pg " << pgid
+                << " features " << m->get_connection()->get_features()
+                << "\n";
+    if (g_conf->osd_enxio_on_misdirected_op) {
+      service.reply_op_error(op, -ENXIO);
+    }
     return;
   }