From 7df69486f4ac1efd00442ecfb9a44f463783801c Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Wed, 14 Mar 2018 16:29:02 -0400
Subject: [PATCH] rgw: Cache notify fault injection

Allow a configured probability of dropping a notify message on the
floor without processing or acking it.

Fixes: http://tracker.ceph.com/issues/24962

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/common/options.cc | 16 ++++++++++++++++
 src/rgw/rgw_rados.cc  | 17 +++++++++++++++++
 src/rgw/rgw_rados.h   |  2 ++
 3 files changed, 35 insertions(+)

diff --git a/src/common/options.cc b/src/common/options.cc
index b2f43f0d1fb..998d21b1b43 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -6248,6 +6248,22 @@ std::vector<Option> get_rgw_options() {
 			  "of RGW instances under heavy use. If you would like "
 			  "to turn off cache expiry, set this value to zero."),
 
+
+    Option("rgw_inject_notify_timeout_probability", Option::TYPE_FLOAT,
+	   Option::LEVEL_DEV)
+    .set_default(0)
+    .add_tag("fault injection")
+    .add_tag("testing")
+    .add_service("rgw")
+    .set_min_max(0.0, 1.0)
+    .set_description("Likelihood of ignoring a notify")
+    .set_long_description("This is the probability that the RGW cache will "
+			  "ignore a cache notify message. It exists to help "
+			  "with the development and testing of cache "
+			  "consistency and recovery improvements. Please "
+			  "do not set it in a production cluster, as it "
+			  "actively causes failures. Set this to a floating "
+			  "point value between 0 and 1."),
   });
 }
 
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 717fbfe4c30..aea7448c6ec 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -2945,6 +2945,20 @@ public:
 			    << " cookie " << cookie
 			    << " notifier " << notifier_id
 			    << " bl.length()=" << bl.length() << dendl;
+
+    if (unlikely(rados->inject_notify_timeout_probability == 1) ||
+	(rados->inject_notify_timeout_probability > 0 &&
+	 (rados->inject_notify_timeout_probability >
+	  ceph::util::generate_random_number(0.0, 1.0)))) {
+      ldout(rados->ctx(), 0)
+	<< "RGWWatcher::handle_notify() dropping notification! "
+	<< "If this isn't what you want, set "
+	<< "rgw_inject_notify_timeout_probability to zero!" << dendl;
+      return;
+    }
+
+
+
     rados->watch_cb(notify_id, cookie, notifier_id, bl);
 
     bufferlist reply_bl; // empty reply payload
@@ -4739,6 +4753,9 @@ int RGWRados::initialize()
 {
   int ret;
 
+  inject_notify_timeout_probability =
+    cct->_conf.get_val<double>("rgw_inject_notify_timeout_probability");
+
   ret = init_rados();
   if (ret < 0)
     return ret;
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 4fc4504dd60..b5c01025b7b 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -2320,6 +2320,8 @@ class RGWRados : public AdminSocketHook
   librados::IoCtx control_pool_ctx;   // .rgw.control
   bool watch_initialized;
 
+  double inject_notify_timeout_probability = 0;
+
   friend class RGWWatcher;
 
   Mutex bucket_id_lock;
-- 
2.39.5