From: Or Friedmann <ofriedma@redhat.com>
Date: Wed, 16 Dec 2020 16:09:24 +0000 (+0200)
Subject: rgw: distribute cache for exclusive put
X-Git-Tag: v15.2.9~42^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F38971%2Fhead;p=ceph.git

rgw: distribute cache for exclusive put

Distribute cache for exclusive put

Fixes: https://tracker.ceph.com/issues/48632

Thanks to Casey Bodley for finding the solution

Signed-off-by: Or Friedmann <ofriedma@redhat.com>
(cherry picked from commit 4fb51bc125b4ec23c2f47e7326d67f885f7d268c)
---

diff --git a/src/rgw/services/svc_notify.cc b/src/rgw/services/svc_notify.cc
index e6c0b58fbd2a3..8e911e577d5ac 100644
--- a/src/rgw/services/svc_notify.cc
+++ b/src/rgw/services/svc_notify.cc
@@ -151,6 +151,7 @@ string RGWSI_Notify::get_control_oid(int i)
   return string(buf);
 }
 
+// do not call pick_obj_control before init_watch
 RGWSI_RADOS::Obj RGWSI_Notify::pick_control_obj(const string& key)
 {
   uint32_t r = ceph_str_hash_linux(key.c_str(), key.size());
@@ -360,11 +361,20 @@ void RGWSI_Notify::_set_enabled(bool status)
 int RGWSI_Notify::distribute(const string& key, bufferlist& bl,
                              optional_yield y)
 {
-  RGWSI_RADOS::Obj notify_obj = pick_control_obj(key);
-
-  ldout(cct, 10) << "distributing notification oid=" << notify_obj.get_ref().obj
-      << " bl.length()=" << bl.length() << dendl;
-  return robust_notify(notify_obj, bl, y);
+  /* The RGW uses the control pool to store the watch notify objects.
+    The precedence in RGWSI_Notify::do_start is to call to zone_svc->start and later to init_watch().
+    The first time, RGW starts in the cluster, the RGW will try to create zone and zonegroup system object.
+    In that case RGW will try to distribute the cache before it ran init_watch,
+    which will lead to division by 0 in pick_obj_control (num_watchers is 0).
+  */
+  if (num_watchers > 0) {
+    RGWSI_RADOS::Obj notify_obj = pick_control_obj(key);
+
+    ldout(cct, 10) << "distributing notification oid=" << notify_obj.get_ref().obj
+        << " bl.length()=" << bl.length() << dendl;
+    return robust_notify(notify_obj, bl, y);
+  }
+  return 0;
 }
 
 int RGWSI_Notify::robust_notify(RGWSI_RADOS::Obj& notify_obj, bufferlist& bl,
diff --git a/src/rgw/services/svc_sys_obj_cache.cc b/src/rgw/services/svc_sys_obj_cache.cc
index 6943542b2a873..5854d76547c44 100644
--- a/src/rgw/services/svc_sys_obj_cache.cc
+++ b/src/rgw/services/svc_sys_obj_cache.cc
@@ -304,18 +304,9 @@ int RGWSI_SysObj_Cache::write(const rgw_raw_obj& obj,
   string name = normal_name(pool, oid);
   if (ret >= 0) {
     cache.put(name, info, NULL);
-    // Only distribute the cache information if we did not just create
-    // the object with the exclusive flag. Note: PUT_OBJ_EXCL implies
-    // PUT_OBJ_CREATE. Generally speaking, when successfully creating
-    // a system object with the exclusive flag it is not necessary to
-    // call distribute_cache, as a) it's unclear whether other RGWs
-    // will need that system object in the near-term and b) it
-    // generates additional network traffic.
-    if (!exclusive) {
-      int r = distribute_cache(name, obj, info, UPDATE_OBJ, y);
-      if (r < 0)
-	ldout(cct, 0) << "ERROR: failed to distribute cache for " << obj << dendl;
-    }
+    int r = distribute_cache(name, obj, info, UPDATE_OBJ, y);
+    if (r < 0)
+      ldout(cct, 0) << "ERROR: failed to distribute cache for " << obj << dendl;
   } else {
     cache.remove(name);
   }