From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Fri, 22 Jan 2021 14:14:00 +0000 (+0200)
Subject: rgw: fix valgrind errors when protected_fixedsize_stack is used
X-Git-Tag: v17.1.0~3125^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F39025%2Fhead;p=ceph.git

rgw: fix valgrind errors when protected_fixedsize_stack is used

Fixes: https://tracker.ceph.com/issues/48963

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b925881ed734..f167ef6dfaff 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -350,6 +350,13 @@ else(ALLOCATOR)
   endif(gperftools_FOUND)
 endif(ALLOCATOR)
 
+if(${ALLOCATOR} STREQUAL "libc" AND WITH_SYSTEM_BOOST)
+  # valgrind can only work with libc allocator
+  # if system boost is not used, valgrind use should
+  #   be indicated via the WITH_BOOST_VALGRIND parameter
+  add_definitions(-DBOOST_USE_VALGRIND)
+endif()
+
 # Mingw generates incorrect entry points when using "-pie".
 if(WIN32 OR (HAVE_LIBTCMALLOC AND WITH_STATIC_LIBSTDCXX))
   set(EXE_LINKER_USE_PIE FALSE)
diff --git a/src/rgw/rgw_notify.cc b/src/rgw/rgw_notify.cc
index 5d6b1759f4f7..7e9c05391c4e 100644
--- a/src/rgw/rgw_notify.cc
+++ b/src/rgw/rgw_notify.cc
@@ -6,6 +6,7 @@
 #include "cls/lock/cls_lock_client.h"
 #include <memory>
 #include <boost/algorithm/hex.hpp>
+#include <boost/context/protected_fixedsize_stack.hpp>
 #include <spawn/spawn.hpp>
 #include "rgw_pubsub.h"
 #include "rgw_pubsub_push.h"
@@ -46,6 +47,11 @@ WRITE_CLASS_ENCODER(event_entry_t)
 
 using queues_t = std::set<std::string>;
 
+// use mmap/mprotect to allocate 128k coroutine stacks
+auto make_stack_allocator() {
+  return boost::context::protected_fixedsize_stack{128*1024};
+}
+
 class Manager : public DoutPrefixProvider {
   const size_t max_queue_size;
   const uint32_t queues_update_period_ms;
@@ -231,7 +237,7 @@ class Manager : public DoutPrefixProvider {
     // start a the cleanup coroutine for the queue
     spawn::spawn(io_context, [this, queue_name](spawn::yield_context yield) {
             cleanup_queue(queue_name, yield);
-            });
+            }, make_stack_allocator());
     
     while (true) {
       // if queue was empty the last time, sleep for idle timeout
@@ -318,7 +324,7 @@ class Manager : public DoutPrefixProvider {
               ldpp_dout(this, 20) << "INFO: processing of entry: " << 
                 entry.marker << " (" << entry_idx << "/" << total_entries << ") from: " << queue_name << " failed" << dendl;
             } 
-        });
+        }, make_stack_allocator());
         ++entry_idx;
       }
 
@@ -437,7 +443,8 @@ class Manager : public DoutPrefixProvider {
             std::lock_guard lock_guard(queue_gc_lock);
             queue_gc.push_back(queue_name);
             ldpp_dout(this, 10) << "INFO: queue: " << queue_name << " marked for removal" << dendl;
-          });
+            ldout(cct, 10) << "INFO: queue: " << queue_name << " marked for removal" << dendl;
+          }, make_stack_allocator());
         } else {
           ldpp_dout(this, 20) << "INFO: queue: " << queue_name << " ownership (lock) renewed" << dendl;
         }
@@ -482,7 +489,7 @@ public:
     {
       spawn::spawn(io_context, [this](spawn::yield_context yield) {
             process_queues(yield);
-          });
+          }, make_stack_allocator());
 
       // start the worker threads to do the actual queue processing
       const std::string WORKER_THREAD_NAME = "notif-worker";