rgw/posix: fix Inotify member initialization order race

wfd and efd were initialized in the Inotify constructor body, but
the inotify thread was started in the member initializer list via
thrd(&Inotify::ev_loop, this). Since C++ initializes members in
declaration order (wfd, efd, thrd), the thread could start before
the constructor body ran, causing ev_loop() to capture indeterminate
fd values into its local pollfd array.

When the destructor later signaled shutdown via the real efd, the
thread never woke because it was polling the wrong fd, causing
thrd.join() to block indefinitely. This predominantly affected arm64
builds due to the weaker memory model widening the race window.

Fix by moving wfd/efd initialization into the member initializer
list so they are set before the thread starts. Also make shutdown
std::atomic<bool> to eliminate the data race, close wfd/efd in the
destructor to fix the fd leak, and add error checking for eventfd().

Fixes: https://tracker.ceph.com/issues/75601
Signed-off-by: Kefu Chai <k.chai@proxmox.com>

diff --git a/src/rgw/driver/posix/notify.h b/src/rgw/driver/posix/notify.h
index 517615193d4f01c34fbaa2b80f4d127f053a8c4e..bdfd5ec582ab04922c402c45a749d83f3f54dcd3 100644 (file)
--- a/src/rgw/driver/posix/notify.h
+++ b/src/rgw/driver/posix/notify.h
@@ -3,6 +3,7 @@
 
 #pragma once
 
+#include <atomic>
 #include <iostream>
 #include <string>
 #include <memory>
@@ -115,7 +116,7 @@ namespace file::listing {
     std::mutex map_mutex;  // protects wd_callback_map and wd_remove_map
     wd_callback_map_t wd_callback_map;
     wd_remove_map_t wd_remove_map;
-    bool shutdown{false};
+    std::atomic<bool> shutdown{false};
 
     class AlignedBuf
     {
@@ -147,9 +148,9 @@ namespace file::listing {
       struct pollfd fds[2] = {{wfd, POLLIN}, {efd, POLLIN}};
 
     restart:
-      while(! shutdown) {
+      while(! shutdown.load(std::memory_order_acquire)) {
        npoll = poll(fds, nfds, -1); /* for up to 10 fds, poll is fast as epoll */
-       if (shutdown) {
+       if (shutdown.load(std::memory_order_acquire)) {
          return;
        }
        if (npoll == -1) {
@@ -211,14 +212,18 @@ namespace file::listing {
 
     Inotify(Notifiable* n, const std::string& bucket_root)
       : Notify(n, bucket_root),
+       wfd(inotify_init1(IN_NONBLOCK)),
+       efd(eventfd(0, EFD_NONBLOCK)),
        thrd(&Inotify::ev_loop, this)
       {
-       wfd = inotify_init1(IN_NONBLOCK);
        if (wfd == -1) {
          std::cerr << fmt::format("{} inotify_init1 failed with {}", __func__, wfd) << std::endl;
          exit(1);
        }
-       efd = eventfd(0, EFD_NONBLOCK);
+       if (efd == -1) {
+         std::cerr << fmt::format("{} eventfd failed", __func__) << std::endl;
+         exit(1);
+       }
       }
 
     void signal_shutdown() {
@@ -258,9 +263,11 @@ namespace file::listing {
     }
 
     virtual ~Inotify() {
-      shutdown = true;
+      shutdown.store(true, std::memory_order_release);
       signal_shutdown();
       thrd.join();
+      close(wfd);
+      close(efd);
     }
   };
 #endif /* linux */