From 7f6685383dfdab1b685c9c4df609feb7fdc97a91 Mon Sep 17 00:00:00 2001
From: Jason Dillaman <dillaman@redhat.com>
Date: Tue, 14 May 2019 09:34:04 -0400
Subject: [PATCH] librbd: use custom allocator for aligned
 boost::lockfree::queue

If tcmalloc is in-use as the allocator and its version is less than 2.6.2,
it might be missing support for 'aligned_alloc'. This can result in the
glibc version of 'aligned_alloc' being used to allocate memory that is
then freed by tcmalloc -- resulting in a crash.

Fixes: http://tracker.ceph.com/issues/39703
Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---
 CMakeLists.txt                |  4 +++
 src/common/allocator.h        | 66 +++++++++++++++++++++++++++++++++++
 src/include/config-h.in.cmake |  1 +
 src/librbd/ImageCtx.h         |  6 +++-
 4 files changed, 76 insertions(+), 1 deletion(-)
 create mode 100644 src/common/allocator.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f97f96b5bf32a..bb68ffa83cccb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -386,6 +386,10 @@ else()
   set(EXE_LINKER_USE_PIE ${ENABLE_SHARED})
 endif()
 
+if (HAVE_LIBTCMALLOC AND TCMALLOC_VERSION_STRING VERSION_LESS 2.6.2)
+  set(LIBTCMALLOC_MISSING_ALIGNED_ALLOC ON)
+endif()
+
 if(WITH_LIBCEPHFS OR WITH_KRBD)
   find_package(keyutils REQUIRED)
 endif()
diff --git a/src/common/allocator.h b/src/common/allocator.h
new file mode 100644
index 0000000000000..977a25c5d72d9
--- /dev/null
+++ b/src/common/allocator.h
@@ -0,0 +1,66 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_COMMON_ALLOCATOR_H
+#define CEPH_COMMON_ALLOCATOR_H
+
+#include <malloc.h>
+#include <memory>
+#include <new>
+#include "acconfig.h"
+
+namespace ceph {
+
+#ifdef LIBTCMALLOC_MISSING_ALIGNED_ALLOC
+
+// If libtcmalloc is missing 'aligned_alloc', provide a new allocator class that
+// uses memalign which is what newer versions of tcmalloc do internally. C++17
+// will automatically use 'operator new(size_t, align_val_t)' for aligned
+// structures, which will invoke the missing 'aligned_alloc' tcmalloc function.
+// This method was added to tcmalloc (gperftools) in commit d406f228 after
+// the 2.6.1 release was tagged.
+template <typename T>
+struct allocator : public std::allocator<T> {
+  using pointer = typename std::allocator<T>::pointer;
+  using size_type = typename std::allocator<T>::size_type;
+
+  template<class U>
+  struct rebind {
+    typedef allocator<U> other;
+  };
+
+  allocator() noexcept {
+  }
+
+  allocator(const allocator& other) noexcept : std::allocator<T>(other) {
+  }
+
+  template <class U>
+  allocator(const allocator<U>& other) noexcept : std::allocator<T>(other) {
+  }
+
+  pointer allocate(size_type n, const void* hint = nullptr) {
+    if (n > this->max_size()) {
+      throw std::bad_alloc();
+    }
+
+    if (alignof(T) > __STDCPP_DEFAULT_NEW_ALIGNMENT__) {
+      return static_cast<T*>(memalign(alignof(T), n * sizeof(T)));
+    }
+    return std::allocator<T>::allocate(n, hint);
+  }
+};
+
+#else  // LIBTCMALLOC_MISSING_ALIGNED_ALLOC
+
+// re-use the full std::allocator implementation if tcmalloc is functional
+
+template <typename T>
+using allocator = std::allocator<T>;
+
+#endif // LIBTCMALLOC_MISSING_ALIGNED_ALLOC
+
+} // namespace ceph
+
+#endif // CEPH_COMMON_ALLOCATOR_H
+
diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake
index 326facb9e253b..29580042f241a 100644
--- a/src/include/config-h.in.cmake
+++ b/src/include/config-h.in.cmake
@@ -119,6 +119,7 @@
 
 /* Define if you have tcmalloc */
 #cmakedefine HAVE_LIBTCMALLOC
+#cmakedefine LIBTCMALLOC_MISSING_ALIGNED_ALLOC
 
 /* Define if have curl_multi_wait() */
 #cmakedefine HAVE_CURL_MULTI_WAIT 1
diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index 85afc5aecbbf4..32f5f5f2b9655 100644
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -12,6 +12,7 @@
 #include <string>
 #include <vector>
 
+#include "common/allocator.h"
 #include "common/config_proxy.h"
 #include "common/event_socket.h"
 #include "common/Mutex.h"
@@ -31,6 +32,7 @@
 #include "librbd/AsyncRequest.h"
 #include "librbd/Types.h"
 
+#include <boost/lockfree/policies.hpp>
 #include <boost/lockfree/queue.hpp>
 
 class CephContext;
@@ -170,7 +172,9 @@ namespace librbd {
 
     ContextWQ *op_work_queue;
 
-    boost::lockfree::queue<io::AioCompletion*> completed_reqs;
+    boost::lockfree::queue<
+      io::AioCompletion*,
+      boost::lockfree::allocator<ceph::allocator<void>>> completed_reqs;
     EventSocket event_socket;
 
     bool ignore_migrating = false;
-- 
2.47.3