rbd-mirror: disable librbd caching for replicated images

Each image has its own cache and each cache uses its own thread. With
a large replicated cluster, this could result in thousands of extra
threads and gigabytes of extra memory.

Fixes: http://tracker.ceph.com/issues/15930
Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc
index 84e4afc22a4f53d8363bba46f48928bb0f00a7df..db854beca035dbd0559339f7f4e68d9a6b3fcabd 100644 (file)
--- a/src/test/rbd_mirror/test_ImageReplayer.cc
+++ b/src/test/rbd_mirror/test_ImageReplayer.cc
@@ -74,6 +74,7 @@ public:
   TestImageReplayer() : m_watch_handle(0)
   {
     EXPECT_EQ("", connect_cluster_pp(m_local_cluster));
+    EXPECT_EQ(0, m_local_cluster.conf_set("rbd_cache", "false"));
 
     m_local_pool_name = get_temp_pool_name();
     EXPECT_EQ(0, m_local_cluster.pool_create(m_local_pool_name.c_str()));
@@ -81,6 +82,7 @@ public:
                                              m_local_ioctx));
 
     EXPECT_EQ("", connect_cluster_pp(m_remote_cluster));
+    EXPECT_EQ(0, m_remote_cluster.conf_set("rbd_cache", "false"));
 
     m_remote_pool_name = get_temp_pool_name();
     EXPECT_EQ(0, m_remote_cluster.pool_create(m_remote_pool_name.c_str()));

diff --git a/src/test/rbd_mirror/test_fixture.cc b/src/test/rbd_mirror/test_fixture.cc
index 7ec0ab3284f5742d933cfb65b618d0fda9188f25..b99b9fc6bd83bab5ef11e8dccf71eac04186fb39 100644 (file)
--- a/src/test/rbd_mirror/test_fixture.cc
+++ b/src/test/rbd_mirror/test_fixture.cc
@@ -23,6 +23,7 @@ TestFixture::TestFixture() {
 
 void TestFixture::SetUpTestCase() {
   ASSERT_EQ("", connect_cluster_pp(_rados));
+  ASSERT_EQ(0, _rados.conf_set("rbd_cache", "false"));
 
   _local_pool_name = get_temp_pool_name("test-rbd-mirror-");
   ASSERT_EQ(0, _rados.pool_create(_local_pool_name.c_str()));

diff --git a/src/tools/rbd_mirror/Replayer.cc b/src/tools/rbd_mirror/Replayer.cc
index 54a003d7044d7240fa9c515abc5d814552707d40..9c22d5a2c131deca65ef5970567c49976d4846de 100644 (file)
--- a/src/tools/rbd_mirror/Replayer.cc
+++ b/src/tools/rbd_mirror/Replayer.cc
@@ -304,6 +304,8 @@ int Replayer::init()
     }
   }
 
+  // disable unnecessary librbd cache
+  cct->_conf->set_val_or_die("rbd_cache", "false");
   cct->_conf->apply_changes(nullptr);
   cct->_conf->complain_about_parse_errors(cct);
 

diff --git a/src/tools/rbd_mirror/main.cc b/src/tools/rbd_mirror/main.cc
index 017f5a4b2b4a912a4735588769a53a5fc7895dbd..0bdafe8767586cdb079773d106d863703ecb27f9 100644 (file)
--- a/src/tools/rbd_mirror/main.cc
+++ b/src/tools/rbd_mirror/main.cc
@@ -61,6 +61,9 @@ int main(int argc, const char **argv)
   std::vector<const char*> cmd_args;
   argv_to_vec(argc, argv, cmd_args);
 
+  // disable unnecessary librbd cache
+  g_ceph_context->_conf->set_val_or_die("rbd_cache", "false");
+
   mirror = new rbd::mirror::Mirror(g_ceph_context, cmd_args);
   int r = mirror->init();
   if (r < 0) {