From ea35f148257282fe3f3ae02fe7a26cf245cda952 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 18 May 2016 20:53:26 -0400 Subject: [PATCH] rbd-mirror: disable librbd caching for replicated images Each image has its own cache and each cache uses its own thread. With a large replicated cluster, this could result in thousands of extra threads and gigabytes of extra memory. Fixes: http://tracker.ceph.com/issues/15930 Signed-off-by: Jason Dillaman --- src/test/rbd_mirror/test_ImageReplayer.cc | 2 ++ src/test/rbd_mirror/test_fixture.cc | 1 + src/tools/rbd_mirror/Replayer.cc | 2 ++ src/tools/rbd_mirror/main.cc | 3 +++ 4 files changed, 8 insertions(+) diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index 84e4afc22a4f5..db854beca035d 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -74,6 +74,7 @@ public: TestImageReplayer() : m_watch_handle(0) { EXPECT_EQ("", connect_cluster_pp(m_local_cluster)); + EXPECT_EQ(0, m_local_cluster.conf_set("rbd_cache", "false")); m_local_pool_name = get_temp_pool_name(); EXPECT_EQ(0, m_local_cluster.pool_create(m_local_pool_name.c_str())); @@ -81,6 +82,7 @@ public: m_local_ioctx)); EXPECT_EQ("", connect_cluster_pp(m_remote_cluster)); + EXPECT_EQ(0, m_remote_cluster.conf_set("rbd_cache", "false")); m_remote_pool_name = get_temp_pool_name(); EXPECT_EQ(0, m_remote_cluster.pool_create(m_remote_pool_name.c_str())); diff --git a/src/test/rbd_mirror/test_fixture.cc b/src/test/rbd_mirror/test_fixture.cc index 7ec0ab3284f57..b99b9fc6bd83b 100644 --- a/src/test/rbd_mirror/test_fixture.cc +++ b/src/test/rbd_mirror/test_fixture.cc @@ -23,6 +23,7 @@ TestFixture::TestFixture() { void TestFixture::SetUpTestCase() { ASSERT_EQ("", connect_cluster_pp(_rados)); + ASSERT_EQ(0, _rados.conf_set("rbd_cache", "false")); _local_pool_name = get_temp_pool_name("test-rbd-mirror-"); ASSERT_EQ(0, _rados.pool_create(_local_pool_name.c_str())); diff --git a/src/tools/rbd_mirror/Replayer.cc b/src/tools/rbd_mirror/Replayer.cc index 54a003d7044d7..9c22d5a2c131d 100644 --- a/src/tools/rbd_mirror/Replayer.cc +++ b/src/tools/rbd_mirror/Replayer.cc @@ -304,6 +304,8 @@ int Replayer::init() } } + // disable unnecessary librbd cache + cct->_conf->set_val_or_die("rbd_cache", "false"); cct->_conf->apply_changes(nullptr); cct->_conf->complain_about_parse_errors(cct); diff --git a/src/tools/rbd_mirror/main.cc b/src/tools/rbd_mirror/main.cc index 017f5a4b2b4a9..0bdafe8767586 100644 --- a/src/tools/rbd_mirror/main.cc +++ b/src/tools/rbd_mirror/main.cc @@ -61,6 +61,9 @@ int main(int argc, const char **argv) std::vector cmd_args; argv_to_vec(argc, argv, cmd_args); + // disable unnecessary librbd cache + g_ceph_context->_conf->set_val_or_die("rbd_cache", "false"); + mirror = new rbd::mirror::Mirror(g_ceph_context, cmd_args); int r = mirror->init(); if (r < 0) { -- 2.39.5