test_mock_InstanceReplayer.cc
test_mock_InstanceWatcher.cc
test_mock_LeaderWatcher.cc
+ test_mock_NamespaceReplayer.cc
test_mock_PoolReplayer.cc
test_mock_PoolWatcher.cc
image_deleter/test_mock_RemoveRequest.cc
#include "librbd/io/ImageRequestWQ.h"
#include "librbd/io/ReadResult.h"
#include "tools/rbd_mirror/ImageReplayer.h"
+#include "tools/rbd_mirror/ImageSyncThrottler.h"
#include "tools/rbd_mirror/InstanceWatcher.h"
-#include "tools/rbd_mirror/ServiceDaemon.h"
#include "tools/rbd_mirror/Threads.h"
#include "tools/rbd_mirror/Types.h"
m_remote_image_id = get_image_id(m_remote_ioctx, m_image_name);
m_global_image_id = get_global_image_id(m_remote_ioctx, m_remote_image_id);
- m_threads.reset(new rbd::mirror::Threads<>(reinterpret_cast<CephContext*>(
- m_local_ioctx.cct())));
+ auto cct = reinterpret_cast<CephContext*>(m_local_ioctx.cct());
+ m_threads.reset(new rbd::mirror::Threads<>(cct));
- m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context,
- m_local_cluster,
- m_threads.get()));
+ m_image_sync_throttler.reset(new rbd::mirror::ImageSyncThrottler<>(cct));
m_instance_watcher = rbd::mirror::InstanceWatcher<>::create(
- m_local_ioctx, m_threads->work_queue, nullptr);
+ m_local_ioctx, m_threads->work_queue, nullptr,
+ m_image_sync_throttler.get());
m_instance_watcher->handle_acquire_leader();
}
template <typename ImageReplayerT = rbd::mirror::ImageReplayer<> >
void create_replayer() {
- m_replayer = new ImageReplayerT(
- m_threads.get(), m_instance_watcher, nullptr,
- rbd::mirror::RadosRef(new librados::Rados(m_local_ioctx)),
- m_local_mirror_uuid, m_local_ioctx.get_id(), m_global_image_id);
+ m_replayer = new ImageReplayerT(m_local_ioctx, m_local_mirror_uuid,
+ m_global_image_id, m_threads.get(),
+ m_instance_watcher, nullptr);
m_replayer->add_peer("peer uuid", m_remote_ioctx);
}
std::shared_ptr<librados::Rados> m_local_cluster;
std::unique_ptr<rbd::mirror::Threads<>> m_threads;
- std::unique_ptr<rbd::mirror::ServiceDaemon<>> m_service_daemon;
+ std::unique_ptr<rbd::mirror::ImageSyncThrottler<>> m_image_sync_throttler;
librados::Rados m_remote_cluster;
rbd::mirror::InstanceWatcher<> *m_instance_watcher;
std::string m_local_mirror_uuid = "local mirror uuid";
#include "librbd/io/ReadResult.h"
#include "librbd/journal/Types.h"
#include "tools/rbd_mirror/ImageSync.h"
+#include "tools/rbd_mirror/ImageSyncThrottler.h"
#include "tools/rbd_mirror/InstanceWatcher.h"
#include "tools/rbd_mirror/Threads.h"
create_and_open(m_local_io_ctx, &m_local_image_ctx);
create_and_open(m_remote_io_ctx, &m_remote_image_ctx);
+ auto cct = reinterpret_cast<CephContext*>(m_local_io_ctx.cct());
+ m_image_sync_throttler = rbd::mirror::ImageSyncThrottler<>::create(cct);
+
m_instance_watcher = rbd::mirror::InstanceWatcher<>::create(
- m_local_io_ctx, m_threads->work_queue, nullptr);
+ m_local_io_ctx, m_threads->work_queue, nullptr, m_image_sync_throttler);
m_instance_watcher->handle_acquire_leader();
m_remote_journaler = new ::journal::Journaler(
}
void TearDown() override {
- TestFixture::TearDown();
-
m_instance_watcher->handle_release_leader();
delete m_remote_journaler;
delete m_instance_watcher;
+ delete m_image_sync_throttler;
+
+ TestFixture::TearDown();
}
void create_and_open(librados::IoCtx &io_ctx, librbd::ImageCtx **image_ctx) {
librbd::ImageCtx *m_remote_image_ctx;
librbd::ImageCtx *m_local_image_ctx;
+ rbd::mirror::ImageSyncThrottler<> *m_image_sync_throttler;
rbd::mirror::InstanceWatcher<> *m_instance_watcher;
::journal::Journaler *m_remote_journaler;
librbd::journal::MirrorPeerClientMeta m_client_meta;
TEST_F(TestInstanceWatcher, InitShutdown)
{
InstanceWatcher<> instance_watcher(m_local_io_ctx, m_threads->work_queue,
- nullptr, m_instance_id);
+ nullptr, nullptr, m_instance_id);
std::vector<std::string> instance_ids;
get_instances(&instance_ids);
ASSERT_EQ(0U, instance_ids.size());
ASSERT_EQ("", connect_cluster_pp(cluster));
ASSERT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx));
InstanceWatcher<> instance_watcher(m_local_io_ctx, m_threads->work_queue,
- nullptr, "instance_id");
+ nullptr, nullptr, "instance_id");
// Init
ASSERT_EQ(0, instance_watcher.init());
void create_image_replayer(MockThreads &mock_threads) {
m_image_replayer = new MockImageReplayer(
- &mock_threads, &m_instance_watcher, nullptr,
- rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)),
- "local_mirror_uuid", m_local_io_ctx.get_id(), "global image id");
+ m_local_io_ctx, "local_mirror_uuid", "global image id",
+ &mock_threads, &m_instance_watcher, nullptr);
m_image_replayer->add_peer("peer_uuid", m_remote_io_ctx);
}
TEST_F(TestMockImageSyncThrottler, Single_Sync) {
MockImageSyncThrottler throttler(g_ceph_context);
C_SaferCond on_start;
- throttler.start_op("id", &on_start);
+ throttler.start_op("ns", "id", &on_start);
ASSERT_EQ(0, on_start.wait());
- throttler.finish_op("id");
+ throttler.finish_op("ns", "id");
}
TEST_F(TestMockImageSyncThrottler, Multiple_Syncs) {
throttler.set_max_concurrent_syncs(2);
C_SaferCond on_start1;
- throttler.start_op("id1", &on_start1);
+ throttler.start_op("ns", "id1", &on_start1);
C_SaferCond on_start2;
- throttler.start_op("id2", &on_start2);
+ throttler.start_op("ns", "id2", &on_start2);
C_SaferCond on_start3;
- throttler.start_op("id3", &on_start3);
+ throttler.start_op("ns", "id3", &on_start3);
C_SaferCond on_start4;
- throttler.start_op("id4", &on_start4);
+ throttler.start_op("ns", "id4", &on_start4);
ASSERT_EQ(0, on_start2.wait());
- throttler.finish_op("id2");
+ throttler.finish_op("ns", "id2");
ASSERT_EQ(0, on_start3.wait());
- throttler.finish_op("id3");
+ throttler.finish_op("ns", "id3");
ASSERT_EQ(0, on_start1.wait());
- throttler.finish_op("id1");
+ throttler.finish_op("ns", "id1");
ASSERT_EQ(0, on_start4.wait());
- throttler.finish_op("id4");
+ throttler.finish_op("ns", "id4");
}
TEST_F(TestMockImageSyncThrottler, Cancel_Running_Sync) {
MockImageSyncThrottler throttler(g_ceph_context);
C_SaferCond on_start;
- throttler.start_op("id", &on_start);
+ throttler.start_op("ns", "id", &on_start);
ASSERT_EQ(0, on_start.wait());
- ASSERT_FALSE(throttler.cancel_op("id"));
- throttler.finish_op("id");
+ ASSERT_FALSE(throttler.cancel_op("ns", "id"));
+ throttler.finish_op("ns", "id");
}
TEST_F(TestMockImageSyncThrottler, Cancel_Waiting_Sync) {
throttler.set_max_concurrent_syncs(1);
C_SaferCond on_start1;
- throttler.start_op("id1", &on_start1);
+ throttler.start_op("ns", "id1", &on_start1);
C_SaferCond on_start2;
- throttler.start_op("id2", &on_start2);
+ throttler.start_op("ns", "id2", &on_start2);
ASSERT_EQ(0, on_start1.wait());
- ASSERT_TRUE(throttler.cancel_op("id2"));
+ ASSERT_TRUE(throttler.cancel_op("ns", "id2"));
ASSERT_EQ(-ECANCELED, on_start2.wait());
- throttler.finish_op("id1");
+ throttler.finish_op("ns", "id1");
}
-
TEST_F(TestMockImageSyncThrottler, Cancel_Running_Sync_Start_Waiting) {
MockImageSyncThrottler throttler(g_ceph_context);
throttler.set_max_concurrent_syncs(1);
C_SaferCond on_start1;
- throttler.start_op("id1", &on_start1);
+ throttler.start_op("ns", "id1", &on_start1);
C_SaferCond on_start2;
- throttler.start_op("id2", &on_start2);
+ throttler.start_op("ns", "id2", &on_start2);
ASSERT_EQ(0, on_start1.wait());
- ASSERT_FALSE(throttler.cancel_op("id1"));
- throttler.finish_op("id1");
+ ASSERT_FALSE(throttler.cancel_op("ns", "id1"));
+ throttler.finish_op("ns", "id1");
ASSERT_EQ(0, on_start2.wait());
- throttler.finish_op("id2");
+ throttler.finish_op("ns", "id2");
}
TEST_F(TestMockImageSyncThrottler, Duplicate) {
throttler.set_max_concurrent_syncs(1);
C_SaferCond on_start1;
- throttler.start_op("id1", &on_start1);
+ throttler.start_op("ns", "id1", &on_start1);
ASSERT_EQ(0, on_start1.wait());
C_SaferCond on_start2;
- throttler.start_op("id1", &on_start2);
+ throttler.start_op("ns", "id1", &on_start2);
ASSERT_EQ(0, on_start2.wait());
C_SaferCond on_start3;
- throttler.start_op("id2", &on_start3);
+ throttler.start_op("ns", "id2", &on_start3);
C_SaferCond on_start4;
- throttler.start_op("id2", &on_start4);
+ throttler.start_op("ns", "id2", &on_start4);
ASSERT_EQ(-ENOENT, on_start3.wait());
- throttler.finish_op("id1");
+ throttler.finish_op("ns", "id1");
ASSERT_EQ(0, on_start4.wait());
- throttler.finish_op("id2");
+ throttler.finish_op("ns", "id2");
}
TEST_F(TestMockImageSyncThrottler, Duplicate2) {
throttler.set_max_concurrent_syncs(2);
C_SaferCond on_start1;
- throttler.start_op("id1", &on_start1);
+ throttler.start_op("ns", "id1", &on_start1);
ASSERT_EQ(0, on_start1.wait());
C_SaferCond on_start2;
- throttler.start_op("id2", &on_start2);
+ throttler.start_op("ns", "id2", &on_start2);
ASSERT_EQ(0, on_start2.wait());
C_SaferCond on_start3;
- throttler.start_op("id3", &on_start3);
+ throttler.start_op("ns", "id3", &on_start3);
C_SaferCond on_start4;
- throttler.start_op("id3", &on_start4); // dup
+ throttler.start_op("ns", "id3", &on_start4); // dup
ASSERT_EQ(-ENOENT, on_start3.wait());
C_SaferCond on_start5;
- throttler.start_op("id4", &on_start5);
+ throttler.start_op("ns", "id4", &on_start5);
- throttler.finish_op("id1");
+ throttler.finish_op("ns", "id1");
ASSERT_EQ(0, on_start4.wait());
- throttler.finish_op("id2");
+ throttler.finish_op("ns", "id2");
ASSERT_EQ(0, on_start5.wait());
C_SaferCond on_start6;
- throttler.start_op("id5", &on_start6);
+ throttler.start_op("ns", "id5", &on_start6);
- throttler.finish_op("id3");
+ throttler.finish_op("ns", "id3");
ASSERT_EQ(0, on_start6.wait());
- throttler.finish_op("id4");
- throttler.finish_op("id5");
+ throttler.finish_op("ns", "id4");
+ throttler.finish_op("ns", "id5");
}
TEST_F(TestMockImageSyncThrottler, Increase_Max_Concurrent_Syncs) {
throttler.set_max_concurrent_syncs(2);
C_SaferCond on_start1;
- throttler.start_op("id1", &on_start1);
+ throttler.start_op("ns", "id1", &on_start1);
C_SaferCond on_start2;
- throttler.start_op("id2", &on_start2);
+ throttler.start_op("ns", "id2", &on_start2);
C_SaferCond on_start3;
- throttler.start_op("id3", &on_start3);
+ throttler.start_op("ns", "id3", &on_start3);
C_SaferCond on_start4;
- throttler.start_op("id4", &on_start4);
+ throttler.start_op("ns", "id4", &on_start4);
C_SaferCond on_start5;
- throttler.start_op("id5", &on_start5);
+ throttler.start_op("ns", "id5", &on_start5);
ASSERT_EQ(0, on_start1.wait());
ASSERT_EQ(0, on_start2.wait());
ASSERT_EQ(0, on_start3.wait());
ASSERT_EQ(0, on_start4.wait());
- throttler.finish_op("id4");
+ throttler.finish_op("ns", "id4");
ASSERT_EQ(0, on_start5.wait());
- throttler.finish_op("id1");
- throttler.finish_op("id2");
- throttler.finish_op("id3");
- throttler.finish_op("id5");
+ throttler.finish_op("ns", "id1");
+ throttler.finish_op("ns", "id2");
+ throttler.finish_op("ns", "id3");
+ throttler.finish_op("ns", "id5");
}
TEST_F(TestMockImageSyncThrottler, Decrease_Max_Concurrent_Syncs) {
throttler.set_max_concurrent_syncs(4);
C_SaferCond on_start1;
- throttler.start_op("id1", &on_start1);
+ throttler.start_op("ns", "id1", &on_start1);
C_SaferCond on_start2;
- throttler.start_op("id2", &on_start2);
+ throttler.start_op("ns", "id2", &on_start2);
C_SaferCond on_start3;
- throttler.start_op("id3", &on_start3);
+ throttler.start_op("ns", "id3", &on_start3);
C_SaferCond on_start4;
- throttler.start_op("id4", &on_start4);
+ throttler.start_op("ns", "id4", &on_start4);
C_SaferCond on_start5;
- throttler.start_op("id5", &on_start5);
+ throttler.start_op("ns", "id5", &on_start5);
ASSERT_EQ(0, on_start1.wait());
ASSERT_EQ(0, on_start2.wait());
throttler.set_max_concurrent_syncs(2);
- throttler.finish_op("id1");
- throttler.finish_op("id2");
- throttler.finish_op("id3");
+ throttler.finish_op("ns", "id1");
+ throttler.finish_op("ns", "id2");
+ throttler.finish_op("ns", "id3");
ASSERT_EQ(0, on_start5.wait());
- throttler.finish_op("id4");
- throttler.finish_op("id5");
+ throttler.finish_op("ns", "id4");
+ throttler.finish_op("ns", "id5");
+}
+
+TEST_F(TestMockImageSyncThrottler, Drain) {
+ MockImageSyncThrottler throttler(g_ceph_context);
+ throttler.set_max_concurrent_syncs(1);
+
+ C_SaferCond on_start1;
+ throttler.start_op("ns", "id1", &on_start1);
+ C_SaferCond on_start2;
+ throttler.start_op("ns", "id2", &on_start2);
+
+ ASSERT_EQ(0, on_start1.wait());
+ throttler.drain("ns", -ESTALE);
+ ASSERT_EQ(-ESTALE, on_start2.wait());
}
} // namespace mirror
} // namespace rbd
-
std::string global_image_id;
static ImageReplayer *create(
- Threads<librbd::MockTestImageCtx> *threads,
- InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher,
- journal::CacheManagerHandler *cache_manager_handler,
- RadosRef local, const std::string &local_mirror_uuid, int64_t local_pool_id,
- const std::string &global_image_id) {
+ librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid,
+ const std::string &global_image_id,
+ Threads<librbd::MockTestImageCtx> *threads,
+ InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher,
+ journal::CacheManagerHandler *cache_manager_handler) {
ceph_assert(s_instance != nullptr);
s_instance->global_image_id = global_image_id;
return s_instance;
MockInstanceWatcher mock_instance_watcher;
MockImageReplayer mock_image_replayer;
MockInstanceReplayer instance_replayer(
- &mock_threads, &mock_service_daemon, nullptr,
- rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)),
- "local_mirror_uuid", m_local_io_ctx.get_id());
+ m_local_io_ctx, "local_mirror_uuid",
+ &mock_threads, &mock_service_daemon, nullptr);
std::string global_image_id("global_image_id");
EXPECT_CALL(mock_image_replayer, get_global_image_id())
MockInstanceWatcher mock_instance_watcher;
MockImageReplayer mock_image_replayer;
MockInstanceReplayer instance_replayer(
- &mock_threads, &mock_service_daemon, nullptr,
- rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)),
- "local_mirror_uuid", m_local_io_ctx.get_id());
+ m_local_io_ctx, "local_mirror_uuid",
+ &mock_threads, &mock_service_daemon, nullptr);
std::string global_image_id("global_image_id");
EXPECT_CALL(mock_image_replayer, get_global_image_id())
MockInstanceWatcher mock_instance_watcher;
MockImageReplayer mock_image_replayer;
MockInstanceReplayer instance_replayer(
- &mock_threads, &mock_service_daemon, nullptr,
- rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)),
- "local_mirror_uuid", m_local_io_ctx.get_id());
+ m_local_io_ctx, "local_mirror_uuid",
+ &mock_threads, &mock_service_daemon, nullptr);
std::string global_image_id("global_image_id");
EXPECT_CALL(mock_image_replayer, get_global_image_id())
#include "test/librbd/mock/MockImageCtx.h"
#include "test/rbd_mirror/test_mock_fixture.h"
#include "tools/rbd_mirror/InstanceReplayer.h"
-#include "tools/rbd_mirror/ImageSyncThrottler.h"
#include "tools/rbd_mirror/InstanceWatcher.h"
#include "tools/rbd_mirror/Threads.h"
struct ImageSyncThrottler<librbd::MockTestImageCtx> {
static ImageSyncThrottler* s_instance;
- static ImageSyncThrottler *create(CephContext *cct) {
- ceph_assert(s_instance != nullptr);
- return s_instance;
- }
-
ImageSyncThrottler() {
ceph_assert(s_instance == nullptr);
s_instance = this;
s_instance = nullptr;
}
- MOCK_METHOD0(destroy, void());
- MOCK_METHOD1(drain, void(int));
- MOCK_METHOD2(start_op, void(const std::string &, Context *));
- MOCK_METHOD1(finish_op, void(const std::string &));
+ MOCK_METHOD3(start_op, void(const std::string &, const std::string &,
+ Context *));
+ MOCK_METHOD2(finish_op, void(const std::string &, const std::string &));
+ MOCK_METHOD2(drain, void(const std::string &, int));
};
ImageSyncThrottler<librbd::MockTestImageCtx>* ImageSyncThrottler<librbd::MockTestImageCtx>::s_instance = nullptr;
librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
auto instance_watcher = new MockInstanceWatcher(
- m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id);
+ m_local_io_ctx, m_mock_threads->work_queue, nullptr, nullptr,
+ m_instance_id);
InSequence seq;
// Init
librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
auto instance_watcher = new MockInstanceWatcher(
- m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id);
+ m_local_io_ctx, m_mock_threads->work_queue, nullptr, nullptr,
+ m_instance_id);
InSequence seq;
expect_register_instance(mock_io_ctx, 0);
librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
auto instance_watcher = new MockInstanceWatcher(
- m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id);
+ m_local_io_ctx, m_mock_threads->work_queue, nullptr, nullptr,
+ m_instance_id);
InSequence seq;
// Init
librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1));
MockInstanceReplayer mock_instance_replayer1;
auto instance_watcher1 = MockInstanceWatcher::create(
- io_ctx1, m_mock_threads->work_queue, &mock_instance_replayer1);
+ io_ctx1, m_mock_threads->work_queue, &mock_instance_replayer1, nullptr);
librados::Rados cluster;
librados::IoCtx io_ctx2;
librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2));
MockInstanceReplayer mock_instance_replayer2;
auto instance_watcher2 = MockInstanceWatcher::create(
- io_ctx2, m_mock_threads->work_queue, &mock_instance_replayer2);
+ io_ctx2, m_mock_threads->work_queue, &mock_instance_replayer2, nullptr);
InSequence seq;
librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1));
MockInstanceReplayer mock_instance_replayer1;
auto instance_watcher1 = MockInstanceWatcher::create(
- io_ctx1, m_mock_threads->work_queue, &mock_instance_replayer1);
+ io_ctx1, m_mock_threads->work_queue, &mock_instance_replayer1, nullptr);
librados::Rados cluster;
librados::IoCtx io_ctx2;
librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2));
MockInstanceReplayer mock_instance_replayer2;
auto instance_watcher2 = MockInstanceWatcher::create(
- io_ctx2, m_mock_threads->work_queue, &mock_instance_replayer2);
+ io_ctx2, m_mock_threads->work_queue, &mock_instance_replayer2, nullptr);
InSequence seq;
librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
auto instance_watcher = new MockInstanceWatcher(
- m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id);
+ m_local_io_ctx, m_mock_threads->work_queue, nullptr, nullptr,
+ m_instance_id);
InSequence seq;
// Init
MockInstanceReplayer mock_instance_replayer;
auto instance_watcher = new MockInstanceWatcher(
- m_local_io_ctx, m_mock_threads->work_queue, &mock_instance_replayer,
- m_instance_id);
+ m_local_io_ctx, m_mock_threads->work_queue, &mock_instance_replayer,
+ nullptr, m_instance_id);
InSequence seq;
// Init
MockInstanceReplayer mock_instance_replayer;
auto instance_watcher = new MockInstanceWatcher(
- m_local_io_ctx, m_mock_threads->work_queue, &mock_instance_replayer,
- m_instance_id);
+ m_local_io_ctx, m_mock_threads->work_queue, &mock_instance_replayer,
+ nullptr, m_instance_id);
InSequence seq;
// Init
librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
auto instance_watcher = new MockInstanceWatcher(
- m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id);
+ m_local_io_ctx, m_mock_threads->work_queue, nullptr, nullptr,
+ m_instance_id);
InSequence seq;
// Init
librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1));
instance_watcher1 = MockInstanceWatcher::create(io_ctx1,
m_mock_threads->work_queue,
- nullptr);
+ nullptr,
+ &mock_image_sync_throttler);
EXPECT_EQ("", connect_cluster_pp(cluster));
EXPECT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx2));
instance_id2 = stringify(io_ctx2.get_instance_id());
librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2));
instance_watcher2 = MockInstanceWatcher::create(io_ctx2,
m_mock_threads->work_queue,
- nullptr);
+ nullptr,
+ &mock_image_sync_throttler);
InSequence seq;
// Init instance watcher 1 (leader)
InSequence seq;
- expect_throttler_destroy();
+ expect_throttler_drain();
instance_watcher1->handle_release_leader();
// Shutdown instance watcher 1
TestMockInstanceWatcher::TearDown();
}
- void expect_throttler_destroy(
- std::vector<Context *> *throttler_queue = nullptr) {
- EXPECT_CALL(mock_image_sync_throttler, drain(-ESTALE))
- .WillOnce(Invoke([throttler_queue] (int r) {
- if (throttler_queue != nullptr) {
- for (auto ctx : *throttler_queue) {
- ctx->complete(r);
- }
- }
- }));
- EXPECT_CALL(mock_image_sync_throttler, destroy());
- }
-
void expect_throttler_start_op(const std::string &sync_id,
Context *on_call = nullptr,
Context **on_start_ctx = nullptr) {
- EXPECT_CALL(mock_image_sync_throttler, start_op(sync_id, _))
+ EXPECT_CALL(mock_image_sync_throttler, start_op("", sync_id, _))
.WillOnce(Invoke([on_call, on_start_ctx] (const std::string &,
+ const std::string &,
Context *ctx) {
if (on_start_ctx != nullptr) {
*on_start_ctx = ctx;
void expect_throttler_finish_op(const std::string &sync_id,
Context *on_finish) {
- EXPECT_CALL(mock_image_sync_throttler, finish_op("sync_id"))
- .WillOnce(Invoke([on_finish](const std::string &) {
+ EXPECT_CALL(mock_image_sync_throttler, finish_op("", "sync_id"))
+ .WillOnce(Invoke([on_finish](const std::string &, const std::string &) {
on_finish->complete(0);
}));
}
+
+ void expect_throttler_drain() {
+ EXPECT_CALL(mock_image_sync_throttler, drain("", -ESTALE));
+ }
};
TEST_F(TestMockInstanceWatcher_NotifySync, StartStopOnLeader) {
ASSERT_EQ(0, on_start1.wait());
C_SaferCond on_start2;
- EXPECT_CALL(mock_image_sync_throttler, finish_op("sync_id"))
- .WillOnce(Invoke([this, &on_start2](const std::string &) {
+ EXPECT_CALL(mock_image_sync_throttler, finish_op("", "sync_id"))
+ .WillOnce(Invoke([this, &on_start2](const std::string &,
+ const std::string &) {
instance_watcher2->notify_sync_request("sync_id", &on_start2);
}));
expect_throttler_start_op("sync_id");
TEST_F(TestMockInstanceWatcher_NotifySync, NoInFlightReleaseAcquireLeader) {
InSequence seq;
- expect_throttler_destroy();
+ expect_throttler_drain();
instance_watcher1->handle_release_leader();
instance_watcher1->handle_acquire_leader();
}
TEST_F(TestMockInstanceWatcher_NotifySync, StartedOnLeaderReleaseLeader) {
InSequence seq;
- expect_throttler_destroy();
+ expect_throttler_drain();
instance_watcher1->handle_release_leader();
instance_watcher2->handle_acquire_leader();
C_SaferCond on_start;
instance_watcher2->notify_sync_request("sync_id", &on_start);
ASSERT_EQ(0, on_start.wait());
- expect_throttler_destroy();
+ expect_throttler_drain();
instance_watcher2->handle_release_leader();
instance_watcher2->notify_sync_complete("sync_id");
instance_watcher1->notify_sync_request("sync_id", &on_start);
ASSERT_EQ(0, on_start_op_called.wait());
- std::vector<Context *> throttler_queue = {on_start_ctx};
- expect_throttler_destroy(&throttler_queue);
+ expect_throttler_drain();
instance_watcher1->handle_release_leader();
+ // emulate throttler queue drain on leader release
+ on_start_ctx->complete(-ESTALE);
+
expect_throttler_start_op("sync_id");
instance_watcher2->handle_acquire_leader();
instance_watcher1->handle_update_leader(instance_id2);
instance_watcher1->notify_sync_complete("sync_id");
ASSERT_EQ(0, on_finish.wait());
- expect_throttler_destroy();
+ expect_throttler_drain();
instance_watcher2->handle_release_leader();
instance_watcher1->handle_acquire_leader();
}
TEST_F(TestMockInstanceWatcher_NotifySync, StartedOnNonLeaderAcquireLeader) {
InSequence seq;
- expect_throttler_destroy();
+ expect_throttler_drain();
instance_watcher1->handle_release_leader();
instance_watcher2->handle_acquire_leader();
instance_watcher1->handle_update_leader(instance_id2);
instance_watcher1->notify_sync_request("sync_id", &on_start);
ASSERT_EQ(0, on_start.wait());
- expect_throttler_destroy();
+ expect_throttler_drain();
instance_watcher2->handle_release_leader();
instance_watcher1->handle_acquire_leader();
instance_watcher2->handle_update_leader(instance_id1);
instance_watcher2->notify_sync_request("sync_id", &on_start);
ASSERT_EQ(0, on_start_op_called.wait());
- std::vector<Context *> throttler_queue = {on_start_ctx};
- expect_throttler_destroy(&throttler_queue);
+ expect_throttler_drain();
instance_watcher1->handle_release_leader();
+ // emulate throttler queue drain on leader release
+ on_start_ctx->complete(-ESTALE);
- EXPECT_CALL(mock_image_sync_throttler, start_op("sync_id", _))
- .WillOnce(WithArg<1>(CompleteContext(0)));
+ EXPECT_CALL(mock_image_sync_throttler, start_op("", "sync_id", _))
+ .WillOnce(WithArg<2>(CompleteContext(0)));
instance_watcher2->handle_acquire_leader();
instance_watcher1->handle_update_leader(instance_id2);
instance_watcher2->notify_sync_complete("sync_id");
ASSERT_EQ(0, on_finish.wait());
- expect_throttler_destroy();
+ expect_throttler_drain();
instance_watcher2->handle_release_leader();
instance_watcher1->handle_acquire_leader();
}
}
};
-template <>
-struct MirrorStatusWatcher<librbd::MockTestImageCtx> {
- static MirrorStatusWatcher* s_instance;
-
- static MirrorStatusWatcher *create(librados::IoCtx &io_ctx,
- ContextWQ *work_queue) {
- ceph_assert(s_instance != nullptr);
- return s_instance;
- }
-
- MirrorStatusWatcher() {
- ceph_assert(s_instance == nullptr);
- s_instance = this;
- }
-
- ~MirrorStatusWatcher() {
- ceph_assert(s_instance == this);
- s_instance = nullptr;
- }
-
- MOCK_METHOD0(destroy, void());
- MOCK_METHOD1(init, void(Context *));
- MOCK_METHOD1(shut_down, void(Context *));
-};
-
-MirrorStatusWatcher<librbd::MockTestImageCtx> *MirrorStatusWatcher<librbd::MockTestImageCtx>::s_instance = nullptr;
-
template <>
struct Instances<librbd::MockTestImageCtx> {
static Instances* s_instance;
class TestMockLeaderWatcher : public TestMockFixture {
public:
- typedef MirrorStatusWatcher<librbd::MockTestImageCtx> MockMirrorStatusWatcher;
typedef Instances<librbd::MockTestImageCtx> MockInstances;
typedef LeaderWatcher<librbd::MockTestImageCtx> MockLeaderWatcher;
typedef Threads<librbd::MockTestImageCtx> MockThreads;
Return(true)));
}
- void expect_destroy(MockMirrorStatusWatcher &mock_mirror_status_watcher) {
- EXPECT_CALL(mock_mirror_status_watcher, destroy());
- }
-
- void expect_init(MockMirrorStatusWatcher &mock_mirror_status_watcher, int r) {
- EXPECT_CALL(mock_mirror_status_watcher, init(_))
- .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
- }
-
- void expect_shut_down(MockMirrorStatusWatcher &mock_mirror_status_watcher, int r) {
- EXPECT_CALL(mock_mirror_status_watcher, shut_down(_))
- .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
- expect_destroy(mock_mirror_status_watcher);
- }
-
void expect_destroy(MockInstances &mock_instances) {
EXPECT_CALL(mock_instances, destroy());
}
TEST_F(TestMockLeaderWatcher, InitShutdown) {
MockManagedLock mock_managed_lock;
- MockMirrorStatusWatcher mock_mirror_status_watcher;
MockInstances mock_instances;
MockListener listener;
MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
// Init
- expect_init(mock_mirror_status_watcher, 0);
C_SaferCond on_heartbeat_finish;
expect_is_leader(mock_managed_lock, false, false);
expect_try_acquire_lock(mock_managed_lock, 0);
expect_shut_down(mock_instances, 0);
expect_release_lock(mock_managed_lock, 0);
expect_shut_down(mock_managed_lock, true, 0);
- expect_shut_down(mock_mirror_status_watcher, 0);
expect_is_leader(mock_managed_lock, false, false);
leader_watcher.shut_down();
TEST_F(TestMockLeaderWatcher, InitReleaseShutdown) {
MockManagedLock mock_managed_lock;
- MockMirrorStatusWatcher mock_mirror_status_watcher;
MockInstances mock_instances;
MockListener listener;
MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
// Init
- expect_init(mock_mirror_status_watcher, 0);
C_SaferCond on_heartbeat_finish;
expect_is_leader(mock_managed_lock, false, false);
expect_try_acquire_lock(mock_managed_lock, 0);
// Shutdown
expect_shut_down(mock_managed_lock, false, 0);
- expect_shut_down(mock_mirror_status_watcher, 0);
- expect_is_leader(mock_managed_lock, false, false);
-
- leader_watcher.shut_down();
-}
-
-TEST_F(TestMockLeaderWatcher, InitStatusWatcherError) {
- MockManagedLock mock_managed_lock;
- MockMirrorStatusWatcher mock_mirror_status_watcher;
- MockInstances mock_instances;
- MockListener listener;
-
- expect_is_shutdown(mock_managed_lock);
- expect_is_leader(mock_managed_lock);
- expect_destroy(mock_managed_lock);
-
- InSequence seq;
-
- expect_construct(mock_managed_lock);
- MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
-
- // Init
- expect_init(mock_mirror_status_watcher, -EINVAL);
- ASSERT_EQ(-EINVAL, leader_watcher.init());
-
- // Shutdown
- expect_shut_down(mock_managed_lock, false, 0);
- expect_shut_down(mock_mirror_status_watcher, 0);
expect_is_leader(mock_managed_lock, false, false);
leader_watcher.shut_down();
TEST_F(TestMockLeaderWatcher, AcquireError) {
MockManagedLock mock_managed_lock;
- MockMirrorStatusWatcher mock_mirror_status_watcher;
MockInstances mock_instances;
MockListener listener;
MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
// Init
- expect_init(mock_mirror_status_watcher, 0);
C_SaferCond on_heartbeat_finish;
expect_is_leader(mock_managed_lock, false, false);
expect_try_acquire_lock(mock_managed_lock, -EAGAIN);
expect_shut_down(mock_instances, 0);
expect_release_lock(mock_managed_lock, 0);
expect_shut_down(mock_managed_lock, true, 0);
- expect_shut_down(mock_mirror_status_watcher, 0);
expect_is_leader(mock_managed_lock, false, false);
leader_watcher.shut_down();
"rbd_mirror_leader_max_acquire_attempts_before_break");
MockManagedLock mock_managed_lock;
- MockMirrorStatusWatcher mock_mirror_status_watcher;
MockInstances mock_instances;
MockListener listener;
librbd::managed_lock::Locker
MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
// Init
- expect_init(mock_mirror_status_watcher, 0);
expect_is_leader(mock_managed_lock, false, false);
for (int i = 0; i < max_acquire_attempts; i++) {
expect_try_acquire_lock(mock_managed_lock, -EAGAIN);
expect_shut_down(mock_instances, 0);
expect_release_lock(mock_managed_lock, 0);
expect_shut_down(mock_managed_lock, true, 0);
- expect_shut_down(mock_mirror_status_watcher, 0);
expect_is_leader(mock_managed_lock, false, false);
leader_watcher.shut_down();
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/api/Config.h"
+#include "test/librbd/mock/MockImageCtx.h"
+#include "test/rbd_mirror/test_mock_fixture.h"
+#include "test/rbd_mirror/mock/MockContextWQ.h"
+#include "test/rbd_mirror/mock/MockSafeTimer.h"
+#include "tools/rbd_mirror/NamespaceReplayer.h"
+#include "tools/rbd_mirror/ImageDeleter.h"
+#include "tools/rbd_mirror/ImageMap.h"
+#include "tools/rbd_mirror/InstanceWatcher.h"
+#include "tools/rbd_mirror/InstanceReplayer.h"
+#include "tools/rbd_mirror/PoolWatcher.h"
+#include "tools/rbd_mirror/ServiceDaemon.h"
+#include "tools/rbd_mirror/Threads.h"
+
+namespace librbd {
+
+namespace {
+
+struct MockTestImageCtx : public MockImageCtx {
+ MockTestImageCtx(librbd::ImageCtx &image_ctx)
+ : librbd::MockImageCtx(image_ctx) {
+ }
+};
+
+} // anonymous namespace
+
+} // namespace librbd
+
+namespace rbd {
+namespace mirror {
+
+template <>
+struct ImageDeleter<librbd::MockTestImageCtx> {
+ static ImageDeleter* s_instance;
+
+ static ImageDeleter* create(
+ librados::IoCtx &ioctx, Threads<librbd::MockTestImageCtx> *threads,
+ ServiceDaemon<librbd::MockTestImageCtx> *service_daemon) {
+ ceph_assert(s_instance != nullptr);
+ return s_instance;
+ }
+
+ MOCK_METHOD1(init, void(Context*));
+ MOCK_METHOD1(shut_down, void(Context*));
+ MOCK_METHOD2(print_status, void(Formatter*, std::stringstream*));
+
+ ImageDeleter() {
+ s_instance = this;
+ }
+};
+
+ImageDeleter<librbd::MockTestImageCtx>* ImageDeleter<librbd::MockTestImageCtx>::s_instance = nullptr;
+
+template<>
+struct ImageMap<librbd::MockTestImageCtx> {
+ static ImageMap* s_instance;
+
+ static ImageMap *create(librados::IoCtx &ioctx,
+ Threads<librbd::MockTestImageCtx> *threads,
+ const std::string& instance_id,
+ image_map::Listener &listener) {
+ ceph_assert(s_instance != nullptr);
+ return s_instance;
+ }
+
+ MOCK_METHOD1(init, void(Context*));
+ MOCK_METHOD1(shut_down, void(Context*));
+
+ MOCK_METHOD1(update_instances_added, void(const std::vector<std::string>&));
+ MOCK_METHOD1(update_instances_removed, void(const std::vector<std::string>&));
+
+ MOCK_METHOD3(update_images_mock, void(const std::string&,
+ const std::set<std::string>&,
+ const std::set<std::string>&));
+ void update_images(const std::string& mirror_uuid,
+ std::set<std::string>&& added,
+ std::set<std::string>&& removed) {
+ update_images_mock(mirror_uuid, added, removed);
+ }
+
+ ImageMap() {
+ s_instance = this;
+ }
+};
+
+ImageMap<librbd::MockTestImageCtx>* ImageMap<librbd::MockTestImageCtx>::s_instance = nullptr;
+
+template<>
+struct InstanceReplayer<librbd::MockTestImageCtx> {
+ static InstanceReplayer* s_instance;
+
+ static InstanceReplayer* create(
+ librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid,
+ Threads<librbd::MockTestImageCtx> *threads,
+ ServiceDaemon<librbd::MockTestImageCtx> *service_daemon,
+ journal::CacheManagerHandler *cache_manager_handler) {
+ ceph_assert(s_instance != nullptr);
+ return s_instance;
+ }
+
+ MOCK_METHOD0(start, void());
+ MOCK_METHOD0(stop, void());
+ MOCK_METHOD0(restart, void());
+ MOCK_METHOD0(flush, void());
+
+ MOCK_METHOD1(stop, void(Context *));
+
+ MOCK_METHOD2(print_status, void(Formatter*, std::stringstream*));
+
+ MOCK_METHOD2(add_peer, void(const std::string&, librados::IoCtx&));
+
+ MOCK_METHOD1(init, void(Context*));
+ MOCK_METHOD1(shut_down, void(Context*));
+ MOCK_METHOD1(release_all, void(Context*));
+
+ InstanceReplayer() {
+ s_instance = this;
+ }
+};
+
+InstanceReplayer<librbd::MockTestImageCtx>* InstanceReplayer<librbd::MockTestImageCtx>::s_instance = nullptr;
+
+template<>
+struct InstanceWatcher<librbd::MockTestImageCtx> {
+ static InstanceWatcher* s_instance;
+
+ static InstanceWatcher* create(
+ librados::IoCtx &ioctx, ContextWQ* work_queue,
+ InstanceReplayer<librbd::MockTestImageCtx>* instance_replayer,
+ ImageSyncThrottler<librbd::MockTestImageCtx> *image_sync_throttler) {
+ ceph_assert(s_instance != nullptr);
+ return s_instance;
+ }
+
+ MOCK_METHOD0(handle_acquire_leader, void());
+ MOCK_METHOD0(handle_release_leader, void());
+
+ MOCK_METHOD0(get_instance_id, std::string());
+
+ MOCK_METHOD2(print_sync_status, void(Formatter*, std::stringstream*));
+
+ MOCK_METHOD1(init, void(Context *));
+ MOCK_METHOD1(shut_down, void(Context *));
+
+ MOCK_METHOD3(notify_image_acquire, void(const std::string&,
+ const std::string&,
+ Context*));
+ MOCK_METHOD3(notify_image_release, void(const std::string&,
+ const std::string&,
+ Context*));
+ MOCK_METHOD4(notify_peer_image_removed, void(const std::string&,
+ const std::string&,
+ const std::string&,
+ Context*));
+
+ MOCK_METHOD1(handle_update_leader, void(const std::string&));
+
+ InstanceWatcher() {
+ s_instance = this;
+ }
+
+};
+
+InstanceWatcher<librbd::MockTestImageCtx>* InstanceWatcher<librbd::MockTestImageCtx>::s_instance = nullptr;
+
+template <>
+struct MirrorStatusWatcher<librbd::MockTestImageCtx> {
+ static MirrorStatusWatcher* s_instance;
+
+ static MirrorStatusWatcher *create(librados::IoCtx &io_ctx,
+ ContextWQ *work_queue) {
+ ceph_assert(s_instance != nullptr);
+ return s_instance;
+ }
+
+ MirrorStatusWatcher() {
+ ceph_assert(s_instance == nullptr);
+ s_instance = this;
+ }
+
+ ~MirrorStatusWatcher() {
+ ceph_assert(s_instance == this);
+ s_instance = nullptr;
+ }
+
+ MOCK_METHOD1(init, void(Context *));
+ MOCK_METHOD1(shut_down, void(Context *));
+};
+
+MirrorStatusWatcher<librbd::MockTestImageCtx> *MirrorStatusWatcher<librbd::MockTestImageCtx>::s_instance = nullptr;
+
+template<>
+struct PoolWatcher<librbd::MockTestImageCtx> {
+ static std::map<int64_t, PoolWatcher *> s_instances;
+
+ static PoolWatcher *create(Threads<librbd::MockTestImageCtx> *threads,
+ librados::IoCtx &ioctx,
+ pool_watcher::Listener& listener) {
+ auto pool_id = ioctx.get_id();
+ ceph_assert(s_instances.count(pool_id));
+ return s_instances[pool_id];
+ }
+
+ MOCK_METHOD0(is_blacklisted, bool());
+
+ MOCK_METHOD0(get_image_count, uint64_t());
+
+ MOCK_METHOD1(init, void(Context*));
+ MOCK_METHOD1(shut_down, void(Context*));
+
+ PoolWatcher(int64_t pool_id) {
+ ceph_assert(!s_instances.count(pool_id));
+ s_instances[pool_id] = this;
+ }
+};
+
+std::map<int64_t, PoolWatcher<librbd::MockTestImageCtx> *> PoolWatcher<librbd::MockTestImageCtx>::s_instances;
+
+template<>
+struct ServiceDaemon<librbd::MockTestImageCtx> {
+ MOCK_METHOD3(add_or_update_attribute,
+ void(int64_t, const std::string&,
+ const service_daemon::AttributeValue&));
+ MOCK_METHOD2(remove_attribute,
+ void(int64_t, const std::string&));
+
+ MOCK_METHOD4(add_or_update_callout, uint64_t(int64_t, uint64_t,
+ service_daemon::CalloutLevel,
+ const std::string&));
+ MOCK_METHOD2(remove_callout, void(int64_t, uint64_t));
+};
+
+template <>
+struct Threads<librbd::MockTestImageCtx> {
+ ceph::mutex &timer_lock;
+ SafeTimer *timer;
+ ContextWQ *work_queue;
+
+ Threads(Threads<librbd::ImageCtx> *threads)
+ : timer_lock(threads->timer_lock), timer(threads->timer),
+ work_queue(threads->work_queue) {
+ }
+};
+
+} // namespace mirror
+} // namespace rbd
+
+// template definitions
+#include "tools/rbd_mirror/NamespaceReplayer.cc"
+
+namespace rbd {
+namespace mirror {
+
+using ::testing::_;
+using ::testing::DoAll;
+using ::testing::InSequence;
+using ::testing::Invoke;
+using ::testing::Return;
+using ::testing::StrEq;
+using ::testing::WithArg;
+
+class TestMockNamespaceReplayer : public TestMockFixture {
+public:
+ typedef NamespaceReplayer<librbd::MockTestImageCtx> MockNamespaceReplayer;
+ typedef ImageDeleter<librbd::MockTestImageCtx> MockImageDeleter;
+ typedef ImageMap<librbd::MockTestImageCtx> MockImageMap;
+ typedef InstanceReplayer<librbd::MockTestImageCtx> MockInstanceReplayer;
+ typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher;
+ typedef MirrorStatusWatcher<librbd::MockTestImageCtx> MockMirrorStatusWatcher;
+ typedef PoolWatcher<librbd::MockTestImageCtx> MockPoolWatcher;
+ typedef ServiceDaemon<librbd::MockTestImageCtx> MockServiceDaemon;
+ typedef Threads<librbd::MockTestImageCtx> MockThreads;
+
+ void SetUp() override {
+ TestMockFixture::SetUp();
+ m_mock_threads = new MockThreads(m_threads);
+ }
+
+ void TearDown() override {
+ delete m_mock_threads;
+ TestMockFixture::TearDown();
+ }
+
+ void expect_mirror_status_watcher_init(
+ MockMirrorStatusWatcher &mock_mirror_status_watcher, int r) {
+ EXPECT_CALL(mock_mirror_status_watcher, init(_))
+ .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
+ }
+
+ void expect_mirror_status_watcher_shut_down(
+ MockMirrorStatusWatcher &mock_mirror_status_watcher) {
+ EXPECT_CALL(mock_mirror_status_watcher, shut_down(_))
+ .WillOnce(CompleteContext(m_mock_threads->work_queue, 0));
+ }
+
+ void expect_instance_replayer_init(
+ MockInstanceReplayer& mock_instance_replayer, int r) {
+ EXPECT_CALL(mock_instance_replayer, init(_))
+ .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
+ }
+
+ void expect_instance_replayer_shut_down(
+ MockInstanceReplayer& mock_instance_replayer) {
+ EXPECT_CALL(mock_instance_replayer, shut_down(_))
+ .WillOnce(CompleteContext(m_mock_threads->work_queue, 0));
+ }
+
+ void expect_instance_replayer_stop(
+ MockInstanceReplayer& mock_instance_replayer) {
+ EXPECT_CALL(mock_instance_replayer, stop(_))
+ .WillOnce(CompleteContext(m_mock_threads->work_queue, 0));
+ }
+
+ void expect_instance_replayer_add_peer(
+ MockInstanceReplayer& mock_instance_replayer, const std::string& uuid) {
+ EXPECT_CALL(mock_instance_replayer, add_peer(uuid, _));
+ }
+
+ void expect_instance_replayer_release_all(
+ MockInstanceReplayer& mock_instance_replayer) {
+ EXPECT_CALL(mock_instance_replayer, release_all(_))
+ .WillOnce(CompleteContext(m_mock_threads->work_queue, 0));
+ }
+
+ void expect_instance_watcher_get_instance_id(
+ MockInstanceWatcher& mock_instance_watcher,
+ const std::string &instance_id) {
+ EXPECT_CALL(mock_instance_watcher, get_instance_id())
+ .WillOnce(Return(instance_id));
+ }
+
+ void expect_instance_watcher_init(
+ MockInstanceWatcher& mock_instance_watcher, int r) {
+ EXPECT_CALL(mock_instance_watcher, init(_))
+ .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
+ }
+
+ void expect_instance_watcher_shut_down(
+ MockInstanceWatcher& mock_instance_watcher) {
+ EXPECT_CALL(mock_instance_watcher, shut_down(_))
+ .WillOnce(CompleteContext(m_mock_threads->work_queue, 0));
+ }
+
+ void expect_instance_watcher_handle_acquire_leader(
+ MockInstanceWatcher& mock_instance_watcher) {
+ EXPECT_CALL(mock_instance_watcher, handle_acquire_leader());
+ }
+
+ void expect_instance_watcher_handle_release_leader(
+ MockInstanceWatcher& mock_instance_watcher) {
+ EXPECT_CALL(mock_instance_watcher, handle_release_leader());
+ }
+
+ void expect_image_map_init(MockInstanceWatcher &mock_instance_watcher,
+ MockImageMap& mock_image_map, int r) {
+ expect_instance_watcher_get_instance_id(mock_instance_watcher, "1234");
+ EXPECT_CALL(mock_image_map, init(_))
+ .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
+ }
+
+ void expect_image_map_shut_down(MockImageMap& mock_image_map) {
+ EXPECT_CALL(mock_image_map, shut_down(_))
+ .WillOnce(CompleteContext(m_mock_threads->work_queue, 0));
+ }
+
+ void expect_pool_watcher_init(MockPoolWatcher& mock_pool_watcher, int r) {
+ EXPECT_CALL(mock_pool_watcher, init(_))
+ .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
+ }
+
+ void expect_pool_watcher_shut_down(MockPoolWatcher& mock_pool_watcher) {
+ EXPECT_CALL(mock_pool_watcher, shut_down(_))
+ .WillOnce(CompleteContext(m_mock_threads->work_queue, 0));
+ }
+
+ void expect_image_deleter_init(MockImageDeleter& mock_image_deleter, int r) {
+ EXPECT_CALL(mock_image_deleter, init(_))
+ .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
+ }
+
+ void expect_image_deleter_shut_down(MockImageDeleter& mock_image_deleter) {
+ EXPECT_CALL(mock_image_deleter, shut_down(_))
+ .WillOnce(CompleteContext(m_mock_threads->work_queue, 0));
+ }
+
+ void expect_service_daemon_add_or_update_attribute(
+ MockServiceDaemon &mock_service_daemon, const std::string& key,
+ const service_daemon::AttributeValue& value) {
+ EXPECT_CALL(mock_service_daemon, add_or_update_attribute(_, key, value));
+ }
+
+ void expect_service_daemon_add_or_update_instance_id_attribute(
+ MockInstanceWatcher &mock_instance_watcher,
+ MockServiceDaemon &mock_service_daemon) {
+ expect_instance_watcher_get_instance_id(mock_instance_watcher, "1234");
+ expect_service_daemon_add_or_update_attribute(
+ mock_service_daemon, "instance_id", {std::string("1234")});
+ }
+
+ MockThreads *m_mock_threads;
+};
+
+TEST_F(TestMockNamespaceReplayer, Init_MirrorStatusWatcherError) {
+ InSequence seq;
+
+ auto mock_mirror_status_watcher = new MockMirrorStatusWatcher;
+ expect_mirror_status_watcher_init(*mock_mirror_status_watcher, -EINVAL);
+
+ MockNamespaceReplayer namespace_replayer(
+ {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
+ "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr);
+
+ C_SaferCond on_init;
+ namespace_replayer.init(&on_init);
+ ASSERT_EQ(-EINVAL, on_init.wait());
+}
+
+TEST_F(TestMockNamespaceReplayer, Init_InstanceReplayerError) {
+ InSequence seq;
+
+ auto mock_mirror_status_watcher = new MockMirrorStatusWatcher;
+ expect_mirror_status_watcher_init(*mock_mirror_status_watcher, 0);
+
+ auto mock_instance_replayer = new MockInstanceReplayer();
+ expect_instance_replayer_init(*mock_instance_replayer, -EINVAL);
+
+ expect_mirror_status_watcher_shut_down(*mock_mirror_status_watcher);
+
+ MockNamespaceReplayer namespace_replayer(
+ {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
+ "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr);
+
+ C_SaferCond on_init;
+ namespace_replayer.init(&on_init);
+ ASSERT_EQ(-EINVAL, on_init.wait());
+}
+
+TEST_F(TestMockNamespaceReplayer, Init_InstanceWatcherError) {
+ InSequence seq;
+
+ auto mock_mirror_status_watcher = new MockMirrorStatusWatcher;
+ expect_mirror_status_watcher_init(*mock_mirror_status_watcher, 0);
+
+ auto mock_instance_replayer = new MockInstanceReplayer();
+ expect_instance_replayer_init(*mock_instance_replayer, 0);
+ expect_instance_replayer_add_peer(*mock_instance_replayer,
+ "remote mirror uuid");
+
+ auto mock_instance_watcher = new MockInstanceWatcher();
+ expect_instance_watcher_init(*mock_instance_watcher, -EINVAL);
+
+ expect_instance_replayer_shut_down(*mock_instance_replayer);
+ expect_mirror_status_watcher_shut_down(*mock_mirror_status_watcher);
+
+ MockNamespaceReplayer namespace_replayer(
+ {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
+ "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr);
+
+ C_SaferCond on_init;
+ namespace_replayer.init(&on_init);
+ ASSERT_EQ(-EINVAL, on_init.wait());
+}
+
+TEST_F(TestMockNamespaceReplayer, Init) {
+ InSequence seq;
+
+ auto mock_mirror_status_watcher = new MockMirrorStatusWatcher;
+ expect_mirror_status_watcher_init(*mock_mirror_status_watcher, 0);
+
+ auto mock_instance_replayer = new MockInstanceReplayer();
+ expect_instance_replayer_init(*mock_instance_replayer, 0);
+ expect_instance_replayer_add_peer(*mock_instance_replayer,
+ "remote mirror uuid");
+
+ auto mock_instance_watcher = new MockInstanceWatcher();
+ expect_instance_watcher_init(*mock_instance_watcher, 0);
+
+ MockServiceDaemon mock_service_daemon;
+ expect_service_daemon_add_or_update_instance_id_attribute(
+ *mock_instance_watcher, mock_service_daemon);
+
+ MockNamespaceReplayer namespace_replayer(
+ {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
+ "remote mirror uuid", m_mock_threads, nullptr, &mock_service_daemon,
+ nullptr);
+
+ C_SaferCond on_init;
+ namespace_replayer.init(&on_init);
+ ASSERT_EQ(0, on_init.wait());
+
+ expect_instance_replayer_stop(*mock_instance_replayer);
+ expect_instance_watcher_shut_down(*mock_instance_watcher);
+ expect_instance_replayer_shut_down(*mock_instance_replayer);
+ expect_mirror_status_watcher_shut_down(*mock_mirror_status_watcher);
+
+ C_SaferCond on_shut_down;
+ namespace_replayer.shut_down(&on_shut_down);
+ ASSERT_EQ(0, on_shut_down.wait());
+}
+
+TEST_F(TestMockNamespaceReplayer, AcuqireLeader) {
+ InSequence seq;
+
+ // init
+
+ auto mock_mirror_status_watcher = new MockMirrorStatusWatcher;
+ expect_mirror_status_watcher_init(*mock_mirror_status_watcher, 0);
+
+ auto mock_instance_replayer = new MockInstanceReplayer();
+ expect_instance_replayer_init(*mock_instance_replayer, 0);
+ expect_instance_replayer_add_peer(*mock_instance_replayer,
+ "remote mirror uuid");
+
+ auto mock_instance_watcher = new MockInstanceWatcher();
+ expect_instance_watcher_init(*mock_instance_watcher, 0);
+
+ MockServiceDaemon mock_service_daemon;
+ expect_service_daemon_add_or_update_instance_id_attribute(
+ *mock_instance_watcher, mock_service_daemon);
+
+ MockNamespaceReplayer namespace_replayer(
+ {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
+ "remote mirror uuid", m_mock_threads, nullptr, &mock_service_daemon,
+ nullptr);
+
+ C_SaferCond on_init;
+ namespace_replayer.init(&on_init);
+ ASSERT_EQ(0, on_init.wait());
+
+ // acquire leader
+
+ expect_instance_watcher_handle_acquire_leader(*mock_instance_watcher);
+
+ auto mock_image_map = new MockImageMap();
+ expect_image_map_init(*mock_instance_watcher, *mock_image_map, 0);
+
+ auto mock_local_pool_watcher = new MockPoolWatcher(m_local_io_ctx.get_id());
+ expect_pool_watcher_init(*mock_local_pool_watcher, 0);
+
+ auto mock_remote_pool_watcher = new MockPoolWatcher(m_remote_io_ctx.get_id());
+ expect_pool_watcher_init(*mock_remote_pool_watcher, 0);
+
+ auto mock_image_deleter = new MockImageDeleter();
+ expect_image_deleter_init(*mock_image_deleter, 0);
+
+ C_SaferCond on_acquire;
+ namespace_replayer.handle_acquire_leader(&on_acquire);
+ ASSERT_EQ(0, on_acquire.wait());
+
+ // release leader
+
+ expect_instance_watcher_handle_release_leader(*mock_instance_watcher);
+ expect_image_deleter_shut_down(*mock_image_deleter);
+ expect_pool_watcher_shut_down(*mock_local_pool_watcher);
+ expect_pool_watcher_shut_down(*mock_remote_pool_watcher);
+ expect_image_map_shut_down(*mock_image_map);
+ expect_instance_replayer_release_all(*mock_instance_replayer);
+
+ // shut down
+
+ expect_instance_replayer_stop(*mock_instance_replayer);
+ expect_instance_watcher_shut_down(*mock_instance_watcher);
+ expect_instance_replayer_shut_down(*mock_instance_replayer);
+ expect_mirror_status_watcher_shut_down(*mock_mirror_status_watcher);
+
+ C_SaferCond on_shut_down;
+ namespace_replayer.shut_down(&on_shut_down);
+ ASSERT_EQ(0, on_shut_down.wait());
+}
+
+} // namespace mirror
+} // namespace rbd
// vim: ts=8 sw=2 smarttab
#include "librbd/api/Config.h"
+#include "librbd/api/Namespace.h"
#include "test/librbd/mock/MockImageCtx.h"
#include "test/librados_test_stub/MockTestMemCluster.h"
#include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
#include "test/rbd_mirror/test_mock_fixture.h"
#include "test/rbd_mirror/mock/MockContextWQ.h"
#include "test/rbd_mirror/mock/MockSafeTimer.h"
-#include "tools/rbd_mirror/PoolReplayer.h"
-#include "tools/rbd_mirror/ImageDeleter.h"
-#include "tools/rbd_mirror/ImageMap.h"
-#include "tools/rbd_mirror/InstanceWatcher.h"
-#include "tools/rbd_mirror/InstanceReplayer.h"
+#include "tools/rbd_mirror/ImageSyncThrottler.h"
#include "tools/rbd_mirror/LeaderWatcher.h"
-#include "tools/rbd_mirror/PoolWatcher.h"
+#include "tools/rbd_mirror/NamespaceReplayer.h"
+#include "tools/rbd_mirror/PoolReplayer.h"
#include "tools/rbd_mirror/ServiceDaemon.h"
#include "tools/rbd_mirror/Threads.h"
}
};
-}
-
-} // namespace librbd
-
-namespace rbd {
-namespace mirror {
-
template <>
-struct ImageDeleter<librbd::MockTestImageCtx> {
- static ImageDeleter* s_instance;
+class Namespace<MockTestImageCtx> {
+public:
+ static Namespace* s_instance;
- static ImageDeleter* create(librados::IoCtx &ioctx,
- Threads<librbd::MockTestImageCtx> *threads,
- ServiceDaemon<librbd::MockTestImageCtx> *service_daemon) {
- ceph_assert(s_instance != nullptr);
- return s_instance;
- }
+ static int list(librados::IoCtx& io_ctx, std::vector<std::string> *names) {
+ if (s_instance) {
+ return s_instance->list(names);
+ }
- MOCK_METHOD1(init, void(Context*));
- MOCK_METHOD1(shut_down, void(Context*));
- MOCK_METHOD2(print_status, void(Formatter*, std::stringstream*));
+ return 0;
+ }
- ImageDeleter() {
+ Namespace() {
s_instance = this;
}
-};
-ImageDeleter<librbd::MockTestImageCtx>* ImageDeleter<librbd::MockTestImageCtx>::s_instance = nullptr;
-
-template<>
-struct ImageMap<librbd::MockTestImageCtx> {
- static ImageMap* s_instance;
+ void add(const std::string &name) {
+ std::lock_guard locker{m_lock};
- static ImageMap *create(librados::IoCtx &ioctx,
- Threads<librbd::MockTestImageCtx> *threads,
- const std::string& instance_id,
- image_map::Listener &listener) {
- ceph_assert(s_instance != nullptr);
- return s_instance;
+ m_names.insert(name);
}
- MOCK_METHOD1(init, void(Context*));
- MOCK_METHOD1(shut_down, void(Context*));
+ void remove(const std::string &name) {
+ std::lock_guard locker{m_lock};
- MOCK_METHOD1(update_instances_added, void(const std::vector<std::string>&));
- MOCK_METHOD1(update_instances_removed, void(const std::vector<std::string>&));
+ m_names.erase(name);
+ }
+
+ void clear() {
+ std::lock_guard locker{m_lock};
- MOCK_METHOD3(update_images_mock, void(const std::string&,
- const std::set<std::string>&,
- const std::set<std::string>&));
- void update_images(const std::string& mirror_uuid,
- std::set<std::string>&& added,
- std::set<std::string>&& removed) {
- update_images_mock(mirror_uuid, added, removed);
+ m_names.clear();
}
- ImageMap() {
- s_instance = this;
+private:
+ ceph::mutex m_lock = ceph::make_mutex("Namespace");
+ std::set<std::string> m_names;
+
+ int list(std::vector<std::string> *names) {
+ std::lock_guard locker{m_lock};
+
+ names->clear();
+ names->insert(names->begin(), m_names.begin(), m_names.end());
+ return 0;
}
};
-ImageMap<librbd::MockTestImageCtx>* ImageMap<librbd::MockTestImageCtx>::s_instance = nullptr;
+Namespace<librbd::MockTestImageCtx>* Namespace<librbd::MockTestImageCtx>::s_instance = nullptr;
-template<>
-struct InstanceReplayer<librbd::MockTestImageCtx> {
- static InstanceReplayer* s_instance;
-
- static InstanceReplayer* create(Threads<librbd::MockTestImageCtx> *threads,
- ServiceDaemon<librbd::MockTestImageCtx> *service_daemon,
- journal::CacheManagerHandler *cache_manager_handler,
- RadosRef rados, const std::string& uuid,
- int64_t pool_id) {
- ceph_assert(s_instance != nullptr);
- return s_instance;
- }
+} // namespace api
- MOCK_METHOD0(start, void());
- MOCK_METHOD0(stop, void());
- MOCK_METHOD0(restart, void());
- MOCK_METHOD0(flush, void());
+} // namespace librbd
- MOCK_METHOD2(print_status, void(Formatter*, std::stringstream*));
+namespace rbd {
+namespace mirror {
- MOCK_METHOD2(add_peer, void(const std::string&, librados::IoCtx&));
+template <>
+struct ImageSyncThrottler<librbd::MockTestImageCtx> {
+ static ImageSyncThrottler* s_instance;
- MOCK_METHOD0(init, void());
- MOCK_METHOD0(shut_down, void());
- MOCK_METHOD1(release_all, void(Context*));
+ static ImageSyncThrottler *create(CephContext *cct) {
+ return s_instance;
+ }
- InstanceReplayer() {
+ ImageSyncThrottler() {
+ ceph_assert(s_instance == nullptr);
s_instance = this;
}
-};
-InstanceReplayer<librbd::MockTestImageCtx>* InstanceReplayer<librbd::MockTestImageCtx>::s_instance = nullptr;
+ virtual ~ImageSyncThrottler() {
+ ceph_assert(s_instance == this);
+ s_instance = nullptr;
+ }
+
+ MOCK_METHOD2(print_status, void(Formatter*, std::stringstream*));
+};
-template<>
-struct InstanceWatcher<librbd::MockTestImageCtx> {
- static InstanceWatcher* s_instance;
+ImageSyncThrottler<librbd::MockTestImageCtx>* ImageSyncThrottler<librbd::MockTestImageCtx>::s_instance = nullptr;
- static InstanceWatcher* create(librados::IoCtx &ioctx,
- MockContextWQ* work_queue,
- InstanceReplayer<librbd::MockTestImageCtx>* instance_replayer) {
- ceph_assert(s_instance != nullptr);
- return s_instance;
+template <>
+struct NamespaceReplayer<librbd::MockTestImageCtx> {
+ static std::map<std::string, NamespaceReplayer *> s_instances;
+
+ static NamespaceReplayer *create(
+ const std::string &name,
+ librados::IoCtx &local_ioctx,
+ librados::IoCtx &remote_ioctx,
+ const std::string &local_mirror_uuid,
+ const std::string &remote_mirror_uuid,
+ Threads<librbd::MockTestImageCtx> *threads,
+ ImageSyncThrottler<librbd::MockTestImageCtx> *image_sync_throttler,
+ ServiceDaemon<librbd::MockTestImageCtx> *service_daemon,
+ journal::CacheManagerHandler *cache_manager_handler) {
+ ceph_assert(s_instances.count(name));
+ auto namespace_replayer = s_instances[name];
+ s_instances.erase(name);
+ return namespace_replayer;
}
- MOCK_METHOD0(handle_acquire_leader, void());
- MOCK_METHOD0(handle_release_leader, void());
-
+ MOCK_METHOD0(is_blacklisted, bool());
MOCK_METHOD0(get_instance_id, std::string());
- MOCK_METHOD2(print_sync_status, void(Formatter*, std::stringstream*));
+ MOCK_METHOD1(init, void(Context*));
+ MOCK_METHOD1(shut_down, void(Context*));
- MOCK_METHOD0(init, int());
- MOCK_METHOD0(shut_down, void());
-
- MOCK_METHOD3(notify_image_acquire, void(const std::string&,
- const std::string&,
- Context*));
- MOCK_METHOD3(notify_image_release, void(const std::string&,
- const std::string&,
- Context*));
- MOCK_METHOD4(notify_peer_image_removed, void(const std::string&,
- const std::string&,
- const std::string&,
- Context*));
-
- MOCK_METHOD1(handle_update_leader, void(const std::string&));
-
- InstanceWatcher() {
- s_instance = this;
- }
+ MOCK_METHOD1(handle_acquire_leader, void(Context *));
+ MOCK_METHOD1(handle_release_leader, void(Context *));
+ MOCK_METHOD1(handle_update_leader, void(const std::string &));
+ MOCK_METHOD1(handle_instances_added, void(const std::vector<std::string> &));
+ MOCK_METHOD1(handle_instances_removed, void(const std::vector<std::string> &));
+ MOCK_METHOD2(print_status, void(Formatter*, std::stringstream*));
+ MOCK_METHOD0(start, void());
+ MOCK_METHOD0(stop, void());
+ MOCK_METHOD0(restart, void());
+ MOCK_METHOD0(flush, void());
+
+ NamespaceReplayer(const std::string &name = "") {
+ ceph_assert(!s_instances.count(name));
+ s_instances[name] = this;
+ }
};
-InstanceWatcher<librbd::MockTestImageCtx>* InstanceWatcher<librbd::MockTestImageCtx>::s_instance = nullptr;
+std::map<std::string, NamespaceReplayer<librbd::MockTestImageCtx> *> NamespaceReplayer<librbd::MockTestImageCtx>::s_instances;
template<>
struct LeaderWatcher<librbd::MockTestImageCtx> {
static LeaderWatcher* s_instance;
+ leader_watcher::Listener* listener = nullptr;
static LeaderWatcher *create(Threads<librbd::MockTestImageCtx> *threads,
librados::IoCtx &ioctx,
leader_watcher::Listener* listener) {
ceph_assert(s_instance != nullptr);
+ s_instance->listener = listener;
return s_instance;
}
MOCK_METHOD0(is_leader, bool());
MOCK_METHOD0(release_leader, void());
- MOCK_METHOD1(get_leader_instance_id, void(std::string*));
+ MOCK_METHOD1(get_leader_instance_id, bool(std::string*));
MOCK_METHOD1(list_instances, void(std::vector<std::string>*));
MOCK_METHOD0(init, int());
LeaderWatcher<librbd::MockTestImageCtx>* LeaderWatcher<librbd::MockTestImageCtx>::s_instance = nullptr;
-template<>
-struct PoolWatcher<librbd::MockTestImageCtx> {
- static PoolWatcher* s_instance;
-
- static PoolWatcher *create(Threads<librbd::MockTestImageCtx> *threads,
- librados::IoCtx &ioctx,
- pool_watcher::Listener& listener) {
- ceph_assert(s_instance != nullptr);
- return s_instance;
- }
-
- MOCK_METHOD0(is_blacklisted, bool());
-
- MOCK_METHOD0(get_image_count, uint64_t());
-
- MOCK_METHOD1(init, void(Context*));
- MOCK_METHOD1(shut_down, void(Context*));
-
- PoolWatcher() {
- s_instance = this;
- }
-
-};
-
-PoolWatcher<librbd::MockTestImageCtx>* PoolWatcher<librbd::MockTestImageCtx>::s_instance = nullptr;
-
template<>
struct ServiceDaemon<librbd::MockTestImageCtx> {
MOCK_METHOD3(add_or_update_attribute,
namespace mirror {
using ::testing::_;
+using ::testing::AtLeast;
using ::testing::DoAll;
using ::testing::InSequence;
using ::testing::Invoke;
class TestMockPoolReplayer : public TestMockFixture {
public:
+ typedef librbd::api::Namespace<librbd::MockTestImageCtx> MockNamespace;
typedef PoolReplayer<librbd::MockTestImageCtx> MockPoolReplayer;
- typedef ImageMap<librbd::MockTestImageCtx> MockImageMap;
- typedef InstanceReplayer<librbd::MockTestImageCtx> MockInstanceReplayer;
- typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher;
+ typedef ImageSyncThrottler<librbd::MockTestImageCtx> MockImageSyncThrottler;
+ typedef NamespaceReplayer<librbd::MockTestImageCtx> MockNamespaceReplayer;
typedef LeaderWatcher<librbd::MockTestImageCtx> MockLeaderWatcher;
- typedef PoolWatcher<librbd::MockTestImageCtx> MockPoolWatcher;
typedef ServiceDaemon<librbd::MockTestImageCtx> MockServiceDaemon;
typedef Threads<librbd::MockTestImageCtx> MockThreads;
+ void expect_work_queue(MockThreads &mock_threads) {
+ EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
+ .WillRepeatedly(Invoke([this](Context *ctx, int r) {
+ m_threads->work_queue->queue(ctx, r);
+ }));
+ }
+
void expect_connect(librados::MockTestMemCluster& mock_cluster,
librados::MockTestMemRadosClient* mock_rados_client,
const std::string& cluster_name, CephContext** cct_ref) {
cct->get();
*cct_ref = cct;
}
-
return mock_rados_client;
}));
}
Return(r)));
}
- void expect_instance_replayer_init(MockInstanceReplayer& mock_instance_replayer) {
- EXPECT_CALL(mock_instance_replayer, init());
+ void expect_mirror_mode_get(librados::MockTestMemIoCtxImpl *io_ctx_impl,
+ cls::rbd::MirrorMode mirror_mode, int r) {
+ bufferlist out_bl;
+ encode(mirror_mode, out_bl);
+
+ EXPECT_CALL(*io_ctx_impl,
+ exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_mode_get"),
+ _, _, _))
+ .WillOnce(DoAll(WithArg<5>(Invoke([out_bl](bufferlist *bl) {
+ *bl = out_bl;
+ })),
+ Return(r)));
+ }
+
+ void expect_mirror_mode_get(librados::MockTestMemIoCtxImpl *io_ctx_impl) {
+ EXPECT_CALL(*io_ctx_impl,
+ exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_mode_get"),
+ _, _, _))
+ .WillRepeatedly(DoAll(WithArg<5>(Invoke([](bufferlist *bl) {
+ encode(cls::rbd::MIRROR_MODE_POOL, *bl);
+ })),
+ Return(0)));
+ }
+
+ void expect_leader_watcher_init(MockLeaderWatcher& mock_leader_watcher,
+ int r) {
+ EXPECT_CALL(mock_leader_watcher, init())
+ .WillOnce(Return(r));
+ }
+
+ void expect_leader_watcher_shut_down(MockLeaderWatcher& mock_leader_watcher) {
+ EXPECT_CALL(mock_leader_watcher, shut_down());
}
- void expect_instance_replayer_shut_down(MockInstanceReplayer& mock_instance_replayer) {
- EXPECT_CALL(mock_instance_replayer, shut_down());
+ void expect_leader_watcher_get_leader_instance_id(
+ MockLeaderWatcher& mock_leader_watcher) {
+ EXPECT_CALL(mock_leader_watcher, get_leader_instance_id(_))
+ .WillRepeatedly(Return(true));
}
- void expect_instance_replayer_stop(MockInstanceReplayer& mock_instance_replayer) {
- EXPECT_CALL(mock_instance_replayer, stop());
+ void expect_leader_watcher_list_instances(
+ MockLeaderWatcher& mock_leader_watcher) {
+ EXPECT_CALL(mock_leader_watcher, list_instances(_))
+ .Times(AtLeast(0));
}
- void expect_instance_replayer_add_peer(MockInstanceReplayer& mock_instance_replayer,
- const std::string& uuid) {
- EXPECT_CALL(mock_instance_replayer, add_peer(uuid, _));
+ void expect_namespace_replayer_is_blacklisted(
+ MockNamespaceReplayer &mock_namespace_replayer,
+ bool blacklisted) {
+ EXPECT_CALL(mock_namespace_replayer, is_blacklisted())
+ .WillRepeatedly(Return(blacklisted));
}
- void expect_instance_watcher_get_instance_id(
- MockInstanceWatcher& mock_instance_watcher,
+ void expect_namespace_replayer_get_instance_id(
+ MockNamespaceReplayer &mock_namespace_replayer,
const std::string &instance_id) {
- EXPECT_CALL(mock_instance_watcher, get_instance_id())
+ EXPECT_CALL(mock_namespace_replayer, get_instance_id())
.WillOnce(Return(instance_id));
}
- void expect_instance_watcher_init(MockInstanceWatcher& mock_instance_watcher,
- int r) {
- EXPECT_CALL(mock_instance_watcher, init())
- .WillOnce(Return(r));
+ void expect_namespace_replayer_init(
+ MockNamespaceReplayer &mock_namespace_replayer, int r,
+ Context *on_init = nullptr) {
+
+ EXPECT_CALL(mock_namespace_replayer, init(_))
+ .WillOnce(Invoke([this, r, on_init](Context* ctx) {
+ m_threads->work_queue->queue(ctx, r);
+ if (on_init != nullptr) {
+ m_threads->work_queue->queue(on_init, r);
+ }
+ }));
}
- void expect_instance_watcher_shut_down(MockInstanceWatcher& mock_instance_watcher) {
- EXPECT_CALL(mock_instance_watcher, shut_down());
+ void expect_namespace_replayer_shut_down(
+ MockNamespaceReplayer &mock_namespace_replayer,
+ Context *on_shut_down = nullptr) {
+ EXPECT_CALL(mock_namespace_replayer, shut_down(_))
+ .WillOnce(Invoke([this, on_shut_down](Context* ctx) {
+ m_threads->work_queue->queue(ctx);
+ if (on_shut_down != nullptr) {
+ m_threads->work_queue->queue(on_shut_down);
+ }
+ }));
}
- void expect_leader_watcher_init(MockLeaderWatcher& mock_leader_watcher,
- int r) {
- EXPECT_CALL(mock_leader_watcher, init())
- .WillOnce(Return(r));
+ void expect_namespace_replayer_handle_acquire_leader(
+ MockNamespaceReplayer &mock_namespace_replayer, int r,
+ Context *on_acquire = nullptr) {
+ EXPECT_CALL(mock_namespace_replayer, handle_acquire_leader(_))
+ .WillOnce(Invoke([this, r, on_acquire](Context* ctx) {
+ m_threads->work_queue->queue(ctx, r);
+ if (on_acquire != nullptr) {
+ m_threads->work_queue->queue(on_acquire, r);
+ }
+ }));
}
- void expect_leader_watcher_shut_down(MockLeaderWatcher& mock_leader_watcher) {
- EXPECT_CALL(mock_leader_watcher, shut_down());
+ void expect_namespace_replayer_handle_release_leader(
+ MockNamespaceReplayer &mock_namespace_replayer, int r,
+ Context *on_release = nullptr) {
+ EXPECT_CALL(mock_namespace_replayer, handle_release_leader(_))
+ .WillOnce(Invoke([this, r, on_release](Context* ctx) {
+ m_threads->work_queue->queue(ctx, r);
+ if (on_release != nullptr) {
+ m_threads->work_queue->queue(on_release, r);
+ }
+ }));
+ }
+
+ void expect_namespace_replayer_handle_update_leader(
+ MockNamespaceReplayer &mock_namespace_replayer,
+ const std::string &leader_instance_id,
+ Context *on_update = nullptr) {
+ EXPECT_CALL(mock_namespace_replayer,
+ handle_update_leader(leader_instance_id))
+ .WillOnce(Invoke([on_update](const std::string &) {
+ if (on_update != nullptr) {
+ on_update->complete(0);
+ }
+ }));
+ }
+
+ void expect_namespace_replayer_handle_instances_added(
+ MockNamespaceReplayer &mock_namespace_replayer) {
+ EXPECT_CALL(mock_namespace_replayer, handle_instances_added(_));
+ }
+
+ void expect_namespace_replayer_handle_instances_removed(
+ MockNamespaceReplayer &mock_namespace_replayer) {
+ EXPECT_CALL(mock_namespace_replayer, handle_instances_removed(_));
}
void expect_service_daemon_add_or_update_attribute(
MockServiceDaemon &mock_service_daemon, const std::string& key,
const service_daemon::AttributeValue& value) {
- EXPECT_CALL(mock_service_daemon, add_or_update_attribute(_, _, _));
+ EXPECT_CALL(mock_service_daemon, add_or_update_attribute(_, key, value));
}
- void expect_service_daemon_add_or_update_instance_id_attribute(
- MockInstanceWatcher& mock_instance_watcher,
- MockServiceDaemon &mock_service_daemon) {
- expect_instance_watcher_get_instance_id(mock_instance_watcher, "1234");
- expect_service_daemon_add_or_update_attribute(mock_service_daemon,
- "instance_id", "1234");
+ void expect_service_daemon_remove_attribute(
+ MockServiceDaemon &mock_service_daemon, const std::string& key) {
+ EXPECT_CALL(mock_service_daemon, remove_attribute(_, key));
}
};
peer_spec.mon_host = "123";
peer_spec.key = "234";
+ auto mock_default_namespace_replayer = new MockNamespaceReplayer();
+ expect_namespace_replayer_is_blacklisted(*mock_default_namespace_replayer,
+ false);
+
+ MockThreads mock_threads(m_threads);
+ expect_work_queue(mock_threads);
+
+ auto mock_leader_watcher = new MockLeaderWatcher();
+ expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher);
+
InSequence seq;
auto& mock_cluster = get_mock_cluster();
expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx);
expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0);
+ expect_namespace_replayer_init(*mock_default_namespace_replayer, 0);
+ expect_leader_watcher_init(*mock_leader_watcher, 0);
+
+ MockServiceDaemon mock_service_daemon;
+ MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr,
+ m_local_io_ctx.get_id(), peer_spec, {});
+ pool_replayer.init();
+
+ ASSERT_TRUE(remote_cct != nullptr);
+ ASSERT_EQ("123", remote_cct->_conf.get_val<std::string>("mon_host"));
+ ASSERT_EQ("234", remote_cct->_conf.get_val<std::string>("key"));
+ remote_cct->put();
+
+ expect_leader_watcher_shut_down(*mock_leader_watcher);
+ expect_namespace_replayer_shut_down(*mock_default_namespace_replayer);
+
+ pool_replayer.shut_down();
+}
+
+TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) {
+ PeerSpec peer_spec{"uuid", "cluster name", "client.name"};
+ peer_spec.mon_host = "123";
+ peer_spec.key = "234";
+
+ auto mock_default_namespace_replayer = new MockNamespaceReplayer();
+ expect_namespace_replayer_is_blacklisted(*mock_default_namespace_replayer,
+ false);
+
+ MockThreads mock_threads(m_threads);
+ expect_work_queue(mock_threads);
+
+ auto mock_leader_watcher = new MockLeaderWatcher();
+ expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher);
+ expect_leader_watcher_list_instances(*mock_leader_watcher);
+
+ InSequence seq;
- auto mock_instance_replayer = new MockInstanceReplayer();
- expect_instance_replayer_init(*mock_instance_replayer);
- expect_instance_replayer_add_peer(*mock_instance_replayer, "uuid");
+ auto& mock_cluster = get_mock_cluster();
+ auto mock_local_rados_client = mock_cluster.do_create_rados_client(
+ g_ceph_context);
+ expect_connect(mock_cluster, mock_local_rados_client, "ceph", nullptr);
- auto mock_instance_watcher = new MockInstanceWatcher();
- expect_instance_watcher_init(*mock_instance_watcher, 0);
+ auto mock_remote_rados_client = mock_cluster.do_create_rados_client(
+ g_ceph_context);
+ expect_connect(mock_cluster, mock_remote_rados_client, "cluster name",
+ nullptr);
+
+ auto mock_local_io_ctx = mock_local_rados_client->do_create_ioctx(
+ m_local_io_ctx.get_id(), m_local_io_ctx.get_pool_name());
+ expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx);
+
+ expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0);
+ expect_namespace_replayer_init(*mock_default_namespace_replayer, 0);
+ expect_leader_watcher_init(*mock_leader_watcher, 0);
MockServiceDaemon mock_service_daemon;
- expect_service_daemon_add_or_update_instance_id_attribute(
- *mock_instance_watcher, mock_service_daemon);
+ MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr,
+ m_local_io_ctx.get_id(), peer_spec, {});
+ pool_replayer.init();
+
+ expect_service_daemon_add_or_update_attribute(
+ mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true);
+ expect_namespace_replayer_handle_acquire_leader(
+ *mock_default_namespace_replayer, 0);
+
+ C_SaferCond on_acquire;
+ mock_leader_watcher->listener->post_acquire_handler(&on_acquire);
+ ASSERT_EQ(0, on_acquire.wait());
+
+ expect_service_daemon_remove_attribute(mock_service_daemon,
+ SERVICE_DAEMON_LEADER_KEY);
+ expect_namespace_replayer_handle_release_leader(
+ *mock_default_namespace_replayer, 0);
+
+ C_SaferCond on_release;
+ mock_leader_watcher->listener->pre_release_handler(&on_release);
+ ASSERT_EQ(0, on_release.wait());
+
+ expect_leader_watcher_shut_down(*mock_leader_watcher);
+ expect_namespace_replayer_shut_down(*mock_default_namespace_replayer);
+
+ pool_replayer.shut_down();
+}
+
+TEST_F(TestMockPoolReplayer, Namespaces) {
+ PeerSpec peer_spec{"uuid", "cluster name", "client.name"};
+ peer_spec.mon_host = "123";
+ peer_spec.key = "234";
+
+ g_ceph_context->_conf.set_val(
+ "rbd_mirror_pool_replayers_refresh_interval", "1");
+
+ MockNamespace mock_namespace;
+
+ auto mock_default_namespace_replayer = new MockNamespaceReplayer();
+ expect_namespace_replayer_is_blacklisted(*mock_default_namespace_replayer,
+ false);
+
+ auto mock_ns1_namespace_replayer = new MockNamespaceReplayer("ns1");
+ expect_namespace_replayer_is_blacklisted(*mock_ns1_namespace_replayer,
+ false);
+
+ auto mock_ns2_namespace_replayer = new MockNamespaceReplayer("ns2");
+ expect_namespace_replayer_is_blacklisted(*mock_ns2_namespace_replayer,
+ false);
+
+ MockThreads mock_threads(m_threads);
+ expect_work_queue(mock_threads);
auto mock_leader_watcher = new MockLeaderWatcher();
+ expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher);
+ expect_leader_watcher_list_instances(*mock_leader_watcher);
+
+ auto& mock_cluster = get_mock_cluster();
+ auto mock_local_rados_client = mock_cluster.do_create_rados_client(
+ g_ceph_context);
+ auto mock_local_io_ctx = mock_local_rados_client->do_create_ioctx(
+ m_local_io_ctx.get_id(), m_local_io_ctx.get_pool_name());
+ auto mock_remote_rados_client = mock_cluster.do_create_rados_client(
+ g_ceph_context);
+
+ expect_mirror_mode_get(mock_local_io_ctx);
+
+ InSequence seq;
+
+ expect_connect(mock_cluster, mock_local_rados_client, "ceph", nullptr);
+ expect_connect(mock_cluster, mock_remote_rados_client, "cluster name",
+ nullptr);
+ expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx);
+ expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0);
+ expect_namespace_replayer_init(*mock_default_namespace_replayer, 0);
expect_leader_watcher_init(*mock_leader_watcher, 0);
+ MockServiceDaemon mock_service_daemon;
+ MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr,
+ m_local_io_ctx.get_id(), peer_spec, {});
+ pool_replayer.init();
+
+ C_SaferCond on_ns1_init;
+ expect_namespace_replayer_init(*mock_ns1_namespace_replayer, 0);
+ expect_namespace_replayer_handle_update_leader(*mock_ns1_namespace_replayer,
+ "", &on_ns1_init);
+
+ mock_namespace.add("ns1");
+ ASSERT_EQ(0, on_ns1_init.wait());
+
+ expect_service_daemon_add_or_update_attribute(
+ mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true);
+ expect_namespace_replayer_handle_acquire_leader(
+ *mock_default_namespace_replayer, 0);
+ expect_namespace_replayer_handle_acquire_leader(
+ *mock_ns1_namespace_replayer, 0);
+
+ C_SaferCond on_acquire;
+ mock_leader_watcher->listener->post_acquire_handler(&on_acquire);
+ ASSERT_EQ(0, on_acquire.wait());
+
+ expect_namespace_replayer_init(*mock_ns2_namespace_replayer, 0);
+ C_SaferCond on_ns2_acquire;
+ expect_namespace_replayer_handle_acquire_leader(
+ *mock_ns2_namespace_replayer, 0, &on_ns2_acquire);
+ expect_namespace_replayer_handle_instances_added(
+ *mock_ns2_namespace_replayer);
+
+ mock_namespace.add("ns2");
+ ASSERT_EQ(0, on_ns2_acquire.wait());
+
+ C_SaferCond on_ns2_shut_down;
+ expect_namespace_replayer_shut_down(*mock_ns2_namespace_replayer,
+ &on_ns2_shut_down);
+ mock_namespace.remove("ns2");
+ ASSERT_EQ(0, on_ns2_shut_down.wait());
+
+ expect_service_daemon_remove_attribute(mock_service_daemon,
+ SERVICE_DAEMON_LEADER_KEY);
+ expect_namespace_replayer_handle_release_leader(
+ *mock_default_namespace_replayer, 0);
+ expect_namespace_replayer_handle_release_leader(
+ *mock_ns1_namespace_replayer, 0);
+
+ C_SaferCond on_release;
+ mock_leader_watcher->listener->pre_release_handler(&on_release);
+ ASSERT_EQ(0, on_release.wait());
+
+ expect_namespace_replayer_shut_down(*mock_ns1_namespace_replayer);
+ expect_leader_watcher_shut_down(*mock_leader_watcher);
+ expect_namespace_replayer_shut_down(*mock_default_namespace_replayer);
+
+ pool_replayer.shut_down();
+}
+
+TEST_F(TestMockPoolReplayer, NamespacesError) {
+ PeerSpec peer_spec{"uuid", "cluster name", "client.name"};
+ peer_spec.mon_host = "123";
+ peer_spec.key = "234";
+
+ g_ceph_context->_conf.set_val(
+ "rbd_mirror_pool_replayers_refresh_interval", "1");
+
+ MockNamespace mock_namespace;
+
+ auto mock_default_namespace_replayer = new MockNamespaceReplayer();
+ expect_namespace_replayer_is_blacklisted(*mock_default_namespace_replayer,
+ false);
+ auto mock_ns1_namespace_replayer = new MockNamespaceReplayer("ns1");
+ auto mock_ns2_namespace_replayer = new MockNamespaceReplayer("ns2");
+ expect_namespace_replayer_is_blacklisted(*mock_ns2_namespace_replayer,
+ false);
+ auto mock_ns3_namespace_replayer = new MockNamespaceReplayer("ns3");
+
MockThreads mock_threads(m_threads);
+ expect_work_queue(mock_threads);
+
+ auto mock_leader_watcher = new MockLeaderWatcher();
+ expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher);
+ expect_leader_watcher_list_instances(*mock_leader_watcher);
+
+ auto& mock_cluster = get_mock_cluster();
+ auto mock_local_rados_client = mock_cluster.do_create_rados_client(
+ g_ceph_context);
+ auto mock_local_io_ctx = mock_local_rados_client->do_create_ioctx(
+ m_local_io_ctx.get_id(), m_local_io_ctx.get_pool_name());
+ auto mock_remote_rados_client = mock_cluster.do_create_rados_client(
+ g_ceph_context);
+
+ expect_mirror_mode_get(mock_local_io_ctx);
+
+ InSequence seq;
+
+ expect_connect(mock_cluster, mock_local_rados_client, "ceph", nullptr);
+ expect_connect(mock_cluster, mock_remote_rados_client, "cluster name",
+ nullptr);
+ expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx);
+ expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0);
+ expect_namespace_replayer_init(*mock_default_namespace_replayer, 0);
+ expect_leader_watcher_init(*mock_leader_watcher, 0);
+
+ MockServiceDaemon mock_service_daemon;
MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr,
m_local_io_ctx.get_id(), peer_spec, {});
pool_replayer.init();
- ASSERT_TRUE(remote_cct != nullptr);
- ASSERT_EQ("123", remote_cct->_conf.get_val<std::string>("mon_host"));
- ASSERT_EQ("234", remote_cct->_conf.get_val<std::string>("key"));
- remote_cct->put();
+ // test namespace replayer init fails for non leader
+
+ C_SaferCond on_ns1_init;
+ auto ctx = new FunctionContext(
+ [&mock_namespace, &on_ns1_init](int r) {
+ mock_namespace.remove("ns1");
+ on_ns1_init.complete(r);
+ });
+ expect_namespace_replayer_init(*mock_ns1_namespace_replayer, -EINVAL, ctx);
+ mock_namespace.add("ns1");
+ ASSERT_EQ(-EINVAL, on_ns1_init.wait());
+
+ // test acquire leader fails when default namespace replayer fails
+
+ expect_service_daemon_add_or_update_attribute(
+ mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true);
+ expect_namespace_replayer_handle_acquire_leader(
+ *mock_default_namespace_replayer, -EINVAL);
+
+ C_SaferCond on_acquire1;
+ mock_leader_watcher->listener->post_acquire_handler(&on_acquire1);
+ ASSERT_EQ(-EINVAL, on_acquire1.wait());
+
+ // test acquire leader succeeds when non-default namespace replayer fails
+
+ C_SaferCond on_ns2_init;
+ expect_namespace_replayer_init(*mock_ns2_namespace_replayer, 0);
+ expect_namespace_replayer_handle_update_leader(*mock_ns2_namespace_replayer,
+ "", &on_ns2_init);
+ mock_namespace.add("ns2");
+ ASSERT_EQ(0, on_ns2_init.wait());
+
+ expect_service_daemon_add_or_update_attribute(
+ mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true);
+ expect_namespace_replayer_handle_acquire_leader(
+ *mock_default_namespace_replayer, 0);
+
+ expect_namespace_replayer_handle_acquire_leader(*mock_ns2_namespace_replayer,
+ -EINVAL);
+ ctx = new FunctionContext(
+ [&mock_namespace](int) {
+ mock_namespace.remove("ns2");
+ });
+ expect_namespace_replayer_shut_down(*mock_ns2_namespace_replayer, ctx);
+ mock_namespace.add("ns2");
+
+ C_SaferCond on_acquire2;
+ mock_leader_watcher->listener->post_acquire_handler(&on_acquire2);
+ ASSERT_EQ(0, on_acquire2.wait());
+
+ // test namespace replayer init fails on acquire leader
+
+ C_SaferCond on_ns3_shut_down;
+ ctx = new FunctionContext(
+ [&mock_namespace, &on_ns3_shut_down](int) {
+ mock_namespace.remove("ns3");
+ on_ns3_shut_down.complete(0);
+ });
+ expect_namespace_replayer_init(*mock_ns3_namespace_replayer, 0);
+ expect_namespace_replayer_handle_acquire_leader(*mock_ns3_namespace_replayer,
+ -EINVAL);
+ expect_namespace_replayer_shut_down(*mock_ns3_namespace_replayer, ctx);
+ mock_namespace.add("ns3");
+ ASSERT_EQ(0, on_ns3_shut_down.wait());
- expect_instance_replayer_stop(*mock_instance_replayer);
expect_leader_watcher_shut_down(*mock_leader_watcher);
- expect_instance_watcher_shut_down(*mock_instance_watcher);
- expect_instance_replayer_shut_down(*mock_instance_replayer);
+ expect_namespace_replayer_shut_down(*mock_default_namespace_replayer);
pool_replayer.shut_down();
}
LeaderWatcher.cc
Mirror.cc
MirrorStatusWatcher.cc
+ NamespaceReplayer.cc
PoolReplayer.cc
PoolWatcher.cc
ServiceDaemon.cc
template <typename I>
ImageReplayer<I>::ImageReplayer(
- Threads<I> *threads, InstanceWatcher<I> *instance_watcher,
- journal::CacheManagerHandler *cache_manager_handler, RadosRef local,
- const std::string &local_mirror_uuid, int64_t local_pool_id,
- const std::string &global_image_id) :
- m_threads(threads),
+ librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid,
+ const std::string &global_image_id, Threads<I> *threads,
+ InstanceWatcher<I> *instance_watcher,
+ journal::CacheManagerHandler *cache_manager_handler) :
+ m_local_io_ctx(local_io_ctx), m_local_mirror_uuid(local_mirror_uuid),
+ m_global_image_id(global_image_id), m_threads(threads),
m_instance_watcher(instance_watcher),
m_cache_manager_handler(cache_manager_handler),
- m_local(local),
- m_local_mirror_uuid(local_mirror_uuid),
- m_local_pool_id(local_pool_id),
- m_global_image_id(global_image_id), m_local_image_name(global_image_id),
- m_lock(ceph::make_mutex("rbd::mirror::ImageReplayer " + stringify(local_pool_id) + " " +
- global_image_id)),
+ m_local_image_name(global_image_id),
+ m_lock(ceph::make_mutex("rbd::mirror::ImageReplayer " +
+ stringify(local_io_ctx.get_id()) + " " + global_image_id)),
m_progress_cxt(this),
m_journal_listener(new JournalListener(this)),
m_remote_listener(this)
// name. When the image name becomes known on start the asok commands will be
// re-registered using "remote_pool_name/remote_image_name" name.
- std::string pool_name;
- int r = m_local->pool_reverse_lookup(m_local_pool_id, &pool_name);
- if (r < 0) {
- derr << "error resolving local pool " << m_local_pool_id
- << ": " << cpp_strerror(r) << dendl;
- pool_name = stringify(m_local_pool_id);
- }
-
- m_name = pool_name + "/" + m_global_image_id;
+ m_name = admin_socket_hook_name(global_image_id);
register_admin_socket_hook();
}
return;
}
- m_local_ioctx.reset(new librados::IoCtx{});
- r = m_local->ioctx_create2(m_local_pool_id, *m_local_ioctx);
- if (r < 0) {
- m_local_ioctx.reset();
-
- derr << "error opening ioctx for local pool " << m_local_pool_id
- << ": " << cpp_strerror(r) << dendl;
- on_start_fail(r, "error opening local pool");
- return;
- }
-
prepare_local_image();
}
Context *ctx = create_context_callback<
ImageReplayer, &ImageReplayer<I>::handle_prepare_local_image>(this);
auto req = PrepareLocalImageRequest<I>::create(
- *m_local_ioctx, m_global_image_id, &m_local_image_id, &m_local_image_name,
+ m_local_io_ctx, m_global_image_id, &m_local_image_id, &m_local_image_name,
&m_local_image_tag_owner, m_threads->work_queue, ctx);
req->send();
}
ceph_assert(!m_peers.empty());
m_remote_image = {*m_peers.begin()};
- auto cct = static_cast<CephContext *>(m_local->cct());
+ auto cct = static_cast<CephContext *>(m_local_io_ctx.cct());
journal::Settings journal_settings;
journal_settings.commit_interval = cct->_conf.get_val<double>(
"rbd_mirror_journal_commit_age");
auto ctx = create_context_callback<
ImageReplayer, &ImageReplayer<I>::handle_bootstrap>(this);
request = BootstrapRequest<I>::create(
- m_threads, *m_local_ioctx, m_remote_image.io_ctx, m_instance_watcher,
+ m_threads, m_local_io_ctx, m_remote_image.io_ctx, m_instance_watcher,
&m_local_image_ctx, m_local_image_id, m_remote_image.image_id,
m_global_image_id, m_local_mirror_uuid, m_remote_image.mirror_uuid,
m_remote_journaler, &m_client_state, &m_client_meta, ctx,
}
{
- CephContext *cct = static_cast<CephContext *>(m_local->cct());
+ CephContext *cct = static_cast<CephContext *>(m_local_io_ctx.cct());
double poll_seconds = cct->_conf.get_val<double>(
"rbd_mirror_journal_poll_age");
}
set_state_description(r, desc);
- if (m_local_ioctx) {
- update_mirror_image_status(false, boost::none);
- }
+ update_mirror_image_status(false, boost::none);
reschedule_update_status_task(-1);
shut_down(r);
});
librados::ObjectWriteOperation op;
librbd::cls_client::mirror_image_status_set(&op, m_global_image_id, status);
- ceph_assert(m_local_ioctx);
librados::AioCompletion *aio_comp = create_rados_callback<
ImageReplayer<I>, &ImageReplayer<I>::handle_mirror_status_update>(this);
- int r = m_local_ioctx->aio_operate(RBD_MIRRORING, aio_comp, &op);
+ int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op);
ceph_assert(r == 0);
aio_comp->release();
}
// chain the shut down sequence (reverse order)
Context *ctx = new FunctionContext(
[this, r](int _r) {
- if (m_local_ioctx) {
- update_mirror_image_status(true, STATE_STOPPED);
- }
+ update_mirror_image_status(true, STATE_STOPPED);
handle_shut_down(r);
});
auto ctx = new FunctionContext([this, r](int) {
handle_shut_down(r);
});
- ImageDeleter<I>::trash_move(*m_local_ioctx, m_global_image_id,
+ ImageDeleter<I>::trash_move(m_local_io_ctx, m_global_image_id,
resync_requested, m_threads->work_queue, ctx);
return;
}
if (r == 0) {
m_asok_hook = asok_hook;
- CephContext *cct = static_cast<CephContext *>(m_local->cct());
+ CephContext *cct = static_cast<CephContext *>(m_local_io_ctx.cct());
auto prio = cct->_conf.get_val<int64_t>("rbd_mirror_perf_stats_prio");
PerfCountersBuilder plb(g_ceph_context, "rbd_mirror_" + m_name,
l_rbd_mirror_first, l_rbd_mirror_last);
void ImageReplayer<I>::reregister_admin_socket_hook() {
{
std::lock_guard locker{m_lock};
- auto name = m_local_ioctx->get_pool_name() + "/" + m_local_image_name;
+
+ auto name = admin_socket_hook_name(m_local_image_name);
if (m_asok_hook != nullptr && m_name == name) {
return;
}
register_admin_socket_hook();
}
+template <typename I>
+std::string ImageReplayer<I>::admin_socket_hook_name(
+ const std::string &image_name) const {
+ std::string name = m_local_io_ctx.get_namespace();
+ if (!name.empty()) {
+ name += "/";
+ }
+
+ return m_local_io_ctx.get_pool_name() + "/" + name + image_name;
+}
+
template <typename I>
std::ostream &operator<<(std::ostream &os, const ImageReplayer<I> &replayer)
{
class ImageReplayer {
public:
static ImageReplayer *create(
- Threads<ImageCtxT> *threads, InstanceWatcher<ImageCtxT> *instance_watcher,
- journal::CacheManagerHandler *cache_manager_handler, RadosRef local,
- const std::string &local_mirror_uuid, int64_t local_pool_id,
- const std::string &global_image_id) {
- return new ImageReplayer(threads, instance_watcher, cache_manager_handler,
- local, local_mirror_uuid, local_pool_id,
- global_image_id);
+ librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid,
+ const std::string &global_image_id, Threads<ImageCtxT> *threads,
+ InstanceWatcher<ImageCtxT> *instance_watcher,
+ journal::CacheManagerHandler *cache_manager_handler) {
+ return new ImageReplayer(local_io_ctx, local_mirror_uuid, global_image_id,
+ threads, instance_watcher, cache_manager_handler);
}
void destroy() {
delete this;
}
- ImageReplayer(Threads<ImageCtxT> *threads,
+ ImageReplayer(librados::IoCtx &local_io_ctx,
+ const std::string &local_mirror_uuid,
+ const std::string &global_image_id,
+ Threads<ImageCtxT> *threads,
InstanceWatcher<ImageCtxT> *instance_watcher,
- journal::CacheManagerHandler *cache_manager_handler,
- RadosRef local, const std::string &local_mirror_uuid,
- int64_t local_pool_id, const std::string &global_image_id);
+ journal::CacheManagerHandler *cache_manager_handler);
virtual ~ImageReplayer();
ImageReplayer(const ImageReplayer&) = delete;
ImageReplayer& operator=(const ImageReplayer&) = delete;
void add_peer(const std::string &peer_uuid, librados::IoCtx &remote_io_ctx);
inline int64_t get_local_pool_id() const {
- return m_local_pool_id;
+ return m_local_io_ctx.get_id();
}
inline const std::string& get_global_image_id() const {
return m_global_image_id;
ImageReplayer<ImageCtxT> *replayer;
};
+ librados::IoCtx &m_local_io_ctx;
+ std::string m_local_mirror_uuid;
+ std::string m_global_image_id;
Threads<ImageCtxT> *m_threads;
InstanceWatcher<ImageCtxT> *m_instance_watcher;
journal::CacheManagerHandler *m_cache_manager_handler;
Peers m_peers;
RemoteImage m_remote_image;
- RadosRef m_local;
- std::string m_local_mirror_uuid;
- int64_t m_local_pool_id;
std::string m_local_image_id;
- std::string m_global_image_id;
std::string m_local_image_name;
std::string m_name;
image_replayer::EventPreprocessor<ImageCtxT> *m_event_preprocessor = nullptr;
image_replayer::ReplayStatusFormatter<ImageCtxT> *m_replay_status_formatter =
nullptr;
- IoCtxRef m_local_ioctx;
ImageCtxT *m_local_image_ctx = nullptr;
std::string m_local_image_tag_owner;
void register_admin_socket_hook();
void unregister_admin_socket_hook();
void reregister_admin_socket_hook();
+
+ std::string admin_socket_hook_name(const std::string &image_name) const;
};
} // namespace mirror
}
template <typename I>
-void ImageSyncThrottler<I>::start_op(const std::string &id, Context *on_start) {
+void ImageSyncThrottler<I>::start_op(const std::string &ns,
+ const std::string &id_,
+ Context *on_start) {
+ Id id{ns, id_};
+
dout(20) << "id=" << id << dendl;
int r = 0;
}
template <typename I>
-bool ImageSyncThrottler<I>::cancel_op(const std::string &id) {
+bool ImageSyncThrottler<I>::cancel_op(const std::string &ns,
+ const std::string &id_) {
+ Id id{ns, id_};
+
dout(20) << "id=" << id << dendl;
Context *on_start = nullptr;
}
template <typename I>
-void ImageSyncThrottler<I>::finish_op(const std::string &id) {
+void ImageSyncThrottler<I>::finish_op(const std::string &ns,
+ const std::string &id_) {
+ Id id{ns, id_};
+
dout(20) << "id=" << id << dendl;
- if (cancel_op(id)) {
+ if (cancel_op(ns, id_)) {
return;
}
}
template <typename I>
-void ImageSyncThrottler<I>::drain(int r) {
- dout(20) << dendl;
+void ImageSyncThrottler<I>::drain(const std::string &ns, int r) {
+ dout(20) << "ns=" << ns << dendl;
- std::map<std::string, Context *> queued_ops;
+ std::map<Id, Context *> queued_ops;
{
std::lock_guard locker{m_lock};
- std::swap(m_queued_ops, queued_ops);
- m_queue.clear();
- m_inflight_ops.clear();
+ for (auto it = m_queued_ops.begin(); it != m_queued_ops.end(); ) {
+ if (it->first.first == ns) {
+ queued_ops[it->first] = it->second;
+ m_queue.remove(it->first);
+ it = m_queued_ops.erase(it);
+ } else {
+ it++;
+ }
+ }
+ for (auto it = m_inflight_ops.begin(); it != m_inflight_ops.end(); ) {
+ if (it->first == ns) {
+ dout(20) << "inflight_op " << *it << dendl;
+ it = m_inflight_ops.erase(it);
+ } else {
+ it++;
+ }
+ }
}
for (auto &it : queued_ops) {
+ dout(20) << "queued_op " << it.first << dendl;
it.second->complete(r);
}
}
~ImageSyncThrottler() override;
void set_max_concurrent_syncs(uint32_t max);
- void start_op(const std::string &id, Context *on_start);
- bool cancel_op(const std::string &id);
- void finish_op(const std::string &id);
- void drain(int r);
+ void start_op(const std::string &ns, const std::string &id,
+ Context *on_start);
+ bool cancel_op(const std::string &ns, const std::string &id);
+ void finish_op(const std::string &ns, const std::string &id);
+ void drain(const std::string &ns, int r);
void print_status(ceph::Formatter *f, std::stringstream *ss);
private:
+ typedef std::pair<std::string, std::string> Id;
+
CephContext *m_cct;
ceph::mutex m_lock;
uint32_t m_max_concurrent_syncs;
- std::list<std::string> m_queue;
- std::map<std::string, Context *> m_queued_ops;
- std::set<std::string> m_inflight_ops;
+ std::list<Id> m_queue;
+ std::map<Id, Context *> m_queued_ops;
+ std::set<Id> m_inflight_ops;
const char **get_tracked_conf_keys() const override;
void handle_conf_change(const ConfigProxy& conf,
template <typename I>
InstanceReplayer<I>::InstanceReplayer(
+ librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid,
Threads<I> *threads, ServiceDaemon<I>* service_daemon,
- journal::CacheManagerHandler *cache_manager_handler, RadosRef local_rados,
- const std::string &local_mirror_uuid, int64_t local_pool_id)
- : m_threads(threads), m_service_daemon(service_daemon),
- m_cache_manager_handler(cache_manager_handler), m_local_rados(local_rados),
- m_local_mirror_uuid(local_mirror_uuid), m_local_pool_id(local_pool_id),
- m_lock(ceph::make_mutex(
- "rbd::mirror::InstanceReplayer " + stringify(local_pool_id))) {
+ journal::CacheManagerHandler *cache_manager_handler)
+ : m_local_io_ctx(local_io_ctx), m_local_mirror_uuid(local_mirror_uuid),
+ m_threads(threads), m_service_daemon(service_daemon),
+ m_cache_manager_handler(cache_manager_handler),
+ m_lock(ceph::make_mutex("rbd::mirror::InstanceReplayer " +
+ stringify(local_io_ctx.get_id()))) {
}
template <typename I>
auto it = m_image_replayers.find(global_image_id);
if (it == m_image_replayers.end()) {
auto image_replayer = ImageReplayer<I>::create(
- m_threads, instance_watcher, m_cache_manager_handler, m_local_rados,
- m_local_mirror_uuid, m_local_pool_id, global_image_id);
+ m_local_io_ctx, m_local_mirror_uuid, global_image_id,
+ m_threads, instance_watcher, m_cache_manager_handler);
dout(10) << global_image_id << ": creating replayer " << image_replayer
<< dendl;
}
}
+template <typename I>
+void InstanceReplayer<I>::stop(Context *on_finish)
+{
+ dout(10) << dendl;
+
+ auto cct = static_cast<CephContext *>(m_local_io_ctx.cct());
+ auto gather_ctx = new C_Gather(cct, on_finish);
+ {
+ std::lock_guard locker{m_lock};
+
+ m_manual_stop = true;
+
+ for (auto &kv : m_image_replayers) {
+ auto &image_replayer = kv.second;
+ image_replayer->stop(gather_ctx->new_sub(), true);
+ }
+ }
+
+ gather_ctx->activate();
+}
+
template <typename I>
void InstanceReplayer<I>::restart()
{
start_image_replayer(current_it->second);
}
- m_service_daemon->add_or_update_attribute(
- m_local_pool_id, SERVICE_DAEMON_ASSIGNED_COUNT_KEY, image_count);
- m_service_daemon->add_or_update_attribute(
- m_local_pool_id, SERVICE_DAEMON_WARNING_COUNT_KEY, warning_count);
- m_service_daemon->add_or_update_attribute(
- m_local_pool_id, SERVICE_DAEMON_ERROR_COUNT_KEY, error_count);
+ // TODO: add namespace support to service daemon
+ if (m_local_io_ctx.get_namespace().empty()) {
+ m_service_daemon->add_or_update_attribute(
+ m_local_io_ctx.get_id(), SERVICE_DAEMON_ASSIGNED_COUNT_KEY, image_count);
+ m_service_daemon->add_or_update_attribute(
+ m_local_io_ctx.get_id(), SERVICE_DAEMON_WARNING_COUNT_KEY, warning_count);
+ m_service_daemon->add_or_update_attribute(
+ m_local_io_ctx.get_id(), SERVICE_DAEMON_ERROR_COUNT_KEY, error_count);
+ }
m_async_op_tracker.finish_op();
}
queue_start_image_replayers();
});
- auto cct = static_cast<CephContext *>(m_local_rados->cct());
+ auto cct = static_cast<CephContext *>(m_local_io_ctx.cct());
int after = cct->_conf.get_val<uint64_t>(
"rbd_mirror_image_state_check_interval");
class InstanceReplayer {
public:
static InstanceReplayer* create(
- Threads<ImageCtxT> *threads,
- ServiceDaemon<ImageCtxT> *service_daemon,
- journal::CacheManagerHandler *cache_manager_handler,
- RadosRef local_rados, const std::string &local_mirror_uuid,
- int64_t local_pool_id) {
- return new InstanceReplayer(threads, service_daemon, cache_manager_handler,
- local_rados, local_mirror_uuid, local_pool_id);
+ librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid,
+ Threads<ImageCtxT> *threads, ServiceDaemon<ImageCtxT> *service_daemon,
+ journal::CacheManagerHandler *cache_manager_handler) {
+ return new InstanceReplayer(local_io_ctx, local_mirror_uuid, threads,
+ service_daemon, cache_manager_handler);
}
void destroy() {
delete this;
}
- InstanceReplayer(Threads<ImageCtxT> *threads,
+ InstanceReplayer(librados::IoCtx &local_io_ctx,
+ const std::string &local_mirror_uuid,
+ Threads<ImageCtxT> *threads,
ServiceDaemon<ImageCtxT> *service_daemon,
- journal::CacheManagerHandler *cache_manager_handler,
- RadosRef local_rados, const std::string &local_mirror_uuid,
- int64_t local_pool_id);
+ journal::CacheManagerHandler *cache_manager_handler);
~InstanceReplayer();
int init();
void restart();
void flush();
+ void stop(Context *on_finish);
+
private:
/**
* @verbatim
* @endverbatim
*/
+ librados::IoCtx &m_local_io_ctx;
+ std::string m_local_mirror_uuid;
Threads<ImageCtxT> *m_threads;
ServiceDaemon<ImageCtxT> *m_service_daemon;
journal::CacheManagerHandler *m_cache_manager_handler;
- RadosRef m_local_rados;
- std::string m_local_mirror_uuid;
- int64_t m_local_pool_id;
ceph::mutex m_lock;
AsyncOpTracker m_async_op_tracker;
C_RemoveInstanceRequest(librados::IoCtx &io_ctx, ContextWQ *work_queue,
const std::string &instance_id, Context *on_finish)
- : instance_watcher(io_ctx, work_queue, nullptr, instance_id),
+ : instance_watcher(io_ctx, work_queue, nullptr, nullptr, instance_id),
on_finish(on_finish) {
}
template <typename I>
InstanceWatcher<I> *InstanceWatcher<I>::create(
librados::IoCtx &io_ctx, ContextWQ *work_queue,
- InstanceReplayer<I> *instance_replayer) {
+ InstanceReplayer<I> *instance_replayer,
+ ImageSyncThrottler<I> *image_sync_throttler) {
return new InstanceWatcher<I>(io_ctx, work_queue, instance_replayer,
+ image_sync_throttler,
stringify(io_ctx.get_instance_id()));
}
InstanceWatcher<I>::InstanceWatcher(librados::IoCtx &io_ctx,
ContextWQ *work_queue,
InstanceReplayer<I> *instance_replayer,
+ ImageSyncThrottler<I> *image_sync_throttler,
const std::string &instance_id)
: Watcher(io_ctx, work_queue, RBD_MIRROR_INSTANCE_PREFIX + instance_id),
- m_instance_replayer(instance_replayer), m_instance_id(instance_id),
+ m_instance_replayer(instance_replayer),
+ m_image_sync_throttler(image_sync_throttler), m_instance_id(instance_id),
m_lock(ceph::make_mutex(
unique_lock_name("rbd::mirror::InstanceWatcher::m_lock", this))),
m_instance_lock(librbd::ManagedLock<I>::create(
ceph_assert(m_notify_op_tracker.empty());
ceph_assert(m_suspended_ops.empty());
ceph_assert(m_inflight_sync_reqs.empty());
- ceph_assert(m_image_sync_throttler == nullptr);
m_instance_lock->destroy();
}
[this, sync_id] (int r) {
dout(10) << "finish: sync_id=" << sync_id << ", r=" << r << dendl;
std::lock_guard locker{m_lock};
- if (r != -ESTALE && m_image_sync_throttler != nullptr) {
- m_image_sync_throttler->finish_op(sync_id);
+ if (r != -ESTALE && is_leader()) {
+ m_image_sync_throttler->finish_op(m_ioctx.get_namespace(), sync_id);
}
});
auto req = new C_NotifyInstanceRequest(this, instance_id, request_id,
}
}
-template <typename I>
-void InstanceWatcher<I>::print_sync_status(Formatter *f, stringstream *ss) {
- dout(10) << dendl;
-
- std::lock_guard locker{m_lock};
- if (m_image_sync_throttler != nullptr) {
- m_image_sync_throttler->print_status(f, ss);
- }
-}
-
template <typename I>
void InstanceWatcher<I>::handle_acquire_leader() {
dout(10) << dendl;
std::lock_guard locker{m_lock};
- ceph_assert(m_image_sync_throttler == nullptr);
- m_image_sync_throttler = ImageSyncThrottler<I>::create(m_cct);
-
m_leader_instance_id = m_instance_id;
unsuspend_notify_requests();
}
std::lock_guard locker{m_lock};
- ceph_assert(m_image_sync_throttler != nullptr);
-
m_leader_instance_id.clear();
- m_image_sync_throttler->drain(-ESTALE);
- m_image_sync_throttler->destroy();
- m_image_sync_throttler = nullptr;
+ m_image_sync_throttler->drain(m_ioctx.get_namespace(), -ESTALE);
}
template <typename I>
std::lock_guard locker{m_lock};
- if (m_image_sync_throttler == nullptr) {
+ if (!is_leader()) {
dout(10) << "sync request for non-leader" << dendl;
m_work_queue->queue(on_finish, -ESTALE);
return;
}
on_finish->complete(r);
}));
- m_image_sync_throttler->start_op(sync_id, on_start);
+ m_image_sync_throttler->start_op(m_ioctx.get_namespace(), sync_id, on_start);
}
template <typename I>
static InstanceWatcher *create(
librados::IoCtx &io_ctx, ContextWQ *work_queue,
- InstanceReplayer<ImageCtxT> *instance_replayer);
+ InstanceReplayer<ImageCtxT> *instance_replayer,
+ ImageSyncThrottler<ImageCtxT> *image_sync_throttler);
void destroy() {
delete this;
}
InstanceWatcher(librados::IoCtx &io_ctx, ContextWQ *work_queue,
InstanceReplayer<ImageCtxT> *instance_replayer,
+ ImageSyncThrottler<ImageCtxT> *image_sync_throttler,
const std::string &instance_id);
~InstanceWatcher() override;
bool cancel_sync_request(const std::string &sync_id);
void notify_sync_complete(const std::string &sync_id);
- void print_sync_status(Formatter *f, stringstream *ss);
-
void cancel_notify_requests(const std::string &instance_id);
void handle_acquire_leader();
Threads<ImageCtxT> *m_threads;
InstanceReplayer<ImageCtxT> *m_instance_replayer;
+ ImageSyncThrottler<ImageCtxT> *m_image_sync_throttler;
std::string m_instance_id;
mutable ceph::mutex m_lock;
std::set<Request> m_requests;
std::set<C_NotifyInstanceRequest *> m_suspended_ops;
std::map<std::string, C_SyncRequest *> m_inflight_sync_reqs;
- ImageSyncThrottler<ImageCtxT> *m_image_sync_throttler = nullptr;
+
+ inline bool is_leader() const {
+ return m_leader_instance_id == m_instance_id;
+ }
void register_instance();
void handle_register_instance(int r);
template <typename I>
LeaderWatcher<I>::~LeaderWatcher() {
- ceph_assert(m_status_watcher == nullptr);
ceph_assert(m_instances == nullptr);
ceph_assert(m_timer_task == nullptr);
dout(10) << "r=" << r << dendl;
Context *on_finish = nullptr;
- if (r < 0) {
+ {
+ std::lock_guard timer_locker(m_threads->timer_lock);
std::lock_guard locker{m_lock};
- derr << "error registering leader watcher for " << m_oid << " object: "
- << cpp_strerror(r) << dendl;
+
+ if (r < 0) {
+ derr << "error registering leader watcher for " << m_oid << " object: "
+ << cpp_strerror(r) << dendl;
+ } else {
+ schedule_acquire_leader_lock(0);
+ }
+
ceph_assert(m_on_finish != nullptr);
std::swap(on_finish, m_on_finish);
- } else {
- std::lock_guard locker{m_lock};
- init_status_watcher();
- return;
}
on_finish->complete(r);
derr << "error shutting down leader lock: " << cpp_strerror(r) << dendl;
}
- shut_down_status_watcher();
+ unregister_watch();
}
template <typename I>
schedule_acquire_leader_lock(1);
}
-template <typename I>
-void LeaderWatcher<I>::init_status_watcher() {
- dout(10) << dendl;
-
- ceph_assert(ceph_mutex_is_locked(m_lock));
- ceph_assert(m_status_watcher == nullptr);
-
- m_status_watcher = MirrorStatusWatcher<I>::create(m_ioctx, m_work_queue);
-
- Context *ctx = create_context_callback<
- LeaderWatcher<I>, &LeaderWatcher<I>::handle_init_status_watcher>(this);
-
- m_status_watcher->init(ctx);
-}
-
-template <typename I>
-void LeaderWatcher<I>::handle_init_status_watcher(int r) {
- dout(10) << "r=" << r << dendl;
-
- Context *on_finish = nullptr;
- {
- std::scoped_lock locker{m_threads->timer_lock, m_lock};
-
- if (r < 0) {
- derr << "error initializing mirror status watcher: " << cpp_strerror(r)
- << cpp_strerror(r) << dendl;
- } else {
- schedule_acquire_leader_lock(0);
- }
-
- ceph_assert(m_on_finish != nullptr);
- std::swap(on_finish, m_on_finish);
- }
-
- on_finish->complete(r);
-}
-
-template <typename I>
-void LeaderWatcher<I>::shut_down_status_watcher() {
- dout(10) << dendl;
-
- ceph_assert(ceph_mutex_is_locked(m_lock));
- ceph_assert(m_status_watcher != nullptr);
-
- Context *ctx = create_async_context_callback(
- m_work_queue, create_context_callback<LeaderWatcher<I>,
- &LeaderWatcher<I>::handle_shut_down_status_watcher>(this));
-
- m_status_watcher->shut_down(ctx);
-}
-
-template <typename I>
-void LeaderWatcher<I>::handle_shut_down_status_watcher(int r) {
- dout(10) << "r=" << r << dendl;
-
- std::lock_guard locker{m_lock};
- m_status_watcher->destroy();
- m_status_watcher = nullptr;
-
- if (r < 0) {
- derr << "error shutting mirror status watcher down: " << cpp_strerror(r)
- << dendl;
- }
-
- unregister_watch();
-}
-
template <typename I>
void LeaderWatcher<I>::init_instances() {
dout(10) << dendl;
#include "librbd/managed_lock/Types.h"
#include "librbd/watcher/Types.h"
#include "Instances.h"
-#include "MirrorStatusWatcher.h"
#include "tools/rbd_mirror/instances/Types.h"
#include "tools/rbd_mirror/leader_watcher/Types.h"
* CREATE_OBJECT * * * * * (error) UNREGISTER_WATCH
* | * ^
* v * |
- * REGISTER_WATCH * * * * * SHUT_DOWN_STATUS_WATCHER
- * | * ^
- * v * |
- * INIT_STATUS_WATCHER * * SHUT_DOWN_LEADER_LOCK
- * | |
+ * REGISTER_WATCH * * * * * SHUT_DOWN_LEADER_LOCK
+ * | ^
* | (no leader heartbeat and acquire failed) |
* | BREAK_LOCK <-------------------------------------\ |
* | | (no leader heartbeat) | | (shut down)
Context *m_on_shut_down_finish = nullptr;
uint64_t m_acquire_attempts = 0;
int m_ret_val = 0;
- MirrorStatusWatcher<ImageCtxT> *m_status_watcher = nullptr;
Instances<ImageCtxT> *m_instances = nullptr;
librbd::managed_lock::Locker m_locker;
void release_leader_lock();
void handle_release_leader_lock(int r);
- void init_status_watcher();
- void handle_init_status_watcher(int r);
-
- void shut_down_status_watcher();
- void handle_shut_down_status_watcher(int r);
-
void init_instances();
void handle_init_instances(int r);
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "NamespaceReplayer.h"
+#include <boost/bind.hpp>
+#include "common/Formatter.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "librbd/Utils.h"
+#include "librbd/api/Config.h"
+#include "librbd/api/Mirror.h"
+#include "ServiceDaemon.h"
+#include "Threads.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::NamespaceReplayer: " \
+ << this << " " << __func__ << ": "
+
+using librbd::util::create_async_context_callback;
+using librbd::util::create_context_callback;
+
+namespace rbd {
+namespace mirror {
+
+using ::operator<<;
+
+namespace {
+
+const std::string SERVICE_DAEMON_INSTANCE_ID_KEY("instance_id");
+const std::string SERVICE_DAEMON_LOCAL_COUNT_KEY("image_local_count");
+const std::string SERVICE_DAEMON_REMOTE_COUNT_KEY("image_remote_count");
+
+} // anonymous namespace
+
+template <typename I>
+NamespaceReplayer<I>::NamespaceReplayer(
+ const std::string &name,
+ librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx,
+ const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid,
+ Threads<I> *threads, ImageSyncThrottler<I> *image_sync_throttler,
+ ServiceDaemon<I> *service_daemon,
+ journal::CacheManagerHandler *cache_manager_handler) :
+ m_local_mirror_uuid(local_mirror_uuid),
+ m_remote_mirror_uuid(remote_mirror_uuid),
+ m_threads(threads), m_image_sync_throttler(image_sync_throttler),
+ m_service_daemon(service_daemon),
+ m_cache_manager_handler(cache_manager_handler),
+ m_lock(ceph::make_mutex(librbd::util::unique_lock_name(
+ "rbd::mirror::NamespaceReplayer " + name, this))),
+ m_local_pool_watcher_listener(this, true),
+ m_remote_pool_watcher_listener(this, false),
+ m_image_map_listener(this) {
+ dout(10) << name << dendl;
+
+ m_local_io_ctx.dup(local_io_ctx);
+ m_local_io_ctx.set_namespace(name);
+ m_remote_io_ctx.dup(remote_io_ctx);
+ m_remote_io_ctx.set_namespace(name);
+}
+
+template <typename I>
+bool NamespaceReplayer<I>::is_blacklisted() const {
+ std::lock_guard locker{m_lock};
+ return (m_local_pool_watcher &&
+ m_local_pool_watcher->is_blacklisted()) ||
+ (m_remote_pool_watcher &&
+ m_remote_pool_watcher->is_blacklisted());
+}
+
+template <typename I>
+void NamespaceReplayer<I>::init(Context *on_finish) {
+ dout(20) << dendl;
+
+ std::lock_guard locker{m_lock};
+
+ ceph_assert(m_on_finish == nullptr);
+ m_on_finish = on_finish;
+
+ init_status_watcher();
+}
+
+
+template <typename I>
+void NamespaceReplayer<I>::shut_down(Context *on_finish) {
+ dout(20) << dendl;
+
+ {
+ std::lock_guard locker{m_lock};
+
+ ceph_assert(m_on_finish == nullptr);
+ m_on_finish = on_finish;
+
+ if (!m_image_map) {
+ stop_instance_replayer();
+ return;
+ }
+ }
+
+ auto ctx = new FunctionContext(
+ [this] (int r) {
+ std::lock_guard locker{m_lock};
+ stop_instance_replayer();
+ });
+ handle_release_leader(ctx);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::print_status(Formatter *f, stringstream *ss)
+{
+ dout(20) << dendl;
+
+ ceph_assert(f);
+
+ std::lock_guard locker{m_lock};
+
+ m_instance_replayer->print_status(f, ss);
+
+ if (m_image_deleter) {
+ f->open_object_section("image_deleter");
+ m_image_deleter->print_status(f, ss);
+ f->close_section();
+ }
+}
+
+template <typename I>
+void NamespaceReplayer<I>::start()
+{
+ dout(20) << dendl;
+
+ std::lock_guard locker{m_lock};
+
+ m_instance_replayer->start();
+}
+
+template <typename I>
+void NamespaceReplayer<I>::stop()
+{
+ dout(20) << dendl;
+
+ std::lock_guard locker{m_lock};
+
+ m_instance_replayer->stop();
+}
+
+template <typename I>
+void NamespaceReplayer<I>::restart()
+{
+ dout(20) << dendl;
+
+ std::lock_guard locker{m_lock};
+
+ m_instance_replayer->restart();
+}
+
+template <typename I>
+void NamespaceReplayer<I>::flush()
+{
+ dout(20) << dendl;
+
+ std::lock_guard locker{m_lock};
+
+ m_instance_replayer->flush();
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_update(const std::string &mirror_uuid,
+ ImageIds &&added_image_ids,
+ ImageIds &&removed_image_ids) {
+ std::lock_guard locker{m_lock};
+
+ if (!m_image_map) {
+ dout(20) << "not leader" << dendl;
+ return;
+ }
+
+ dout(10) << "mirror_uuid=" << mirror_uuid << ", "
+ << "added_count=" << added_image_ids.size() << ", "
+ << "removed_count=" << removed_image_ids.size() << dendl;
+
+ // TODO: add namespace support to service daemon
+ if (m_local_io_ctx.get_namespace().empty()) {
+ m_service_daemon->add_or_update_attribute(
+ m_local_io_ctx.get_id(), SERVICE_DAEMON_LOCAL_COUNT_KEY,
+ m_local_pool_watcher->get_image_count());
+ if (m_remote_pool_watcher) {
+ m_service_daemon->add_or_update_attribute(
+ m_local_io_ctx.get_id(), SERVICE_DAEMON_REMOTE_COUNT_KEY,
+ m_remote_pool_watcher->get_image_count());
+ }
+ }
+
+ std::set<std::string> added_global_image_ids;
+ for (auto& image_id : added_image_ids) {
+ added_global_image_ids.insert(image_id.global_id);
+ }
+
+ std::set<std::string> removed_global_image_ids;
+ for (auto& image_id : removed_image_ids) {
+ removed_global_image_ids.insert(image_id.global_id);
+ }
+
+ m_image_map->update_images(mirror_uuid,
+ std::move(added_global_image_ids),
+ std::move(removed_global_image_ids));
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_acquire_leader(Context *on_finish) {
+ dout(10) << dendl;
+
+ m_instance_watcher->handle_acquire_leader();
+
+ init_image_map(on_finish);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_release_leader(Context *on_finish) {
+ dout(10) << dendl;
+
+ m_instance_watcher->handle_release_leader();
+ shut_down_image_deleter(on_finish);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_update_leader(
+ const std::string &leader_instance_id) {
+ dout(10) << "leader_instance_id=" << leader_instance_id << dendl;
+
+ m_instance_watcher->handle_update_leader(leader_instance_id);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_instances_added(
+ const std::vector<std::string> &instance_ids) {
+ dout(10) << "instance_ids=" << instance_ids << dendl;
+
+ std::lock_guard locker{m_lock};
+
+ ceph_assert(m_image_map);
+ m_image_map->update_instances_added(instance_ids);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_instances_removed(
+ const std::vector<std::string> &instance_ids) {
+ dout(10) << "instance_ids=" << instance_ids << dendl;
+
+ std::lock_guard locker{m_lock};
+
+ ceph_assert(m_image_map);
+ m_image_map->update_instances_removed(instance_ids);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::init_status_watcher() {
+ dout(10) << dendl;
+
+ ceph_assert(ceph_mutex_is_locked(m_lock));
+ ceph_assert(!m_status_watcher);
+
+ m_status_watcher.reset(MirrorStatusWatcher<I>::create(
+ m_local_io_ctx, m_threads->work_queue));
+ auto ctx = create_context_callback<NamespaceReplayer<I>,
+ &NamespaceReplayer<I>::handle_init_status_watcher>(this);
+
+ m_status_watcher->init(ctx);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_init_status_watcher(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ std::lock_guard locker{m_lock};
+
+ if (r < 0) {
+ derr << "error initializing mirror status watcher: " << cpp_strerror(r)
+ << dendl;
+
+ ceph_assert(m_on_finish != nullptr);
+ m_threads->work_queue->queue(m_on_finish, r);
+ m_on_finish = nullptr;
+ return;
+ }
+
+ init_instance_replayer();
+}
+
+template <typename I>
+void NamespaceReplayer<I>::init_instance_replayer() {
+ dout(10) << dendl;
+
+ ceph_assert(ceph_mutex_is_locked(m_lock));
+ ceph_assert(!m_instance_replayer);
+
+ m_instance_replayer.reset(InstanceReplayer<I>::create(
+ m_local_io_ctx, m_local_mirror_uuid, m_threads, m_service_daemon,
+ m_cache_manager_handler));
+ auto ctx = create_context_callback<NamespaceReplayer<I>,
+ &NamespaceReplayer<I>::handle_init_instance_replayer>(this);
+
+ m_instance_replayer->init(ctx);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_init_instance_replayer(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ std::lock_guard locker{m_lock};
+
+ if (r < 0) {
+ derr << "error initializing instance replayer: " << cpp_strerror(r)
+ << dendl;
+
+ m_instance_replayer.reset();
+ m_ret_val = r;
+ shut_down_status_watcher();
+ return;
+ }
+
+ m_instance_replayer->add_peer(m_remote_mirror_uuid, m_remote_io_ctx);
+
+ init_instance_watcher();
+}
+
+template <typename I>
+void NamespaceReplayer<I>::init_instance_watcher() {
+ dout(10) << dendl;
+
+ ceph_assert(ceph_mutex_is_locked(m_lock));
+ ceph_assert(!m_instance_watcher);
+
+ m_instance_watcher.reset(InstanceWatcher<I>::create(
+ m_local_io_ctx, m_threads->work_queue, m_instance_replayer.get(),
+ m_image_sync_throttler));
+ auto ctx = create_context_callback<NamespaceReplayer<I>,
+ &NamespaceReplayer<I>::handle_init_instance_watcher>(this);
+
+ m_instance_watcher->init(ctx);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_init_instance_watcher(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ std::lock_guard locker{m_lock};
+
+ if (r < 0) {
+ derr << "error initializing instance watcher: " << cpp_strerror(r)
+ << dendl;
+
+ m_instance_watcher.reset();
+ m_ret_val = r;
+ shut_down_instance_replayer();
+ return;
+ }
+
+ // TODO: add namespace support to service daemon
+ if (m_local_io_ctx.get_namespace().empty()) {
+ m_service_daemon->add_or_update_attribute(
+ m_local_io_ctx.get_id(), SERVICE_DAEMON_INSTANCE_ID_KEY,
+ m_instance_watcher->get_instance_id());
+ }
+
+ ceph_assert(m_on_finish != nullptr);
+ m_threads->work_queue->queue(m_on_finish);
+ m_on_finish = nullptr;
+}
+
+template <typename I>
+void NamespaceReplayer<I>::stop_instance_replayer() {
+ dout(10) << dendl;
+
+ ceph_assert(ceph_mutex_is_locked(m_lock));
+
+ Context *ctx = create_async_context_callback(
+ m_threads->work_queue, create_context_callback<NamespaceReplayer<I>,
+ &NamespaceReplayer<I>::handle_stop_instance_replayer>(this));
+
+ m_instance_replayer->stop(ctx);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_stop_instance_replayer(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "error stopping instance replayer: " << cpp_strerror(r) << dendl;
+ }
+
+ std::lock_guard locker{m_lock};
+
+ shut_down_instance_watcher();
+}
+
+template <typename I>
+void NamespaceReplayer<I>::shut_down_instance_watcher() {
+ dout(10) << dendl;
+
+ ceph_assert(ceph_mutex_is_locked(m_lock));
+ ceph_assert(m_instance_watcher);
+
+ Context *ctx = create_async_context_callback(
+ m_threads->work_queue, create_context_callback<NamespaceReplayer<I>,
+ &NamespaceReplayer<I>::handle_shut_down_instance_watcher>(this));
+
+ m_instance_watcher->shut_down(ctx);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_shut_down_instance_watcher(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "error shutting instance watcher down: " << cpp_strerror(r)
+ << dendl;
+ }
+
+ std::lock_guard locker{m_lock};
+
+ m_instance_watcher.reset();
+
+ shut_down_instance_replayer();
+}
+
+template <typename I>
+void NamespaceReplayer<I>::shut_down_instance_replayer() {
+ dout(10) << dendl;
+
+ ceph_assert(ceph_mutex_is_locked(m_lock));
+ ceph_assert(m_instance_replayer);
+
+ Context *ctx = create_async_context_callback(
+ m_threads->work_queue, create_context_callback<NamespaceReplayer<I>,
+ &NamespaceReplayer<I>::handle_shut_down_instance_replayer>(this));
+
+ m_instance_replayer->shut_down(ctx);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_shut_down_instance_replayer(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "error shutting instance replayer down: " << cpp_strerror(r)
+ << dendl;
+ }
+
+ std::lock_guard locker{m_lock};
+
+ m_instance_replayer.reset();
+
+ shut_down_status_watcher();
+}
+
+template <typename I>
+void NamespaceReplayer<I>::shut_down_status_watcher() {
+ dout(10) << dendl;
+
+ ceph_assert(ceph_mutex_is_locked(m_lock));
+ ceph_assert(m_status_watcher);
+
+ Context *ctx = create_async_context_callback(
+ m_threads->work_queue, create_context_callback<NamespaceReplayer<I>,
+ &NamespaceReplayer<I>::handle_shut_down_status_watcher>(this));
+
+ m_status_watcher->shut_down(ctx);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_shut_down_status_watcher(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "error shutting mirror status watcher down: " << cpp_strerror(r)
+ << dendl;
+ }
+
+ std::lock_guard locker{m_lock};
+
+ m_status_watcher.reset();
+
+ ceph_assert(!m_image_map);
+ ceph_assert(!m_image_deleter);
+ ceph_assert(!m_local_pool_watcher);
+ ceph_assert(!m_remote_pool_watcher);
+ ceph_assert(!m_instance_watcher);
+ ceph_assert(!m_instance_replayer);
+
+ ceph_assert(m_on_finish != nullptr);
+ m_threads->work_queue->queue(m_on_finish, m_ret_val);
+ m_on_finish = nullptr;
+ m_ret_val = 0;
+}
+
+template <typename I>
+void NamespaceReplayer<I>::init_image_map(Context *on_finish) {
+ dout(10) << dendl;
+
+ std::lock_guard locker{m_lock};
+ ceph_assert(!m_image_map);
+ m_image_map.reset(ImageMap<I>::create(m_local_io_ctx, m_threads,
+ m_instance_watcher->get_instance_id(),
+ m_image_map_listener));
+
+ auto ctx = new FunctionContext(
+ [this, on_finish](int r) {
+ handle_init_image_map(r, on_finish);
+ });
+ m_image_map->init(create_async_context_callback(
+ m_threads->work_queue, ctx));
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_init_image_map(int r, Context *on_finish) {
+ dout(10) << "r=" << r << dendl;
+ if (r < 0) {
+ derr << "failed to init image map: " << cpp_strerror(r) << dendl;
+ on_finish = new FunctionContext([on_finish, r](int) {
+ on_finish->complete(r);
+ });
+ shut_down_image_map(on_finish);
+ return;
+ }
+
+ init_local_pool_watcher(on_finish);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::init_local_pool_watcher(Context *on_finish) {
+ dout(10) << dendl;
+
+ std::lock_guard locker{m_lock};
+ ceph_assert(!m_local_pool_watcher);
+ m_local_pool_watcher.reset(PoolWatcher<I>::create(
+ m_threads, m_local_io_ctx, m_local_pool_watcher_listener));
+
+ // ensure the initial set of local images is up-to-date
+ // after acquiring the leader role
+ auto ctx = new FunctionContext([this, on_finish](int r) {
+ handle_init_local_pool_watcher(r, on_finish);
+ });
+ m_local_pool_watcher->init(create_async_context_callback(
+ m_threads->work_queue, ctx));
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_init_local_pool_watcher(
+ int r, Context *on_finish) {
+ dout(10) << "r=" << r << dendl;
+ if (r < 0) {
+ derr << "failed to retrieve local images: " << cpp_strerror(r) << dendl;
+ on_finish = new FunctionContext([on_finish, r](int) {
+ on_finish->complete(r);
+ });
+ shut_down_pool_watchers(on_finish);
+ return;
+ }
+
+ init_remote_pool_watcher(on_finish);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::init_remote_pool_watcher(Context *on_finish) {
+ dout(10) << dendl;
+
+ std::lock_guard locker{m_lock};
+ ceph_assert(!m_remote_pool_watcher);
+ m_remote_pool_watcher.reset(PoolWatcher<I>::create(
+ m_threads, m_remote_io_ctx, m_remote_pool_watcher_listener));
+
+ auto ctx = new FunctionContext([this, on_finish](int r) {
+ handle_init_remote_pool_watcher(r, on_finish);
+ });
+ m_remote_pool_watcher->init(create_async_context_callback(
+ m_threads->work_queue, ctx));
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_init_remote_pool_watcher(
+ int r, Context *on_finish) {
+ dout(10) << "r=" << r << dendl;
+ if (r == -ENOENT) {
+ // Technically nothing to do since the other side doesn't
+ // have mirroring enabled. Eventually the remote pool watcher will
+ // detect images (if mirroring is enabled), so no point propagating
+ // an error which would just busy-spin the state machines.
+ dout(0) << "remote peer does not have mirroring configured" << dendl;
+ } else if (r < 0) {
+ derr << "failed to retrieve remote images: " << cpp_strerror(r) << dendl;
+ on_finish = new FunctionContext([on_finish, r](int) {
+ on_finish->complete(r);
+ });
+ shut_down_pool_watchers(on_finish);
+ return;
+ }
+
+ init_image_deleter(on_finish);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::init_image_deleter(Context *on_finish) {
+ dout(10) << dendl;
+
+ std::lock_guard locker{m_lock};
+ ceph_assert(!m_image_deleter);
+
+ on_finish = new FunctionContext([this, on_finish](int r) {
+ handle_init_image_deleter(r, on_finish);
+ });
+ m_image_deleter.reset(ImageDeleter<I>::create(m_local_io_ctx, m_threads,
+ m_service_daemon));
+ m_image_deleter->init(create_async_context_callback(
+ m_threads->work_queue, on_finish));
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_init_image_deleter(
+ int r, Context *on_finish) {
+ dout(10) << "r=" << r << dendl;
+ if (r < 0) {
+ derr << "failed to init image deleter: " << cpp_strerror(r) << dendl;
+ on_finish = new FunctionContext([on_finish, r](int) {
+ on_finish->complete(r);
+ });
+ shut_down_image_deleter(on_finish);
+ return;
+ }
+
+ on_finish->complete(0);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::shut_down_image_deleter(Context* on_finish) {
+ dout(10) << dendl;
+ {
+ std::lock_guard locker{m_lock};
+ if (m_image_deleter) {
+ Context *ctx = new FunctionContext([this, on_finish](int r) {
+ handle_shut_down_image_deleter(r, on_finish);
+ });
+ ctx = create_async_context_callback(m_threads->work_queue, ctx);
+
+ m_image_deleter->shut_down(ctx);
+ return;
+ }
+ }
+ shut_down_pool_watchers(on_finish);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_shut_down_image_deleter(
+ int r, Context* on_finish) {
+ dout(10) << "r=" << r << dendl;
+
+ {
+ std::lock_guard locker{m_lock};
+ ceph_assert(m_image_deleter);
+ m_image_deleter.reset();
+ }
+
+ shut_down_pool_watchers(on_finish);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::shut_down_pool_watchers(Context *on_finish) {
+ dout(10) << dendl;
+
+ {
+ std::lock_guard locker{m_lock};
+ if (m_local_pool_watcher) {
+ Context *ctx = new FunctionContext([this, on_finish](int r) {
+ handle_shut_down_pool_watchers(r, on_finish);
+ });
+ ctx = create_async_context_callback(m_threads->work_queue, ctx);
+
+ auto gather_ctx = new C_Gather(g_ceph_context, ctx);
+ m_local_pool_watcher->shut_down(gather_ctx->new_sub());
+ if (m_remote_pool_watcher) {
+ m_remote_pool_watcher->shut_down(gather_ctx->new_sub());
+ }
+ gather_ctx->activate();
+ return;
+ }
+ }
+
+ on_finish->complete(0);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_shut_down_pool_watchers(
+ int r, Context *on_finish) {
+ dout(10) << "r=" << r << dendl;
+
+ {
+ std::lock_guard locker{m_lock};
+ ceph_assert(m_local_pool_watcher);
+ m_local_pool_watcher.reset();
+
+ if (m_remote_pool_watcher) {
+ m_remote_pool_watcher.reset();
+ }
+ }
+ shut_down_image_map(on_finish);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::shut_down_image_map(Context *on_finish) {
+ dout(5) << dendl;
+
+ std::lock_guard locker{m_lock};
+ if (m_image_map) {
+ on_finish = new FunctionContext(
+ [this, on_finish](int r) {
+ handle_shut_down_image_map(r, on_finish);
+ });
+ m_image_map->shut_down(create_async_context_callback(
+ m_threads->work_queue, on_finish));
+ return;
+ }
+
+ m_threads->work_queue->queue(on_finish);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_shut_down_image_map(int r, Context *on_finish) {
+ dout(5) << "r=" << r << dendl;
+ if (r < 0 && r != -EBLACKLISTED) {
+ derr << "failed to shut down image map: " << cpp_strerror(r) << dendl;
+ }
+
+ std::lock_guard locker{m_lock};
+ ceph_assert(m_image_map);
+ m_image_map.reset();
+
+ m_instance_replayer->release_all(create_async_context_callback(
+ m_threads->work_queue, on_finish));
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_acquire_image(const std::string &global_image_id,
+ const std::string &instance_id,
+ Context* on_finish) {
+ dout(5) << "global_image_id=" << global_image_id << ", "
+ << "instance_id=" << instance_id << dendl;
+
+ m_instance_watcher->notify_image_acquire(instance_id, global_image_id,
+ on_finish);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_release_image(const std::string &global_image_id,
+ const std::string &instance_id,
+ Context* on_finish) {
+ dout(5) << "global_image_id=" << global_image_id << ", "
+ << "instance_id=" << instance_id << dendl;
+
+ m_instance_watcher->notify_image_release(instance_id, global_image_id,
+ on_finish);
+}
+
+template <typename I>
+void NamespaceReplayer<I>::handle_remove_image(const std::string &mirror_uuid,
+ const std::string &global_image_id,
+ const std::string &instance_id,
+ Context* on_finish) {
+ ceph_assert(!mirror_uuid.empty());
+ dout(5) << "mirror_uuid=" << mirror_uuid << ", "
+ << "global_image_id=" << global_image_id << ", "
+ << "instance_id=" << instance_id << dendl;
+
+ m_instance_watcher->notify_peer_image_removed(instance_id, global_image_id,
+ mirror_uuid, on_finish);
+}
+
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::NamespaceReplayer<librbd::ImageCtx>;
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_RBD_MIRROR_NAMESPACE_REPLAYER_H
+#define CEPH_RBD_MIRROR_NAMESPACE_REPLAYER_H
+
+#include "common/AsyncOpTracker.h"
+#include "common/WorkQueue.h"
+#include "common/ceph_mutex.h"
+#include "include/rados/librados.hpp"
+
+#include "tools/rbd_mirror/ImageDeleter.h"
+#include "tools/rbd_mirror/ImageMap.h"
+#include "tools/rbd_mirror/InstanceReplayer.h"
+#include "tools/rbd_mirror/InstanceWatcher.h"
+#include "tools/rbd_mirror/MirrorStatusWatcher.h"
+#include "tools/rbd_mirror/PoolWatcher.h"
+#include "tools/rbd_mirror/Types.h"
+#include "tools/rbd_mirror/image_map/Types.h"
+#include "tools/rbd_mirror/pool_watcher/Types.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+class AdminSocketHook;
+
+namespace journal { struct CacheManagerHandler; }
+
+namespace librbd { class ImageCtx; }
+
+namespace rbd {
+namespace mirror {
+
+template <typename> class ImageSyncThrottler;
+template <typename> class ServiceDaemon;
+template <typename> struct Threads;
+
+/**
+ * Controls mirroring for a single remote cluster.
+ */
+template <typename ImageCtxT = librbd::ImageCtx>
+class NamespaceReplayer {
+public:
+ static NamespaceReplayer *create(
+ const std::string &name,
+ librados::IoCtx &local_ioctx,
+ librados::IoCtx &remote_ioctx,
+ const std::string &local_mirror_uuid,
+ const std::string &remote_mirror_uuid,
+ Threads<ImageCtxT> *threads,
+ ImageSyncThrottler<ImageCtxT> *image_sync_throttler,
+ ServiceDaemon<ImageCtxT> *service_daemon,
+ journal::CacheManagerHandler *cache_manager_handler) {
+ return new NamespaceReplayer(name, local_ioctx, remote_ioctx,
+ local_mirror_uuid, remote_mirror_uuid, threads,
+ image_sync_throttler, service_daemon,
+ cache_manager_handler);
+ }
+
+ NamespaceReplayer(const std::string &name,
+ librados::IoCtx &local_ioctx,
+ librados::IoCtx &remote_ioctx,
+ const std::string &local_mirror_uuid,
+ const std::string &remote_mirror_uuid,
+ Threads<ImageCtxT> *threads,
+ ImageSyncThrottler<ImageCtxT> *image_sync_throttler,
+ ServiceDaemon<ImageCtxT> *service_daemon,
+ journal::CacheManagerHandler *cache_manager_handler);
+ NamespaceReplayer(const NamespaceReplayer&) = delete;
+ NamespaceReplayer& operator=(const NamespaceReplayer&) = delete;
+
+ bool is_blacklisted() const;
+
+ void init(Context *on_finish);
+ void shut_down(Context *on_finish);
+
+ void handle_acquire_leader(Context *on_finish);
+ void handle_release_leader(Context *on_finish);
+ void handle_update_leader(const std::string &leader_instance_id);
+ void handle_instances_added(const std::vector<std::string> &instance_ids);
+ void handle_instances_removed(const std::vector<std::string> &instance_ids);
+
+ void print_status(Formatter *f, stringstream *ss);
+ void start();
+ void stop();
+ void restart();
+ void flush();
+
+private:
+ /**
+ * @verbatim
+ *
+ * <uninitialized> <----------------------\
+ * | (init) ^ (error) |
+ * v * |
+ * INIT_STATUS_WATCHER * * * * * * > SHUT_DOWN_STATUS_WATCHER
+ * | * (error) ^
+ * v * |
+ * INIT_INSTANCE_REPLAYER * * * * > SHUT_DOWN_INSTANCE_REPLAYER
+ * | * ^
+ * v * |
+ * INIT_INSTANCE_WATCHER * * * * * SHUT_DOWN_INSTANCE_WATCHER
+ * | (error) ^
+ * | |
+ * v STOP_INSTANCE_REPLAYER
+ * | ^
+ * | (shut down) |
+ * | /--------------------------------/
+ * v |
+ * <follower> <---------------------------\
+ * . |
+ * . |
+ * v (leader acquired) |
+ * INIT_IMAGE_MAP |
+ * | |
+ * v |
+ * INIT_LOCAL_POOL_WATCHER SHUT_DOWN_IMAGE_MAP
+ * | ^
+ * v |
+ * INIT_REMOTE_POOL_WATCHER SHUT_DOWN_POOL_WATCHERS
+ * | ^
+ * v |
+ * INIT_IMAGE_DELETER SHUT_DOWN_IMAGE_DELETER
+ * | ^
+ * v .
+ * <leader> <-----------\ .
+ * . | .
+ * . (image update) | .
+ * . . > NOTIFY_INSTANCE_WATCHER .
+ * . .
+ * . (leader lost / shut down) .
+ * . . . . . . . . . . . . . . . . . . .
+ *
+ * @endverbatim
+ */
+
+ struct PoolWatcherListener : public pool_watcher::Listener {
+ NamespaceReplayer *namespace_replayer;
+ bool local;
+
+ PoolWatcherListener(NamespaceReplayer *namespace_replayer, bool local)
+ : namespace_replayer(namespace_replayer), local(local) {
+ }
+
+ void handle_update(const std::string &mirror_uuid,
+ ImageIds &&added_image_ids,
+ ImageIds &&removed_image_ids) override {
+ namespace_replayer->handle_update((local ? "" : mirror_uuid),
+ std::move(added_image_ids),
+ std::move(removed_image_ids));
+ }
+ };
+
+ struct ImageMapListener : public image_map::Listener {
+ NamespaceReplayer *namespace_replayer;
+
+ ImageMapListener(NamespaceReplayer *namespace_replayer)
+ : namespace_replayer(namespace_replayer) {
+ }
+
+ void acquire_image(const std::string &global_image_id,
+ const std::string &instance_id,
+ Context* on_finish) override {
+ namespace_replayer->handle_acquire_image(global_image_id, instance_id,
+ on_finish);
+ }
+
+ void release_image(const std::string &global_image_id,
+ const std::string &instance_id,
+ Context* on_finish) override {
+ namespace_replayer->handle_release_image(global_image_id, instance_id,
+ on_finish);
+ }
+
+ void remove_image(const std::string &mirror_uuid,
+ const std::string &global_image_id,
+ const std::string &instance_id,
+ Context* on_finish) override {
+ namespace_replayer->handle_remove_image(mirror_uuid, global_image_id,
+ instance_id, on_finish);
+ }
+ };
+
+ void handle_update(const std::string &mirror_uuid,
+ ImageIds &&added_image_ids,
+ ImageIds &&removed_image_ids);
+
+ int init_rados(const std::string &cluster_name,
+ const std::string &client_name,
+ const std::string &mon_host,
+ const std::string &key,
+ const std::string &description, RadosRef *rados_ref,
+ bool strip_cluster_overrides);
+
+ void init_status_watcher();
+ void handle_init_status_watcher(int r);
+
+ void init_instance_replayer();
+ void handle_init_instance_replayer(int r);
+
+ void init_instance_watcher();
+ void handle_init_instance_watcher(int r);
+
+ void stop_instance_replayer();
+ void handle_stop_instance_replayer(int r);
+
+ void shut_down_instance_watcher();
+ void handle_shut_down_instance_watcher(int r);
+
+ void shut_down_instance_replayer();
+ void handle_shut_down_instance_replayer(int r);
+
+ void shut_down_status_watcher();
+ void handle_shut_down_status_watcher(int r);
+
+ void init_image_map(Context *on_finish);
+ void handle_init_image_map(int r, Context *on_finish);
+
+ void init_local_pool_watcher(Context *on_finish);
+ void handle_init_local_pool_watcher(int r, Context *on_finish);
+
+ void init_remote_pool_watcher(Context *on_finish);
+ void handle_init_remote_pool_watcher(int r, Context *on_finish);
+
+ void init_image_deleter(Context* on_finish);
+ void handle_init_image_deleter(int r, Context* on_finish);
+
+ void shut_down_image_deleter(Context* on_finish);
+ void handle_shut_down_image_deleter(int r, Context* on_finish);
+
+ void shut_down_pool_watchers(Context *on_finish);
+ void handle_shut_down_pool_watchers(int r, Context *on_finish);
+
+ void shut_down_image_map(Context *on_finish);
+ void handle_shut_down_image_map(int r, Context *on_finish);
+
+ void handle_acquire_image(const std::string &global_image_id,
+ const std::string &instance_id,
+ Context* on_finish);
+ void handle_release_image(const std::string &global_image_id,
+ const std::string &instance_id,
+ Context* on_finish);
+ void handle_remove_image(const std::string &mirror_uuid,
+ const std::string &global_image_id,
+ const std::string &instance_id,
+ Context* on_finish);
+
+ librados::IoCtx m_local_io_ctx;
+ librados::IoCtx m_remote_io_ctx;
+ std::string m_local_mirror_uuid;
+ std::string m_remote_mirror_uuid;
+ Threads<ImageCtxT> *m_threads;
+ ImageSyncThrottler<ImageCtxT> *m_image_sync_throttler;
+ ServiceDaemon<ImageCtxT> *m_service_daemon;
+ journal::CacheManagerHandler *m_cache_manager_handler;
+
+ mutable ceph::mutex m_lock;
+
+ int m_ret_val = 0;
+ Context *m_on_finish = nullptr;
+
+ std::unique_ptr<MirrorStatusWatcher<ImageCtxT>> m_status_watcher;
+
+ PoolWatcherListener m_local_pool_watcher_listener;
+ std::unique_ptr<PoolWatcher<ImageCtxT>> m_local_pool_watcher;
+
+ PoolWatcherListener m_remote_pool_watcher_listener;
+ std::unique_ptr<PoolWatcher<ImageCtxT>> m_remote_pool_watcher;
+
+ std::unique_ptr<InstanceReplayer<ImageCtxT>> m_instance_replayer;
+ std::unique_ptr<ImageDeleter<ImageCtxT>> m_image_deleter;
+
+ ImageMapListener m_image_map_listener;
+ std::unique_ptr<ImageMap<ImageCtxT>> m_image_map;
+
+ std::unique_ptr<InstanceWatcher<ImageCtxT>> m_instance_watcher;
+};
+
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::NamespaceReplayer<librbd::ImageCtx>;
+
+#endif // CEPH_RBD_MIRROR_NAMESPACE_REPLAYER_H
#include "PoolReplayer.h"
#include <boost/bind.hpp>
+#include "common/Cond.h"
#include "common/Formatter.h"
#include "common/admin_socket.h"
#include "common/ceph_argparse.h"
#include "common/common_init.h"
#include "common/debug.h"
#include "common/errno.h"
-#include "include/stringify.h"
#include "cls/rbd/cls_rbd_client.h"
#include "global/global_context.h"
-#include "librbd/internal.h"
-#include "librbd/Utils.h"
-#include "librbd/Watcher.h"
#include "librbd/api/Config.h"
-#include "librbd/api/Mirror.h"
-#include "ImageMap.h"
-#include "InstanceReplayer.h"
-#include "InstanceWatcher.h"
-#include "LeaderWatcher.h"
+#include "librbd/api/Namespace.h"
#include "ServiceDaemon.h"
#include "Threads.h"
#define dout_prefix *_dout << "rbd::mirror::PoolReplayer: " \
<< this << " " << __func__ << ": "
-using std::chrono::seconds;
-using std::map;
-using std::string;
-using std::unique_ptr;
-using std::vector;
-
-using librbd::cls_client::dir_get_name;
-using librbd::util::create_async_context_callback;
-
namespace rbd {
namespace mirror {
namespace {
-const std::string SERVICE_DAEMON_INSTANCE_ID_KEY("instance_id");
const std::string SERVICE_DAEMON_LEADER_KEY("leader");
-const std::string SERVICE_DAEMON_LOCAL_COUNT_KEY("image_local_count");
-const std::string SERVICE_DAEMON_REMOTE_COUNT_KEY("image_remote_count");
const std::vector<std::string> UNIQUE_PEER_CONFIG_KEYS {
{"monmap", "mon_host", "mon_dns_srv_name", "key", "keyfile", "keyring"}};
m_local_pool_id(local_pool_id),
m_peer(peer),
m_args(args),
- m_lock(ceph::make_mutex(stringify("rbd::mirror::PoolReplayer ") + stringify(peer))),
- m_local_pool_watcher_listener(this, true),
- m_remote_pool_watcher_listener(this, false),
- m_image_map_listener(this),
+ m_lock(ceph::make_mutex("rbd::mirror::PoolReplayer " + stringify(peer))),
m_pool_replayer_thread(this),
- m_leader_listener(this)
-{
+ m_leader_listener(this) {
}
template <typename I>
PoolReplayer<I>::~PoolReplayer()
{
- delete m_asok_hook;
shut_down();
+
+ ceph_assert(m_asok_hook == nullptr);
}
template <typename I>
}
template <typename I>
-void PoolReplayer<I>::init()
-{
+void PoolReplayer<I>::init() {
ceph_assert(!m_pool_replayer_thread.is_started());
// reset state
auto cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct());
librbd::api::Config<I>::apply_pool_overrides(m_local_io_ctx, &cct->_conf);
- std::string local_mirror_uuid;
r = librbd::cls_client::mirror_uuid_get(&m_local_io_ctx,
- &local_mirror_uuid);
+ &m_local_mirror_uuid);
if (r < 0) {
derr << "failed to retrieve local mirror uuid from pool "
<< m_local_io_ctx.get_pool_name() << ": " << cpp_strerror(r) << dendl;
dout(10) << "connected to " << m_peer << dendl;
- m_instance_replayer.reset(InstanceReplayer<I>::create(
- m_threads, m_service_daemon, m_cache_manager_handler, m_local_rados,
- local_mirror_uuid, m_local_pool_id));
- m_instance_replayer->init();
- m_instance_replayer->add_peer(m_peer.uuid, m_remote_io_ctx);
+ m_image_sync_throttler.reset(ImageSyncThrottler<I>::create(cct));
- m_instance_watcher.reset(InstanceWatcher<I>::create(
- m_local_io_ctx, m_threads->work_queue, m_instance_replayer.get()));
- r = m_instance_watcher->init();
+ m_default_namespace_replayer.reset(NamespaceReplayer<I>::create(
+ "", m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid,
+ m_threads, m_image_sync_throttler.get(), m_service_daemon,
+ m_cache_manager_handler));
+
+ C_SaferCond on_init;
+ m_default_namespace_replayer->init(&on_init);
+ r = on_init.wait();
if (r < 0) {
- derr << "error initializing instance watcher: " << cpp_strerror(r) << dendl;
+ derr << "error initializing default namespace replayer: " << cpp_strerror(r)
+ << dendl;
m_callout_id = m_service_daemon->add_or_update_callout(
m_local_pool_id, m_callout_id, service_daemon::CALLOUT_LEVEL_ERROR,
- "unable to initialize instance messenger object");
+ "unable to initialize default namespace replayer");
+ m_default_namespace_replayer.reset();
return;
}
- m_service_daemon->add_or_update_attribute(
- m_local_pool_id, SERVICE_DAEMON_INSTANCE_ID_KEY,
- m_instance_watcher->get_instance_id());
m_leader_watcher.reset(LeaderWatcher<I>::create(m_threads, m_local_io_ctx,
&m_leader_listener));
m_callout_id = m_service_daemon->add_or_update_callout(
m_local_pool_id, m_callout_id, service_daemon::CALLOUT_LEVEL_ERROR,
"unable to initialize leader messenger object");
+ m_leader_watcher.reset();
return;
}
template <typename I>
void PoolReplayer<I>::shut_down() {
- m_stopping = true;
{
std::lock_guard l{m_lock};
+ m_stopping = true;
m_cond.notify_all();
}
if (m_pool_replayer_thread.is_started()) {
m_pool_replayer_thread.join();
}
+
if (m_leader_watcher) {
m_leader_watcher->shut_down();
}
- if (m_instance_watcher) {
- m_instance_watcher->shut_down();
- }
- if (m_instance_replayer) {
- m_instance_replayer->shut_down();
+ m_leader_watcher.reset();
+
+ if (m_default_namespace_replayer) {
+ C_SaferCond on_shut_down;
+ m_default_namespace_replayer->shut_down(&on_shut_down);
+ on_shut_down.wait();
}
+ m_default_namespace_replayer.reset();
- m_leader_watcher.reset();
- m_instance_watcher.reset();
- m_instance_replayer.reset();
+ m_image_sync_throttler.reset();
- ceph_assert(!m_image_map);
- ceph_assert(!m_image_deleter);
- ceph_assert(!m_local_pool_watcher);
- ceph_assert(!m_remote_pool_watcher);
m_local_rados.reset();
m_remote_rados.reset();
}
}
template <typename I>
-void PoolReplayer<I>::run()
-{
- dout(20) << "enter" << dendl;
+void PoolReplayer<I>::run() {
+ dout(20) << dendl;
- while (!m_stopping) {
+ while (true) {
std::string asok_hook_name = m_local_io_ctx.get_pool_name() + " " +
m_peer.cluster_name;
if (m_asok_hook_name != asok_hook_name || m_asok_hook == nullptr) {
m_asok_hook_name, this);
}
+ with_namespace_replayers([this]() { update_namespace_replayers(); });
+
std::unique_lock locker{m_lock};
- if ((m_local_pool_watcher && m_local_pool_watcher->is_blacklisted()) ||
- (m_remote_pool_watcher && m_remote_pool_watcher->is_blacklisted())) {
+
+ if (m_default_namespace_replayer->is_blacklisted()) {
m_blacklisted = true;
m_stopping = true;
+ }
+
+ for (auto &it : m_namespace_replayers) {
+ if (it.second->is_blacklisted()) {
+ m_blacklisted = true;
+ m_stopping = true;
+ break;
+ }
+ }
+
+ if (m_stopping) {
break;
}
- if (!m_stopping) {
- m_cond.wait_for(locker, 1s);
+ auto seconds = g_ceph_context->_conf.get_val<uint64_t>(
+ "rbd_mirror_pool_replayers_refresh_interval");
+ m_cond.wait_for(locker, ceph::make_timespan(seconds));
+ }
+
+ // shut down namespace replayers
+ with_namespace_replayers([this]() { update_namespace_replayers(); });
+
+ delete m_asok_hook;
+ m_asok_hook = nullptr;
+}
+
+template <typename I>
+void PoolReplayer<I>::update_namespace_replayers() {
+ dout(20) << dendl;
+
+ ceph_assert(ceph_mutex_is_locked(m_lock));
+
+ std::set<std::string> mirroring_namespaces;
+ if (!m_stopping) {
+ int r = list_mirroring_namespaces(&mirroring_namespaces);
+ if (r < 0) {
+ return;
+ }
+ }
+
+ auto cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct());
+ C_SaferCond cond;
+ auto gather_ctx = new C_Gather(cct, &cond);
+ for (auto it = m_namespace_replayers.begin();
+ it != m_namespace_replayers.end(); ) {
+ auto iter = mirroring_namespaces.find(it->first);
+ if (iter == mirroring_namespaces.end()) {
+ auto namespace_replayer = it->second;
+ auto on_shut_down = new FunctionContext(
+ [this, namespace_replayer, ctx=gather_ctx->new_sub()](int r) {
+ delete namespace_replayer;
+ ctx->complete(r);
+ });
+ namespace_replayer->shut_down(on_shut_down);
+ it = m_namespace_replayers.erase(it);
+ } else {
+ mirroring_namespaces.erase(iter);
+ it++;
}
}
- m_instance_replayer->stop();
+ for (auto &name : mirroring_namespaces) {
+ auto namespace_replayer = NamespaceReplayer<I>::create(
+ name, m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid,
+ m_threads, m_image_sync_throttler.get(), m_service_daemon,
+ m_cache_manager_handler);
+ auto on_init = new FunctionContext(
+ [this, namespace_replayer, name, &mirroring_namespaces,
+ ctx=gather_ctx->new_sub()](int r) {
+ if (r < 0) {
+ derr << "failed to initialize namespace replayer for namespace "
+ << name << ": " << cpp_strerror(r) << dendl;
+ delete namespace_replayer;
+ mirroring_namespaces.erase(name);
+ } else {
+ std::lock_guard locker{m_lock};
+ m_namespace_replayers[name] = namespace_replayer;
+ }
+ ctx->complete(r);
+ });
+ namespace_replayer->init(on_init);
+ }
+
+ gather_ctx->activate();
+
+ m_lock.unlock();
+ cond.wait();
+ m_lock.lock();
+
+ if (m_leader) {
+ C_SaferCond acquire_cond;
+ auto acquire_gather_ctx = new C_Gather(cct, &acquire_cond);
+
+ for (auto &name : mirroring_namespaces) {
+ namespace_replayer_acquire_leader(name, acquire_gather_ctx->new_sub());
+ }
+ acquire_gather_ctx->activate();
+
+ m_lock.unlock();
+ acquire_cond.wait();
+ m_lock.lock();
+
+ std::vector<std::string> instance_ids;
+ m_leader_watcher->list_instances(&instance_ids);
+
+ for (auto &name : mirroring_namespaces) {
+ auto it = m_namespace_replayers.find(name);
+ if (it == m_namespace_replayers.end()) {
+ // acuire leader for this namespace replayer failed
+ continue;
+ }
+ it->second->handle_instances_added(instance_ids);
+ }
+ } else {
+ std::string leader_instance_id;
+ if (m_leader_watcher->get_leader_instance_id(&leader_instance_id)) {
+ for (auto &name : mirroring_namespaces) {
+ m_namespace_replayers[name]->handle_update_leader(leader_instance_id);
+ }
+ }
+ }
}
template <typename I>
-void PoolReplayer<I>::print_status(Formatter *f, stringstream *ss)
-{
- dout(20) << "enter" << dendl;
+int PoolReplayer<I>::list_mirroring_namespaces(
+ std::set<std::string> *namespaces) {
+ ceph_assert(ceph_mutex_is_locked(m_lock));
+
+ std::vector<std::string> names;
+
+ int r = librbd::api::Namespace<I>::list(m_local_io_ctx, &names);
+ if (r < 0) {
+ derr << "failed to list namespaces: " << cpp_strerror(r) << dendl;
+ return r;
+ }
+
+ for (auto &name : names) {
+ cls::rbd::MirrorMode mirror_mode = cls::rbd::MIRROR_MODE_DISABLED;
+ int r = librbd::cls_client::mirror_mode_get(&m_local_io_ctx, &mirror_mode);
+ if (r < 0 && r != -ENOENT) {
+ derr << "failed to get namespace mirror mode: " << cpp_strerror(r)
+ << dendl;
+ if (m_namespace_replayers.count(name) == 0) {
+ continue;
+ }
+ } else if (mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) {
+ dout(10) << "mirroring is disabled for namespace " << name << dendl;
+ continue;
+ }
+
+ namespaces->insert(name);
+ }
+
+ return 0;
+}
+
+template <typename I>
+void PoolReplayer<I>::namespace_replayer_acquire_leader(const std::string &name,
+ Context *on_finish) {
+ ceph_assert(ceph_mutex_is_locked(m_lock));
+
+ auto it = m_namespace_replayers.find(name);
+ ceph_assert(it != m_namespace_replayers.end());
+
+ on_finish = new FunctionContext(
+ [this, name, on_finish](int r) {
+ if (r < 0) {
+ derr << "failed to handle acquire leader for namespace: "
+ << name << ": " << cpp_strerror(r) << dendl;
+
+ // remove the namespace replayer -- update_namespace_replayers will
+ // retry to create it and acquire leader.
+
+ std::lock_guard locker{m_lock};
+
+ auto namespace_replayer = m_namespace_replayers[name];
+ m_namespace_replayers.erase(name);
+ auto on_shut_down = new FunctionContext(
+ [this, namespace_replayer, on_finish](int r) {
+ delete namespace_replayer;
+ on_finish->complete(r);
+ });
+ namespace_replayer->shut_down(on_shut_down);
+ return;
+ }
+ on_finish->complete(0);
+ });
+
+ it->second->handle_acquire_leader(on_finish);
+}
+
+template <typename I>
+void PoolReplayer<I>::print_status(Formatter *f, stringstream *ss) {
+ dout(20) << dendl;
if (!f) {
return;
f->open_object_section("pool_replayer_status");
f->dump_string("pool", m_local_io_ctx.get_pool_name());
f->dump_stream("peer") << m_peer;
- f->dump_string("instance_id", m_instance_watcher->get_instance_id());
+ f->dump_stream("instance_id") << m_local_io_ctx.get_instance_id();
std::string state("running");
if (m_manual_stop) {
for (auto instance_id : instance_ids) {
f->dump_string("instance_id", instance_id);
}
- f->close_section();
+ f->close_section(); // instances
}
f->dump_string("local_cluster_admin_socket",
reinterpret_cast<CephContext *>(m_remote_io_ctx.cct())->_conf.
get_val<std::string>("admin_socket"));
- f->open_object_section("sync_throttler");
- m_instance_watcher->print_sync_status(f, ss);
- f->close_section();
+ if (m_image_sync_throttler) {
+ f->open_object_section("sync_throttler");
+ m_image_sync_throttler->print_status(f, ss);
+ f->close_section(); // sync_throttler
+ }
- m_instance_replayer->print_status(f, ss);
+ m_default_namespace_replayer->print_status(f, ss);
- if (m_image_deleter) {
- f->open_object_section("image_deleter");
- m_image_deleter->print_status(f, ss);
- f->close_section();
+ f->open_array_section("namespaces");
+ for (auto &it : m_namespace_replayers) {
+ f->open_object_section("namespace");
+ f->dump_string("name", it.first);
+ it.second->print_status(f, ss);
+ f->close_section(); // namespace
}
+ f->close_section(); // namespaces
- f->close_section();
+ f->close_section(); // pool_replayer_status
f->flush(*ss);
}
template <typename I>
-void PoolReplayer<I>::start()
-{
- dout(20) << "enter" << dendl;
+void PoolReplayer<I>::start() {
+ dout(20) << dendl;
std::lock_guard l{m_lock};
}
m_manual_stop = false;
- m_instance_replayer->start();
+
+ m_default_namespace_replayer->start();
+ for (auto &it : m_namespace_replayers) {
+ it.second->start();
+ }
}
template <typename I>
-void PoolReplayer<I>::stop(bool manual)
-{
+void PoolReplayer<I>::stop(bool manual) {
dout(20) << "enter: manual=" << manual << dendl;
std::lock_guard l{m_lock};
}
m_manual_stop = true;
- m_instance_replayer->stop();
+
+ m_default_namespace_replayer->stop();
+ for (auto &it : m_namespace_replayers) {
+ it.second->stop();
+ }
}
template <typename I>
-void PoolReplayer<I>::restart()
-{
- dout(20) << "enter" << dendl;
+void PoolReplayer<I>::restart() {
+ dout(20) << dendl;
std::lock_guard l{m_lock};
return;
}
- m_instance_replayer->restart();
+ m_default_namespace_replayer->restart();
+ for (auto &it : m_namespace_replayers) {
+ it.second->restart();
+ }
}
template <typename I>
-void PoolReplayer<I>::flush()
-{
- dout(20) << "enter" << dendl;
+void PoolReplayer<I>::flush() {
+ dout(20) << dendl;
std::lock_guard l{m_lock};
return;
}
- m_instance_replayer->flush();
+ m_default_namespace_replayer->flush();
+ for (auto &it : m_namespace_replayers) {
+ it.second->flush();
+ }
}
template <typename I>
-void PoolReplayer<I>::release_leader()
-{
- dout(20) << "enter" << dendl;
+void PoolReplayer<I>::release_leader() {
+ dout(20) << dendl;
std::lock_guard l{m_lock};
m_leader_watcher->release_leader();
}
-template <typename I>
-void PoolReplayer<I>::handle_update(const std::string &mirror_uuid,
- ImageIds &&added_image_ids,
- ImageIds &&removed_image_ids) {
- if (m_stopping) {
- return;
- }
-
- dout(10) << "mirror_uuid=" << mirror_uuid << ", "
- << "added_count=" << added_image_ids.size() << ", "
- << "removed_count=" << removed_image_ids.size() << dendl;
- std::lock_guard locker{m_lock};
- if (!m_leader_watcher->is_leader()) {
- return;
- }
-
- m_service_daemon->add_or_update_attribute(
- m_local_pool_id, SERVICE_DAEMON_LOCAL_COUNT_KEY,
- m_local_pool_watcher->get_image_count());
- if (m_remote_pool_watcher) {
- m_service_daemon->add_or_update_attribute(
- m_local_pool_id, SERVICE_DAEMON_REMOTE_COUNT_KEY,
- m_remote_pool_watcher->get_image_count());
- }
-
- std::set<std::string> added_global_image_ids;
- for (auto& image_id : added_image_ids) {
- added_global_image_ids.insert(image_id.global_id);
- }
-
- std::set<std::string> removed_global_image_ids;
- for (auto& image_id : removed_image_ids) {
- removed_global_image_ids.insert(image_id.global_id);
- }
-
- m_image_map->update_images(mirror_uuid,
- std::move(added_global_image_ids),
- std::move(removed_global_image_ids));
-}
-
template <typename I>
void PoolReplayer<I>::handle_post_acquire_leader(Context *on_finish) {
- dout(10) << dendl;
+ dout(20) << dendl;
- m_service_daemon->add_or_update_attribute(m_local_pool_id,
- SERVICE_DAEMON_LEADER_KEY, true);
- m_instance_watcher->handle_acquire_leader();
- init_image_map(on_finish);
-}
+ with_namespace_replayers(
+ [this](Context *on_finish) {
+ dout(10) << "handle_post_acquire_leader" << dendl;
-template <typename I>
-void PoolReplayer<I>::handle_pre_release_leader(Context *on_finish) {
- dout(10) << dendl;
+ ceph_assert(ceph_mutex_is_locked(m_lock));
- m_service_daemon->remove_attribute(m_local_pool_id,
- SERVICE_DAEMON_LEADER_KEY);
- m_instance_watcher->handle_release_leader();
- shut_down_image_deleter(on_finish);
-}
+ m_service_daemon->add_or_update_attribute(m_local_pool_id,
+ SERVICE_DAEMON_LEADER_KEY,
+ true);
+ auto ctx = new FunctionContext(
+ [this, on_finish](int r) {
+ if (r == 0) {
+ std::lock_guard locker{m_lock};
+ m_leader = true;
+ }
+ on_finish->complete(r);
+ });
-template <typename I>
-void PoolReplayer<I>::init_image_map(Context *on_finish) {
- dout(5) << dendl;
+ auto cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct());
+ auto gather_ctx = new C_Gather(cct, ctx);
- std::lock_guard locker{m_lock};
- ceph_assert(!m_image_map);
- m_image_map.reset(ImageMap<I>::create(m_local_io_ctx, m_threads,
- m_instance_watcher->get_instance_id(),
- m_image_map_listener));
-
- auto ctx = new FunctionContext([this, on_finish](int r) {
- handle_init_image_map(r, on_finish);
- });
- m_image_map->init(create_async_context_callback(
- m_threads->work_queue, ctx));
-}
-
-template <typename I>
-void PoolReplayer<I>::handle_init_image_map(int r, Context *on_finish) {
- dout(5) << "r=" << r << dendl;
- if (r < 0) {
- derr << "failed to init image map: " << cpp_strerror(r) << dendl;
- on_finish = new FunctionContext([on_finish, r](int) {
- on_finish->complete(r);
- });
- shut_down_image_map(on_finish);
- return;
- }
-
- init_local_pool_watcher(on_finish);
-}
-
-template <typename I>
-void PoolReplayer<I>::init_local_pool_watcher(Context *on_finish) {
- dout(10) << dendl;
-
- std::lock_guard locker{m_lock};
- ceph_assert(!m_local_pool_watcher);
- m_local_pool_watcher.reset(PoolWatcher<I>::create(
- m_threads, m_local_io_ctx, m_local_pool_watcher_listener));
-
- // ensure the initial set of local images is up-to-date
- // after acquiring the leader role
- auto ctx = new FunctionContext([this, on_finish](int r) {
- handle_init_local_pool_watcher(r, on_finish);
- });
- m_local_pool_watcher->init(create_async_context_callback(
- m_threads->work_queue, ctx));
-}
-
-template <typename I>
-void PoolReplayer<I>::handle_init_local_pool_watcher(
- int r, Context *on_finish) {
- dout(10) << "r=" << r << dendl;
- if (r < 0) {
- derr << "failed to retrieve local images: " << cpp_strerror(r) << dendl;
- on_finish = new FunctionContext([on_finish, r](int) {
- on_finish->complete(r);
- });
- shut_down_pool_watchers(on_finish);
- return;
- }
-
- init_remote_pool_watcher(on_finish);
-}
-
-template <typename I>
-void PoolReplayer<I>::init_remote_pool_watcher(Context *on_finish) {
- dout(10) << dendl;
-
- std::lock_guard locker{m_lock};
- ceph_assert(!m_remote_pool_watcher);
- m_remote_pool_watcher.reset(PoolWatcher<I>::create(
- m_threads, m_remote_io_ctx, m_remote_pool_watcher_listener));
-
- auto ctx = new FunctionContext([this, on_finish](int r) {
- handle_init_remote_pool_watcher(r, on_finish);
- });
- m_remote_pool_watcher->init(create_async_context_callback(
- m_threads->work_queue, ctx));
-}
-
-template <typename I>
-void PoolReplayer<I>::handle_init_remote_pool_watcher(
- int r, Context *on_finish) {
- dout(10) << "r=" << r << dendl;
- if (r == -ENOENT) {
- // Technically nothing to do since the other side doesn't
- // have mirroring enabled. Eventually the remote pool watcher will
- // detect images (if mirroring is enabled), so no point propagating
- // an error which would just busy-spin the state machines.
- dout(0) << "remote peer does not have mirroring configured" << dendl;
- } else if (r < 0) {
- derr << "failed to retrieve remote images: " << cpp_strerror(r) << dendl;
- on_finish = new FunctionContext([on_finish, r](int) {
- on_finish->complete(r);
- });
- shut_down_pool_watchers(on_finish);
- return;
- }
-
- init_image_deleter(on_finish);
-}
-
-template <typename I>
-void PoolReplayer<I>::init_image_deleter(Context *on_finish) {
- dout(10) << dendl;
-
- std::lock_guard locker{m_lock};
- ceph_assert(!m_image_deleter);
-
- on_finish = new FunctionContext([this, on_finish](int r) {
- handle_init_image_deleter(r, on_finish);
- });
- m_image_deleter.reset(ImageDeleter<I>::create(m_local_io_ctx, m_threads,
- m_service_daemon));
- m_image_deleter->init(create_async_context_callback(
- m_threads->work_queue, on_finish));
-}
-
-template <typename I>
-void PoolReplayer<I>::handle_init_image_deleter(int r, Context *on_finish) {
- dout(10) << "r=" << r << dendl;
- if (r < 0) {
- derr << "failed to init image deleter: " << cpp_strerror(r) << dendl;
- on_finish = new FunctionContext([on_finish, r](int) {
- on_finish->complete(r);
- });
- shut_down_image_deleter(on_finish);
- return;
- }
-
- on_finish->complete(0);
-
- std::lock_guard locker{m_lock};
- m_cond.notify_all();
-}
-
-template <typename I>
-void PoolReplayer<I>::shut_down_image_deleter(Context* on_finish) {
- dout(10) << dendl;
- {
- std::lock_guard locker{m_lock};
- if (m_image_deleter) {
- Context *ctx = new FunctionContext([this, on_finish](int r) {
- handle_shut_down_image_deleter(r, on_finish);
- });
- ctx = create_async_context_callback(m_threads->work_queue, ctx);
-
- m_image_deleter->shut_down(ctx);
- return;
- }
- }
- shut_down_pool_watchers(on_finish);
-}
+ m_default_namespace_replayer->handle_acquire_leader(
+ gather_ctx->new_sub());
-template <typename I>
-void PoolReplayer<I>::handle_shut_down_image_deleter(
- int r, Context* on_finish) {
- dout(10) << "r=" << r << dendl;
-
- {
- std::lock_guard locker{m_lock};
- ceph_assert(m_image_deleter);
- m_image_deleter.reset();
- }
-
- shut_down_pool_watchers(on_finish);
-}
-
-template <typename I>
-void PoolReplayer<I>::shut_down_pool_watchers(Context *on_finish) {
- dout(10) << dendl;
-
- {
- std::lock_guard locker{m_lock};
- if (m_local_pool_watcher) {
- Context *ctx = new FunctionContext([this, on_finish](int r) {
- handle_shut_down_pool_watchers(r, on_finish);
- });
- ctx = create_async_context_callback(m_threads->work_queue, ctx);
-
- auto gather_ctx = new C_Gather(g_ceph_context, ctx);
- m_local_pool_watcher->shut_down(gather_ctx->new_sub());
- if (m_remote_pool_watcher) {
- m_remote_pool_watcher->shut_down(gather_ctx->new_sub());
- }
- gather_ctx->activate();
- return;
- }
- }
-
- on_finish->complete(0);
-}
-
-template <typename I>
-void PoolReplayer<I>::handle_shut_down_pool_watchers(
- int r, Context *on_finish) {
- dout(10) << "r=" << r << dendl;
-
- {
- std::lock_guard locker{m_lock};
- ceph_assert(m_local_pool_watcher);
- m_local_pool_watcher.reset();
-
- if (m_remote_pool_watcher) {
- m_remote_pool_watcher.reset();
- }
- }
- wait_for_update_ops(on_finish);
-}
-
-template <typename I>
-void PoolReplayer<I>::wait_for_update_ops(Context *on_finish) {
- dout(10) << dendl;
-
- std::lock_guard locker{m_lock};
-
- Context *ctx = new FunctionContext([this, on_finish](int r) {
- handle_wait_for_update_ops(r, on_finish);
- });
- ctx = create_async_context_callback(m_threads->work_queue, ctx);
+ for (auto &it : m_namespace_replayers) {
+ namespace_replayer_acquire_leader(it.first, gather_ctx->new_sub());
+ }
- m_update_op_tracker.wait_for_ops(ctx);
+ gather_ctx->activate();
+ }, on_finish);
}
template <typename I>
-void PoolReplayer<I>::handle_wait_for_update_ops(int r, Context *on_finish) {
- dout(10) << "r=" << r << dendl;
- ceph_assert(r == 0);
+void PoolReplayer<I>::handle_pre_release_leader(Context *on_finish) {
+ dout(20) << dendl;
- shut_down_image_map(on_finish);
-}
+ with_namespace_replayers(
+ [this](Context *on_finish) {
+ dout(10) << "handle_pre_release_leader" << dendl;
-template <typename I>
-void PoolReplayer<I>::shut_down_image_map(Context *on_finish) {
- dout(5) << dendl;
+ ceph_assert(ceph_mutex_is_locked(m_lock));
- {
- std::lock_guard locker{m_lock};
- if (m_image_map) {
- on_finish = new FunctionContext([this, on_finish](int r) {
- handle_shut_down_image_map(r, on_finish);
- });
- m_image_map->shut_down(create_async_context_callback(
- m_threads->work_queue, on_finish));
- return;
- }
- }
+ m_leader = false;
+ m_service_daemon->remove_attribute(m_local_pool_id,
+ SERVICE_DAEMON_LEADER_KEY);
- on_finish->complete(0);
-}
+ auto cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct());
+ auto gather_ctx = new C_Gather(cct, on_finish);
-template <typename I>
-void PoolReplayer<I>::handle_shut_down_image_map(int r, Context *on_finish) {
- dout(5) << "r=" << r << dendl;
- if (r < 0 && r != -EBLACKLISTED) {
- derr << "failed to shut down image map: " << cpp_strerror(r) << dendl;
- }
+ m_default_namespace_replayer->handle_release_leader(
+ gather_ctx->new_sub());
- std::lock_guard locker{m_lock};
- ceph_assert(m_image_map);
- m_image_map.reset();
+ for (auto &it : m_namespace_replayers) {
+ it.second->handle_release_leader(gather_ctx->new_sub());
+ }
- m_instance_replayer->release_all(on_finish);
+ gather_ctx->activate();
+ }, on_finish);
}
template <typename I>
const std::string &leader_instance_id) {
dout(10) << "leader_instance_id=" << leader_instance_id << dendl;
- m_instance_watcher->handle_update_leader(leader_instance_id);
-}
-
-template <typename I>
-void PoolReplayer<I>::handle_acquire_image(const std::string &global_image_id,
- const std::string &instance_id,
- Context* on_finish) {
- dout(5) << "global_image_id=" << global_image_id << ", "
- << "instance_id=" << instance_id << dendl;
-
- m_instance_watcher->notify_image_acquire(instance_id, global_image_id,
- on_finish);
-}
+ std::lock_guard locker{m_lock};
-template <typename I>
-void PoolReplayer<I>::handle_release_image(const std::string &global_image_id,
- const std::string &instance_id,
- Context* on_finish) {
- dout(5) << "global_image_id=" << global_image_id << ", "
- << "instance_id=" << instance_id << dendl;
-
- m_instance_watcher->notify_image_release(instance_id, global_image_id,
- on_finish);
-}
+ m_default_namespace_replayer->handle_update_leader(leader_instance_id);
-template <typename I>
-void PoolReplayer<I>::handle_remove_image(const std::string &mirror_uuid,
- const std::string &global_image_id,
- const std::string &instance_id,
- Context* on_finish) {
- ceph_assert(!mirror_uuid.empty());
- dout(5) << "mirror_uuid=" << mirror_uuid << ", "
- << "global_image_id=" << global_image_id << ", "
- << "instance_id=" << instance_id << dendl;
-
- m_instance_watcher->notify_peer_image_removed(instance_id, global_image_id,
- mirror_uuid, on_finish);
+ for (auto &it : m_namespace_replayers) {
+ it.second->handle_update_leader(leader_instance_id);
+ }
}
template <typename I>
-void PoolReplayer<I>::handle_instances_added(const InstanceIds &instance_ids) {
+void PoolReplayer<I>::handle_instances_added(
+ const std::vector<std::string> &instance_ids) {
dout(5) << "instance_ids=" << instance_ids << dendl;
+
std::lock_guard locker{m_lock};
if (!m_leader_watcher->is_leader()) {
return;
}
- ceph_assert(m_image_map);
- m_image_map->update_instances_added(instance_ids);
+ m_default_namespace_replayer->handle_instances_added(instance_ids);
+
+ for (auto &it : m_namespace_replayers) {
+ it.second->handle_instances_added(instance_ids);
+ }
}
template <typename I>
void PoolReplayer<I>::handle_instances_removed(
- const InstanceIds &instance_ids) {
+ const std::vector<std::string> &instance_ids) {
dout(5) << "instance_ids=" << instance_ids << dendl;
+
std::lock_guard locker{m_lock};
if (!m_leader_watcher->is_leader()) {
return;
}
- ceph_assert(m_image_map);
- m_image_map->update_instances_removed(instance_ids);
+ m_default_namespace_replayer->handle_instances_removed(instance_ids);
+
+ for (auto &it : m_namespace_replayers) {
+ it.second->handle_instances_removed(instance_ids);
+ }
}
} // namespace mirror
#ifndef CEPH_RBD_MIRROR_POOL_REPLAYER_H
#define CEPH_RBD_MIRROR_POOL_REPLAYER_H
-#include "common/AsyncOpTracker.h"
-#include "common/ceph_mutex.h"
+#include "common/Cond.h"
#include "common/WorkQueue.h"
+#include "common/ceph_mutex.h"
#include "include/rados/librados.hpp"
+#include "librbd/Utils.h"
-#include "ClusterWatcher.h"
-#include "LeaderWatcher.h"
-#include "PoolWatcher.h"
-#include "ImageDeleter.h"
+#include "tools/rbd_mirror/ImageSyncThrottler.h"
+#include "tools/rbd_mirror/LeaderWatcher.h"
+#include "tools/rbd_mirror/NamespaceReplayer.h"
#include "tools/rbd_mirror/Types.h"
-#include "tools/rbd_mirror/image_map/Types.h"
#include "tools/rbd_mirror/leader_watcher/Types.h"
-#include "tools/rbd_mirror/pool_watcher/Types.h"
#include "tools/rbd_mirror/service_daemon/Types.h"
-#include <set>
#include <map>
#include <memory>
-#include <atomic>
#include <string>
#include <vector>
namespace rbd {
namespace mirror {
-template <typename> class ImageMap;
-template <typename> class InstanceReplayer;
-template <typename> class InstanceWatcher;
template <typename> class ServiceDaemon;
template <typename> struct Threads;
* INIT
* |
* v
- * <follower> <-------------------------\
- * . |
- * . |
- * v (leader acquired) |
- * INIT_IMAGE_MAP SHUT_DOWN_IMAGE_MAP
- * | ^
- * v |
- * INIT_LOCAL_POOL_WATCHER WAIT_FOR_NOTIFICATIONS
- * | ^
- * v |
- * INIT_REMOTE_POOL_WATCHER SHUT_DOWN_POOL_WATCHERS
- * | ^
- * v |
- * INIT_IMAGE_DELETER SHUT_DOWN_IMAGE_DELETER
- * | ^
- * v .
- * <leader> <-----------\ .
- * . | .
- * . (image update) | .
- * . . > NOTIFY_INSTANCE_WATCHER .
- * . .
- * . (leader lost / shut down) .
- * . . . . . . . . . . . . . . . . . .
+ * <follower> <---------------------\
+ * . |
+ * . (leader acquired) |
+ * v |
+ * NOTIFY_NAMESPACE_WATCHERS NOTIFY_NAMESPACE_WATCHERS
+ * | ^
+ * v .
+ * <leader> .
+ * . .
+ * . (leader lost / shut down) .
+ * . . . . . . . . . . . . . . . .
*
* @endverbatim
*/
- typedef std::vector<std::string> InstanceIds;
-
- struct PoolWatcherListener : public pool_watcher::Listener {
- PoolReplayer *pool_replayer;
- bool local;
-
- PoolWatcherListener(PoolReplayer *pool_replayer, bool local)
- : pool_replayer(pool_replayer), local(local) {
- }
-
- void handle_update(const std::string &mirror_uuid,
- ImageIds &&added_image_ids,
- ImageIds &&removed_image_ids) override {
- pool_replayer->handle_update((local ? "" : mirror_uuid),
- std::move(added_image_ids),
- std::move(removed_image_ids));
- }
- };
-
- struct ImageMapListener : public image_map::Listener {
- PoolReplayer *pool_replayer;
-
- ImageMapListener(PoolReplayer *pool_replayer)
- : pool_replayer(pool_replayer) {
- }
-
- void acquire_image(const std::string &global_image_id,
- const std::string &instance_id,
- Context* on_finish) override {
- pool_replayer->handle_acquire_image(global_image_id, instance_id,
- on_finish);
- }
-
- void release_image(const std::string &global_image_id,
- const std::string &instance_id,
- Context* on_finish) override {
- pool_replayer->handle_release_image(global_image_id, instance_id,
- on_finish);
- }
-
- void remove_image(const std::string &mirror_uuid,
- const std::string &global_image_id,
- const std::string &instance_id,
- Context* on_finish) override {
- pool_replayer->handle_remove_image(mirror_uuid, global_image_id,
- instance_id, on_finish);
- }
- };
-
- void handle_update(const std::string &mirror_uuid,
- ImageIds &&added_image_ids,
- ImageIds &&removed_image_ids);
-
int init_rados(const std::string &cluster_name,
const std::string &client_name,
const std::string &mon_host,
const std::string &description, RadosRef *rados_ref,
bool strip_cluster_overrides);
- void handle_post_acquire_leader(Context *on_finish);
- void handle_pre_release_leader(Context *on_finish);
-
- void init_image_map(Context *on_finish);
- void handle_init_image_map(int r, Context *on_finish);
+ void update_namespace_replayers();
+ int list_mirroring_namespaces(std::set<std::string> *namespaces);
- void init_local_pool_watcher(Context *on_finish);
- void handle_init_local_pool_watcher(int r, Context *on_finish);
+ void namespace_replayer_acquire_leader(const std::string &name,
+ Context *on_finish);
- void init_remote_pool_watcher(Context *on_finish);
- void handle_init_remote_pool_watcher(int r, Context *on_finish);
-
- void init_image_deleter(Context* on_finish);
- void handle_init_image_deleter(int r, Context* on_finish);
-
- void shut_down_image_deleter(Context* on_finish);
- void handle_shut_down_image_deleter(int r, Context* on_finish);
+ void handle_post_acquire_leader(Context *on_finish);
+ void handle_pre_release_leader(Context *on_finish);
- void shut_down_pool_watchers(Context *on_finish);
- void handle_shut_down_pool_watchers(int r, Context *on_finish);
+ void handle_update_leader(const std::string &leader_instance_id);
- void wait_for_update_ops(Context *on_finish);
- void handle_wait_for_update_ops(int r, Context *on_finish);
+ void handle_instances_added(const std::vector<std::string> &instance_ids);
+ void handle_instances_removed(const std::vector<std::string> &instance_ids);
+
+ // sync version, executed in the caller thread
+ template <typename L>
+ void with_namespace_replayers(L &&callback) {
+ std::lock_guard locker{m_lock};
+
+ if (m_namespace_replayers_locked) {
+ ceph_assert(m_on_namespace_replayers_unlocked == nullptr);
+ C_SaferCond cond;
+ m_on_namespace_replayers_unlocked = &cond;
+ m_lock.unlock();
+ cond.wait();
+ m_lock.lock();
+ } else {
+ m_namespace_replayers_locked = true;
+ }
- void shut_down_image_map(Context *on_finish);
- void handle_shut_down_image_map(int r, Context *on_finish);
+ ceph_assert(m_namespace_replayers_locked);
+ callback(); // may temporary release the lock
+ ceph_assert(m_namespace_replayers_locked);
- void handle_update_leader(const std::string &leader_instance_id);
+ if (m_on_namespace_replayers_unlocked == nullptr) {
+ m_namespace_replayers_locked = false;
+ return;
+ }
- void handle_acquire_image(const std::string &global_image_id,
- const std::string &instance_id,
- Context* on_finish);
- void handle_release_image(const std::string &global_image_id,
- const std::string &instance_id,
- Context* on_finish);
- void handle_remove_image(const std::string &mirror_uuid,
- const std::string &global_image_id,
- const std::string &instance_id,
- Context* on_finish);
+ m_threads->work_queue->queue(m_on_namespace_replayers_unlocked);
+ m_on_namespace_replayers_unlocked = nullptr;
+ }
+
+ // async version
+ template <typename L>
+ void with_namespace_replayers(L &&callback, Context *on_finish) {
+ std::lock_guard locker{m_lock};
+
+ on_finish = librbd::util::create_async_context_callback(
+ m_threads->work_queue, new FunctionContext(
+ [this, on_finish](int r) {
+ {
+ std::lock_guard locker{m_lock};
+ ceph_assert(m_namespace_replayers_locked);
+
+ m_namespace_replayers_locked = false;
+
+ if (m_on_namespace_replayers_unlocked != nullptr) {
+ m_namespace_replayers_locked = true;
+ m_threads->work_queue->queue(m_on_namespace_replayers_unlocked);
+ m_on_namespace_replayers_unlocked = nullptr;
+ }
+ }
+ on_finish->complete(r);
+ }));
+
+ auto on_lock = new FunctionContext(
+ [this, callback, on_finish](int) {
+ std::lock_guard locker{m_lock};
+ ceph_assert(m_namespace_replayers_locked);
+
+ callback(on_finish);
+ });
+
+ if (m_namespace_replayers_locked) {
+ ceph_assert(m_on_namespace_replayers_unlocked == nullptr);
+ m_on_namespace_replayers_unlocked = on_lock;
+ return;
+ }
- void handle_instances_added(const InstanceIds &instance_ids);
- void handle_instances_removed(const InstanceIds &instance_ids);
+ m_namespace_replayers_locked = true;
+ m_threads->work_queue->queue(on_lock);
+ }
Threads<ImageCtxT> *m_threads;
ServiceDaemon<ImageCtxT> *m_service_daemon;
mutable ceph::mutex m_lock;
ceph::condition_variable m_cond;
- std::atomic<bool> m_stopping = { false };
+ bool m_stopping = false;
bool m_manual_stop = false;
bool m_blacklisted = false;
librados::IoCtx m_local_io_ctx;
librados::IoCtx m_remote_io_ctx;
- PoolWatcherListener m_local_pool_watcher_listener;
- std::unique_ptr<PoolWatcher<ImageCtxT>> m_local_pool_watcher;
-
- PoolWatcherListener m_remote_pool_watcher_listener;
- std::unique_ptr<PoolWatcher<ImageCtxT>> m_remote_pool_watcher;
+ std::string m_local_mirror_uuid;
- std::unique_ptr<InstanceReplayer<ImageCtxT>> m_instance_replayer;
- std::unique_ptr<ImageDeleter<ImageCtxT>> m_image_deleter;
-
- ImageMapListener m_image_map_listener;
- std::unique_ptr<ImageMap<ImageCtxT>> m_image_map;
+ std::unique_ptr<NamespaceReplayer<ImageCtxT>> m_default_namespace_replayer;
+ std::map<std::string, NamespaceReplayer<ImageCtxT> *> m_namespace_replayers;
std::string m_asok_hook_name;
AdminSocketHook *m_asok_hook = nullptr;
service_daemon::CalloutId m_callout_id = service_daemon::CALLOUT_ID_NONE;
+ bool m_leader = false;
+ bool m_namespace_replayers_locked = false;
+ Context *m_on_namespace_replayers_unlocked = nullptr;
+
class PoolReplayerThread : public Thread {
PoolReplayer *m_pool_replayer;
public:
} m_leader_listener;
std::unique_ptr<LeaderWatcher<ImageCtxT>> m_leader_watcher;
- std::unique_ptr<InstanceWatcher<ImageCtxT>> m_instance_watcher;
- AsyncOpTracker m_update_op_tracker;
+ std::unique_ptr<ImageSyncThrottler<ImageCtxT>> m_image_sync_throttler;
};
} // namespace mirror
projects / ceph-ci.git / commitdiff