From 8e8deb35a213a64ff645f9a09779a890cd97132d Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Tue, 22 Mar 2016 18:52:45 -0400 Subject: [PATCH] rbd-mirror: bootstrap now has initial support for existing images Signed-off-by: Jason Dillaman --- src/test/rbd_mirror/image_replay.cc | 3 +- src/test/rbd_mirror/test_ImageReplayer.cc | 67 +---- src/tools/rbd_mirror/ImageReplayer.cc | 258 ++---------------- src/tools/rbd_mirror/ImageReplayer.h | 76 ++---- .../image_replayer/BootstrapRequest.cc | 208 +++++++++++--- .../image_replayer/BootstrapRequest.h | 54 +++- 6 files changed, 269 insertions(+), 397 deletions(-) diff --git a/src/test/rbd_mirror/image_replay.cc b/src/test/rbd_mirror/image_replay.cc index 3abe40cb7a57b..a51a200d8e3f2 100644 --- a/src/test/rbd_mirror/image_replay.cc +++ b/src/test/rbd_mirror/image_replay.cc @@ -103,8 +103,7 @@ int main(int argc, const char **argv) << local_pool_name << ", remote_pool_name=" << remote_pool_name << ", image_name=" << image_name << dendl; - rbd::mirror::ImageReplayer<>::BootstrapParams bootstap_params(local_pool_name, - image_name); + rbd::mirror::ImageReplayer<>::BootstrapParams bootstap_params(image_name); int64_t local_pool_id; int64_t remote_pool_id; std::string remote_image_id; diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index 35c4c8e3f13d3..a4d597319aff8 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -151,7 +151,7 @@ public: create_replayer<>(); rbd::mirror::ImageReplayer<>::BootstrapParams - bootstap_params(m_local_pool_name, m_image_name); + bootstap_params(m_image_name); start(&bootstap_params); wait_for_replay_complete(); stop(); @@ -342,17 +342,6 @@ TEST_F(TestImageReplayer, Bootstrap) bootstrap(); } -TEST_F(TestImageReplayer, BootstrapErrorInvalidPool) -{ - create_replayer<>(); - - rbd::mirror::ImageReplayer<>::BootstrapParams - bootstap_params("INVALID_LOCAL_POOL_NAME", m_image_name); - C_SaferCond cond; - m_replayer->start(&cond, &bootstap_params); - ASSERT_EQ(-ENOENT, cond.wait()); -} - TEST_F(TestImageReplayer, BootstrapErrorLocalImageExists) { int order = 0; @@ -361,7 +350,7 @@ TEST_F(TestImageReplayer, BootstrapErrorLocalImageExists) create_replayer<>(); rbd::mirror::ImageReplayer<>::BootstrapParams - bootstap_params(m_local_pool_name, m_image_name); + bootstap_params(m_image_name); C_SaferCond cond; m_replayer->start(&cond, &bootstap_params); ASSERT_EQ(-EEXIST, cond.wait()); @@ -379,7 +368,7 @@ TEST_F(TestImageReplayer, BootstrapErrorNoJournal) create_replayer<>(); rbd::mirror::ImageReplayer<>::BootstrapParams - bootstap_params(m_local_pool_name, m_image_name); + bootstap_params(m_image_name); C_SaferCond cond; m_replayer->start(&cond, &bootstap_params); ASSERT_EQ(-ENOENT, cond.wait()); @@ -389,7 +378,7 @@ TEST_F(TestImageReplayer, StartInterrupted) { create_replayer<>(); rbd::mirror::ImageReplayer<>::BootstrapParams - bootstap_params(m_local_pool_name, m_image_name); + bootstap_params(m_image_name); C_SaferCond start_cond, stop_cond; m_replayer->start(&start_cond, &bootstap_params); m_replayer->stop(&stop_cond); @@ -426,7 +415,7 @@ TEST_F(TestImageReplayer, ErrorNoJournal) close_image(ictx); rbd::mirror::ImageReplayer<>::BootstrapParams - bootstap_params(m_local_pool_name, m_image_name); + bootstap_params(m_image_name); C_SaferCond cond; m_replayer->start(&cond, &bootstap_params); ASSERT_EQ(-ENOENT, cond.wait()); @@ -558,44 +547,6 @@ public: } protected: - virtual void on_start_get_registered_client_status_finish(int r, - const std::set ®istered_clients, - const BootstrapParams &bootstrap_params) { - rbd::mirror::ImageReplayer<>::on_start_get_registered_client_status_finish( - get_error("on_start_get_registered_client_status"), registered_clients, - bootstrap_params); - } - - virtual void on_start_remote_journaler_init_finish(int r) { - ASSERT_EQ(0, r); - rbd::mirror::ImageReplayer<>::on_start_remote_journaler_init_finish( - get_error("on_start_remote_journaler_init")); - } - - virtual void on_start_local_image_open_finish(int r) { - int test_r = get_error("on_start_local_image_open"); - if (!test_r) { - rbd::mirror::ImageReplayer<>::on_start_local_image_open_finish(r); - return; - } - - // The image open error was imitated, so we need to close the image back - // before propagating the error. - ASSERT_EQ(0, r); - set_error("on_start_local_image_open", 0); - FunctionContext *ctx = new FunctionContext( - [this, test_r](int r) { - on_start_local_image_open_finish(test_r); - }); - close_local_image(ctx); - } - - virtual void on_start_wait_for_local_journal_ready_finish(int r) { - ASSERT_EQ(0, r); - rbd::mirror::ImageReplayer<>::on_start_wait_for_local_journal_ready_finish( - get_error("on_start_wait_for_local_journal_ready")); - } - virtual void on_stop_journal_replay_shut_down_finish(int r) { ASSERT_EQ(0, r); rbd::mirror::ImageReplayer<>::on_stop_journal_replay_shut_down_finish( @@ -619,7 +570,7 @@ TEST_F(TestImageReplayer, Error_on_start_##state) \ reinterpret_cast(m_replayer)-> \ set_error("on_start_" #state, -1); \ rbd::mirror::ImageReplayer<>::BootstrapParams \ - bootstap_params(m_local_pool_name, m_image_name); \ + bootstap_params(m_image_name); \ C_SaferCond cond; \ m_replayer->start(&cond, &bootstap_params); \ ASSERT_EQ(-1, cond.wait()); \ @@ -632,7 +583,7 @@ TEST_F(TestImageReplayer, Error_on_stop_##state) \ reinterpret_cast(m_replayer)-> \ set_error("on_stop_" #state, -1); \ rbd::mirror::ImageReplayer<>::BootstrapParams \ - bootstap_params(m_local_pool_name, m_image_name); \ + bootstap_params(m_image_name); \ start(&bootstap_params); \ /* TODO: investigate: without wait below I observe: */ \ /* librbd/journal/Replay.cc: 70: FAILED assert(m_op_events.empty()) */\ @@ -642,10 +593,6 @@ TEST_F(TestImageReplayer, Error_on_stop_##state) \ ASSERT_EQ(0, cond.wait()); \ } -TEST_ON_START_ERROR(get_registered_client_status); -TEST_ON_START_ERROR(remote_journaler_init); -TEST_ON_START_ERROR(wait_for_local_journal_ready); - TEST_ON_STOP_ERROR(journal_replay_shut_down); TEST_ON_STOP_ERROR(no_error); diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index fa87a0d67dfb9..b9762f655d58c 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -173,14 +173,14 @@ private: template ImageReplayer::ImageReplayer(Threads *threads, RadosRef local, RadosRef remote, - const std::string &client_id, + const std::string &mirror_uuid, int64_t local_pool_id, int64_t remote_pool_id, const std::string &remote_image_id) : m_threads(threads), m_local(local), m_remote(remote), - m_client_id(client_id), + m_mirror_uuid(mirror_uuid), m_remote_pool_id(remote_pool_id), m_local_pool_id(local_pool_id), m_remote_image_id(remote_image_id), @@ -210,7 +210,7 @@ ImageReplayer::~ImageReplayer() template void ImageReplayer::start(Context *on_finish, - const BootstrapParams *bootstrap_params) + const BootstrapParams *bootstrap_params) { dout(20) << "on_finish=" << on_finish << ", m_on_finish=" << m_on_finish << dendl; @@ -234,14 +234,7 @@ void ImageReplayer::start(Context *on_finish, } if (bootstrap_params != nullptr && !bootstrap_params->empty()) { - r = m_local->pool_lookup(bootstrap_params->local_pool_name.c_str()); - if (r < 0) { - derr << "error finding local pool " << bootstrap_params->local_pool_name - << ": " << cpp_strerror(r) << dendl; - on_start_fail_start(r); - return; - } - m_local_pool_id = r; + m_local_image_name = bootstrap_params->local_image_name; } r = m_local->ioctx_create2(m_local_pool_id, m_local_ioctx); @@ -253,124 +246,28 @@ void ImageReplayer::start(Context *on_finish, } CephContext *cct = static_cast(m_local->cct()); - double commit_interval = cct->_conf->rbd_journal_commit_age; m_remote_journaler = new Journaler(m_threads->work_queue, m_threads->timer, &m_threads->timer_lock, m_remote_ioctx, - m_remote_image_id, m_client_id, + m_remote_image_id, m_mirror_uuid, commit_interval); - on_start_get_registered_client_status_start(bootstrap_params); -} - -template -void ImageReplayer::on_start_get_registered_client_status_start( - const BootstrapParams *bootstrap_params) -{ - dout(20) << "enter" << dendl; - - struct Metadata { - uint64_t minimum_set; - uint64_t active_set; - std::set registered_clients; - BootstrapParams bootstrap_params; - } *m = new Metadata(); - - if (bootstrap_params) { - m->bootstrap_params = *bootstrap_params; - } - - FunctionContext *ctx = new FunctionContext( - [this, m, bootstrap_params](int r) { - on_start_get_registered_client_status_finish(r, m->registered_clients, - m->bootstrap_params); - delete m; - }); - - m_remote_journaler->get_mutable_metadata(&m->minimum_set, &m->active_set, - &m->registered_clients, ctx); -} - -template -void ImageReplayer::on_start_get_registered_client_status_finish(int r, - const std::set ®istered_clients, - const BootstrapParams &bootstrap_params) -{ - dout(20) << "r=" << r << dendl; - - if (r < 0) { - derr << "error obtaining registered client status: " - << cpp_strerror(r) << dendl; - on_start_fail_start(r); - return; - } - if (on_start_interrupted()) { - return; - } - - for (auto c : registered_clients) { - if (c.id == m_client_id) { - librbd::journal::ClientData client_data; - bufferlist::iterator bl = c.data.begin(); - try { - ::decode(client_data, bl); - } catch (const buffer::error &err) { - derr << "failed to decode client meta data: " << err.what() << dendl; - on_start_fail_start(-EINVAL); - return; - } - - // TODO: unsafe cast - m_client_meta = - boost::get(client_data.client_meta); - m_local_image_id = m_client_meta.image_id; - - dout(20) << "client found, pool_id=" << m_local_pool_id << ", image_id=" - << m_local_image_id << dendl; - - if (!bootstrap_params.empty()) { - dout(0) << "ignoring bootsrap params: client already registered" << dendl; - } - - on_start_remote_journaler_init_start(); - return; - } - } - - dout(20) << "client not found" << dendl; - bootstrap(bootstrap_params); + bootstrap(); } template -void ImageReplayer::bootstrap(const BootstrapParams &bootstrap_params) { - int r; - BootstrapParams params; - - if (!bootstrap_params.empty()) { - dout(20) << "using external bootstrap params" << dendl; - params = bootstrap_params; - } else { - r = get_bootstrap_params(¶ms); - if (r < 0) { - derr << "error obtaining bootstrap parameters: " - << cpp_strerror(r) << dendl; - on_start_fail_start(r); - return; - } - } - +void ImageReplayer::bootstrap() { dout(20) << "bootstrap params: " - << "local_pool_name=" << params.local_pool_name << ", " - << "local_image_name=" << params.local_image_name << dendl; + << "local_image_name=" << m_local_image_name << dendl; // TODO: add a new bootstrap state and support canceling Context *ctx = create_context_callback< ImageReplayer, &ImageReplayer::handle_bootstrap>(this); BootstrapRequest *request = BootstrapRequest::create( m_local_ioctx, m_remote_ioctx, &m_local_image_ctx, - params.local_image_name, m_remote_image_id, m_threads->work_queue, - m_threads->timer, &m_threads->timer_lock, m_client_id, m_remote_journaler, + m_local_image_name, m_remote_image_id, m_threads->work_queue, + m_threads->timer, &m_threads->timer_lock, m_mirror_uuid, m_remote_journaler, &m_client_meta, ctx); request->send(); } @@ -382,124 +279,51 @@ void ImageReplayer::handle_bootstrap(int r) { if (r < 0) { on_start_fail_start(r); return; - } - if (on_start_interrupted()) { + } else if (on_start_interrupted()) { return; } - on_start_remote_journaler_init_start(); -} - -template -void ImageReplayer::on_start_remote_journaler_init_start() -{ - if (on_start_interrupted()) { - return; - } - - dout(20) << "enter" << dendl; - - FunctionContext *ctx = new FunctionContext( - [this](int r) { - on_start_remote_journaler_init_finish(r); - }); - - m_remote_journaler->init(ctx); -} - -template -void ImageReplayer::on_start_remote_journaler_init_finish(int r) -{ - dout(20) << "r=" << r << dendl; + { + Mutex::Locker locker(m_lock); + m_name = m_local_ioctx.get_pool_name() + "/" + m_local_image_ctx->name; - if (r < 0) { - derr << "error initializing journal: " << cpp_strerror(r) << dendl; - on_start_fail_start(r); - return; - } - if (on_start_interrupted()) { - return; + CephContext *cct = static_cast(m_local->cct()); + delete m_asok_hook; + m_asok_hook = new ImageReplayerAdminSocketHook(cct, m_name, this); } - - on_start_local_image_open_start(); + init_remote_journaler(); } template -void ImageReplayer::on_start_local_image_open_start() -{ - dout(20) << "enter" << dendl; - if (m_local_image_ctx != nullptr) { - // already opened during bootstrap - on_start_wait_for_local_journal_ready_start(); - return; - } +void ImageReplayer::init_remote_journaler() { + dout(20) << dendl; - // open and lock the local image Context *ctx = create_context_callback< - ImageReplayer, &ImageReplayer::on_start_local_image_open_finish>(this); - OpenLocalImageRequest *request = OpenLocalImageRequest::create( - m_local_ioctx, &m_local_image_ctx, "", m_local_image_id, - m_threads->work_queue, ctx); - request->send(); + ImageReplayer, &ImageReplayer::handle_init_remote_journaler>(this); + m_remote_journaler->init(ctx); } template -void ImageReplayer::on_start_local_image_open_finish(int r) -{ +void ImageReplayer::handle_init_remote_journaler(int r) { dout(20) << "r=" << r << dendl; if (r < 0) { - derr << "error opening local image " << m_local_image_id - << ": " << cpp_strerror(r) << dendl; + derr << "failed to initialize remote journal: " << cpp_strerror(r) << dendl; on_start_fail_start(r); return; - } - if (on_start_interrupted()) { + } else if (on_start_interrupted()) { return; } - on_start_wait_for_local_journal_ready_start(); -} - -template -void ImageReplayer::on_start_wait_for_local_journal_ready_start() -{ - dout(20) << "enter" << dendl; - - if (!m_asok_hook) { - Mutex::Locker locker(m_lock); - - m_name = m_local_ioctx.get_pool_name() + "/" + m_local_image_ctx->name; - - CephContext *cct = static_cast(m_local->cct()); - - m_asok_hook = new ImageReplayerAdminSocketHook(cct, m_name, this); - } - - FunctionContext *ctx = new FunctionContext( - [this](int r) { - on_start_wait_for_local_journal_ready_finish(r); - }); - m_local_image_ctx->journal->wait_for_journal_ready(ctx); + start_replay(); } template -void ImageReplayer::on_start_wait_for_local_journal_ready_finish(int r) -{ - dout(20) << "r=" << r << dendl; - - if (r < 0) { - derr << "error when waiting for local journal ready: " << cpp_strerror(r) - << dendl; - on_start_fail_start(r); - return; - } - if (on_start_interrupted()) { - return; - } +void ImageReplayer::start_replay() { + dout(20) << dendl; - r = m_local_image_ctx->journal->start_external_replay(&m_local_replay); + int r = m_local_image_ctx->journal->start_external_replay(&m_local_replay); if (r < 0) { derr << "error starting external replay on local image " << m_local_image_id << ": " << cpp_strerror(r) << dendl; @@ -508,7 +332,6 @@ void ImageReplayer::on_start_wait_for_local_journal_ready_finish(int r) } m_replay_handler = new ReplayHandler(this); - m_remote_journaler->start_live_replay(m_replay_handler, 1 /* TODO: configurable */); @@ -517,10 +340,8 @@ void ImageReplayer::on_start_wait_for_local_journal_ready_finish(int r) assert(r == 0); Context *on_finish(nullptr); - { Mutex::Locker locker(m_lock); - if (m_state == STATE_STOPPING) { on_start_fail_start(-EINTR); return; @@ -528,12 +349,10 @@ void ImageReplayer::on_start_wait_for_local_journal_ready_finish(int r) assert(m_state == STATE_STARTING); m_state = STATE_REPLAYING; - std::swap(m_on_finish, on_finish); } dout(20) << "start succeeded" << dendl; - if (on_finish) { dout(20) << "on finish complete, r=" << r << dendl; on_finish->complete(r); @@ -708,7 +527,7 @@ void ImageReplayer::on_stop_local_image_close_start() Context *ctx = create_context_callback< ImageReplayer, &ImageReplayer::on_stop_local_image_close_finish>(this); CloseImageRequest *request = CloseImageRequest::create( - &m_local_image_ctx, m_threads->work_queue, ctx); + &m_local_image_ctx, m_threads->work_queue, false, ctx); request->send(); } @@ -958,23 +777,6 @@ void ImageReplayer::handle_replay_committed(ReplayEntry *replay_entry, int r) m_remote_journaler->committed(*replay_entry); } -template -int ImageReplayer::get_bootstrap_params(BootstrapParams *params) -{ - int r = librbd::cls_client::dir_get_name(&m_remote_ioctx, RBD_DIRECTORY, - m_remote_image_id, - ¶ms->local_image_name); - if (r < 0) { - derr << "error looking up name for remote image id " << m_remote_image_id - << ": " << cpp_strerror(r) << dendl; - return r; - } - - params->local_pool_name = m_remote_ioctx.get_pool_name(); - - return 0; -} - template void ImageReplayer::shut_down_journal_replay(bool cancel_ops) { diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h index 261f570c5eb87..d7d1b2a0eecc3 100644 --- a/src/tools/rbd_mirror/ImageReplayer.h +++ b/src/tools/rbd_mirror/ImageReplayer.h @@ -55,22 +55,19 @@ public: }; struct BootstrapParams { - std::string local_pool_name; std::string local_image_name; BootstrapParams() {} - BootstrapParams(const std::string &local_pool_name, - const std::string local_image_name) : - local_pool_name(local_pool_name), + BootstrapParams(const std::string local_image_name) : local_image_name(local_image_name) {} bool empty() const { - return local_pool_name.empty() && local_image_name.empty(); + return local_image_name.empty(); } }; ImageReplayer(Threads *threads, RadosRef local, RadosRef remote, - const std::string &client_id, int64_t local_pool_id, + const std::string &mirror_uuid, int64_t local_pool_id, int64_t remote_pool_id, const std::string &remote_image_id); virtual ~ImageReplayer(); ImageReplayer(const ImageReplayer&) = delete; @@ -111,27 +108,17 @@ protected: * * * | * * v (error) * - * GET_REGISTERED_CLIENT_STATUS * * * * * * * * - * | * - * | (sync required) * - * |\-----\ * - * | | * - * | v (error) * - * | BOOTSTRAP_IMAGE * * * * * * * * * * * - * | | * - * | v * - * |/-----/ * - * | * - * v (no sync required) (error) * - * REMOTE_JOURNALER_INIT * * * * * * * * * * * + * BOOTSTRAP_IMAGE * * * * * * * * * * * * * * * | * * v (error) * - * LOCAL_IMAGE_OPEN (skip if not * * * * * * * - * | needed * + * INIT_REMOTE_JOURNALER * * * * * * * * * * * + * | * * v (error) * - * WAIT_FOR_LOCAL_JOURNAL_READY * * * * * * * * + * START_REPLAY * * * * * * * * * * * * * * * * * | - * v-----------------------------------------------\ + * | /-------------------------------------------\ + * | | | + * v v | * --------------> | * | | | * v v | @@ -149,21 +136,6 @@ protected: * @endverbatim */ - virtual void on_start_get_registered_client_status_start( - const BootstrapParams *bootstrap_params); - virtual void on_start_get_registered_client_status_finish(int r, - const std::set ®istered_clients, - const BootstrapParams &bootstrap_params); - - void bootstrap(const BootstrapParams ¶ms); - void handle_bootstrap(int r); - - virtual void on_start_remote_journaler_init_start(); - virtual void on_start_remote_journaler_init_finish(int r); - virtual void on_start_local_image_open_start(); - virtual void on_start_local_image_open_finish(int r); - virtual void on_start_wait_for_local_journal_ready_start(); - virtual void on_start_wait_for_local_journal_ready_finish(int r); virtual void on_start_fail_start(int r); virtual void on_start_fail_finish(int r); virtual bool on_start_interrupted(); @@ -184,24 +156,15 @@ protected: private: typedef typename librbd::journal::TypeTraits::Journaler Journaler; - State get_state_() const { return m_state; } - bool is_stopped_() const { return m_state == STATE_UNINITIALIZED || - m_state == STATE_STOPPED; } - bool is_running_() const { return !is_stopped_() && m_state != STATE_STOPPING; } - - int get_bootstrap_params(BootstrapParams *params); - - void shut_down_journal_replay(bool cancel_ops); - Threads *m_threads; RadosRef m_local, m_remote; - std::string m_client_id; + std::string m_mirror_uuid; int64_t m_remote_pool_id, m_local_pool_id; std::string m_remote_image_id, m_local_image_id; + std::string m_local_image_name; std::string m_name; Mutex m_lock; State m_state; - std::string m_local_pool_name, m_remote_pool_name; librados::IoCtx m_local_ioctx, m_remote_ioctx; ImageCtxT *m_local_image_ctx; librbd::journal::Replay *m_local_replay; @@ -211,6 +174,21 @@ private: AdminSocketHook *m_asok_hook; librbd::journal::MirrorPeerClientMeta m_client_meta; + + State get_state_() const { return m_state; } + bool is_stopped_() const { return m_state == STATE_UNINITIALIZED || + m_state == STATE_STOPPED; } + bool is_running_() const { return !is_stopped_() && m_state != STATE_STOPPING; } + + void shut_down_journal_replay(bool cancel_ops); + + void bootstrap(); + void handle_bootstrap(int r); + + void init_remote_journaler(); + void handle_init_remote_journaler(int r); + + void start_replay(); }; } // namespace mirror diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc index 267de3850a468..77fea4fef6396 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc @@ -86,6 +86,66 @@ BootstrapRequest::~BootstrapRequest() { template void BootstrapRequest::send() { + get_client(); +} + +template +void BootstrapRequest::get_client() { + dout(20) << dendl; + + Context *ctx = create_context_callback< + BootstrapRequest, &BootstrapRequest::handle_get_client>( + this); + m_journaler->get_client(m_mirror_uuid, &m_client, ctx); +} + +template +void BootstrapRequest::handle_get_client(int r) { + dout(20) << ": r=" << r << dendl; + + if (r == -ENOENT) { + dout(10) << ": client not registered" << dendl; + } else if (r < 0) { + derr << ": failed to retreive client: " << cpp_strerror(r) << dendl; + finish(r); + return; + } else if (decode_client_meta()) { + // skip registration if it already exists + open_remote_image(); + return; + } + + register_client(); +} + +template +void BootstrapRequest::register_client() { + dout(20) << dendl; + + // record an empty place-holder record + librbd::journal::ClientData client_data{ + librbd::journal::MirrorPeerClientMeta{}}; + bufferlist client_data_bl; + ::encode(client_data, client_data_bl); + + Context *ctx = create_context_callback< + BootstrapRequest, &BootstrapRequest::handle_register_client>( + this); + m_journaler->register_client(client_data_bl, ctx); +} + +template +void BootstrapRequest::handle_register_client(int r) { + dout(20) << ": r=" << r << dendl; + + if (r < 0) { + derr << ": failed to register with remote journal: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + *m_client_meta = librbd::journal::MirrorPeerClientMeta(); open_remote_image(); } @@ -108,106 +168,135 @@ void BootstrapRequest::handle_open_remote_image(int r) { dout(20) << ": r=" << r << dendl; if (r < 0) { - derr << "failed to open remote image: " << cpp_strerror(r) << dendl; + derr << ": failed to open remote image: " << cpp_strerror(r) << dendl; m_ret_val = r; close_remote_image(); return; } - create_local_image(); + // default local image name to the remote image name if not provided + if (m_local_image_name.empty()) { + m_local_image_name = m_remote_image_ctx->name; + } + + if (m_local_image_id.empty()) { + create_local_image(); + return; + } + + open_local_image(); } template -void BootstrapRequest::create_local_image() { +void BootstrapRequest::open_local_image() { dout(20) << dendl; - // TODO: local image might already exist (e.g. interrupted sync) - // need to determine what type of bootstrap we are performing - - // TODO: librbd should provide an AIO image creation method -- this is - // blocking so we execute in our worker thread Context *ctx = create_context_callback< - BootstrapRequest, &BootstrapRequest::handle_create_local_image>( + BootstrapRequest, &BootstrapRequest::handle_open_local_image>( this); - m_work_queue->queue(new C_CreateImage(m_local_io_ctx, m_local_image_name, - m_remote_image_ctx, ctx), 0); + OpenLocalImageRequest *request = OpenLocalImageRequest::create( + m_local_io_ctx, m_local_image_ctx, + (!m_local_image_id.empty() ? std::string() : m_local_image_name), + m_local_image_id, m_work_queue, ctx); + request->send(); } template -void BootstrapRequest::handle_create_local_image(int r) { +void BootstrapRequest::handle_open_local_image(int r) { dout(20) << ": r=" << r << dendl; - if (r < 0) { - derr << "failed to create local image: " << cpp_strerror(r) << dendl; + if (r == -ENOENT) { + assert(*m_local_image_ctx == nullptr); + dout(10) << ": local image missing" << dendl; + create_local_image(); + return; + } else if (r < 0) { + assert(*m_local_image_ctx == nullptr); + derr << ": failed to open local image: " << cpp_strerror(r) << dendl; m_ret_val = r; close_remote_image(); return; } - open_local_image(); + update_client(); } template -void BootstrapRequest::open_local_image() { +void BootstrapRequest::remove_local_image() { + dout(20) << dendl; + + // TODO +} + +template +void BootstrapRequest::handle_remove_local_image(int r) { + dout(20) << ": r=" << r << dendl; + + // TODO +} + +template +void BootstrapRequest::create_local_image() { dout(20) << dendl; + // TODO: librbd should provide an AIO image creation method -- this is + // blocking so we execute in our worker thread Context *ctx = create_context_callback< - BootstrapRequest, &BootstrapRequest::handle_open_local_image>( + BootstrapRequest, &BootstrapRequest::handle_create_local_image>( this); - OpenLocalImageRequest *request = OpenLocalImageRequest::create( - m_local_io_ctx, m_local_image_ctx, m_local_image_name, "", m_work_queue, - ctx); - request->send(); + m_work_queue->queue(new C_CreateImage(m_local_io_ctx, m_local_image_name, + m_remote_image_ctx, ctx), 0); } template -void BootstrapRequest::handle_open_local_image(int r) { +void BootstrapRequest::handle_create_local_image(int r) { dout(20) << ": r=" << r << dendl; if (r < 0) { - assert(*m_local_image_ctx == nullptr); - derr << "failed to open local image: " << cpp_strerror(r) << dendl; + derr << ": failed to create local image: " << cpp_strerror(r) << dendl; m_ret_val = r; close_remote_image(); return; } - register_client(); + open_local_image(); } template -void BootstrapRequest::register_client() { - dout(20) << dendl; +void BootstrapRequest::update_client() { + if (m_local_image_id == (*m_local_image_ctx)->id) { + image_sync(); + return; + } + m_local_image_id = (*m_local_image_ctx)->id; - // TODO: if client fails to register newly created image to journal, - // need to ensure we can recover (i.e. see if image of the same - // name already exists) + dout(20) << dendl; - librbd::journal::MirrorPeerClientMeta client_meta(*m_client_meta); - client_meta.image_id = (*m_local_image_ctx)->id; + librbd::journal::MirrorPeerClientMeta client_meta; + client_meta.image_id = m_local_image_id; librbd::journal::ClientData client_data(client_meta); - bufferlist client_data_bl; - ::encode(client_data, client_data_bl); + bufferlist data_bl; + ::encode(client_data, data_bl); Context *ctx = create_context_callback< - BootstrapRequest, &BootstrapRequest::handle_register_client>( + BootstrapRequest, &BootstrapRequest::handle_update_client>( this); - m_journaler->register_client(client_data_bl, ctx); + m_journaler->update_client(data_bl, ctx); } template -void BootstrapRequest::handle_register_client(int r) { +void BootstrapRequest::handle_update_client(int r) { dout(20) << ": r=" << r << dendl; if (r < 0) { - derr << "failed to register with remote journal: " << cpp_strerror(r) - << dendl; + derr << ": failed to update client: " << cpp_strerror(r) << dendl; + m_ret_val = r; close_local_image(); return; } - m_client_meta->image_id = (*m_local_image_ctx)->id; + m_client_meta->image_id = m_local_image_id; image_sync(); } @@ -231,7 +320,7 @@ void BootstrapRequest::handle_image_sync(int r) { dout(20) << ": r=" << r << dendl; if (r < 0) { - derr << "failed to sync remote image: " << cpp_strerror(r) << dendl; + derr << ": failed to sync remote image: " << cpp_strerror(r) << dendl; m_ret_val = r; close_local_image(); return; @@ -248,7 +337,7 @@ void BootstrapRequest::close_local_image() { BootstrapRequest, &BootstrapRequest::handle_close_local_image>( this); CloseImageRequest *request = CloseImageRequest::create( - m_local_image_ctx, m_work_queue, ctx); + m_local_image_ctx, m_work_queue, false, ctx); request->send(); } @@ -257,7 +346,7 @@ void BootstrapRequest::handle_close_local_image(int r) { dout(20) << ": r=" << r << dendl; if (r < 0) { - derr << "error encountered closing local image: " << cpp_strerror(r) + derr << ": error encountered closing local image: " << cpp_strerror(r) << dendl; } @@ -272,7 +361,7 @@ void BootstrapRequest::close_remote_image() { BootstrapRequest, &BootstrapRequest::handle_close_remote_image>( this); CloseImageRequest *request = CloseImageRequest::create( - &m_remote_image_ctx, m_work_queue, ctx); + &m_remote_image_ctx, m_work_queue, false, ctx); request->send(); } @@ -281,7 +370,7 @@ void BootstrapRequest::handle_close_remote_image(int r) { dout(20) << ": r=" << r << dendl; if (r < 0) { - derr << "error encountered closing remote image: " << cpp_strerror(r) + derr << ": error encountered closing remote image: " << cpp_strerror(r) << dendl; } @@ -296,6 +385,35 @@ void BootstrapRequest::finish(int r) { delete this; } +template +bool BootstrapRequest::decode_client_meta() { + dout(20) << dendl; + + librbd::journal::ClientData client_data; + bufferlist::iterator it = m_client.data.begin(); + try { + ::decode(client_data, it); + } catch (const buffer::error &err) { + derr << ": failed to decode client meta data: " << err.what() << dendl; + return true; + } + + librbd::journal::MirrorPeerClientMeta *client_meta = + boost::get(&client_data.client_meta); + if (client_meta == nullptr) { + derr << ": unknown peer registration" << dendl; + return true; + } else if (!client_meta->image_id.empty()) { + // have an image id -- use that to open the image + m_local_image_id = client_meta->image_id; + } + + *m_client_meta = *client_meta; + + dout(20) << ": client found: image_id=" << m_local_image_id << dendl; + return true; +} + } // namespace image_replayer } // namespace mirror } // namespace rbd diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h index 12fffb7e9c36a..955195161305a 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h @@ -6,6 +6,7 @@ #include "include/int_types.h" #include "include/rados/librados.hpp" +#include "cls/journal/cls_journal_types.h" #include "librbd/journal/TypeTraits.h" #include @@ -64,21 +65,36 @@ private: * * | * v - * OPEN_REMOTE_IMAGE * * * * * * * * * * * * + * GET_CLIENT * * * * * * * * * * * * * * * * + * | * + * v (skip if not needed) * (error) + * REGISTER_CLIENT * * * * * * * * * * * * * * | * * v * - * CREATE_LOCAL_IMAGE * * * * * * * * * * * * (error) + * OPEN_REMOTE_IMAGE * * * * * * * * * * * * * | * * v * * OPEN_LOCAL_IMAGE * * * * * * * * * * * * * + * | . ^ * + * | . | * + * | . \-----------------------\ * + * | . | * + * | . (image sync requested) | * + * | . . > REMOVE_LOCAL_IMAGE * * * * * + * | . | | * + * | . (image doesn't | | * + * | . exist) v | * + * | . . > CREATE_LOCAL_IMAGE * * * * * + * | | | * + * | \-----------------/ * * | * - * v * - * REGISTER_CLIENT * * * * * - * | * * - * v v * + * v (skip if not needed) * + * UPDATE_CLIENT * + * | * + * v (skip if not needed) * * IMAGE_SYNC * * * > CLOSE_LOCAL_IMAGE * - * | | * - * | /-------------/ * + * | | * + * | /-------------------/ * * | | * * v v * * CLOSE_REMOTE_IMAGE < * * * * * * * * * * * @@ -92,6 +108,7 @@ private: librados::IoCtx &m_remote_io_ctx; ImageCtxT **m_local_image_ctx; std::string m_local_image_name; + std::string m_local_image_id; std::string m_remote_image_id; ContextWQ *m_work_queue; SafeTimer *m_timer; @@ -101,20 +118,30 @@ private: MirrorPeerClientMeta *m_client_meta; Context *m_on_finish; + cls::journal::Client m_client; ImageCtxT *m_remote_image_ctx = nullptr; int m_ret_val = 0; + void get_client(); + void handle_get_client(int r); + + void register_client(); + void handle_register_client(int r); + void open_remote_image(); void handle_open_remote_image(int r); - void create_local_image(); - void handle_create_local_image(int r); - void open_local_image(); void handle_open_local_image(int r); - void register_client(); - void handle_register_client(int r); + void remove_local_image(); + void handle_remove_local_image(int r); + + void create_local_image(); + void handle_create_local_image(int r); + + void update_client(); + void handle_update_client(int r); void image_sync(); void handle_image_sync(int r); @@ -127,6 +154,7 @@ private: void finish(int r); + bool decode_client_meta(); }; } // namespace image_replayer -- 2.39.5