testlog "TEST: stop/start/restart group via admin socket"
admin_daemons ${CLUSTER1} rbd mirror group stop ${POOL}/${group1}
- wait_for_group_replay_stopped ${CLUSTER1} ${POOL}/${group1}
- wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group1} 'up+stopped' 0
+ wait_for_group_replay_stopped ${CLUSTER1} ${POOL}/${group1} 1
+ wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group1} 'up+stopped' 1
admin_daemons ${CLUSTER1} rbd mirror group start ${POOL}/${group1}
wait_for_group_replay_started ${CLUSTER1} ${POOL}/${group1} 1
testlog " - demote and promote same cluster"
mirror_group_demote ${CLUSTER2} ${POOL}/${group1}
test_fields_in_group_info ${CLUSTER2} ${POOL}/${group1} 'snapshot' 'enabled' 'false'
-wait_for_group_replay_stopped ${CLUSTER1} ${POOL}/${group1}
+wait_for_group_replay_stopped ${CLUSTER1} ${POOL}/${group1} 0
wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group1} 'up+unknown' 0
wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL}/${group1} 'up+unknown' 0
testlog " - failover (unmodified)"
mirror_group_demote ${CLUSTER2} ${POOL}/${group}
test_fields_in_group_info ${CLUSTER2} ${POOL}/${group} 'snapshot' 'enabled' 'false'
-wait_for_group_replay_stopped ${CLUSTER1} ${POOL}/${group}
+wait_for_group_replay_stopped ${CLUSTER1} ${POOL}/${group} 0
wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group} 'up+unknown' 0
wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL}/${group} 'up+unknown' 0
mirror_group_promote ${CLUSTER1} ${POOL}/${group}
testlog " - failback (unmodified)"
mirror_group_demote ${CLUSTER1} ${POOL}/${group}
test_fields_in_group_info ${CLUSTER1} ${POOL}/${group} 'snapshot' 'enabled' 'false'
-wait_for_group_replay_stopped ${CLUSTER2} ${POOL}/${group}
+wait_for_group_replay_stopped ${CLUSTER2} ${POOL}/${group} 0
wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group} 'up+unknown' 0
wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL}/${group} 'up+unknown' 0
mirror_group_promote ${CLUSTER2} ${POOL}/${group}
wait_for_group_replay_started ${CLUSTER1} ${POOL}/${group} 1
mirror_group_snapshot_and_wait_for_sync_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${group}
wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group} 'up+replaying' 1
-wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL}/${group} 'up+stopped' 0
+wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL}/${group} 'up+stopped' 1
compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image}
testlog " - failover"
mirror_group_demote ${CLUSTER2} ${POOL}/${group1}
test_fields_in_group_info ${CLUSTER2} ${POOL}/${group1} 'snapshot' 'enabled' 'false'
-wait_for_group_replay_stopped ${CLUSTER1} ${POOL}/${group1}
+wait_for_group_replay_stopped ${CLUSTER1} ${POOL}/${group1} 0
wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group1} 'up+unknown' 0
wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL}/${group1} 'up+unknown' 0
mirror_group_promote ${CLUSTER1} ${POOL}/${group1}
wait_for_group_replay_started ${CLUSTER2} ${POOL}/${group1} 1
write_image ${CLUSTER1} ${POOL} ${image1} 100
mirror_group_snapshot_and_wait_for_sync_complete ${CLUSTER2} ${CLUSTER1} ${POOL}/${group1}
-wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group1} 'up+stopped' 0
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group1} 'up+stopped' 1
wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL}/${group1} 'up+replaying' 1
compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image1}
testlog " - failback to cluster2"
mirror_group_demote ${CLUSTER1} ${POOL}/${group1}
test_fields_in_group_info ${CLUSTER1} ${POOL}/${group1} 'snapshot' 'enabled' 'false'
-wait_for_group_replay_stopped ${CLUSTER2} ${POOL}/${group1}
+wait_for_group_replay_stopped ${CLUSTER2} ${POOL}/${group1} 0
wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group1} 'up+unknown' 0
wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL}/${group1} 'up+unknown' 0
mirror_group_promote ${CLUSTER2} ${POOL}/${group1}
write_image ${CLUSTER2} ${POOL} ${image1} 100
mirror_group_snapshot_and_wait_for_sync_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${group1}
wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group1} 'up+replaying' 1
-wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL}/${group1} 'up+stopped' 0
+wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL}/${group1} 'up+stopped' 1
compare_images ${CLUSTER1} ${CLUSTER2} ${POOL} ${POOL} ${image1}
testlog " - force promote cluster1"
test_fields_in_group_info ${CLUSTER1} ${POOL}/${group} 'snapshot' 'enabled' 'true'
test_fields_in_group_info ${CLUSTER2} ${POOL}/${group} 'snapshot' 'enabled' 'true'
-wait_for_group_replay_stopped ${CLUSTER1} ${POOL}/${group}
-wait_for_group_replay_stopped ${CLUSTER2} ${POOL}/${group}
-wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group} 'up+stopped' 0
-wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL}/${group} 'up+stopped' 0
+wait_for_group_replay_stopped ${CLUSTER1} ${POOL}/${group} 1
+wait_for_group_replay_stopped ${CLUSTER2} ${POOL}/${group} 1
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group} 'up+stopped' 1
+wait_for_group_status_in_pool_dir ${CLUSTER2} ${POOL}/${group} 'up+stopped' 1
write_image ${CLUSTER1} ${POOL} ${image} 100
write_image ${CLUSTER2} ${POOL} ${image} 100
wait_for_group_present ${CLUSTER1} ${POOL} ${group} 1
mirror_group_promote ${CLUSTER1} ${POOL}/${group} --force
test_fields_in_group_info ${CLUSTER1} ${POOL}/${group} 'snapshot' 'enabled' 'true'
test_fields_in_group_info ${CLUSTER2} ${POOL}/${group} 'snapshot' 'enabled' 'true'
-wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group} 'up+stopped' 0
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group} 'up+stopped' 1
write_image ${CLUSTER1} ${POOL} ${image} 10
mirror_group_demote ${CLUSTER1} ${POOL}/${group}
test_fields_in_group_info ${CLUSTER1} ${POOL}/${group} 'snapshot' 'enabled' 'false'
wait_for_group_status_in_pool_dir "${secondary_cluster}" "${pool}"/"${group0}" 'up+replaying' "${image_count}"
if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
- wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group0}" 'up+stopped' 0
+ wait_for_group_status_in_pool_dir "${primary_cluster}" "${pool}"/"${group0}" 'up+stopped' "${image_count}"
fi
wait_for_group_synced "${primary_cluster}" "${pool}"/"${group0}"
mirror_group_promote "${secondary_cluster}" "${pool}/${group0}" '--force'
wait_for_group_replay_stopped ${secondary_cluster} ${pool}/${group0}
wait_for_group_replay_stopped ${primary_cluster} ${pool}/${group0}
- wait_for_group_status_in_pool_dir ${secondary_cluster} ${pool}/${group0} 'up+stopped' 0
- wait_for_group_status_in_pool_dir ${primary_cluster} ${pool}/${group0} 'up+stopped' 0
+ wait_for_group_status_in_pool_dir ${secondary_cluster} ${pool}/${group0} 'up+stopped' "${image_count}"
+ wait_for_group_status_in_pool_dir ${primary_cluster} ${pool}/${group0} 'up+stopped' "${image_count}"
wait_for_group_present "${primary_cluster}" "${pool}" "${group0}" "${image_count}"
wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}"
wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" "${image_count}"
# group still exists on original primary
- wait_for_group_status_in_pool_dir ${primary_cluster} ${pool}/${group0} 'up+stopped' 0
+ wait_for_group_status_in_pool_dir ${primary_cluster} ${pool}/${group0} 'up+stopped' "${image_count}"
group_image_remove ${secondary_cluster} ${pool}/${group0} ${pool}/${image_prefix}0
wait_for_group_present "${primary_cluster}" "${pool}" "${group0}" "${image_count}"
wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" $(("${image_count}"-1))
test_fields_in_group_info ${primary_cluster} ${pool}/${group0} 'snapshot' 'enabled' 'true'
- wait_for_group_status_in_pool_dir ${primary_cluster} ${pool}/${group0} 'up+stopped' 0
+ wait_for_group_status_in_pool_dir ${primary_cluster} ${pool}/${group0} 'up+stopped' "${image_count}"
mirror_group_enable "${secondary_cluster}" "${pool}/${group0}"
test_fields_in_group_info ${primary_cluster} ${pool}/${group0} 'snapshot' 'enabled' 'true'
test_fields_in_group_info ${secondary_cluster} ${pool}/${group0} 'snapshot' 'enabled' 'true'
- wait_for_group_status_in_pool_dir ${primary_cluster} ${pool}/${group0} 'up+stopped' 0
- wait_for_group_status_in_pool_dir ${secondary_cluster} ${pool}/${group0} 'up+stopped' 0
+ wait_for_group_status_in_pool_dir ${primary_cluster} ${pool}/${group0} 'up+stopped' "${image_count}"
+ wait_for_group_status_in_pool_dir ${secondary_cluster} ${pool}/${group0} 'up+stopped' $(("${image_count}"-1))
# TODO - test normally fails on next line with missing images
wait_for_group_present "${secondary_cluster}" "${pool}" "${group0}" $(("${image_count}"-1))
for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16; do
sleep ${s}
if [ 'true' = "${asok_query}" ]; then
- test_group_replay_state "${cluster}" "${group_spec}" "${state}" "${image_count}" && return 0
+ test_group_replay_state "${cluster}" "${group_spec}" "${state}" && return 0
else
test_group_replay_state_cli "${cluster}" "${group_spec}" "${state}" "${image_count}" && return 0
fi
{
local cluster=$1
local group_spec=$2
+ local image_count=$3
- # Image_count will be 0 if the group is stopped
- # Query the state via daemon socket and also via the cli to confirm that they agree
+ # Image_count will be 0 if the group is stopped except when the group is primary
+ # Query the state via daemon socket and also via the cli.
+ # The admin socket status does not include image information
wait_for_group_replay_state "${cluster}" "${group_spec}" 'stopped' 0 'true'
- wait_for_group_replay_state "${cluster}" "${group_spec}" 'stopped' 0 'false'
+ wait_for_group_replay_state "${cluster}" "${group_spec}" 'stopped' "${image_count}" 'false'
}
get_newest_group_snapshot_id()
unregister_admin_socket_hook();
ceph_assert(m_on_start_finish == nullptr);
ceph_assert(m_bootstrap_request == nullptr);
- ceph_assert(m_bootstrap_request == nullptr);
ceph_assert(m_replayer == nullptr);
ceph_assert(m_image_replayers.empty());
ceph_assert(m_on_stop_contexts.empty());
+ ceph_assert(m_replayer_check_task == nullptr);
+}
+
+template <typename I>
+void GroupReplayer<I>::destroy() {
+ {
+ std::lock_guard locker{m_lock};
+
+ for (auto &it : m_image_replayers) {
+ ceph_assert(it.second->is_stopped());
+ it.second->destroy();
+ }
+ m_image_replayers.clear();
+ }
+ delete this;
}
template <typename I>
m_last_r = 0;
m_state_desc.clear();
m_local_group_snaps.clear();
+ ceph_assert(m_replayer_check_task == nullptr);
ceph_assert(m_replayer == nullptr);
- ceph_assert(m_image_replayers.empty());
- m_image_replayers.clear();
+ if (m_destroy_replayers) {
+ ceph_assert(m_image_replayers.empty());
+ }
+ m_destroy_replayers = false;
+ // FIXME: replayer index is not used
m_image_replayer_index.clear();
m_manual_stop = false;
m_finished = false;
state = STATE_STOPPED;
}
f->dump_string("state", state_to_string(state));
-// TODO: remove the image_replayers section
+/*
f->open_array_section("image_replayers");
for (auto &[_, image_replayer] : m_image_replayers) {
image_replayer->print_status(f);
}
f->close_section(); // image_replayers
+*/
f->close_section(); // group_replayer
}
m_state_desc = "";
}
+ cancel_image_replayers_check();
shut_down(r);
}
}
ceph_assert(m_replayer == nullptr);
- ceph_assert(m_image_replayers.empty());
auto ctx = create_context_callback<
GroupReplayer,
reregister_admin_socket_hook();
}
+//FIXME : Relook at this once the Bootstrap no longer reuses values.
+// Since a rerun may end up with variables populated by a previous run,
+// it is safer to destroy the image_replayers now.
+ m_destroy_replayers = true;
if (finish_start_if_interrupted()) {
return;
} else if (r == -ENOENT) {
finish_start_fail(r, "group removed");
return;
} else if (r == -EREMOTEIO) {
+ m_destroy_replayers = true;
finish_start_fail(r, "remote group is non-primary");
return;
} else if (r == -EEXIST) {
return;
}
+ // Start the image replayers in case the group is primary
+ // in order to have the mirror pool health status set to ok.
+ m_destroy_replayers = false;
+
+/*
if (m_local_group_ctx.primary) { // XXXMG
set_mirror_group_status_update(
cls::rbd::MIRROR_GROUP_STATUS_STATE_STOPPED,
"local group is primary");
finish_start_fail(0, "local group is primary");
return;
- } else if (m_remote_group_id.empty()) { // m_remote_group_id matter for
- // secondary cluster case.
- finish_start_fail(-EINVAL, "remote is not ready yet");
- return;
}
-
+*/
m_local_group_ctx.listener = &m_listener;
- create_group_replayer();
+ // Start the image replayers first
+ start_image_replayers();
}
template <typename I>
return;
}
- start_image_replayers();
+ Context *on_finish = nullptr;
+ {
+ std::unique_lock locker{m_lock};
+ ceph_assert(m_state == STATE_STARTING);
+ m_state = STATE_REPLAYING;
+ std::swap(m_on_start_finish, on_finish);
+
+ std::unique_lock timer_locker{m_threads->timer_lock};
+ schedule_image_replayers_check();
+ }
+
+ set_mirror_group_status_update(
+ cls::rbd::MIRROR_GROUP_STATUS_STATE_REPLAYING, "replaying");
+
+ if (on_finish) {
+ on_finish->complete(0);
+ }
}
// TODO: Move this to group_replayer::Replayer?
return;
}
- Context *on_finish = nullptr;
+ create_group_replayer();
+}
+
+template <typename I>
+void GroupReplayer<I>::check_image_replayers_running() {
+ dout(10) << dendl;
+ bool stopped = false;
{
- std::unique_lock locker{m_lock};
- ceph_assert(m_state == STATE_STARTING);
- m_state = STATE_REPLAYING;
- std::swap(m_on_start_finish, on_finish);
+ std::lock_guard locker{m_lock};
+ if (m_state == STATE_STOPPING || m_state == STATE_STOPPED) {
+ dout(10) << "stopping" <<dendl;
+ return;
+ }
+
+ for (auto &it : m_image_replayers) {
+ if (it.second->is_stopped()) {
+ dout(10) << "image replayer stopped for global_id : "
+ << it.second->get_global_image_id() << dendl;
+ stopped = true;
+ break;
+ }
+ }
}
- set_mirror_group_status_update(
- cls::rbd::MIRROR_GROUP_STATUS_STATE_REPLAYING, "replaying");
+ if (stopped) {
+ // shut down
+ dout(10) << " stopping group replayer" << dendl;
+ //TODO: determine the error
+ on_stop_replay();
+ return;
+ }
+}
- if(on_finish) {
- on_finish->complete(0);
+template <typename I>
+void GroupReplayer<I>::schedule_image_replayers_check() {
+ ceph_assert(ceph_mutex_is_locked_by_me(m_lock));
+ ceph_assert(ceph_mutex_is_locked_by_me(m_threads->timer_lock));
+
+ if (m_state != STATE_REPLAYING) {
+ return;
+ }
+
+ dout(10) << dendl;
+
+ ceph_assert(m_replayer_check_task == nullptr);
+ m_replayer_check_task = create_context_callback<
+ GroupReplayer<I>,
+ &GroupReplayer<I>::handle_image_replayers_check>(this);
+ m_threads->timer->add_event_after(10, m_replayer_check_task);
+}
+
+template <typename I>
+void GroupReplayer<I>::handle_image_replayers_check(int r) {
+ dout(10) << dendl;
+
+ ceph_assert(ceph_mutex_is_locked_by_me(m_threads->timer_lock));
+
+ ceph_assert(m_replayer_check_task != nullptr);
+ m_replayer_check_task = nullptr;
+
+ auto ctx = new LambdaContext([this](int) {
+ check_image_replayers_running();
+ {
+ std::unique_lock locker{m_lock};
+ std::unique_lock timer_locker{m_threads->timer_lock};
+
+ schedule_image_replayers_check();
+ }
+ m_in_flight_op_tracker.finish_op();
+ });
+
+ m_in_flight_op_tracker.start_op();
+ m_threads->work_queue->queue(ctx, 0);
+}
+
+template <typename I>
+void GroupReplayer<I>::cancel_image_replayers_check() {
+ std::unique_lock timer_locker{m_threads->timer_lock};
+ if (m_replayer_check_task != nullptr) {
+ dout(10) << dendl;
+
+ if (m_threads->timer->cancel_event(m_replayer_check_task)) {
+ m_replayer_check_task = nullptr;
+ }
}
}
ceph_assert(m_state == STATE_STOPPING);
}
+ if (!m_in_flight_op_tracker.empty()) {
+ dout(15) << "waiting for in-flight operations to complete" << dendl;
+ m_in_flight_op_tracker.wait_for_ops(new LambdaContext([this, r](int) {
+ shut_down(r);
+ }));
+ return;
+ }
+
// chain the shut down sequence (reverse order)
Context *ctx = new LambdaContext(
[this, r](int _r) {
});
// stop and destroy the replayers
- ctx = new LambdaContext([this, ctx](int r) {
- {
- std::lock_guard locker{m_lock};
+ if (m_destroy_replayers) {
+ ctx = new LambdaContext([this, ctx](int r) {
+ {
+ std::lock_guard locker{m_lock};
- for (auto &it : m_image_replayers) {
- ceph_assert(it.second->is_stopped());
- it.second->destroy();
+ for (auto &it : m_image_replayers) {
+ ceph_assert(it.second->is_stopped());
+ it.second->destroy();
+ }
+ m_image_replayers.clear();
}
- m_image_replayers.clear();
- }
- ctx->complete(0);
- });
-
+ ctx->complete(0);
+ });
+ }
ctx = new LambdaContext([this, ctx](int r) {
C_Gather *gather_ctx = new C_Gather(g_ceph_context, ctx);
{
threads, instance_watcher, local_status_updater,
cache_manager_handler, pool_meta_cache);
}
- void destroy() {
- delete this;
- }
+ void destroy();
GroupReplayer(librados::IoCtx &local_io_ctx,
const std::string &local_mirror_uuid,
std::string m_local_group_id;
std::string m_remote_group_id;
+ bool m_destroy_replayers = false;
+
mutable ceph::mutex m_lock;
- AsyncOpTracker m_async_op_tracker;
State m_state = STATE_STOPPED;
std::string m_state_desc;
cls::rbd::MirrorGroupStatusState m_status_state;
AdminSocketHook *m_asok_hook = nullptr;
+ AsyncOpTracker m_in_flight_op_tracker;
+ Context* m_replayer_check_task = nullptr;
+
group_replayer::BootstrapRequest<ImageCtxT> *m_bootstrap_request = nullptr;
group_replayer::Replayer<ImageCtxT> *m_replayer = nullptr;
std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> m_image_replayers;
void start_image_replayers();
void handle_start_image_replayers(int r);
+ void check_image_replayers_running();
+ void schedule_image_replayers_check();
+ void handle_image_replayers_check(int r);
+ void cancel_image_replayers_check();
+
bool finish_start_if_interrupted();
bool finish_start_if_interrupted(ceph::mutex &lock);
void finish_start_fail(int r, const std::string &desc);
int BootstrapRequest<I>::create_replayers() {
dout(10) << dendl;
+ //TODO: check that the images have not changed
+ if (!m_image_replayers->empty()) {
+ dout(10) << "image replayers already exist."<< dendl;
+ return 0;
+ }
+
int r = 0;
if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
m_remote_mirror_group_primary) {
if (ns->state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) {
break;
}
- // this is primary, IDLE the group replayer
- m_state = STATE_IDLE;
+ m_state = STATE_COMPLETE;
notify_group_listener_stop();
return;
}
projects / ceph-ci.git / commitdiff