From: Dhairya Parmar Date: Mon, 25 May 2026 12:01:33 +0000 (+0530) Subject: src/test/reclaim: test session reclaim after mds failover X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F69037%2Fhead;p=ceph.git src/test/reclaim: test session reclaim after mds failover ensure that the new active MDS reads the auth_name from the ESession event and assigns it to the new session that MDS creates during journal replay. NOTE: the mds failover is carried by sending "respawn" command to active MDS using libcephfs's ceph_mds_command(). Fixes: https://tracker.ceph.com/issues/76728 Signed-off-by: Dhairya Parmar --- diff --git a/src/test/libcephfs/reclaim.cc b/src/test/libcephfs/reclaim.cc index 428f3d1cc40..4ff541dcea9 100644 --- a/src/test/libcephfs/reclaim.cc +++ b/src/test/libcephfs/reclaim.cc @@ -16,29 +16,30 @@ #include #include #include +#include #include #include #include #include #include +#include #ifdef __linux__ #include #include #endif -#ifdef __FreeBSD__ -#include -#endif - - #include +#include #include #include #include #define CEPHFS_RECLAIM_TIMEOUT 60 +/* execl argv[1] for ReclaimResetAfterMDSFailover child */ +static const char DYING_FAILOVER[] = "--failover"; + static int dying_client(int argc, char **argv) { struct ceph_mount_info *cmount; @@ -84,6 +85,58 @@ static int dying_client(int argc, char **argv) return 0; } +static int dying_client_failover(const char *uuid) +{ + struct ceph_mount_info *cmount; + + if (ceph_create(&cmount, nullptr) != 0) { + return 1; + } + if (ceph_conf_read_file(cmount, nullptr) != 0) { + return 1; + } + if (ceph_conf_parse_env(cmount, nullptr) != 0) { + return 1; + } + if (ceph_init(cmount) != 0) { + return 1; + } + + ceph_set_session_timeout(cmount, 300); + + if (ceph_start_reclaim(cmount, uuid, CEPH_RECLAIM_RESET) != -ENOENT) { + return 1; + } + + ceph_set_uuid(cmount, uuid); + + if (ceph_mount(cmount, "/") != 0) { + return 1; + } + + if (ceph_mkdir(cmount, "/reclaim_failover_test", 0755) != 0 && errno != EEXIST) { + return 1; + } + + int fd = ceph_open(cmount, "/reclaim_failover_test/testfile", + O_RDWR|O_CREAT, 0644); + if (fd < 0) { + return 1; + } + + if (ceph_write(cmount, fd, "hello", 5, 0) < 0) { + return 1; + } + + if (ceph_fsync(cmount, fd, 0) != 0) { + return 1; + } + + /*since this is called from the child inside an execl(), returning from here + without cloing fd would mimic client crash*/ + return 0; +} + TEST(LibCephFS, ReclaimReset) { pid_t pid; char uuid[256]; @@ -129,6 +182,78 @@ TEST(LibCephFS, ReclaimReset) { ceph_release(cmount); } +TEST(LibCephFS, ReclaimResetAfterMDSFailover) { + char uuid[256]; + const char *exe = "/proc/self/exe"; + + sprintf(uuid, "reclaim_failover:%x", getpid()); + + pid_t pid = fork(); + ASSERT_GE(pid, 0); + if (pid == 0) { + errno = 0; + execl(exe, exe, DYING_FAILOVER, uuid, nullptr); + ASSERT_EQ(errno, 0); + } + + int status = 0; + ASSERT_EQ(waitpid(pid, &status, 0), pid); + ASSERT_TRUE(WIFEXITED(status)); + ASSERT_EQ(WEXITSTATUS(status), 0); + + /* Get a separate client to kill mds because the same client cannot be used + to reclaim since it needs to be done before mounting. */ + { + struct ceph_mount_info *admin; + + ASSERT_EQ(ceph_create(&admin, nullptr), 0); + ASSERT_EQ(ceph_conf_read_file(admin, nullptr), 0); + ASSERT_EQ(ceph_conf_parse_env(admin, nullptr), 0); + ASSERT_EQ(ceph_init(admin), 0); + ASSERT_EQ(ceph_mount(admin, "/"), 0); + + /* Respawn rank 0 so that once it boots up it replays the ESession events + in journal and recreates the old open session. */ + const char *cmd[1] = {"{\"prefix\": \"respawn\"}"}; + char *outbuf = nullptr, *outs = nullptr; + size_t outbuf_len = 0, outs_len = 0; + int ret = ceph_mds_command(admin, "0", cmd, 1, nullptr, 0, + &outbuf, &outbuf_len, &outs, &outs_len); + + bool exiting = (ret == 0 && outbuf && outbuf_len > 0 && + std::string(outbuf, outbuf_len).find("Respawning") != std::string::npos && + outs_len == 0); + ASSERT_EQ(exiting, true); + + if (outbuf) ceph_buffer_free(outbuf); + if (outs) ceph_buffer_free(outs); + ceph_unmount(admin); + ceph_release(admin); + } + + sleep(60); + + struct ceph_mount_info *cmount; + ASSERT_EQ(ceph_create(&cmount, nullptr), 0); + ASSERT_EQ(ceph_conf_read_file(cmount, nullptr), 0); + ASSERT_EQ(ceph_conf_parse_env(cmount, nullptr), 0); + ASSERT_EQ(ceph_init(cmount), 0); + ceph_set_session_timeout(cmount, 300); + /* Now reclaim the dead client's session. If auth_name was not preserved + across journal replay, this will fail with -EPERM. */ + ASSERT_EQ(ceph_start_reclaim(cmount, uuid, CEPH_RECLAIM_RESET), 0); + ceph_finish_reclaim(cmount); + ceph_set_uuid(cmount, uuid); + ASSERT_EQ(ceph_mount(cmount, "/"), 0); + + /* Verify we can still see the file the dead client had created */ + struct ceph_statx stx; + ASSERT_EQ(ceph_statx(cmount, "/reclaim_failover_test/testfile", &stx, 0, 0), 0); + + ceph_unmount(cmount); + ceph_release(cmount); +} + static int update_root_mode() { struct ceph_mount_info *admin; @@ -155,9 +280,18 @@ int main(int argc, char **argv) ::testing::InitGoogleTest(&argc, argv); - if (argc > 1) + if (argc == 2) return dying_client(argc, argv); + if (argc == 3 && strcmp(argv[1], DYING_FAILOVER) == 0) { + r = dying_client_failover(argv[2]); + if (r == 0) { + return r; + } else { + exit(1); + } + } + srand(getpid()); return RUN_ALL_TESTS();