From f2c0ef40fd674fecc6e3e97cd6155b976e6759b4 Mon Sep 17 00:00:00 2001 From: shun-s Date: Wed, 2 Dec 2015 10:36:35 +0800 Subject: [PATCH] global/pidfile: do not start two daemons with a single pid-file add functions named pidfile_open and pidfile_verify to avoid starting two daemons by a single pid-file Fixes: #13422 Signed-off-by: shun song --- src/global/global_init.cc | 17 ++++- src/global/pidfile.cc | 150 ++++++++++++++++++++++++++++---------- src/global/pidfile.h | 7 +- 3 files changed, 131 insertions(+), 43 deletions(-) diff --git a/src/global/global_init.cc b/src/global/global_init.cc index c0df0ee3129ca..e721abae5ecce 100644 --- a/src/global/global_init.cc +++ b/src/global/global_init.cc @@ -261,14 +261,19 @@ int global_init_prefork(CephContext *cct) { if (g_code_env != CODE_ENVIRONMENT_DAEMON) return -1; + const md_config_t *conf = cct->_conf; if (!conf->daemonize) { + + if (pidfile_open(g_conf) < 0) { + exit(1); + } + if (atexit(pidfile_remove_void)) { derr << "global_init_daemonize: failed to set pidfile_remove function " << "to run at exit." << dendl; } - - pidfile_write(g_conf); + pidfile_write(); return -1; } @@ -292,7 +297,7 @@ void global_init_daemonize(CephContext *cct) << cpp_strerror(ret) << dendl; exit(1); } - + global_init_postfork_start(cct); global_init_postfork_finish(cct); #else @@ -305,6 +310,10 @@ void global_init_postfork_start(CephContext *cct) // restart log thread g_ceph_context->_log->start(); + if (pidfile_open(g_conf) < 0) { + exit(1); + } + if (atexit(pidfile_remove_void)) { derr << "global_init_daemonize: failed to set pidfile_remove function " << "to run at exit." << dendl; @@ -333,7 +342,7 @@ void global_init_postfork_start(CephContext *cct) exit(1); } - pidfile_write(g_conf); + pidfile_write(); } void global_init_postfork_finish(CephContext *cct) diff --git a/src/global/pidfile.cc b/src/global/pidfile.cc index 3b8962a0ea34b..c9452647ae99b 100644 --- a/src/global/pidfile.cc +++ b/src/global/pidfile.cc @@ -31,68 +31,144 @@ #define dout_prefix *_dout -static char pid_file[PATH_MAX] = ""; +struct pidfh { + int pf_fd; + char pf_path[PATH_MAX + 1]; + dev_t pf_dev; + ino_t pf_ino; + + pidfh() : pf_fd(-1), pf_dev(0), pf_ino(0) { + memset(pf_path, 0, sizeof(pf_path)); + } + + void close() { + pf_fd = -1; + pf_path[0] = '\0'; + pf_dev = 0; + pf_ino = 0; + } +}; +static pidfh pfh; -int pidfile_write(const md_config_t *conf) -{ - int ret, fd; +static int pidfile_verify() { + struct stat st; - if (conf->pid_file.empty()) { - return pidfile_remove(); + if (pfh.pf_fd == -1) { + return -EINVAL; + } + /* + * Check remembered descriptor + */ + if (fstat(pfh.pf_fd, &st) == -1) { + return -errno; } - snprintf(pid_file, PATH_MAX, "%s", conf->pid_file.c_str()); - fd = TEMP_FAILURE_RETRY(::open(pid_file, - O_CREAT|O_TRUNC|O_WRONLY, 0644)); - if (fd < 0) { - int err = errno; - derr << "write_pid_file: failed to open pid file '" - << pid_file << "': " << cpp_strerror(err) << dendl; - return err; + if (st.st_dev != pfh.pf_dev || st.st_ino != pfh.pf_ino) { + return -ESTALE; + } + return 0; +} + +int pidfile_write() +{ + if (!pfh.pf_path[0]) { + return 0; } - char buf[20]; + int ret; + if ((ret = pidfile_verify()) < 0) { + return ret; + } + + char buf[32]; int len = snprintf(buf, sizeof(buf), "%d\n", getpid()); - ret = safe_write(fd, buf, len); + ret = safe_write(pfh.pf_fd, buf, len); if (ret < 0) { derr << "write_pid_file: failed to write to pid file '" - << pid_file << "': " << cpp_strerror(ret) << dendl; - VOID_TEMP_FAILURE_RETRY(::close(fd)); + << pfh.pf_path << "': " << cpp_strerror(ret) << dendl; + pfh.close(); return ret; } - if (TEMP_FAILURE_RETRY(::close(fd))) { - ret = errno; - derr << "SimpleMessenger::write_pid_file: failed to close to pid file '" - << pid_file << "': " << cpp_strerror(ret) << dendl; - return -ret; - } return 0; } -int pidfile_remove(void) +int pidfile_remove() { - if (!pid_file[0]) + if (!pfh.pf_path[0]) { return 0; + } + + int ret; + if ((ret = pidfile_verify()) < 0) { + if (pfh.pf_fd != -1) { + ::close(pfh.pf_fd); + } + return ret; + } // only remove it if it has OUR pid in it! - int fd = TEMP_FAILURE_RETRY(::open(pid_file, O_RDONLY)); - if (fd < 0) - return -errno; char buf[32]; memset(buf, 0, sizeof(buf)); - ssize_t res = safe_read(fd, buf, sizeof(buf)); - VOID_TEMP_FAILURE_RETRY(::close(fd)); - if (res < 0) + ssize_t res = safe_read(pfh.pf_fd, buf, sizeof(buf)); + if (pfh.pf_fd != -1) { + ::close(pfh.pf_fd); + } + + if (res < 0) { return res; + } + int a = atoi(buf); - if (a != getpid()) + if (a != getpid()) { return -EDOM; - - res = ::unlink(pid_file); - if (res) + } + res = ::unlink(pfh.pf_path); + if (res) { return res; + } + pfh.close(); + return 0; +} + +int pidfile_open(const md_config_t *conf) +{ + if (conf->pid_file.empty()) { + return 0; + } + int len = snprintf(pfh.pf_path, sizeof(pfh.pf_path), + "%s", conf->pid_file.c_str()); + + if (len >= (int)sizeof(pfh.pf_path)) { + return -ENAMETOOLONG; + } - pid_file[0] = '\0'; + int fd; + fd = ::open(pfh.pf_path, O_CREAT|O_WRONLY, 0644); + if (fd < 0) { + int err = errno; + derr << "write_pid_file: failed to open pid file '" + << pfh.pf_path << "': " << cpp_strerror(err) << dendl; + pfh.close(); + return -errno; + } + struct stat st; + if (fstat(fd, &st) == -1) { + close(fd); + pfh.close(); + return -errno; + } + pfh.pf_fd = fd; + pfh.pf_dev = st.st_dev; + pfh.pf_ino = st.st_ino; + + struct flock l = { F_WRLCK, SEEK_SET, 0, 0, 0 }; + int r = ::fcntl(pfh.pf_fd, F_SETLK, &l); + if (r < 0) { + derr << "failed to lock pidfile - " << pfh.pf_path << ". is there another process in use?" << dendl; + close(pfh.pf_fd); + pfh.close(); + return -errno; + } return 0; } diff --git a/src/global/pidfile.h b/src/global/pidfile.h index 6b60a5a16b50f..49ea5416ac148 100644 --- a/src/global/pidfile.h +++ b/src/global/pidfile.h @@ -19,10 +19,13 @@ struct md_config_t; // Write a pidfile with the current pid, using the configuration in the // provided conf structure. -int pidfile_write(const md_config_t *conf); +int pidfile_write(); // Remove the pid file that was previously written by pidfile_write. // This is safe to call in a signal handler context. -int pidfile_remove(void); +int pidfile_remove(); + +// test whether the pid_file is being used by another process +int pidfile_open(const md_config_t *conf); #endif -- 2.39.5