From 5f7aaf074e3d0372f8d0fb5a1b721f440a9c028f Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Tue, 30 Apr 2019 17:43:01 +0200 Subject: [PATCH] global/global_init: do first transport connection after setuid() uverbs kernel module forbids access to a file descriptor after credentials change, that leads to -EACCESS on each following ibv_*() call. Why it matters? Infiniband transport stops working after the following syscalls: o setuid() o fork() Originally the problem was described here [1] and here [2]. This patch targets only setuid() syscall and moves the first transport initialization after setuid() has been done. fork() is used to daemonize ceph services (when systemd is not used for any reason) and probably the easiest way is to rip the whole lagacy daemonization code out, so this patch does not target this problem. [1] https://tracker.ceph.com/issues/39238 [2] https://www.spinics.net/lists/ceph-devel/msg45083.html Signed-off-by: Roman Penyaev (cherry picked from commit 02c0a20a09fc8f678d0f9caf7f580b3dbc1f0f91) --- src/global/global_init.cc | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/global/global_init.cc b/src/global/global_init.cc index 8f6a60f54ebe4..e05386583e413 100644 --- a/src/global/global_init.cc +++ b/src/global/global_init.cc @@ -150,18 +150,6 @@ void global_pre_init( cct->_log->start(); } - if (!conf->no_mon_config) { - // make sure our mini-session gets legacy values - conf.apply_changes(nullptr); - - MonClient mc_bootstrap(g_ceph_context); - if (mc_bootstrap.get_monmap_and_config() < 0) { - cct->_log->flush(); - cerr << "failed to fetch mon config (--no-mon-config to skip)" - << std::endl; - _exit(1); - } - } if (!cct->_log->is_started()) { cct->_log->start(); } @@ -334,6 +322,28 @@ global_init(const std::map *defaults, # endif #endif + // + // Utterly important to run first network connection after setuid(). + // In case of rdma transport uverbs kernel module starts returning + // -EACCESS on each operation if credentials has been changed, see + // callers of ib_safe_file_access() for details. + // + // fork() syscall also matters, so daemonization won't work in case + // of rdma. + // + if (!g_conf()->no_mon_config) { + // make sure our mini-session gets legacy values + g_conf().apply_changes(nullptr); + + MonClient mc_bootstrap(g_ceph_context); + if (mc_bootstrap.get_monmap_and_config() < 0) { + g_ceph_context->_log->flush(); + cerr << "failed to fetch mon config (--no-mon-config to skip)" + << std::endl; + _exit(1); + } + } + // Expand metavariables. Invoke configuration observers. Open log file. g_conf().apply_changes(nullptr); -- 2.39.5