From d4cb49d5c91d972cb803b9f316855ad6093efd17 Mon Sep 17 00:00:00 2001 From: Neeraj Pratap Singh Date: Wed, 8 Dec 2021 14:12:59 +0530 Subject: [PATCH] client: Inode::hold_caps_until is time from monotonic clock now. Inode::hold_caps_until storing time from ceph::coarse_mono_clock now. Fixes: https://tracker.ceph.com/issues/52982 Signed-off-by: Neeraj Pratap Singh (cherry picked from commit 983b10506dc8466a0e47ff0d320d480dd09999ec) --- src/client/Client.cc | 35 ++++++++++++++++++++------- src/client/Client.h | 6 ++--- src/client/Inode.h | 2 +- src/client/MetaRequest.h | 1 + src/common/options/mds-client.yaml.in | 6 ++--- src/librados/RadosClient.cc | 2 +- src/mon/MonClient.cc | 2 +- src/mon/MonClient.h | 2 +- src/neorados/RADOSImpl.cc | 2 +- src/tools/cephfs_mirror/Mirror.cc | 2 +- 10 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src/client/Client.cc b/src/client/Client.cc index 33253ed10c0d1..a008a2daeaf75 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -357,6 +357,12 @@ Client::Client(Messenger *m, MonClient *mc, Objecter *objecter_) _collect_and_send_global_metrics = cct->_conf.get_val( "client_collect_and_send_global_metrics"); + mount_timeout = cct->_conf.get_val( + "client_mount_timeout"); + + caps_release_delay = cct->_conf.get_val( + "client_caps_release_delay"); + if (cct->_conf->client_acl_type == "posix_acl") acl_type = POSIX_ACL; @@ -1791,6 +1797,7 @@ int Client::make_request(MetaRequest *request, // and timestamp request->op_stamp = ceph_clock_now(); + request->created = ceph::coarse_mono_clock::now(); // make note mds_requests[tid] = request->get(); @@ -3542,8 +3549,8 @@ int Client::get_caps_used(Inode *in) void Client::cap_delay_requeue(Inode *in) { ldout(cct, 10) << __func__ << " on " << *in << dendl; - in->hold_caps_until = ceph_clock_now(); - in->hold_caps_until += cct->_conf->client_caps_release_delay; + + in->hold_caps_until = ceph::coarse_mono_clock::now() + caps_release_delay; delayed_list.push_back(&in->delay_cap_item); } @@ -5937,7 +5944,7 @@ int Client::authenticate() } client_lock.unlock(); - int r = monclient->authenticate(cct->_conf->client_mount_timeout); + int r = monclient->authenticate(std::chrono::duration(mount_timeout).count()); client_lock.lock(); if (r < 0) { return r; @@ -6551,8 +6558,8 @@ void Client::renew_and_flush_cap_releases() if (!mount_aborted && mdsmap->get_epoch()) { // renew caps? - utime_t el = ceph_clock_now() - last_cap_renew; - if (unlikely(el > mdsmap->get_session_timeout() / 3.0)) + auto el = ceph::coarse_mono_clock::now() - last_cap_renew; + if (unlikely(utime_t(el) > mdsmap->get_session_timeout() / 3.0)) renew_caps(); flush_cap_releases(); @@ -6563,7 +6570,7 @@ void Client::tick() { ldout(cct, 20) << "tick" << dendl; - utime_t now = ceph_clock_now(); + auto now = ceph::coarse_mono_clock::now(); /* * If the mount() is not finished @@ -6571,7 +6578,7 @@ void Client::tick() if (is_mounting() && !mds_requests.empty()) { MetaRequest *req = mds_requests.begin()->second; - if (req->op_stamp + cct->_conf->client_mount_timeout < now) { + if (req->created + mount_timeout < now) { req->abort(-CEPHFS_ETIMEDOUT); if (req->caller_cond) { req->kick = true; @@ -6605,7 +6612,7 @@ void Client::tick() trim_cache(true); if (blocklisted && (is_mounted() || is_unmounting()) && - last_auto_reconnect + 30 * 60 < now && + last_auto_reconnect + std::chrono::seconds(30 * 60) < now && cct->_conf.get_val("client_reconnect_stale")) { messenger->client_reset(); fd_gen++; // invalidate open files @@ -6757,7 +6764,7 @@ void Client::collect_and_send_global_metrics() { void Client::renew_caps() { ldout(cct, 10) << "renew_caps()" << dendl; - last_cap_renew = ceph_clock_now(); + last_cap_renew = ceph::coarse_mono_clock::now(); for (auto &p : mds_sessions) { ldout(cct, 15) << "renew_caps requesting from mds." << p.first << dendl; @@ -15715,6 +15722,8 @@ const char** Client::get_tracked_conf_keys() const "client_oc_max_dirty", "client_oc_target_dirty", "client_oc_max_dirty_age", + "client_caps_release_delay", + "client_mount_timeout", NULL }; return keys; @@ -15752,6 +15761,14 @@ void Client::handle_conf_change(const ConfigProxy& conf, _collect_and_send_global_metrics = cct->_conf.get_val( "client_collect_and_send_global_metrics"); } + if (changed.count("client_caps_release_delay")) { + caps_release_delay = cct->_conf.get_val( + "client_caps_release_delay"); + } + if (changed.count("client_mount_timeout")) { + mount_timeout = cct->_conf.get_val( + "client_mount_timeout"); + } } void intrusive_ptr_add_ref(Inode *in) diff --git a/src/client/Client.h b/src/client/Client.h index 84fc15d3fe879..858b4d825e907 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -1475,7 +1475,7 @@ private: Finisher async_ino_releasor; Finisher objecter_finisher; - utime_t last_cap_renew; + ceph::coarse_mono_time last_cap_renew; CommandHook m_command_hook; @@ -1536,8 +1536,8 @@ private: ceph::unordered_map snap_realms; std::map metadata; - utime_t last_auto_reconnect; - + ceph::coarse_mono_time last_auto_reconnect; + std::chrono::seconds caps_release_delay, mount_timeout; // trace generation std::ofstream traceout; diff --git a/src/client/Inode.h b/src/client/Inode.h index eddfb6da9a69a..665f4b76af88c 100644 --- a/src/client/Inode.h +++ b/src/client/Inode.h @@ -115,6 +115,7 @@ struct CapSnap { #define I_ERROR_FILELOCK (1 << 5) struct Inode : RefCountedObject { + ceph::coarse_mono_time hold_caps_until; Client *client; // -- the actual inode -- @@ -211,7 +212,6 @@ struct Inode : RefCountedObject { int cache_gen = 0; int snap_caps = 0; int snap_cap_refs = 0; - utime_t hold_caps_until; xlist::item delay_cap_item, dirty_cap_item, flushing_cap_item; SnapRealm *snaprealm = 0; diff --git a/src/client/MetaRequest.h b/src/client/MetaRequest.h index 630f5ee15bc87..ec05a38335d75 100644 --- a/src/client/MetaRequest.h +++ b/src/client/MetaRequest.h @@ -25,6 +25,7 @@ private: Dentry *_old_dentry = NULL; //associated with path2 int abort_rc = 0; public: + ceph::coarse_mono_time created = ceph::coarse_mono_clock::zero(); uint64_t tid = 0; utime_t op_stamp; ceph_mds_request_head head; diff --git a/src/common/options/mds-client.yaml.in b/src/common/options/mds-client.yaml.in index 1881130f8316f..eb92f1b21abf9 100644 --- a/src/common/options/mds-client.yaml.in +++ b/src/common/options/mds-client.yaml.in @@ -31,14 +31,13 @@ options: - mds_client with_legacy: true - name: client_mount_timeout - type: float + type: secs level: advanced desc: timeout for mounting CephFS (seconds) fmt_desc: Set the timeout for CephFS mount in seconds. default: 5_min services: - mds_client - with_legacy: true - name: client_tick_interval type: secs level: dev @@ -144,7 +143,7 @@ options: - mds_client with_legacy: true - name: client_caps_release_delay - type: int + type: secs level: dev default: 5 services: @@ -153,7 +152,6 @@ options: sets how many seconds a client waits to release capabilities that it no longer needs in case the capabilities are needed for another user space operation. - with_legacy: true - name: client_quota_df type: bool level: advanced diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc index 3d4563c8bf1e6..43914a89c2c18 100644 --- a/src/librados/RadosClient.cc +++ b/src/librados/RadosClient.cc @@ -283,7 +283,7 @@ int librados::RadosClient::connect() goto out; } - err = monclient.authenticate(conf->client_mount_timeout); + err = monclient.authenticate(std::chrono::duration(conf.get_val("client_mount_timeout")).count()); if (err) { ldout(cct, 0) << conf->name << " authentication error " << cpp_strerror(-err) << dendl; shutdown(); diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc index 9ffa7d367df98..38ad26f61bdcf 100644 --- a/src/mon/MonClient.cc +++ b/src/mon/MonClient.cc @@ -154,7 +154,7 @@ int MonClient::get_monmap_and_config() if (r < 0) { return r; } - r = authenticate(cct->_conf->client_mount_timeout); + r = authenticate(std::chrono::duration(cct->_conf.get_val("client_mount_timeout")).count()); if (r == -ETIMEDOUT) { shutdown(); continue; diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h index 19aa047c2181f..de6bba574ff1a 100644 --- a/src/mon/MonClient.h +++ b/src/mon/MonClient.h @@ -158,7 +158,7 @@ struct MonClientPinger : public Dispatcher, int wait_for_reply(double timeout = 0.0) { std::unique_lock locker{lock}; if (timeout <= 0) { - timeout = cct->_conf->client_mount_timeout; + timeout = std::chrono::duration(cct->_conf.get_val("client_mount_timeout")).count(); } done = false; if (ping_recvd_cond.wait_for(locker, diff --git a/src/neorados/RADOSImpl.cc b/src/neorados/RADOSImpl.cc index bddd62c2f8ba1..6c9c210a85cde 100644 --- a/src/neorados/RADOSImpl.cc +++ b/src/neorados/RADOSImpl.cc @@ -59,7 +59,7 @@ RADOS::RADOS(boost::asio::io_context& ioctx, if (err) { throw boost::system::system_error(ceph::to_error_code(err)); } - err = monclient.authenticate(cct->_conf->client_mount_timeout); + err = monclient.authenticate(std::chrono::duration(cct->_conf.get_val("client_mount_timeout")).count()); if (err) { throw boost::system::system_error(ceph::to_error_code(err)); } diff --git a/src/tools/cephfs_mirror/Mirror.cc b/src/tools/cephfs_mirror/Mirror.cc index c20a1fc659b3a..890805764c048 100644 --- a/src/tools/cephfs_mirror/Mirror.cc +++ b/src/tools/cephfs_mirror/Mirror.cc @@ -238,7 +238,7 @@ int Mirror::init_mon_client() { return r; } - r = m_monc->authenticate(m_cct->_conf->client_mount_timeout); + r = m_monc->authenticate(std::chrono::duration(m_cct->_conf.get_val("client_mount_timeout")).count()); if (r < 0) { derr << ": failed to authenticate to monitor: " << cpp_strerror(r) << dendl; return r; -- 2.39.5