From a2fb2535c8bc80cafbefa7cc4a4b553cf98a25fd Mon Sep 17 00:00:00 2001
From: wanghongxu <x.ken@outlook.com>
Date: Wed, 15 Apr 2020 19:52:40 +0800
Subject: [PATCH] bluestore: Support flock retry

When the block changes, systemd-udevd will open the block,
read some information and close it. Then a failure occurs here.
So we need to try again here.

Fixes: https://tracker.ceph.com/issues/46124
Signed-off-by: wanghongxu <x.ken@outlook.com>
(cherry picked from commit 23f8c323d2ba9133c3ef5834ae93115425c862d1)
---
 src/common/legacy_config_opts.h  |  2 ++
 src/common/options.cc            |  8 ++++++++
 src/os/bluestore/KernelDevice.cc | 23 ++++++++++++++++++-----
 3 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index e59913b37bf..a5fc6004b0b 100644
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -897,6 +897,8 @@ OPTION(bdev_nvme_unbind_from_kernel, OPT_BOOL)
 OPTION(bdev_nvme_retry_count, OPT_INT) // -1 means by default which is 4
 OPTION(bdev_enable_discard, OPT_BOOL)
 OPTION(bdev_async_discard, OPT_BOOL)
+OPTION(bdev_flock_retry_interval, OPT_FLOAT)
+OPTION(bdev_flock_retry, OPT_INT)
 
 OPTION(objectstore_blackhole, OPT_BOOL)
 
diff --git a/src/common/options.cc b/src/common/options.cc
index e79be858e95..ffd22addf29 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -3973,6 +3973,14 @@ std::vector<Option> get_global_options() {
     Option("bdev_async_discard", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(false)
     .set_description(""),
+    
+    Option("bdev_flock_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
+    .set_default(0.1)
+    .set_description("interval to retry the flock"),
+    
+    Option("bdev_flock_retry", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(3)
+    .set_description("times to retry the flock"),
 
     Option("bluefs_alloc_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
     .set_default(1_M)
diff --git a/src/os/bluestore/KernelDevice.cc b/src/os/bluestore/KernelDevice.cc
index dd6e667804f..a50a7aa0eb4 100644
--- a/src/os/bluestore/KernelDevice.cc
+++ b/src/os/bluestore/KernelDevice.cc
@@ -74,12 +74,25 @@ KernelDevice::KernelDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, ai
 int KernelDevice::_lock()
 {
   dout(10) << __func__ << " " << fd_directs[WRITE_LIFE_NOT_SET] << dendl;
-  int r = ::flock(fd_directs[WRITE_LIFE_NOT_SET], LOCK_EX | LOCK_NB);
-  if (r < 0) {
-    derr << __func__ << " flock failed on " << path << dendl;
-    return -errno;
+  utime_t sleeptime;
+  sleeptime.set_from_double(cct->_conf->bdev_flock_retry_interval);
+
+  // When the block changes, systemd-udevd will open the block,
+  // read some information and close it. Then a failure occurs here.
+  // So we need to try again here.
+  for (int i = 0; i < cct->_conf->bdev_flock_retry + 1; i++) {
+    int r = ::flock(fd_directs[WRITE_LIFE_NOT_SET], LOCK_EX | LOCK_NB);
+    if (r < 0 && errno == EAGAIN) {
+      dout(1) << __func__ << " flock busy on " << path << dendl;    
+      sleeptime.sleep();
+    } else if (r < 0) {
+      derr << __func__ << " flock failed on " << path << dendl;    
+      break;
+    } else {
+      return 0;
+    }
   }
-  return 0;
+  return -errno;
 }
 
 int KernelDevice::open(const string& p)
-- 
2.47.3