From: Kefu Chai <kchai@redhat.com>
Date: Tue, 15 Sep 2020 07:38:36 +0000 (+0800)
Subject: blk/kernel: use open file description lock if available
X-Git-Tag: v16.1.0~978^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=ec7fa396247ecf20e58eeb3c6caa1e5f5f05ae98;p=ceph.git

blk/kernel: use open file description lock if available

* use OFD lock if available. OFD is Linux specific, and only available
  on 3.15 kernels. OFD is able to synchronize both threads and
  processes. and has simpler semantics. this is just a cleanup.
  as we don't create threads for acquiring the flock.
* use BSD flock(2) as a fallback
* return the errno right away, without printing logging messages.
  for two reasons:
  - writing logging messages would reset the errno.
  - the caller of _lock() also prints the logging messages along
    with strerror(errno)

Fixes: https://tracker.ceph.com/issues/46124
Signed-off-by: Kefu Chai <kchai@redhat.com>
---

diff --git a/src/blk/kernel/KernelDevice.cc b/src/blk/kernel/KernelDevice.cc
index f79fae351c1..7684ad60b5f 100644
--- a/src/blk/kernel/KernelDevice.cc
+++ b/src/blk/kernel/KernelDevice.cc
@@ -85,24 +85,36 @@ KernelDevice::KernelDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, ai
 int KernelDevice::_lock()
 {
   dout(10) << __func__ << " " << fd_directs[WRITE_LIFE_NOT_SET] << dendl;
-  double retry_interval = cct->_conf.get_val<double>("bdev_flock_retry_interval");
-  uint64_t max_retry = cct->_conf.get_val<uint64_t>("bdev_flock_retry");
   // When the block changes, systemd-udevd will open the block,
   // read some information and close it. Then a failure occurs here.
   // So we need to try again here.
-  for (uint64_t i = 0; i < max_retry + 1; i++) {
-    int r = ::flock(fd_directs[WRITE_LIFE_NOT_SET], LOCK_EX | LOCK_NB);
-    if (r < 0 && errno == EAGAIN) {
-      dout(1) << __func__ << " flock busy on " << path << dendl;
-      std::this_thread::sleep_for(ceph::make_timespan(retry_interval));
-    } else if (r < 0) {
-      derr << __func__ << " flock failed on " << path << dendl;    
-      break;
-    } else {
+  int fd = fd_directs[WRITE_LIFE_NOT_SET];
+  uint64_t nr_tries = 0;
+  for (;;) {
+    struct flock fl = { F_WRLCK,
+                        SEEK_SET };
+    int r = ::fcntl(fd, F_OFD_SETLK, &fl);
+    if (r < 0) {
+      if (errno == EINVAL) {
+        r = ::flock(fd, LOCK_EX | LOCK_NB);
+      }
+    }
+    if (r == 0) {
       return 0;
     }
+    if (errno != EAGAIN) {
+      return -errno;
+    }
+    dout(1) << __func__ << " flock busy on " << path << dendl;
+    if (const uint64_t max_retry =
+	cct->_conf.get_val<uint64_t>("bdev_flock_retry");
+        nr_tries++ == max_retry) {
+      return -EAGAIN;
+    }
+    double retry_interval =
+      cct->_conf.get_val<double>("bdev_flock_retry_interval");
+    std::this_thread::sleep_for(ceph::make_timespan(retry_interval));
   }
-  return -errno;
 }
 
 int KernelDevice::open(const string& p)