librbd: is_exclusive_lock_owner API should ping OSD

This is required to detect if a peer has been silently blacklisted
and is therefore no longer the lock owner.

Fixes: http://tracker.ceph.com/issues/19287
Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/ManagedLock.cc b/src/librbd/ManagedLock.cc
index 3661e2fbdb95b03db1f101f8551a379353a19dfa..71f5dab4df6508bbc7a80c9993ee17f701fa5190 100644 (file)
--- a/src/librbd/ManagedLock.cc
+++ b/src/librbd/ManagedLock.cc
@@ -266,6 +266,42 @@ void ManagedLock<I>::break_lock(const managed_lock::Locker &locker,
   on_finish->complete(r);
 }
 
+template <typename I>
+int ManagedLock<I>::assert_header_locked() {
+  ldout(m_cct, 10) << dendl;
+
+  librados::ObjectReadOperation op;
+  {
+    Mutex::Locker locker(m_lock);
+    rados::cls::lock::assert_locked(&op, RBD_LOCK_NAME,
+                                    (m_mode == EXCLUSIVE ? LOCK_EXCLUSIVE :
+                                                           LOCK_SHARED),
+                                    m_cookie,
+                                    managed_lock::util::get_watcher_lock_tag());
+  }
+
+  int r = m_ioctx.operate(m_oid, &op, nullptr);
+  if (r < 0) {
+    if (r == -EBLACKLISTED) {
+      ldout(m_cct, 5) << "client is not lock owner -- client blacklisted"
+                      << dendl;
+    } else if (r == -ENOENT) {
+      ldout(m_cct, 5) << "client is not lock owner -- no lock detected"
+                      << dendl;
+    } else if (r == -EBUSY) {
+      ldout(m_cct, 5) << "client is not lock owner -- owned by different client"
+                      << dendl;
+    } else {
+      lderr(m_cct) << "failed to verify lock ownership: " << cpp_strerror(r)
+                   << dendl;
+    }
+
+    return r;
+  }
+
+  return 0;
+}
+
 template <typename I>
 void ManagedLock<I>::shutdown_handler(int r, Context *on_finish) {
   on_finish->complete(r);

diff --git a/src/librbd/ManagedLock.h b/src/librbd/ManagedLock.h
index 1684e71a9d9e50ced666812a88d8ae26c95e058f..d18cfbef15602cf418b8be6ddadf5de86388d1b7 100644 (file)
--- a/src/librbd/ManagedLock.h
+++ b/src/librbd/ManagedLock.h
@@ -60,6 +60,8 @@ public:
   void break_lock(const managed_lock::Locker &locker, bool force_break_lock,
                   Context *on_finish);
 
+  int assert_header_locked();
+
   bool is_shutdown() const {
     Mutex::Locker l(m_lock);
     return is_state_shutdown();

diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc
index 3432ae71c4579c3049708f34f6031951cdd46559..65e6a226350433b772fe3388a4f2ca9b37c6d806 100644 (file)
--- a/src/librbd/internal.cc
+++ b/src/librbd/internal.cc
@@ -1289,9 +1289,22 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) {
 
   int is_exclusive_lock_owner(ImageCtx *ictx, bool *is_owner)
   {
-    RWLock::RLocker l(ictx->owner_lock);
-    *is_owner = (ictx->exclusive_lock != nullptr &&
-                ictx->exclusive_lock->is_lock_owner());
+    *is_owner = false;
+
+    RWLock::RLocker owner_locker(ictx->owner_lock);
+    if (ictx->exclusive_lock == nullptr ||
+        !ictx->exclusive_lock->is_lock_owner()) {
+      return 0;
+    }
+
+    // might have been blacklisted by peer -- ensure we still own
+    // the lock by pinging the OSD
+    int r = ictx->exclusive_lock->assert_header_locked();
+    if (r < 0) {
+      return r;
+    }
+
+    *is_owner = true;
     return 0;
   }
 

diff --git a/src/test/pybind/test_rbd.py b/src/test/pybind/test_rbd.py
index e6d4c65915394214908935ac8e2012ae83cf84a0..a0368b64ca6f29da2f61b3942a475bd1ededb0dd 100644 (file)
--- a/src/test/pybind/test_rbd.py
+++ b/src/test/pybind/test_rbd.py
@@ -1281,6 +1281,9 @@ class TestExclusiveLock(object):
                     image.lock_break(RBD_LOCK_MODE_EXCLUSIVE,
                                      lock_owners[0]['owner'])
 
+                    assert_raises(ConnectionShutdown,
+                                  blacklist_image.is_exclusive_lock_owner)
+
                     blacklist_rados.wait_for_latest_osdmap()
                     data = rand_data(256)
                     assert_raises(ConnectionShutdown,