client: add new delegation testcases

Test basic acquire/break functionality from both other clients
and the same client, for different conflicting opens, as well as
changes to the namespace.

Then test delegation timeout behavior. Open file, take delegation in
main thread. Spawn another thread to open the file again, breaking
delegation. Have main thread ignore it, and wait for the spawned thread
to be joined. Once it is, ensure that subsequent access of the cmount
returns -ENOTCONN.

Signed-off-by: Jeff Layton <jlayton@redhat.com>
(cherry picked from commit d9c6a9129eb65031eb1fc6f769cfe59bf3bb1cff)

diff --git a/src/test/libcephfs/CMakeLists.txt b/src/test/libcephfs/CMakeLists.txt
index 2e42c03f0685e0238c0ef8396cca5e8ccbbaebfc..5d3f963484e250248f7b0499de835bfdfca1e28d 100644 (file)
--- a/src/test/libcephfs/CMakeLists.txt
+++ b/src/test/libcephfs/CMakeLists.txt
@@ -8,6 +8,7 @@ if(${WITH_CEPHFS})
     recordlock.cc
     acl.cc
     main.cc
+    deleg.cc
   )
   set_target_properties(ceph_test_libcephfs PROPERTIES COMPILE_FLAGS
     ${UNITTEST_CXX_FLAGS})

diff --git a/src/test/libcephfs/deleg.cc b/src/test/libcephfs/deleg.cc
new file mode 100644 (file)
index 0000000..46b5238
--- /dev/null
+++ b/src/test/libcephfs/deleg.cc
@@ -0,0 +1,304 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Tests for Ceph delegation handling
+ *
+ * (c) 2017, Jeff Layton <jlayton@redhat.com>
+ */
+
+#include "gtest/gtest.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/stat.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <sys/xattr.h>
+#include <sys/uio.h>
+
+#ifdef __linux__
+#include <limits.h>
+#endif
+
+#include <map>
+#include <vector>
+#include <thread>
+#include <atomic>
+
+static int set_default_deleg_timeout(struct ceph_mount_info *cmount)
+{
+  uint32_t session_timeout = ceph_get_cap_return_timeout(cmount);
+  return ceph_set_deleg_timeout(cmount, session_timeout - 1);
+}
+
+static void dummy_deleg_cb(Fh *fh, void *priv)
+{
+  std::atomic_bool *recalled = (std::atomic_bool *)priv;
+  recalled->store(true);
+}
+
+static void open_breaker_func(struct ceph_mount_info *cmount, const char *filename, int flags, std::atomic_bool *opened)
+{
+  bool do_shutdown = false;
+
+  if (!cmount) {
+    ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+    ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+    ASSERT_EQ(ceph_conf_parse_env(cmount, NULL), 0);
+    ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+    ASSERT_EQ(set_default_deleg_timeout(cmount), 0);
+    do_shutdown = true;
+  }
+
+  Inode *root, *file;
+  ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
+
+  Fh *fh;
+  struct ceph_statx stx;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  ASSERT_EQ(ceph_ll_lookup(cmount, root, filename, &file, &stx, 0, 0, perms), 0);
+  int ret;
+  for (;;) {
+    ret = ceph_ll_open(cmount, file, flags, &fh, perms);
+    if (ret != -EAGAIN)
+      break;
+    usleep(1000);
+  }
+  ASSERT_EQ(ret, 0);
+  opened->store(true);
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+
+  if (do_shutdown)
+    ceph_shutdown(cmount);
+}
+
+enum {
+  DelegTestLink,
+  DelegTestRename,
+  DelegTestUnlink
+};
+
+static void namespace_breaker_func(struct ceph_mount_info *cmount, int cmd, const char *oldname, const char *newname)
+{
+  bool do_shutdown = false;
+
+  if (!cmount) {
+    ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+    ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+    ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+    ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+    ASSERT_EQ(set_default_deleg_timeout(cmount), 0);
+    do_shutdown = true;
+  }
+
+  Inode *root, *file = nullptr;
+  ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
+
+  struct ceph_statx stx;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  int ret;
+  for (;;) {
+    switch (cmd) {
+    case DelegTestRename:
+      ret = ceph_ll_rename(cmount, root, oldname, root, newname, perms);
+      break;
+    case DelegTestLink:
+      if (!file) {
+       ASSERT_EQ(ceph_ll_lookup(cmount, root, oldname, &file, &stx, 0, 0, perms), 0);
+      }
+      ret = ceph_ll_link(cmount, file, root, newname, perms);
+      break;
+    case DelegTestUnlink:
+      ret = ceph_ll_unlink(cmount, root, oldname, perms);
+      break;
+    default:
+      // Bad command
+      assert(false);
+    }
+    if (ret != -EAGAIN)
+      break;
+    usleep(1000);
+  }
+  ASSERT_EQ(ret, 0);
+
+  if (do_shutdown)
+    ceph_shutdown(cmount);
+}
+
+static void simple_deleg_test(struct ceph_mount_info *cmount, struct ceph_mount_info *tcmount)
+{
+  Inode *root, *file;
+
+  ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
+
+  char filename[32];
+
+  Fh *fh;
+  struct ceph_statx stx;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  std::atomic_bool recalled(false);
+  std::atomic_bool opened(false);
+
+  // ensure r/w open breaks a r/w delegation
+  sprintf(filename, "deleg.rwrw.%x", getpid());
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+                   O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
+  std::thread breaker1(open_breaker_func, tcmount, filename, O_RDWR, &opened);
+  while (!recalled.load())
+    usleep(1000);
+  ASSERT_EQ(opened.load(), false);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
+  breaker1.join();
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+  ASSERT_EQ(ceph_ll_unlink(cmount, root, filename, perms), 0);
+
+  // ensure r/o open breaks a r/w delegation
+  recalled.store(false);
+  opened.store(false);
+  sprintf(filename, "deleg.rorw.%x", getpid());
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+                   O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
+  std::thread breaker2(open_breaker_func, tcmount, filename, O_RDONLY, &opened);
+  while (!recalled.load())
+    usleep(1000);
+  ASSERT_EQ(opened.load(), false);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
+  breaker2.join();
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+  ASSERT_EQ(ceph_ll_unlink(cmount, root, filename, perms), 0);
+
+  // ensure r/o open does not break a r/o delegation
+  sprintf(filename, "deleg.rwro.%x", getpid());
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+                   O_RDONLY|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+  recalled.store(false);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_RD, dummy_deleg_cb, &recalled), 0);
+  std::thread breaker3(open_breaker_func, tcmount, filename, O_RDONLY, &opened);
+  breaker3.join();
+  ASSERT_EQ(recalled.load(), false);
+
+  // ensure that r/w open breaks r/o delegation
+  opened.store(false);
+  std::thread breaker4(open_breaker_func, tcmount, filename, O_WRONLY, &opened);
+  while (!recalled.load())
+    usleep(1000);
+  usleep(1000);
+  ASSERT_EQ(opened.load(), false);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
+  breaker4.join();
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+  ASSERT_EQ(ceph_ll_unlink(cmount, root, filename, perms), 0);
+
+  // ensure hardlinking breaks a r/w delegation
+  recalled.store(false);
+  char newname[32];
+  sprintf(filename, "deleg.old.%x", getpid());
+  sprintf(newname, "deleg.new.%x", getpid());
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+                   O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
+  std::thread breaker5(namespace_breaker_func, tcmount, DelegTestLink, filename, newname);
+  while (!recalled.load())
+    usleep(1000);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
+  breaker5.join();
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+  ASSERT_EQ(ceph_ll_unlink(cmount, root, filename, perms), 0);
+  ASSERT_EQ(ceph_ll_unlink(cmount, root, newname, perms), 0);
+
+  // ensure renaming breaks a r/w delegation
+  recalled.store(false);
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+                   O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
+  std::thread breaker6(namespace_breaker_func, tcmount, DelegTestRename, filename, newname);
+  while (!recalled.load())
+    usleep(1000);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
+  breaker6.join();
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+  ASSERT_EQ(ceph_ll_unlink(cmount, root, newname, perms), 0);
+
+  // ensure unlinking breaks a r/w delegation
+  recalled.store(false);
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+                   O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
+  std::thread breaker7(namespace_breaker_func, tcmount, DelegTestUnlink, filename, nullptr);
+  while (!recalled.load())
+    usleep(1000);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
+  breaker7.join();
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+}
+
+TEST(LibCephFS, DelegMultiClient) {
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+  ASSERT_EQ(set_default_deleg_timeout(cmount), 0);
+
+  simple_deleg_test(cmount, nullptr);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, DelegSingleClient) {
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+  ASSERT_EQ(set_default_deleg_timeout(cmount), 0);
+
+  simple_deleg_test(cmount, cmount);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, DelegTimeout) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+  // tweak timeout to run quickly, since we don't plan to return it anyway
+  ASSERT_EQ(ceph_set_deleg_timeout(cmount, 2), 0);
+
+  Inode *root, *file;
+  ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
+
+  char filename[32];
+  sprintf(filename, "delegtimeo%x", getpid());
+
+  Fh *fh;
+  struct ceph_statx stx;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+                   O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+
+  /* Reopen read-only */
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+  ASSERT_EQ(ceph_ll_open(cmount, file, O_RDONLY, &fh, perms), 0);
+
+  std::atomic_bool recalled(false);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_RD, dummy_deleg_cb, &recalled), 0);
+  std::atomic_bool opened(false);
+  std::thread breaker1(open_breaker_func, nullptr, filename, O_RDWR, &opened);
+  breaker1.join();
+  ASSERT_EQ(recalled.load(), true);
+  ASSERT_EQ(ceph_ll_getattr(cmount, root, &stx, 0, 0, perms), -ENOTCONN);
+  ceph_release(cmount);
+}