client: add a new inode release request callback

trim_caps() walks the list of caps on the session, and releases
non-auth caps, and attempts to trim dentries until the cache
size is under the max_caps value requested by MDS.

This is fine for FUSE, but doesn't really match the use-case of
nfs-ganesha. Ganesha typically looks up inodes by inode number, not
by dentry. It's quite possible that after a restart, we may have a
ton of outstanding inodes with no dentries associated with them.

Ganesha holds a reference to each inode, so libcephfs can't release
them, and we don't have a way to request that ganesha do so.

Add a new ino_release_callback and finisher. The intent is to allow
libcephfs to "upcall" to the application and request that it release
references to a specific inode.

Signed-off-by: Jeff Layton <jlayton@redhat.com>
(cherry picked from commit e3b9df78fa42add2dfea1f8bff1e00c832a35697)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 5c80d3bdf51d57e717cdc3fdfd0fa6e01cb476d8..3a02b72c2ba2903bcabbc392709a594aef68787a 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -270,6 +270,7 @@ Client::Client(Messenger *m, MonClient *mc, Objecter *objecter_)
     async_dentry_invalidator(m->cct),
     interrupt_finisher(m->cct),
     remount_finisher(m->cct),
+    async_ino_releasor(m->cct),
     objecter_finisher(m->cct),
     m_command_hook(this),
     fscid(0)
@@ -598,6 +599,12 @@ void Client::shutdown()
     remount_finisher.stop();
   }
 
+  if (ino_release_cb) {
+    ldout(cct, 10) << "shutdown stopping inode release finisher" << dendl;
+    async_ino_releasor.wait_for_empty();
+    async_ino_releasor.stop();
+  }
+
   objectcacher->stop();  // outside of client_lock! this does a join.
 
   client_lock.Lock();
@@ -4257,6 +4264,39 @@ void Client::_trim_negative_child_dentries(InodeRef& in)
   }
 }
 
+class C_Client_CacheRelease : public Context  {
+private:
+  Client *client;
+  vinodeno_t ino;
+public:
+  C_Client_CacheRelease(Client *c, Inode *in) :
+    client(c) {
+    if (client->use_faked_inos())
+      ino = vinodeno_t(in->faked_ino, CEPH_NOSNAP);
+    else
+      ino = in->vino();
+  }
+  void finish(int r) override {
+    ceph_assert(ceph_mutex_is_not_locked_by_me(client->client_lock));
+    client->_async_inode_release(ino);
+  }
+};
+
+void Client::_async_inode_release(vinodeno_t ino)
+{
+  if (unmounting)
+    return;
+  ldout(cct, 10) << __func__ << " " << ino << dendl;
+  ino_release_cb(callback_handle, ino);
+}
+
+void Client::_schedule_ino_release_callback(Inode *in) {
+
+  if (ino_release_cb)
+    // we queue the invalidate, which calls the callback and decrements the ref
+    async_ino_releasor.queue(new C_Client_CacheRelease(this, in));
+}
+
 void Client::trim_caps(MetaSession *s, uint64_t max)
 {
   mds_rank_t mds = s->mds_num;
@@ -4311,6 +4351,7 @@ void Client::trim_caps(MetaSession *s, uint64_t max)
       if (all && in->ino != MDS_INO_ROOT) {
         ldout(cct, 20) << __func__ << " counting as trimmed: " << *in << dendl;
        trimmed++;
+       _schedule_ino_release_callback(in.get());
       }
     }
   }
@@ -10407,6 +10448,10 @@ void Client::ll_register_callbacks(struct ceph_client_callback_args *args)
     remount_cb = args->remount_cb;
     remount_finisher.start();
   }
+  if (args->ino_release_cb) {
+    ino_release_cb = args->ino_release_cb;
+    async_ino_releasor.start();
+  }
   if (args->umask_cb)
     umask_cb = args->umask_cb;
 }

diff --git a/src/client/Client.h b/src/client/Client.h
index 66cd5c8d0cec49688dd7448f782343652d1b6215..6e34e4ba1bbb8d000d0b8e95b35e2eb74c3df439 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -233,6 +233,7 @@ public:
   friend class C_Client_Remount;
   friend class C_Client_RequestInterrupt;
   friend class C_Deleg_Timeout; // Asserts on client_lock, called when a delegation is unreturned
+  friend class C_Client_CacheRelease; // Asserts on client_lock
   friend class SyntheticClient;
   friend void intrusive_ptr_release(Inode *in);
 
@@ -672,6 +673,10 @@ public:
   void _invalidate_inode_cache(Inode *in);
   void _invalidate_inode_cache(Inode *in, int64_t off, int64_t len);
   void _async_invalidate(vinodeno_t ino, int64_t off, int64_t len);
+
+  void _schedule_ino_release_callback(Inode *in);
+  void _async_inode_release(vinodeno_t ino);
+
   bool _release(Inode *in);
 
   /**
@@ -1182,6 +1187,7 @@ private:
   client_ino_callback_t ino_invalidate_cb = nullptr;
   client_dentry_callback_t dentry_invalidate_cb = nullptr;
   client_umask_callback_t umask_cb = nullptr;
+  client_ino_release_t ino_release_cb = nullptr;
   void *callback_handle = nullptr;
   bool can_invalidate_dentries = false;
 
@@ -1189,6 +1195,7 @@ private:
   Finisher async_dentry_invalidator;
   Finisher interrupt_finisher;
   Finisher remount_finisher;
+  Finisher async_ino_releasor;
   Finisher objecter_finisher;
 
   Context *tick_event = nullptr;

diff --git a/src/include/cephfs/ceph_ll_client.h b/src/include/cephfs/ceph_ll_client.h
index c1af46dc010f82816d63008827860a58e1e9a1a6..4f3d4235eb59034e64341098699e9a1766a0333c 100644 (file)
--- a/src/include/cephfs/ceph_ll_client.h
+++ b/src/include/cephfs/ceph_ll_client.h
@@ -119,6 +119,9 @@ typedef void (*client_switch_interrupt_callback_t)(void *handle, void *data);
 /* fetch umask of actor */
 typedef mode_t (*client_umask_callback_t)(void *handle);
 
+/* request that application release Inode references */
+typedef void (*client_ino_release_t)(void *handle, vinodeno_t ino);
+
 /*
  * The handle is an opaque value that gets passed to some callbacks. Any fields
  * set to NULL will be left alone. There is no way to unregister callbacks.
@@ -130,6 +133,7 @@ struct ceph_client_callback_args {
   client_switch_interrupt_callback_t switch_intr_cb;
   client_remount_callback_t remount_cb;
   client_umask_callback_t umask_cb;
+  client_ino_release_t ino_release_cb;
 };
 
 #ifdef __cplusplus