From: Jeff Layton <jlayton@redhat.com>
Date: Fri, 17 Apr 2020 13:55:41 +0000 (-0400)
Subject: client: add a new inode release request callback
X-Git-Tag: v16.1.0~2305^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e3b9df78fa42add2dfea1f8bff1e00c832a35697;p=ceph.git

client: add a new inode release request callback

trim_caps() walks the list of caps on the session, and releases
non-auth caps, and attempts to trim dentries until the cache
size is under the max_caps value requested by MDS.

This is fine for FUSE, but doesn't really match the use-case of
nfs-ganesha. Ganesha typically looks up inodes by inode number, not
by dentry. It's quite possible that after a restart, we may have a
ton of outstanding inodes with no dentries associated with them.

Ganesha holds a reference to each inode, so libcephfs can't release
them, and we don't have a way to request that ganesha do so.

Add a new ino_release_callback and finisher. The intent is to allow
libcephfs to "upcall" to the application and request that it release
references to a specific inode.

Signed-off-by: Jeff Layton <jlayton@redhat.com>
---

diff --git a/src/client/Client.cc b/src/client/Client.cc
index c4cae8ae9db..ef99d96feaa 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -270,6 +270,7 @@ Client::Client(Messenger *m, MonClient *mc, Objecter *objecter_)
     async_dentry_invalidator(m->cct),
     interrupt_finisher(m->cct),
     remount_finisher(m->cct),
+    async_ino_releasor(m->cct),
     objecter_finisher(m->cct),
     m_command_hook(this),
     fscid(0)
@@ -589,6 +590,12 @@ void Client::shutdown()
     remount_finisher.stop();
   }
 
+  if (ino_release_cb) {
+    ldout(cct, 10) << "shutdown stopping inode release finisher" << dendl;
+    async_ino_releasor.wait_for_empty();
+    async_ino_releasor.stop();
+  }
+
   objectcacher->stop();  // outside of client_lock! this does a join.
   {
     std::lock_guard l{client_lock};
@@ -4291,6 +4298,39 @@ void Client::_trim_negative_child_dentries(InodeRef& in)
   }
 }
 
+class C_Client_CacheRelease : public Context  {
+private:
+  Client *client;
+  vinodeno_t ino;
+public:
+  C_Client_CacheRelease(Client *c, Inode *in) :
+    client(c) {
+    if (client->use_faked_inos())
+      ino = vinodeno_t(in->faked_ino, CEPH_NOSNAP);
+    else
+      ino = in->vino();
+  }
+  void finish(int r) override {
+    ceph_assert(ceph_mutex_is_not_locked_by_me(client->client_lock));
+    client->_async_inode_release(ino);
+  }
+};
+
+void Client::_async_inode_release(vinodeno_t ino)
+{
+  if (unmounting)
+    return;
+  ldout(cct, 10) << __func__ << " " << ino << dendl;
+  ino_release_cb(callback_handle, ino);
+}
+
+void Client::_schedule_ino_release_callback(Inode *in) {
+
+  if (ino_release_cb)
+    // we queue the invalidate, which calls the callback and decrements the ref
+    async_ino_releasor.queue(new C_Client_CacheRelease(this, in));
+}
+
 void Client::trim_caps(MetaSession *s, uint64_t max)
 {
   mds_rank_t mds = s->mds_num;
@@ -4345,6 +4385,7 @@ void Client::trim_caps(MetaSession *s, uint64_t max)
       if (all && in->ino != MDS_INO_ROOT) {
         ldout(cct, 20) << __func__ << " counting as trimmed: " << *in << dendl;
 	trimmed++;
+	_schedule_ino_release_callback(in.get());
       }
     }
   }
@@ -10506,6 +10547,10 @@ void Client::ll_register_callbacks(struct ceph_client_callback_args *args)
     remount_cb = args->remount_cb;
     remount_finisher.start();
   }
+  if (args->ino_release_cb) {
+    ino_release_cb = args->ino_release_cb;
+    async_ino_releasor.start();
+  }
   if (args->umask_cb)
     umask_cb = args->umask_cb;
 }
diff --git a/src/client/Client.h b/src/client/Client.h
index d0b98b05065..46f3786d590 100644
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -232,6 +232,7 @@ public:
   friend class C_Client_Remount;
   friend class C_Client_RequestInterrupt;
   friend class C_Deleg_Timeout; // Asserts on client_lock, called when a delegation is unreturned
+  friend class C_Client_CacheRelease; // Asserts on client_lock
   friend class SyntheticClient;
   friend void intrusive_ptr_release(Inode *in);
 
@@ -673,6 +674,10 @@ public:
   void _invalidate_inode_cache(Inode *in);
   void _invalidate_inode_cache(Inode *in, int64_t off, int64_t len);
   void _async_invalidate(vinodeno_t ino, int64_t off, int64_t len);
+
+  void _schedule_ino_release_callback(Inode *in);
+  void _async_inode_release(vinodeno_t ino);
+
   bool _release(Inode *in);
 
   /**
@@ -1184,6 +1189,7 @@ private:
   client_ino_callback_t ino_invalidate_cb = nullptr;
   client_dentry_callback_t dentry_invalidate_cb = nullptr;
   client_umask_callback_t umask_cb = nullptr;
+  client_ino_release_t ino_release_cb = nullptr;
   void *callback_handle = nullptr;
   bool can_invalidate_dentries = false;
 
@@ -1191,6 +1197,7 @@ private:
   Finisher async_dentry_invalidator;
   Finisher interrupt_finisher;
   Finisher remount_finisher;
+  Finisher async_ino_releasor;
   Finisher objecter_finisher;
 
   Context *tick_event = nullptr;
diff --git a/src/include/cephfs/ceph_ll_client.h b/src/include/cephfs/ceph_ll_client.h
index c1af46dc010..4f3d4235eb5 100644
--- a/src/include/cephfs/ceph_ll_client.h
+++ b/src/include/cephfs/ceph_ll_client.h
@@ -119,6 +119,9 @@ typedef void (*client_switch_interrupt_callback_t)(void *handle, void *data);
 /* fetch umask of actor */
 typedef mode_t (*client_umask_callback_t)(void *handle);
 
+/* request that application release Inode references */
+typedef void (*client_ino_release_t)(void *handle, vinodeno_t ino);
+
 /*
  * The handle is an opaque value that gets passed to some callbacks. Any fields
  * set to NULL will be left alone. There is no way to unregister callbacks.
@@ -130,6 +133,7 @@ struct ceph_client_callback_args {
   client_switch_interrupt_callback_t switch_intr_cb;
   client_remount_callback_t remount_cb;
   client_umask_callback_t umask_cb;
+  client_ino_release_t ino_release_cb;
 };
 
 #ifdef __cplusplus