FileStore: add fd cache

Signed-off-by: Samuel Just <sam.just@inktank.com>

diff --git a/src/Makefile.am b/src/Makefile.am
index 7a08e1f5a2a83bba3e024475af1066c0da541e60..9525fe63f4f840eae1d7d5f371171963939ba2f1 100644 (file)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1973,6 +1973,7 @@ noinst_HEADERS = \
         os/FileStore.h\
        os/FlatIndex.h\
        os/HashIndex.h\
+       os/FDCache.h\
        os/IndexManager.h\
         os/Journal.h\
         os/JournalingObjectStore.h\

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 8a1da07e036a615feb0f416a259d0d8b154bc575..68c7bef08b35c29f4d4007a0f295f7e992bea8bf 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -514,6 +514,7 @@ OPTION(filestore_merge_threshold, OPT_INT, 10)
 OPTION(filestore_split_multiple, OPT_INT, 2)
 OPTION(filestore_update_to, OPT_INT, 1000)
 OPTION(filestore_blackhole, OPT_BOOL, false)     // drop any new transactions on the floor
+OPTION(filestore_fd_cache_size, OPT_INT, 128)    // FD lru size
 OPTION(filestore_dump_file, OPT_STR, "")         // file onto which store transaction dumps
 OPTION(filestore_kill_at, OPT_INT, 0)            // inject a failure at the n'th opportunity
 OPTION(filestore_inject_stall, OPT_INT, 0)       // artificially stall for N seconds in op queue thread

diff --git a/src/os/FDCache.h b/src/os/FDCache.h
new file mode 100644 (file)
index 0000000..cf07f86
--- /dev/null
+++ b/src/os/FDCache.h
@@ -0,0 +1,95 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_FDCACHE_H
+#define CEPH_FDCACHE_H
+
+#include <memory>
+#include <errno.h>
+#include <cstdio>
+#include "hobject.h"
+#include "common/Mutex.h"
+#include "common/Cond.h"
+#include "common/shared_cache.hpp"
+#include "include/compat.h"
+
+/**
+ * FD Cache
+ */
+class FDCache : public md_config_obs_t {
+  /**
+   * FD
+   *
+   * Wrapper for an fd.  Destructor closes the fd.
+   */
+  class FD {
+  public:
+    const int fd;
+    FD(int _fd) : fd(_fd) {
+      assert(_fd >= 0);
+    }
+    int operator*() const {
+      return fd;
+    }
+    ~FD() {
+      TEMP_FAILURE_RETRY(::close(fd));
+    }
+  };
+
+  SharedLRU<hobject_t, FD> registry;
+  CephContext *cct;
+public:
+  FDCache(CephContext *cct) : cct(cct) {
+    assert(cct);
+    cct->_conf->add_observer(this);
+    registry.set_size(cct->_conf->filestore_fd_cache_size);
+  }
+  ~FDCache() {
+    cct->_conf->remove_observer(this);
+  }
+  typedef std::tr1::shared_ptr<FD> FDRef;
+
+  FDRef lookup(const hobject_t &hoid) {
+    return registry.lookup(hoid);
+  }
+
+  FDRef add(const hobject_t &hoid, int fd) {
+    return registry.add(hoid, new FD(fd));
+  }
+
+  /// clear cached fd for hoid, subsequent lookups will get an empty FD
+  void clear(const hobject_t &hoid) {
+    registry.clear(hoid);
+    assert(!registry.lookup(hoid));
+  }
+
+  /// md_config_obs_t
+  const char** get_tracked_conf_keys() const {
+    static const char* KEYS[] = {
+      "filestore_fd_cache_size",
+      NULL
+    };
+    return KEYS;
+  }
+  void handle_conf_change(const md_config_t *conf,
+                         const std::set<std::string> &changed) {
+    if (changed.count("filestore_fd_cache_size")) {
+      registry.set_size(conf->filestore_fd_cache_size);
+    }
+  }
+
+};
+typedef FDCache::FDRef FDRef;
+
+#endif

diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
index b32f2875f719fb856e079e2a6c8c980e4143eb03..24f7f1eff0806262355759b5504b03f3ad141eee 100644 (file)
--- a/src/os/FileStore.cc
+++ b/src/os/FileStore.cc
@@ -196,10 +196,22 @@ int FileStore::lfn_stat(coll_t cid, const hobject_t& oid, struct stat *buf)
   return r;
 }
 
-int FileStore::lfn_open(coll_t cid, const hobject_t& oid, int flags, mode_t mode,
+int FileStore::lfn_open(coll_t cid,
+                       const hobject_t& oid,
+                       bool create,
+                       FDRef *outfd,
                        IndexedPath *path,
                        Index *index) 
 {
+  assert(outfd);
+  int flags = O_RDWR;
+  if (create)
+    flags |= O_CREAT;
+  Mutex::Locker l(fdcache_lock);
+  *outfd = fdcache.lookup(oid);
+  if (*outfd) {
+    return 0;
+  }
   Index index2;
   IndexedPath path2;
   if (!path)
@@ -224,16 +236,16 @@ int FileStore::lfn_open(coll_t cid, const hobject_t& oid, int flags, mode_t mode
     goto fail;
   }
 
-  r = ::open((*path)->path(), flags, mode);
+  r = ::open((*path)->path(), flags, 0644);
   if (r < 0) {
     r = -errno;
     dout(10) << "error opening file " << (*path)->path() << " with flags="
-            << flags << " and mode=" << mode << ": " << cpp_strerror(-r) << dendl;
+            << flags << ": " << cpp_strerror(-r) << dendl;
     goto fail;
   }
   fd = r;
 
-  if ((flags & O_CREAT) && (!exist)) {
+  if (create && (!exist)) {
     r = (*index)->created(oid, (*path)->path());
     if (r < 0) {
       TEMP_FAILURE_RETRY(::close(fd));
@@ -242,31 +254,16 @@ int FileStore::lfn_open(coll_t cid, const hobject_t& oid, int flags, mode_t mode
       goto fail;
     }
   }
-  return fd;
+  *outfd = fdcache.add(oid, fd);
+  return 0;
 
  fail:
   assert(!m_filestore_fail_eio || r != -EIO);
   return r;
 }
 
-int FileStore::lfn_open(coll_t cid, const hobject_t& oid, int flags, mode_t mode, IndexedPath *path)
-{
-  return lfn_open(cid, oid, flags, mode, path, 0);
-}
-
-int FileStore::lfn_open(coll_t cid, const hobject_t& oid, int flags, mode_t mode)
-{
-  return lfn_open(cid, oid, flags, mode, 0, 0);
-}
-
-int FileStore::lfn_open(coll_t cid, const hobject_t& oid, int flags)
+void FileStore::lfn_close(FDRef fd)
 {
-  return lfn_open(cid, oid, flags, 0);
-}
-
-void FileStore::lfn_close(int fd)
-{
-  TEMP_FAILURE_RETRY(::close(fd));
 }
 
 int FileStore::lfn_link(coll_t c, coll_t cid, const hobject_t& o) 
@@ -324,6 +321,7 @@ int FileStore::lfn_link(coll_t c, coll_t cid, const hobject_t& o)
 int FileStore::lfn_unlink(coll_t cid, const hobject_t& o,
                          const SequencerPosition &spos)
 {
+  Mutex::Locker l(fdcache_lock);
   Index index;
   int r = get_index(cid, &index);
   if (r < 0)
@@ -355,6 +353,7 @@ int FileStore::lfn_unlink(coll_t cid, const hobject_t& o,
       if (g_conf->filestore_debug_inject_read_err) {
        debug_obj_on_delete(o);
       }
+      fdcache.clear(o);
     } else {
       /* Ensure that replay of this op doesn't result in the object_map
        * going away.
@@ -387,6 +386,8 @@ FileStore::FileStore(const std::string &base, const std::string &jdev, const cha
   sync_entry_timeo_lock("sync_entry_timeo_lock"),
   timer(g_ceph_context, sync_entry_timeo_lock),
   stop(false), sync_thread(this),
+  fdcache_lock("fdcache_lock"),
+  fdcache(g_ceph_context),
   default_osr("default"),
   op_queue_len(0), op_queue_bytes(0),
   op_throttle_lock("FileStore::op_throttle_lock"),
@@ -2263,12 +2264,13 @@ int FileStore::_check_replay_guard(coll_t cid, hobject_t oid, const SequencerPos
   if (!replaying || btrfs_stable_commits)
     return 1;
 
-  int fd = lfn_open(cid, oid, 0);
-  if (fd < 0) {
+  FDRef fd;
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
     dout(10) << "_check_replay_guard " << cid << " " << oid << " dne" << dendl;
     return 1;  // if file does not exist, there is no guard, and we can replay.
   }
-  int ret = _check_replay_guard(fd, spos);
+  int ret = _check_replay_guard(**fd, spos);
   lfn_close(fd);
   return ret;
 }
@@ -2762,22 +2764,24 @@ int FileStore::read(
 
   dout(15) << "read " << cid << "/" << oid << " " << offset << "~" << len << dendl;
 
-  int fd = lfn_open(cid, oid, O_RDONLY);
-  if (fd < 0) {
-    dout(10) << "FileStore::read(" << cid << "/" << oid << ") open error: " << cpp_strerror(fd) << dendl;
-    return fd;
+  FDRef fd;
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
+    dout(10) << "FileStore::read(" << cid << "/" << oid << ") open error: "
+            << cpp_strerror(r) << dendl;
+    return r;
   }
 
   if (len == 0) {
     struct stat st;
     memset(&st, 0, sizeof(struct stat));
-    int r = ::fstat(fd, &st);
+    int r = ::fstat(**fd, &st);
     assert(r == 0);
     len = st.st_size;
   }
 
   bufferptr bptr(len);  // prealloc space for entire read
-  got = safe_pread(fd, bptr.c_str(), len, offset);
+  got = safe_pread(**fd, bptr.c_str(), len, offset);
   if (got < 0) {
     dout(10) << "FileStore::read(" << cid << "/" << oid << ") pread error: " << cpp_strerror(got) << dendl;
     lfn_close(fd);
@@ -2815,15 +2819,14 @@ int FileStore::fiemap(coll_t cid, const hobject_t& oid,
 
   dout(15) << "fiemap " << cid << "/" << oid << " " << offset << "~" << len << dendl;
 
-  int r;
-  int fd = lfn_open(cid, oid, O_RDONLY);
-  if (fd < 0) {
-    r = fd;
+  FDRef fd;
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
     dout(10) << "read couldn't open " << cid << "/" << oid << ": " << cpp_strerror(r) << dendl;
   } else {
     uint64_t i;
 
-    r = do_fiemap(fd, offset, len, &fiemap);
+    r = do_fiemap(**fd, offset, len, &fiemap);
     if (r < 0)
       goto done;
 
@@ -2865,10 +2868,10 @@ int FileStore::fiemap(coll_t cid, const hobject_t& oid,
   }
 
 done:
-  if (fd >= 0)
+  if (r >= 0) {
     lfn_close(fd);
-  if (r >= 0)
     ::encode(exomap, bl);
+  }
 
   dout(10) << "fiemap " << cid << "/" << oid << " " << offset << "~" << len << " = " << r << " num_extents=" << exomap.size() << " " << exomap << dendl;
   free(fiemap);
@@ -2899,14 +2902,13 @@ int FileStore::_touch(coll_t cid, const hobject_t& oid)
 {
   dout(15) << "touch " << cid << "/" << oid << dendl;
 
-  int flags = O_WRONLY|O_CREAT;
-  int fd = lfn_open(cid, oid, flags, 0644);
-  int r;
-  if (fd >= 0) {
+  FDRef fd;
+  int r = lfn_open(cid, oid, true, &fd);
+  if (r < 0) {
+    return r;
+  } else {
     lfn_close(fd);
-    r = 0;
-  } else
-    r = fd;
+  }
   dout(10) << "touch " << cid << "/" << oid << " = " << r << dendl;
   return r;
 }
@@ -2920,17 +2922,17 @@ int FileStore::_write(coll_t cid, const hobject_t& oid,
 
   int64_t actual;
 
-  int flags = O_WRONLY|O_CREAT;
-  int fd = lfn_open(cid, oid, flags, 0644);
-  if (fd < 0) {
-    r = fd;
-    dout(0) << "write couldn't open " << cid << "/" << oid << " flags " << flags << ": "
+  FDRef fd;
+  r = lfn_open(cid, oid, true, &fd);
+  if (r < 0) {
+    dout(0) << "write couldn't open " << cid << "/"
+           << oid << ": "
            << cpp_strerror(r) << dendl;
     goto out;
   }
     
   // seek
-  actual = ::lseek64(fd, offset, SEEK_SET);
+  actual = ::lseek64(**fd, offset, SEEK_SET);
   if (actual < 0) {
     r = -errno;
     dout(0) << "write lseek64 to " << offset << " failed: " << cpp_strerror(r) << dendl;
@@ -2945,7 +2947,7 @@ int FileStore::_write(coll_t cid, const hobject_t& oid,
   }
 
   // write
-  r = bl.write_fd(fd);
+  r = bl.write_fd(**fd);
   if (r == 0)
     r = bl.length();
 
@@ -2957,23 +2959,24 @@ int FileStore::_write(coll_t cid, const hobject_t& oid,
     bool async_done = false;
     if (!should_flush ||
        !m_filestore_flusher ||
-       !(async_done = queue_flusher(fd, offset, len, replica))) {
+       !(async_done = queue_flusher(**fd, offset, len, replica))) {
       if (should_flush && m_filestore_sync_flush) {
-       ::sync_file_range(fd, offset, len, SYNC_FILE_RANGE_WRITE);
+       ::sync_file_range(**fd, offset, len, SYNC_FILE_RANGE_WRITE);
        local_flush = true;
       }
     }
+    // TODOSAM: this will be fixed in a subsequent patch
     //Both lfn_close() and possible posix_fadvise() done by flusher
-    if (async_done) fd = -1;
+    //if (async_done) fd = -1;
 #else
     // no sync_file_range; (maybe) flush inline and close.
     if (should_flush && m_filestore_sync_flush) {
-      ::fdatasync(fd);
+      ::fdatasync(**fd);
       local_flush = true;
     }
 #endif
     if (local_flush && replica && m_filestore_replica_fadvise) {
-      int fa_r = posix_fadvise(fd, offset, len, POSIX_FADV_DONTNEED);
+      int fa_r = posix_fadvise(**fd, offset, len, POSIX_FADV_DONTNEED);
       if (fa_r) {
        dout(0) << "posic_fadvise failed: " << cpp_strerror(fa_r) << dendl;
       } else {
@@ -2981,7 +2984,7 @@ int FileStore::_write(coll_t cid, const hobject_t& oid,
       }
     }
   }
-  if (fd >= 0) lfn_close(fd);
+  lfn_close(fd);
 
  out:
   dout(10) << "write " << cid << "/" << oid << " " << offset << "~" << len << " = " << r << dendl;
@@ -2996,14 +2999,14 @@ int FileStore::_zero(coll_t cid, const hobject_t& oid, uint64_t offset, size_t l
 #ifdef CEPH_HAVE_FALLOCATE
 # if !defined(DARWIN) && !defined(__FreeBSD__)
   // first try to punch a hole.
-  int fd = lfn_open(cid, oid, O_RDONLY);
-  if (fd < 0) {
-    ret = -errno;
+  FDRef fd;
+  ret = lfn_open(cid, oid, false, &fd);
+  if (ret < 0) {
     goto out;
   }
 
   // first try fallocate
-  ret = fallocate(fd, FALLOC_FL_PUNCH_HOLE, offset, len);
+  ret = fallocate(**fd, FALLOC_FL_PUNCH_HOLE, offset, len);
   if (ret < 0)
     ret = -errno;
   lfn_close(fd);
@@ -3039,23 +3042,26 @@ int FileStore::_clone(coll_t cid, const hobject_t& oldoid, const hobject_t& newo
   if (_check_replay_guard(cid, newoid, spos) < 0)
     return 0;
 
-  int o, n, r;
+  int r;
+  FDRef o, n;
   {
     Index index;
     IndexedPath from, to;
-    o = lfn_open(cid, oldoid, O_RDONLY, 0, &from, &index);
-    if (o < 0) {
-      r = o;
+    r = lfn_open(cid, oldoid, false, &o, &from, &index);
+    if (r < 0) {
       goto out2;
     }
-    n = lfn_open(cid, newoid, O_CREAT|O_TRUNC|O_WRONLY, 0644, &to, &index);
-    if (n < 0) {
-      r = n;
+    r = lfn_open(cid, newoid, true, &n, &to, &index);
+    if (r < 0) {
+      goto out;
+    }
+    r = ::ftruncate(**n, 0);
+    if (r < 0) {
       goto out;
     }
     struct stat st;
-    ::fstat(o, &st);
-    r = _do_clone_range(o, n, 0, st.st_size, 0);
+    ::fstat(**o, &st);
+    r = _do_clone_range(**o, **n, 0, st.st_size, 0);
     if (r < 0) {
       r = -errno;
       goto out3;
@@ -3068,17 +3074,17 @@ int FileStore::_clone(coll_t cid, const hobject_t& oldoid, const hobject_t& newo
 
   {
     map<string, bufferptr> aset;
-    r = _fgetattrs(o, aset, false);
+    r = _fgetattrs(**o, aset, false);
     if (r < 0)
       goto out3;
 
-    r = _fsetattrs(n, aset);
+    r = _fsetattrs(**n, aset);
     if (r < 0)
       goto out3;
   }
 
   // clone is non-idempotent; record our work.
-  _set_replay_guard(n, spos, &newoid);
+  _set_replay_guard(**n, spos, &newoid);
 
  out3:
   lfn_close(n);
@@ -3248,21 +3254,19 @@ int FileStore::_clone_range(coll_t cid, const hobject_t& oldoid, const hobject_t
     return 0;
 
   int r;
-  int o, n;
-  o = lfn_open(cid, oldoid, O_RDONLY);
-  if (o < 0) {
-    r = o;
+  FDRef o, n;
+  r = lfn_open(cid, oldoid, false, &o);
+  if (r < 0) {
     goto out2;
   }
-  n = lfn_open(cid, newoid, O_CREAT|O_WRONLY, 0644);
-  if (n < 0) {
-    r = n;
+  r = lfn_open(cid, newoid, true, &n);
+  if (r < 0) {
     goto out;
   }
-  r = _do_clone_range(o, n, srcoff, len, dstoff);
+  r = _do_clone_range(**o, **n, srcoff, len, dstoff);
 
   // clone is non-idempotent; record our work.
-  _set_replay_guard(n, spos, &newoid);
+  _set_replay_guard(**n, spos, &newoid);
 
   lfn_close(n);
  out:
@@ -3339,7 +3343,8 @@ void FileStore::flusher_entry()
        } else 
          dout(10) << "flusher_entry JUST closing " << fd << " (stop=" << stop << ", ep=" << ep
                   << ", sync_epoch=" << sync_epoch << ")" << dendl;
-       lfn_close(fd);
+       // TODOSAM: this will be replaced in a subsequent patch
+       //lfn_close(fd);
       }
       lock.Lock();
       flusher_queue_len -= num;   // they're definitely closed, forget
@@ -3856,15 +3861,14 @@ bool FileStore::debug_mdata_eio(const hobject_t &oid) {
 int FileStore::getattr(coll_t cid, const hobject_t& oid, const char *name, bufferptr &bp)
 {
   dout(15) << "getattr " << cid << "/" << oid << " '" << name << "'" << dendl;
-  int r;
-  int fd = lfn_open(cid, oid, 0);
-  if (fd < 0) {
-    r = -errno;
+  FDRef fd;
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
     goto out;
   }
   char n[CHAIN_XATTR_MAX_NAME_LEN];
   get_attrname(name, n, CHAIN_XATTR_MAX_NAME_LEN);
-  r = _fgetattr(fd, n, bp);
+  r = _fgetattr(**fd, n, bp);
   lfn_close(fd);
   if (r == -ENODATA && g_conf->filestore_xattr_use_omap) {
     map<string, bufferlist> got;
@@ -3903,13 +3907,12 @@ int FileStore::getattr(coll_t cid, const hobject_t& oid, const char *name, buffe
 int FileStore::getattrs(coll_t cid, const hobject_t& oid, map<string,bufferptr>& aset, bool user_only) 
 {
   dout(15) << "getattrs " << cid << "/" << oid << dendl;
-  int r;
-  int fd = lfn_open(cid, oid, 0);
-  if (fd < 0) {
-    r = -errno;
+  FDRef fd;
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
     goto out;
   }
-  r = _fgetattrs(fd, aset, user_only);
+  r = _fgetattrs(**fd, aset, user_only);
   lfn_close(fd);
   if (g_conf->filestore_xattr_use_omap) {
     set<string> omap_attrs;
@@ -3967,14 +3970,13 @@ int FileStore::_setattrs(coll_t cid, const hobject_t& oid, map<string,bufferptr>
   set<string> omap_remove;
   map<string, bufferptr> inline_set;
   map<string, bufferptr> inline_to_set;
-  int r = 0;
-  int fd = lfn_open(cid, oid, 0);
-  if (fd < 0) {
-    r = -errno;
+  FDRef fd;
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
     goto out;
   }
   if (g_conf->filestore_xattr_use_omap) {
-    r = _fgetattrs(fd, inline_set, false);
+    r = _fgetattrs(**fd, inline_set, false);
     assert(!m_filestore_fail_eio || r != -EIO);
   }
   dout(15) << "setattrs " << cid << "/" << oid << dendl;
@@ -3988,7 +3990,7 @@ int FileStore::_setattrs(coll_t cid, const hobject_t& oid, map<string,bufferptr>
       if (p->second.length() > g_conf->filestore_max_inline_xattr_size) {
        if (inline_set.count(p->first)) {
          inline_set.erase(p->first);
-         r = chain_fremovexattr(fd, n);
+         r = chain_fremovexattr(**fd, n);
          if (r < 0)
            goto out_close;
        }
@@ -4000,7 +4002,7 @@ int FileStore::_setattrs(coll_t cid, const hobject_t& oid, map<string,bufferptr>
          inline_set.size() >= g_conf->filestore_max_inline_xattrs) {
        if (inline_set.count(p->first)) {
          inline_set.erase(p->first);
-         r = chain_fremovexattr(fd, n);
+         r = chain_fremovexattr(**fd, n);
          if (r < 0)
            goto out_close;
        }
@@ -4015,7 +4017,7 @@ int FileStore::_setattrs(coll_t cid, const hobject_t& oid, map<string,bufferptr>
 
   }
 
-  r = _fsetattrs(fd, inline_to_set);
+  r = _fsetattrs(**fd, inline_to_set);
   if (r < 0)
     goto out_close;
 
@@ -4050,15 +4052,14 @@ int FileStore::_rmattr(coll_t cid, const hobject_t& oid, const char *name,
                       const SequencerPosition &spos)
 {
   dout(15) << "rmattr " << cid << "/" << oid << " '" << name << "'" << dendl;
-  int r = 0;
-  int fd = lfn_open(cid, oid, 0);
-  if (fd < 0) {
-    r = -errno;
+  FDRef fd;
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
     goto out;
   }
   char n[CHAIN_XATTR_MAX_NAME_LEN];
   get_attrname(name, n, CHAIN_XATTR_MAX_NAME_LEN);
-  r = chain_fremovexattr(fd, n);
+  r = chain_fremovexattr(**fd, n);
   if (r == -ENODATA && g_conf->filestore_xattr_use_omap) {
     Index index;
     r = get_index(cid, &index);
@@ -4088,18 +4089,17 @@ int FileStore::_rmattrs(coll_t cid, const hobject_t& oid,
   dout(15) << "rmattrs " << cid << "/" << oid << dendl;
 
   map<string,bufferptr> aset;
-  int r = 0;
-  int fd = lfn_open(cid, oid, 0);
-  if (fd < 0) {
-    r = -errno;
+  FDRef fd;
+  int r = lfn_open(cid, oid, false, &fd);
+  if (r < 0) {
     goto out;
   }
-  r = _fgetattrs(fd, aset, false);
+  r = _fgetattrs(**fd, aset, false);
   if (r >= 0) {
     for (map<string,bufferptr>::iterator p = aset.begin(); p != aset.end(); ++p) {
       char n[CHAIN_XATTR_MAX_NAME_LEN];
       get_attrname(p->first.c_str(), n, CHAIN_XATTR_MAX_NAME_LEN);
-      r = chain_fremovexattr(fd, n);
+      r = chain_fremovexattr(**fd, n);
       if (r < 0)
        break;
     }
@@ -4687,21 +4687,21 @@ int FileStore::_collection_add(coll_t c, coll_t oldcid, const hobject_t& o,
 
   // open guard on object so we don't any previous operations on the
   // new name that will modify the source inode.
-  int fd = lfn_open(oldcid, o, 0);
-  if (fd < 0) {
+  FDRef fd;
+  int r = lfn_open(oldcid, o, 0, &fd);
+  if (r < 0) {
     // the source collection/object does not exist. If we are replaying, we
     // should be safe, so just return 0 and move on.
     assert(replaying);
     dout(10) << "collection_add " << c << "/" << o << " from "
-        << oldcid << "/" << o << " (dne, continue replay) " << dendl;
+            << oldcid << "/" << o << " (dne, continue replay) " << dendl;
     return 0;
   }
-  assert(fd >= 0);
   if (dstcmp > 0) {      // if dstcmp == 0 the guard already says "in-progress"
-    _set_replay_guard(fd, spos, &o, true);
+    _set_replay_guard(**fd, spos, &o, true);
   }
 
-  int r = lfn_link(oldcid, c, o);
+  r = lfn_link(oldcid, c, o);
   if (replaying && !btrfs_stable_commits &&
       r == -EEXIST)    // crashed between link() and set_replay_guard()
     r = 0;
@@ -4710,7 +4710,7 @@ int FileStore::_collection_add(coll_t c, coll_t oldcid, const hobject_t& o,
 
   // close guard on object so we don't do this again
   if (r == 0) {
-    _close_replay_guard(fd, spos);
+    _close_replay_guard(**fd, spos);
   }
   lfn_close(fd);
 

diff --git a/src/os/FileStore.h b/src/os/FileStore.h
index d5ca2a4c237cd6f51061a3fe87c19339f8bab8b9..00249f274c11744e0c261bda07a53fa79b939a56 100644 (file)
--- a/src/os/FileStore.h
+++ b/src/os/FileStore.h
@@ -40,6 +40,7 @@ using namespace __gnu_cxx;
 #include "IndexManager.h"
 #include "ObjectMap.h"
 #include "SequencerPosition.h"
+#include "FDCache.h"
 
 #include "include/uuid.h"
 
@@ -198,6 +199,8 @@ private:
 
   friend ostream& operator<<(ostream& out, const OpSequencer& s);
 
+  Mutex fdcache_lock;
+  FDCache fdcache;
   Sequencer default_osr;
   deque<OpSequencer*> op_queue;
   uint64_t op_queue_len, op_queue_bytes;
@@ -274,13 +277,14 @@ public:
   int lfn_find(coll_t cid, const hobject_t& oid, IndexedPath *path);
   int lfn_truncate(coll_t cid, const hobject_t& oid, off_t length);
   int lfn_stat(coll_t cid, const hobject_t& oid, struct stat *buf);
-  int lfn_open(coll_t cid, const hobject_t& oid, int flags, mode_t mode,
-              IndexedPath *path);
-  int lfn_open(coll_t cid, const hobject_t& oid, int flags, mode_t mode,
-              IndexedPath *path, Index *index);
-  int lfn_open(coll_t cid, const hobject_t& oid, int flags, mode_t mode);
-  int lfn_open(coll_t cid, const hobject_t& oid, int flags);
-  void lfn_close(int fd);
+  int lfn_open(
+    coll_t cid,
+    const hobject_t& oid,
+    bool create,
+    FDRef *outfd,
+    IndexedPath *path = 0,
+    Index *index = 0);
+  void lfn_close(FDRef fd);
   int lfn_link(coll_t c, coll_t cid, const hobject_t& o) ;
   int lfn_unlink(coll_t cid, const hobject_t& o, const SequencerPosition &spos);