filestore: dump open fds when we hit EMFILE

Use a helper to dump /proc/self/fd when we hit EMFILE in the filestore.
Ideally, we should trigger this in other appropriate places, but it is
not immediately clear that there is a sane way to do that.

Fixes: #2330
Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/Makefile.am b/src/Makefile.am
index 82c33f3f245fafeda43563be20d7a9e7a463736a..afeb62ee56967815efc79fbaabcce00ed5030636 100644 (file)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1031,6 +1031,7 @@ libcommon_files = \
        common/ConfUtils.cc \
        common/MemoryModel.cc \
        common/armor.c \
+       common/fd.cc \
        common/xattr.c \
        common/safe_io.c \
        common/snap_types.cc \
@@ -1245,6 +1246,7 @@ noinst_HEADERS = \
        common/debug.h\
        common/dout.h\
        common/escape.h\
+       common/fd.h\
        common/version.h\
        common/hex.h\
        common/entity_name.h\

diff --git a/src/common/fd.cc b/src/common/fd.cc
new file mode 100644 (file)
index 0000000..547e0f8
--- /dev/null
+++ b/src/common/fd.cc
@@ -0,0 +1,57 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2012 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "fd.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <errno.h>
+
+#include "debug.h"
+#include "errno.h"
+
+void dump_open_fds(CephContext *cct)
+{
+  const char *fn = "/proc/self/fd";
+  DIR *d = opendir(fn);
+  if (!d) {
+    lderr(cct) << "dump_open_fds unable to open " << fn << dendl;
+    return;
+  }
+  struct dirent de, *pde = 0;
+
+  int n = 0;
+  while (readdir_r(d, &de, &pde) >= 0) {
+    if (pde == NULL)
+      break;
+    if (de.d_name[0] == '.')
+      continue;
+    char path[PATH_MAX];
+    snprintf(path, sizeof(path), "%s/%s", fn, de.d_name);
+    char target[PATH_MAX];
+    ssize_t r = readlink(path, target, sizeof(target));
+    if (r < 0) {
+      r = -errno;
+      lderr(cct) << "dump_open_fds unable to readlink " << path << ": " << cpp_strerror(r) << dendl;
+      continue;
+    }
+    target[r] = 0;
+    lderr(cct) << "dump_open_fds " << de.d_name << " -> " << target << dendl;
+    n++;
+  }
+  lderr(cct) << "dump_open_fds dumped " << n << " open files" << dendl;
+
+  closedir(d);
+}

diff --git a/src/common/fd.h b/src/common/fd.h
new file mode 100644 (file)
index 0000000..581e1b0
--- /dev/null
+++ b/src/common/fd.h
@@ -0,0 +1,22 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2012 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_COMMON_FD_H
+#define CEPH_COMMON_FD_H
+
+class CephContext;
+
+void dump_open_fds(CephContext *cct);
+
+#endif

diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
index 26c1014faeb89921678607340c28447137d89588..a60ec3632b0fee4973f672a0f6de41797777a805 100644 (file)
--- a/src/os/FileStore.cc
+++ b/src/os/FileStore.cc
@@ -64,6 +64,7 @@
 #include "common/safe_io.h"
 #include "common/perf_counters.h"
 #include "common/sync_filesystem.h"
+#include "common/fd.h"
 #include "HashIndex.h"
 #include "DBObjectMap.h"
 #include "LevelDBStore.h"
@@ -2901,6 +2902,10 @@ unsigned FileStore::_do_transaction(Transaction& t, uint64_t op_seq, int trans_n
        f.flush(*_dout);
        *_dout << dendl;
        assert(0 == "unexpected error");
+
+       if (r == -EMFILE) {
+         dump_open_fds(g_ceph_context);
+       }
       }
     }