mds: return ENOSPC on write ops while osds full

Allow removals and read-only ops, prevent others.  This
is a soft policy aimed at reducing the likelihood of a
"full" (mon_osd_full_ratio) OSD becoming physically full
(where it is unable to accept journal writes from MDS
even though it does not respect the FULL flag)

Signed-off-by: John Spray <john.spray@redhat.com>

diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index 78abb1684652d5bf60c37cf1f5cee6bc49230e27..96c0468de02ccd27df4ab51d34e7b63aa76d0a6d 100644 (file)
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -2359,8 +2359,18 @@ bool MDS::handle_core_message(Message *m)
     break;
   case CEPH_MSG_OSD_MAP:
     ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MON | CEPH_ENTITY_TYPE_OSD);
-    if (is_active() && snapserver)
+
+    if (is_active() && snapserver) {
       snapserver->check_osd_map(true);
+    }
+
+    server->handle_osd_map();
+
+    // By default the objecter only requests OSDMap updates on use,
+    // we would like to always receive the latest maps in order to
+    // apply policy based on the FULL flag.
+    objecter->maybe_request_map();
+
     break;
 
   default:

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index b7b6e70487aacdbb53b3e2d96de062040fe7d21e..1310196e1bda2e305ee62f1dfd9f64bce1e7c4e0 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1299,6 +1299,17 @@ void Server::handle_client_request(MClientRequest *req)
   return;
 }
 
+void Server::handle_osd_map()
+{
+  /* Note that we check the OSDMAP_FULL flag directly rather than
+   * using osdmap_full_flag(), because we want to know "is the flag set"
+   * rather than "does the flag apply to us?" */
+  const OSDMap *osdmap = mds->objecter->get_osdmap_read();
+  is_full = osdmap->test_flag(CEPH_OSDMAP_FULL);
+  dout(7) << __func__ << ": full = " << is_full << " epoch = " << osdmap->get_epoch() << dendl;
+  mds->objecter->put_osdmap_read();
+}
+
 void Server::dispatch_client_request(MDRequestRef& mdr)
 {
   MClientRequest *req = mdr->client_request;
@@ -1323,6 +1334,27 @@ void Server::dispatch_client_request(MDRequestRef& mdr)
     }
   }
   
+  if (is_full) {
+    if (req->get_op() == CEPH_MDS_OP_SETLAYOUT ||
+        req->get_op() == CEPH_MDS_OP_SETDIRLAYOUT ||
+        req->get_op() == CEPH_MDS_OP_SETLAYOUT ||
+        req->get_op() == CEPH_MDS_OP_RMXATTR ||
+        req->get_op() == CEPH_MDS_OP_SETXATTR ||
+        req->get_op() == CEPH_MDS_OP_SETFILELOCK ||
+        req->get_op() == CEPH_MDS_OP_CREATE ||
+        req->get_op() == CEPH_MDS_OP_LINK ||
+        req->get_op() == CEPH_MDS_OP_RENAME ||
+        req->get_op() == CEPH_MDS_OP_SYMLINK ||
+        req->get_op() == CEPH_MDS_OP_MKSNAP) {
+
+      dout(20) << __func__ << ": full, responding ENOSPC to op " << ceph_mds_op_name(req->get_op()) << dendl;
+      respond_to_request(mdr, -ENOSPC);
+      return;
+    } else {
+      dout(20) << __func__ << ": full, permitting op " << ceph_mds_op_name(req->get_op()) << dendl;
+    }
+  }
+
   switch (req->get_op()) {
   case CEPH_MDS_OP_LOOKUPHASH:
   case CEPH_MDS_OP_LOOKUPINO:
@@ -3405,6 +3437,14 @@ void Server::handle_client_setattr(MDRequestRef& mdr)
   inode_t *pi = cur->get_projected_inode();
 
   uint64_t old_size = MAX(pi->size, req->head.args.setattr.old_size);
+
+  // ENOSPC on growing file while full, but allow shrinks
+  if (is_full && req->head.args.setattr.size > old_size) {
+    dout(20) << __func__ << ": full, responding ENOSPC to setattr with larger size" << dendl;
+    respond_to_request(mdr, -ENOSPC);
+    return;
+  }
+
   bool truncating_smaller = false;
   if (mask & CEPH_SETATTR_SIZE) {
     truncating_smaller = req->head.args.setattr.size < old_size;

diff --git a/src/mds/Server.h b/src/mds/Server.h
index 5a90d1394c9f6ffe78b67a9106c2661506438b71..823aa078b298148d89dac89ee4b396d867b91809 100644 (file)
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -50,6 +50,9 @@ private:
   Messenger *messenger;
   PerfCounters *logger;
 
+  // OSDMap full status, used to generate ENOSPC on some operations
+  bool is_full;
+
 public:
   int failed_reconnects;
 
@@ -60,6 +63,7 @@ public:
     mdcache(mds->mdcache), mdlog(mds->mdlog),
     messenger(mds->messenger),
     logger(0),
+    is_full(false),
     failed_reconnects(0),
     terminating_sessions(false) {
   }
@@ -73,6 +77,7 @@ public:
   // message handler
   void dispatch(Message *m);
 
+  void handle_osd_map();
 
   // -- sessions and recovery --
   utime_t  reconnect_start;