mds: track free prealloc_inos and delegated_inos separately

avoid iterating session->info.prealloc_inos to find ino that isn't in
session->delegated_inos.

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/include/interval_set.h b/src/include/interval_set.h
index 3ade9acae1171d814c544e9a7066e853f7ae00a7..f1a21e5f96e1ae290e077386b16692bdf80e038a 100644 (file)
--- a/src/include/interval_set.h
+++ b/src/include/interval_set.h
@@ -519,7 +519,7 @@ class interval_set {
     std::swap(_size, other._size);
   }    
   
-  void erase(iterator &i) {
+  void erase(const iterator &i) {
     _size -= i.get_len();
     m.erase(i._iter);
   }

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 1244fdd2236db3b3c72ecccb139adb93f8356cb1..3a9d1cf28eae3a628798bb66718a078c79c45212 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -1685,6 +1685,8 @@ void MDSRank::validate_sessions()
   // Mitigate bugs like: http://tracker.ceph.com/issues/16842
   for (const auto &i : sessionmap.get_sessions()) {
     Session *session = i.second;
+    ceph_assert(session->info.prealloc_inos == session->free_prealloc_inos);
+
     interval_set<inodeno_t> badones;
     if (inotable->intersects_free(session->info.prealloc_inos, &badones)) {
       clog->error() << "client " << *session

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 232bb6b3d537bbec3fec3fb8cb85ecdd6239d254..19c5d1f8a45a69a2e483dc8a12636a64a14833da 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -811,21 +811,25 @@ void Server::_session_logged(Session *session, uint64_t state_seq, bool open, ve
           << " " << (open ? "open":"close") << " " << pv
           << " inos_to_free " << inos_to_free << " inotablev " << piv
           << " inos_to_purge " << inos_to_purge << dendl;
-  
-  if (inos_to_purge.size()){
-    ceph_assert(ls);
-    session->info.prealloc_inos.subtract(inos_to_purge);
-    ls->purging_inodes.insert(inos_to_purge);
-    mdcache->purge_inodes(inos_to_purge, ls);
-  }
-  
-  if (piv) {
-    ceph_assert(session->is_closing() || session->is_killing() ||
-          session->is_opening()); // re-open closing session
-    session->info.prealloc_inos.subtract(inos_to_free);
+
+  if (!open) {
+    if (inos_to_purge.size()){
+      ceph_assert(ls);
+      session->info.prealloc_inos.subtract(inos_to_purge);
+      ls->purging_inodes.insert(inos_to_purge);
+      mdcache->purge_inodes(inos_to_purge, ls);
+    }
+
+    if (inos_to_free.size()) {
+      ceph_assert(piv);
+      ceph_assert(session->is_closing() || session->is_killing() ||
+         session->is_opening()); // re-open closing session
+      session->info.prealloc_inos.subtract(inos_to_free);
+      mds->inotable->apply_release_ids(inos_to_free);
+      ceph_assert(mds->inotable->get_version() == piv);
+    }
+    session->free_prealloc_inos = session->info.prealloc_inos;
     session->delegated_inos.clear();
-    mds->inotable->apply_release_ids(inos_to_free);
-    ceph_assert(mds->inotable->get_version() == piv);
   }
 
   mds->sessionmap.mark_dirty(session);
@@ -3251,8 +3255,7 @@ CInode* Server::prepare_new_inode(MDRequestRef& mdr, CDir *dir, inodeno_t useino
   if (allow_prealloc_inos && (mdr->used_prealloc_ino = _inode->ino = mdr->session->take_ino(useino))) {
     mds->sessionmap.mark_projected(mdr->session);
     dout(10) << "prepare_new_inode used_prealloc " << mdr->used_prealloc_ino
-            << " (" << mdr->session->info.prealloc_inos
-            << ", " << mdr->session->info.prealloc_inos.size() << " left)"
+            << " (" << mdr->session->info.prealloc_inos.size() << " left)"
             << dendl;
   } else {
     mdr->alloc_ino = 
@@ -3363,13 +3366,14 @@ void Server::apply_allocated_inos(MDRequestRef& mdr, Session *session)
   if (mdr->prealloc_inos.size()) {
     ceph_assert(session);
     session->pending_prealloc_inos.subtract(mdr->prealloc_inos);
+    session->free_prealloc_inos.insert(mdr->prealloc_inos);
     session->info.prealloc_inos.insert(mdr->prealloc_inos);
     mds->sessionmap.mark_dirty(session, !mdr->used_prealloc_ino);
     mds->inotable->apply_alloc_ids(mdr->prealloc_inos);
   }
   if (mdr->used_prealloc_ino) {
     ceph_assert(session);
-    session->info.used_inos.erase(mdr->used_prealloc_ino);
+    session->info.prealloc_inos.erase(mdr->used_prealloc_ino);
     mds->sessionmap.mark_dirty(session);
   }
 }

diff --git a/src/mds/SessionMap.cc b/src/mds/SessionMap.cc
index 93e957829e178f5c9449f9f09df3b1dbfb824544..a321156acf8d008d6ab23d07bdbbed74244dfd39 100644 (file)
--- a/src/mds/SessionMap.cc
+++ b/src/mds/SessionMap.cc
@@ -78,9 +78,8 @@ void SessionMap::dump()
     dout(10) << p->first << " " << p->second
             << " state " << p->second->get_state_name()
             << " completed " << p->second->info.completed_requests
-            << " prealloc_inos " << p->second->info.prealloc_inos
+            << " free_prealloc_inos " << p->second->free_prealloc_inos
             << " delegated_inos " << p->second->delegated_inos
-            << " used_inos " << p->second->info.used_inos
             << dendl;
 }
 
@@ -598,6 +597,16 @@ void Session::dump(Formatter *f, bool cap_dump) const
   f->dump_object("recall_caps_throttle2o", recall_caps_throttle2o);
   f->dump_object("session_cache_liveness", session_cache_liveness);
   f->dump_object("cap_acquisition", cap_acquisition);
+
+  f->open_array_section("delegated_inos");
+  for (const auto& [start, len] : delegated_inos) {
+    f->open_object_section("ino_range");
+    f->dump_stream("start") << start;
+    f->dump_unsigned("length", len);
+    f->close_section();
+  }
+  f->close_section();
+
   info.dump(f);
 }
 
@@ -636,9 +645,9 @@ void SessionMap::wipe_ino_prealloc()
        p != session_map.end(); 
        ++p) {
     p->second->pending_prealloc_inos.clear();
+    p->second->free_prealloc_inos.clear();
     p->second->delegated_inos.clear();
     p->second->info.prealloc_inos.clear();
-    p->second->info.used_inos.clear();
   }
   projected = ++version;
 }
@@ -983,6 +992,8 @@ void Session::decode(bufferlist::const_iterator &p)
 {
   info.decode(p);
 
+  free_prealloc_inos = info.prealloc_inos;
+
   _update_human_name();
 }
 

diff --git a/src/mds/SessionMap.h b/src/mds/SessionMap.h
index 1d97f5d0917bdaef920511becb8146791a0fabb5..e59f7f26484501083d98d76ea31e5415c75feb25 100644 (file)
--- a/src/mds/SessionMap.h
+++ b/src/mds/SessionMap.h
@@ -175,55 +175,45 @@ public:
   inodeno_t take_ino(inodeno_t ino = 0) {
     if (ino) {
       if (!info.prealloc_inos.contains(ino))
-       return 0;
-      info.prealloc_inos.erase(ino);
-      if (delegated_inos.contains(ino))
+        return 0;
+      if (delegated_inos.contains(ino)) {
        delegated_inos.erase(ino);
-    } else {
-      /* Grab first prealloc_ino that isn't delegated */
-      for (const auto& [start, len] : info.prealloc_inos) {
-       for (auto i = start ; i < start + len ; i += 1) {
-         inodeno_t dstart, dlen;
-         if (!delegated_inos.contains(i, &dstart, &dlen)) {
-           ino = i;
-           info.prealloc_inos.erase(ino);
-           break;
-         }
-         /* skip to end of delegated interval */
-         i = dstart + dlen - 1;
-       }
-       if (ino)
-         break;
+      } else if (free_prealloc_inos.contains(ino)) {
+       free_prealloc_inos.erase(ino);
+      } else {
+       ceph_assert(0);
       }
+    } else if (!free_prealloc_inos.empty()) {
+      ino = free_prealloc_inos.range_start();
+      free_prealloc_inos.erase(ino);
     }
-    if (ino)
-      info.used_inos.insert(ino, 1);
     return ino;
   }
-  void delegate_inos(int want, interval_set<inodeno_t>& newinos) {
+
+  void delegate_inos(int want, interval_set<inodeno_t>& inos) {
     want -= (int)delegated_inos.size();
     if (want <= 0)
       return;
 
-    for (const auto& [start, len] : info.prealloc_inos) {
-      for (auto i = start ; i < start + len ; i += 1) {
-       inodeno_t dstart, dlen;
-       if (!delegated_inos.contains(i, &dstart, &dlen)) {
-         delegated_inos.insert(i);
-         newinos.insert(i);
-         if (--want == 0)
-            return;
-       } else {
-         /* skip to end of delegated interval */
-         i = dstart + dlen - 1;
-       }
+    for (auto it = free_prealloc_inos.begin(); it != free_prealloc_inos.end(); ) {
+      if (want < (int)it.get_len()) {
+       inos.insert(it.get_start(), (inodeno_t)want);
+       delegated_inos.insert(it.get_start(), (inodeno_t)want);
+       free_prealloc_inos.erase(it.get_start(), (inodeno_t)want);
+       break;
       }
+      want -= (int)it.get_len();
+      inos.insert(it.get_start(), it.get_len());
+      delegated_inos.insert(it.get_start(), it.get_len());
+      free_prealloc_inos.erase(it++);
+      if (want <= 0)
+       break;
     }
   }
 
   // sans any delegated ones
   int get_num_prealloc_inos() const {
-    return info.prealloc_inos.size() - delegated_inos.size();
+    return free_prealloc_inos.size();
   }
 
   int get_num_projected_prealloc_inos() const {
@@ -412,6 +402,7 @@ public:
 
   void clear() {
     pending_prealloc_inos.clear();
+    free_prealloc_inos.clear();
     delegated_inos.clear();
     info.clear_meta();
 
@@ -433,6 +424,7 @@ public:
   mutable elist<MDRequestImpl*> requests;
 
   interval_set<inodeno_t> pending_prealloc_inos; // journaling prealloc, will be added to prealloc_inos
+  interval_set<inodeno_t> free_prealloc_inos; //
   interval_set<inodeno_t> delegated_inos; // hand these out to client
 
   xlist<Capability*> caps;     // inodes with caps; front=most recently used

diff --git a/src/mds/journal.cc b/src/mds/journal.cc
index 67a9d6375a00ae8a0d5e7ee8059429a41a5a2500..ae64a4232a4940ca29fbd5646a9acb4b34b95570 100644 (file)
--- a/src/mds/journal.cc
+++ b/src/mds/journal.cc
@@ -1591,13 +1591,14 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDPeerUpdate *peerup)
        dout(20) << " (session prealloc " << session->info.prealloc_inos << ")" << dendl;
        if (used_preallocated_ino) {
          if (!session->info.prealloc_inos.empty()) {
-           inodeno_t i = session->take_ino(used_preallocated_ino);
-           ceph_assert(i == used_preallocated_ino);
-           session->info.used_inos.clear();
+           inodeno_t ino = session->take_ino(used_preallocated_ino);
+           session->info.prealloc_inos.erase(ino);
+           ceph_assert(ino == used_preallocated_ino);
          }
           mds->sessionmap.replay_dirty_session(session);
        }
        if (!preallocated_inos.empty()) {
+         session->free_prealloc_inos.insert(preallocated_inos);
          session->info.prealloc_inos.insert(preallocated_inos);
           mds->sessionmap.replay_dirty_session(session);
        }

diff --git a/src/mds/mdstypes.cc b/src/mds/mdstypes.cc
index 86fc0156522b8756be6858c8c227dca2993af9f7..9d46c8d867924da8688659441b196794473eda1c 100644 (file)
--- a/src/mds/mdstypes.cc
+++ b/src/mds/mdstypes.cc
@@ -565,7 +565,7 @@ void session_info_t::encode(bufferlist& bl, uint64_t features) const
   encode(inst, bl, features);
   encode(completed_requests, bl);
   encode(prealloc_inos, bl);   // hacky, see below.
-  encode(used_inos, bl);
+  encode((__u32)0, bl); // used_inos
   encode(completed_flushes, bl);
   encode(auth_name, bl);
   encode(client_metadata, bl);
@@ -587,9 +587,11 @@ void session_info_t::decode(bufferlist::const_iterator& p)
     decode(completed_requests, p);
   }
   decode(prealloc_inos, p);
-  decode(used_inos, p);
-  prealloc_inos.insert(used_inos);
-  used_inos.clear();
+  {
+    interval_set<inodeno_t> used_inos;
+    decode(used_inos, p);
+    prealloc_inos.insert(used_inos);
+  }
   if (struct_v >= 4 && struct_v < 7) {
     decode(client_metadata.kv_map, p);
   }
@@ -627,15 +629,6 @@ void session_info_t::dump(Formatter *f) const
   }
   f->close_section();
 
-  f->open_array_section("used_inos");
-  for (const auto& [start, len] : used_inos) {
-    f->open_object_section("ino_range");
-    f->dump_stream("start") << start;
-    f->dump_unsigned("length", len);
-    f->close_section();
-  }
-  f->close_section();
-
   f->dump_object("client_metadata", client_metadata);
 }
 

diff --git a/src/mds/mdstypes.h b/src/mds/mdstypes.h
index 158a447d96aaf570a7c99745cec4401607d6a9b7..91dce0ad6982f725aca7f93f3b44a97d8d4f27e4 100644 (file)
--- a/src/mds/mdstypes.h
+++ b/src/mds/mdstypes.h
@@ -1333,7 +1333,6 @@ struct session_info_t {
 
   void clear_meta() {
     prealloc_inos.clear();
-    used_inos.clear();
     completed_requests.clear();
     completed_flushes.clear();
     client_metadata.clear();
@@ -1347,7 +1346,6 @@ struct session_info_t {
   entity_inst_t inst;
   std::map<ceph_tid_t,inodeno_t> completed_requests;
   interval_set<inodeno_t> prealloc_inos;   // preallocated, ready to use.
-  interval_set<inodeno_t> used_inos;       // journaling use
   client_metadata_t client_metadata;
   std::set<ceph_tid_t> completed_flushes;
   EntityName auth_name;