osd/OSDMap: more efficient PGMapTemp

Use a flat_map with pointers into a buffer with the actual data.  For a
decoded mapping, we have just two allocations (one for flat_map and one
for the encoded buffer).

This can get slow if you make lots of incremental changes after the fact
since flat_map is not efficient for modifications at large sizes.  :/

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/include/buffer.h b/src/include/buffer.h
index 177e95f8a8753e3441c1d0dae8a440ac4b345218..3cd069fa4abde7cb4bd03422fe0f7afa14ecd69c 100644 (file)
--- a/src/include/buffer.h
+++ b/src/include/buffer.h
@@ -688,6 +688,10 @@ namespace buffer CEPH_BUFFER_API {
     const ptr& front() const { return _buffers.front(); }
     const ptr& back() const { return _buffers.back(); }
 
+    size_t get_append_buffer_unused_tail_length() const {
+      return append_buffer.unused_tail_length();
+    }
+
     unsigned get_memcopy_count() const {return _memcopy_count; }
     const std::list<ptr>& buffers() const { return _buffers; }
     void swap(list& other);

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index e73a63d985e06f0b673e2ff19cbfb81253b53f5f..4a83c0af07e9ba0362c608f98e6257461b5360f6 100644 (file)
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -1618,6 +1618,10 @@ int OSDMap::apply_incremental(const Incremental &inc)
     else
       pg_temp->set(pg.first, pg.second);
   }
+  if (!inc.new_pg_temp.empty()) {
+    // make sure pg_temp is efficiently stored
+    pg_temp->rebuild();
+  }
 
   for (const auto &pg : inc.new_primary_temp) {
     if (pg.second == -1)

diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index 374b0a06afbde94145a3672e71ab7adb5e848446..46d23ea71215bcb207ca8913dc61aa5c9a8452a7 100644 (file)
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -19,6 +19,8 @@
 #ifndef CEPH_OSDMAP_H
 #define CEPH_OSDMAP_H
 
+#include "include/cpp-btree/btree_map.h"
+
 /*
  * describe properties of the OSD cluster.
  *   disks, disk groups, total # osds,
@@ -115,6 +117,154 @@ ostream& operator<<(ostream& out, const osd_xinfo_t& xi);
 
 
 struct PGTempMap {
+#if 1
+  bufferlist data;
+  typedef btree::btree_map<pg_t,int32_t*> map_t;
+  map_t map;
+
+  void encode(bufferlist& bl) const {
+    uint32_t n = map.size();
+    ::encode(n, bl);
+    for (auto &p : map) {
+      ::encode(p.first, bl);
+      bl.append((char*)p.second, (*p.second + 1) * sizeof(int32_t));
+    }
+  }
+  void decode(bufferlist::iterator& p) {
+    data.clear();
+    map.clear();
+    uint32_t n;
+    ::decode(n, p);
+    if (!n)
+      return;
+    bufferlist::iterator pstart = p;
+    size_t start_off = pstart.get_off();
+    vector<pair<pg_t,size_t>> offsets;
+    offsets.resize(n);
+    for (unsigned i=0; i<n; ++i) {
+      pg_t pgid;
+      ::decode(pgid, p);
+      offsets[i].first = pgid;
+      offsets[i].second = p.get_off() - start_off;
+      uint32_t vn;
+      ::decode(vn, p);
+      p.advance(vn * sizeof(int32_t));
+    }
+    size_t len = p.get_off() - start_off;
+    pstart.copy(len, data);
+    if (data.get_num_buffers() > 1) {
+      data.rebuild();
+    }
+    //map.reserve(n);
+    char *start = data.c_str();
+    for (auto i : offsets) {
+      map.insert(map.end(), make_pair(i.first, (int32_t*)(start + i.second)));
+    }
+  }
+  void rebuild() {
+    bufferlist bl;
+    encode(bl);
+    auto p = bl.begin();
+    decode(p);
+  }
+  friend bool operator==(const PGTempMap& l, const PGTempMap& r) {
+    return
+      l.map.size() == r.map.size() &&
+      l.data.contents_equal(r.data);
+  }
+
+  class iterator {
+    map_t::const_iterator it;
+    map_t::const_iterator end;
+    pair<pg_t,vector<int32_t>> current;
+    void init_current() {
+      if (it != end) {
+       current.first = it->first;
+       assert(it->second);
+       current.second.resize(*it->second);
+       int32_t *p = it->second + 1;
+       for (int n = 0; n < *it->second; ++n, ++p) {
+         current.second[n] = *p;
+       }
+      }
+    }
+  public:
+    iterator(map_t::const_iterator p,
+            map_t::const_iterator e)
+      : it(p), end(e) {
+      init_current();
+    }
+
+    const pair<pg_t,vector<int32_t>>& operator*() const {
+      return current;
+    }
+    const pair<pg_t,vector<int32_t>>* operator->() const {
+      return &current;
+    }
+    friend bool operator==(const iterator& l, const iterator& r) {
+      return l.it == r.it;
+    }
+    friend bool operator!=(const iterator& l, const iterator& r) {
+      return l.it != r.it;
+    }
+    iterator& operator++() {
+      ++it;
+      if (it != end)
+       init_current();
+      return *this;
+    }
+    iterator operator++(int) {
+      iterator r = *this;
+      ++it;
+      if (it != end)
+       init_current();
+      return r;
+    }
+  };
+  iterator begin() const {
+    return iterator(map.begin(), map.end());
+  }
+  iterator end() const {
+    return iterator(map.end(), map.end());
+  }
+  iterator find(pg_t pgid) const {
+    return iterator(map.find(pgid), map.end());
+  }
+  size_t size() const {
+    return map.size();
+  }
+  size_t count(pg_t pgid) const {
+    return map.count(pgid);
+  }
+  void erase(pg_t pgid) {
+    map.erase(pgid);
+  }
+  void clear() {
+    map.clear();
+    data.clear();
+  }
+  void set(pg_t pgid, const mempool::osdmap::vector<int32_t>& v) {
+    size_t need = sizeof(int32_t) * (1 + v.size());
+    if (need < data.get_append_buffer_unused_tail_length()) {
+      bufferptr z(data.get_append_buffer_unused_tail_length());
+      z.zero();
+      data.append(z.c_str(), z.length());
+    }
+    ::encode(v, data);
+    map[pgid] = (int32_t*)(data.back().end_c_str()) - (1 + v.size());
+  }
+  mempool::osdmap::vector<int32_t> get(pg_t pgid) {
+    mempool::osdmap::vector<int32_t> v;
+    int32_t *p = map[pgid];
+    size_t n = *p++;
+    v.resize(n);
+    for (size_t i = 0; i < n; ++i, ++p) {
+      v[i] = *p;
+    }
+    return v;
+  }
+#else
+  // trivial implementation
   mempool::osdmap::map<pg_t,mempool::osdmap::vector<int32_t> > pg_temp;
 
   void encode(bufferlist& bl) const {
@@ -185,6 +335,7 @@ struct PGTempMap {
   const mempool::osdmap::vector<int32_t>& get(pg_t pgid) {
     return pg_temp.at(pgid);
   }
+#endif
   void dump(Formatter *f) const {
     for (const auto &pg : *this) {
       f->open_object_section("osds");