common/LRUSet: combine lru and hash-based lookup

Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/src/common/LRUSet.h b/src/common/LRUSet.h
new file mode 100644 (file)
index 0000000..b62956b
--- /dev/null
+++ b/src/common/LRUSet.h
@@ -0,0 +1,145 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <functional>
+#include <boost/intrusive/list.hpp>
+#include <boost/intrusive/unordered_set.hpp>
+#include "include/encoding.h"
+
+/// Combination of an LRU with fast hash-based membership lookup
+template<class T, int NUM_BUCKETS=128>
+class LRUSet {
+  /// internal node
+  struct Node
+    : boost::intrusive::unordered_set_base_hook<> {
+    // actual payload
+    T value;
+
+    // for the lru
+    boost::intrusive::list_member_hook<> lru_item;
+
+    Node(const T& v) : value(v) {}
+
+    friend std::size_t hash_value(const Node &node) {
+      return std::hash<T>{}(node.value);
+    }
+    friend bool operator<(const Node &a, const Node &b) {
+      return a.value < b.value;
+    }
+    friend bool operator>(const Node &a, const Node &b) {
+      return a.value > b.value;
+    }
+    friend bool operator==(const Node &a, const Node &b) {
+      return a.value == b.value;
+    }
+  };
+
+  struct NodeDeleteDisposer {
+    void operator()(Node *n) { delete n; }
+  };
+
+  // lru
+  boost::intrusive::list<
+    Node,
+    boost::intrusive::member_hook<Node,
+                                 boost::intrusive::list_member_hook<>,
+                                 &Node::lru_item>
+    > lru;
+
+  // hash-based set
+  typename boost::intrusive::unordered_set<Node>::bucket_type base_buckets[NUM_BUCKETS];
+  boost::intrusive::unordered_set<Node> set;
+
+ public:
+  LRUSet()
+    : set(typename boost::intrusive::unordered_set<Node>::bucket_traits(base_buckets,
+                                                                       NUM_BUCKETS))
+    {}
+  ~LRUSet() {
+    clear();
+  }
+
+  LRUSet(const LRUSet& other)
+    : set(typename boost::intrusive::unordered_set<Node>::bucket_traits(base_buckets,
+                                                                       NUM_BUCKETS)) {
+    for (auto & i : other.lru) {
+      insert(i.value);
+    }
+  }
+  const LRUSet& operator=(const LRUSet& other) {
+    clear();
+    for (auto& i : other.lru) {
+      insert(i.value);
+    }
+    return *this;
+  }
+
+  size_t size() const {
+    return set.size();
+  }
+
+  bool empty() const {
+    return set.empty();
+  }
+
+  bool contains(const T& item) const {
+    return set.count(item) > 0;
+  }
+
+  void clear() {
+    prune(0);
+  }
+
+  void insert(const T& item) {
+    erase(item);
+    Node *n = new Node(item);
+    lru.push_back(*n);
+    set.insert(*n);
+  }
+
+  bool erase(const T& item) {
+    auto p = set.find(item);
+    if (p == set.end()) {
+      return false;
+    }
+    lru.erase(lru.iterator_to(*p));
+    set.erase_and_dispose(p, NodeDeleteDisposer());
+    return true;
+  }
+
+  void prune(size_t max) {
+    while (set.size() > max) {
+      auto p = lru.begin();
+      set.erase(*p);
+      lru.erase_and_dispose(p, NodeDeleteDisposer());
+    }
+  }
+
+  void encode(bufferlist& bl) const {
+    using ceph::encode;
+    ENCODE_START(1, 1, bl);
+    uint32_t n = set.size();
+    encode(n, bl);
+    auto p = set.begin();
+    while (n--) {
+      encode(p->value, bl);
+      ++p;
+    }
+    ENCODE_FINISH(bl);
+  }
+  
+  void decode(bufferlist::const_iterator& p) {
+    using ceph::decode;
+    DECODE_START(1, p);
+    uint32_t n;
+    decode(n, p);
+    while (n--) {
+      T v;
+      decode(v, p);
+      insert(v);
+    }
+    DECODE_FINISH(p);
+  }
+};

diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt
index 679acd24c82f1f58606ef960dca02f03c5b51abb..95981510d8ede0801218a20146c952409f3b4060 100644 (file)
--- a/src/test/common/CMakeLists.txt
+++ b/src/test/common/CMakeLists.txt
@@ -55,6 +55,13 @@ add_executable(unittest_bloom_filter
 add_ceph_unittest(unittest_bloom_filter)
 target_link_libraries(unittest_bloom_filter ceph-common)
 
+# unittest_lruset
+add_executable(unittest_lruset
+  test_lruset.cc
+  )
+add_ceph_unittest(unittest_lruset)
+target_link_libraries(unittest_lruset)
+
 # unittest_histogram
 add_executable(unittest_histogram
   histogram.cc

diff --git a/src/test/common/test_lruset.cc b/src/test/common/test_lruset.cc
new file mode 100644 (file)
index 0000000..64e823e
--- /dev/null
+++ b/src/test/common/test_lruset.cc
@@ -0,0 +1,109 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank <info@inktank.com>
+ *
+ * LGPL-2.1 (see COPYING-LGPL2.1) or later
+ */
+
+#include <iostream>
+#include <gtest/gtest.h>
+
+#include "common/LRUSet.h"
+
+struct thing {
+  int a;
+  thing(int i) : a(i) {}
+  friend bool operator==(const thing &a, const thing &b) {
+    return a.a == b.a;
+  }
+  friend std::size_t hash_value(const thing &value) {
+    return value.a;
+  }
+};
+
+namespace std {
+  template<> struct hash<thing> {
+    size_t operator()(const thing& r) const {
+      return r.a;
+    }
+  };
+}
+
+
+TEST(LRUSet, insert_complex) {
+  LRUSet<thing> s;
+  s.insert(thing(1));
+  s.insert(thing(2));
+
+  ASSERT_TRUE(s.contains(thing(1)));
+  ASSERT_TRUE(s.contains(thing(2)));
+  ASSERT_FALSE(s.contains(thing(3)));
+}
+
+TEST(LRUSet, insert) {
+  LRUSet<int> s;
+  s.insert(1);
+  s.insert(2);
+
+  ASSERT_TRUE(s.contains(1));
+  ASSERT_TRUE(s.contains(2));
+  ASSERT_FALSE(s.contains(3));
+}
+
+TEST(LRUSet, erase) {
+  LRUSet<int> s;
+  s.insert(1);
+  s.insert(2);
+  s.insert(3);
+
+  s.erase(2);
+  ASSERT_TRUE(s.contains(1));
+  ASSERT_FALSE(s.contains(2));
+  ASSERT_TRUE(s.contains(3));
+  s.prune(1);
+  ASSERT_TRUE(s.contains(3));
+  ASSERT_FALSE(s.contains(1));
+}
+
+TEST(LRUSet, prune) {
+  LRUSet<int> s;
+  int max = 1000;
+  for (int i=0; i<max; ++i) {
+    s.insert(i);
+    s.prune(max / 10);
+  }
+  s.prune(0);
+  ASSERT_TRUE(s.empty());
+}
+
+TEST(LRUSet, lru) {
+  LRUSet<int> s;
+  s.insert(1);
+  s.insert(2);
+  s.insert(3);
+  s.prune(2);
+  ASSERT_FALSE(s.contains(1));
+  ASSERT_TRUE(s.contains(2));
+  ASSERT_TRUE(s.contains(3));
+
+  s.insert(2);
+  s.insert(4);
+  s.prune(2);
+  ASSERT_FALSE(s.contains(3));
+  ASSERT_TRUE(s.contains(2));
+  ASSERT_TRUE(s.contains(4));
+}
+
+TEST(LRUSet, copy) {
+  LRUSet<int> a, b;
+  a.insert(1);
+  b.insert(2);
+  b.insert(3);
+  a = b;
+  ASSERT_FALSE(a.contains(1));
+  ASSERT_TRUE(a.contains(2));
+  ASSERT_TRUE(a.contains(3));
+}