From: Sage Weil Date: Mon, 21 Jun 2021 21:10:31 +0000 (-0400) Subject: common/LRUSet: combine lru and hash-based lookup X-Git-Tag: v17.1.0~1423^2~9 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=344278ecced622c7edec593eda454aef33797a37;p=ceph.git common/LRUSet: combine lru and hash-based lookup Signed-off-by: Sage Weil --- diff --git a/src/common/LRUSet.h b/src/common/LRUSet.h new file mode 100644 index 000000000000..b62956ba460f --- /dev/null +++ b/src/common/LRUSet.h @@ -0,0 +1,145 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +#include +#include +#include +#include "include/encoding.h" + +/// Combination of an LRU with fast hash-based membership lookup +template +class LRUSet { + /// internal node + struct Node + : boost::intrusive::unordered_set_base_hook<> { + // actual payload + T value; + + // for the lru + boost::intrusive::list_member_hook<> lru_item; + + Node(const T& v) : value(v) {} + + friend std::size_t hash_value(const Node &node) { + return std::hash{}(node.value); + } + friend bool operator<(const Node &a, const Node &b) { + return a.value < b.value; + } + friend bool operator>(const Node &a, const Node &b) { + return a.value > b.value; + } + friend bool operator==(const Node &a, const Node &b) { + return a.value == b.value; + } + }; + + struct NodeDeleteDisposer { + void operator()(Node *n) { delete n; } + }; + + // lru + boost::intrusive::list< + Node, + boost::intrusive::member_hook, + &Node::lru_item> + > lru; + + // hash-based set + typename boost::intrusive::unordered_set::bucket_type base_buckets[NUM_BUCKETS]; + boost::intrusive::unordered_set set; + + public: + LRUSet() + : set(typename boost::intrusive::unordered_set::bucket_traits(base_buckets, + NUM_BUCKETS)) + {} + ~LRUSet() { + clear(); + } + + LRUSet(const LRUSet& other) + : set(typename boost::intrusive::unordered_set::bucket_traits(base_buckets, + NUM_BUCKETS)) { + for (auto & i : other.lru) { + insert(i.value); + } + } + const LRUSet& operator=(const LRUSet& other) { + clear(); + for (auto& i : other.lru) { + insert(i.value); + } + return *this; + } + + size_t size() const { + return set.size(); + } + + bool empty() const { + return set.empty(); + } + + bool contains(const T& item) const { + return set.count(item) > 0; + } + + void clear() { + prune(0); + } + + void insert(const T& item) { + erase(item); + Node *n = new Node(item); + lru.push_back(*n); + set.insert(*n); + } + + bool erase(const T& item) { + auto p = set.find(item); + if (p == set.end()) { + return false; + } + lru.erase(lru.iterator_to(*p)); + set.erase_and_dispose(p, NodeDeleteDisposer()); + return true; + } + + void prune(size_t max) { + while (set.size() > max) { + auto p = lru.begin(); + set.erase(*p); + lru.erase_and_dispose(p, NodeDeleteDisposer()); + } + } + + void encode(bufferlist& bl) const { + using ceph::encode; + ENCODE_START(1, 1, bl); + uint32_t n = set.size(); + encode(n, bl); + auto p = set.begin(); + while (n--) { + encode(p->value, bl); + ++p; + } + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& p) { + using ceph::decode; + DECODE_START(1, p); + uint32_t n; + decode(n, p); + while (n--) { + T v; + decode(v, p); + insert(v); + } + DECODE_FINISH(p); + } +}; diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt index 679acd24c82f..95981510d8ed 100644 --- a/src/test/common/CMakeLists.txt +++ b/src/test/common/CMakeLists.txt @@ -55,6 +55,13 @@ add_executable(unittest_bloom_filter add_ceph_unittest(unittest_bloom_filter) target_link_libraries(unittest_bloom_filter ceph-common) +# unittest_lruset +add_executable(unittest_lruset + test_lruset.cc + ) +add_ceph_unittest(unittest_lruset) +target_link_libraries(unittest_lruset) + # unittest_histogram add_executable(unittest_histogram histogram.cc diff --git a/src/test/common/test_lruset.cc b/src/test/common/test_lruset.cc new file mode 100644 index 000000000000..64e823e2f5bb --- /dev/null +++ b/src/test/common/test_lruset.cc @@ -0,0 +1,109 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Inktank + * + * LGPL-2.1 (see COPYING-LGPL2.1) or later + */ + +#include +#include + +#include "common/LRUSet.h" + +struct thing { + int a; + thing(int i) : a(i) {} + friend bool operator==(const thing &a, const thing &b) { + return a.a == b.a; + } + friend std::size_t hash_value(const thing &value) { + return value.a; + } +}; + +namespace std { + template<> struct hash { + size_t operator()(const thing& r) const { + return r.a; + } + }; +} + + +TEST(LRUSet, insert_complex) { + LRUSet s; + s.insert(thing(1)); + s.insert(thing(2)); + + ASSERT_TRUE(s.contains(thing(1))); + ASSERT_TRUE(s.contains(thing(2))); + ASSERT_FALSE(s.contains(thing(3))); +} + +TEST(LRUSet, insert) { + LRUSet s; + s.insert(1); + s.insert(2); + + ASSERT_TRUE(s.contains(1)); + ASSERT_TRUE(s.contains(2)); + ASSERT_FALSE(s.contains(3)); +} + +TEST(LRUSet, erase) { + LRUSet s; + s.insert(1); + s.insert(2); + s.insert(3); + + s.erase(2); + ASSERT_TRUE(s.contains(1)); + ASSERT_FALSE(s.contains(2)); + ASSERT_TRUE(s.contains(3)); + s.prune(1); + ASSERT_TRUE(s.contains(3)); + ASSERT_FALSE(s.contains(1)); +} + +TEST(LRUSet, prune) { + LRUSet s; + int max = 1000; + for (int i=0; i s; + s.insert(1); + s.insert(2); + s.insert(3); + s.prune(2); + ASSERT_FALSE(s.contains(1)); + ASSERT_TRUE(s.contains(2)); + ASSERT_TRUE(s.contains(3)); + + s.insert(2); + s.insert(4); + s.prune(2); + ASSERT_FALSE(s.contains(3)); + ASSERT_TRUE(s.contains(2)); + ASSERT_TRUE(s.contains(4)); +} + +TEST(LRUSet, copy) { + LRUSet a, b; + a.insert(1); + b.insert(2); + b.insert(3); + a = b; + ASSERT_FALSE(a.contains(1)); + ASSERT_TRUE(a.contains(2)); + ASSERT_TRUE(a.contains(3)); +}