From: Jesse F. Williamson Date: Fri, 12 Jun 2026 16:28:11 +0000 (-0700) Subject: Improve Concept vocabulary for libfdb X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=5853a7edde732a3e5254a44b77ae398987c8b483;p=ceph.git Improve Concept vocabulary for libfdb Extend a stronger vocabulary of C++ Concepts and helpful functions via an internal collection of some Concepts not currently in the C++ Standard along with useful predicates for compile-time capability queries and runtime feature invocation. The mini-library provides general capabilities meant for writing clean algorithms, doing metaprogramming, or offering better support for general operations involving sequences, and so on: Concepts for common container categories (like contiguous sequence, ordered associative, etc.); Capability queries for container operations such as append, prepend, range insertion, etc.; Aggregate capability checks such as “can_append”, “can_append_range”, “can_erase_if”, etc.; Utility helpers that select appropriate operations for containers, like “push_back”, "insert_front", etc.; The tests cover standard STL containers, custom minimal test containers for fallback behavior, negative concept edge cases, and an example container-generic algorithm that composes several helpers. Assisted-by: Codex:GPT-5 Signed-off-by: Jesse F. Williamson --- diff --git a/src/common/container_concepts.h b/src/common/container_concepts.h new file mode 100644 index 000000000000..d873662ed057 --- /dev/null +++ b/src/common/container_concepts.h @@ -0,0 +1,665 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- +// vim: ts=8 sw=2 sts=2 expandtab ft=cpp + +#ifndef CEPH_COMMON_CONTAINER_CONCEPTS_H + #define CEPH_COMMON_CONTAINER_CONCEPTS_H + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2026 International Business Machines Corp. (IBM) + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +/* A helpful collection of C++ Concepts that aren't available in the C++23 +* standard. The goal here is to provide common and practical concepts and +* capability queries. +* +* Some of the material is derived from: +* - https://eel.is/c++draft/range.utility.conv +* - https://en.cppreference.com/w/cpp/ranges/to +* +* The facilities provided by this mini-library include: +* - some Concepts not in C++ (such as associative_container and +* unordered_associative_container); +* - a local name for some exposition-only C++ concepts used by the standard (such as +* container_compatible_range); +* +* - capability query predicates, which make metaprogramming tasks more straightforward +* and are very useful in a variety of algorithms; +* +* - end()/begin() style function forms for some container operations (like clear()); +* +* - utility functions for capturing common cases like optionally clearing a container +* based on whether or not the operation is supported by its concrete type; +* +* - append helpers that construct values through the container's natural +* insertion operation, including stateful appenders for forward-only containers; +* +* - range materialization helpers for building containers without requiring +* std::ranges::to support from the local standard library; +* +* Note that a number of the things you might wish were in here are already in implemented +* in the Standard Library-- as a guide (depending on your mental model!) this may be helpful: +* - ceph::concepts::sequence -> std::ranges::input_range +* - ceph::concepts::contiguous_sequence -> std::ranges::contiguous_range +* - ceph::util::erase_if -> std::erase_if +* - ceph::util::size -> std::ranges::size +* - ceph::util::empty -> std::ranges::empty +* +* Capability predicates such as can_erase_if and has_size are not direct +* operation wrappers; keep them when they make generic constraints clearer. +* +* This header also keeps local names for useful C++23 exposition-only +* constraints from ranges::to, including reservable_container and +* container_appendable. +*/ + +// General Concepts: +namespace ceph::concepts { + +template +concept same_as_any = (std::same_as || ...); + +template +concept associative_container = requires(ContainerT& c, const ContainerT& cc, + const typename ContainerT::key_type& key) { + requires std::ranges::range; + + typename ContainerT::key_type; + typename ContainerT::value_type; + typename ContainerT::key_compare; + typename ContainerT::value_compare; + typename ContainerT::iterator; + typename ContainerT::const_iterator; + typename ContainerT::size_type; + + { cc.key_comp() } -> std::same_as; + { cc.value_comp() } -> std::same_as; + { c.find(key) } -> std::same_as; + { cc.find(key) } -> std::same_as; + { c.count(key) } -> std::convertible_to; + { c.lower_bound(key) } -> std::same_as; + { cc.lower_bound(key) } -> std::same_as; + { c.upper_bound(key) } -> std::same_as; + { cc.upper_bound(key) } -> std::same_as; + { c.equal_range(key) }; + { cc.equal_range(key) }; +}; + +template +concept unordered_associative_container = requires(ContainerT& c, const ContainerT& cc, + const typename ContainerT::key_type& key) { + requires std::ranges::range; + + typename ContainerT::key_type; + typename ContainerT::value_type; + typename ContainerT::hasher; + typename ContainerT::key_equal; + typename ContainerT::iterator; + typename ContainerT::const_iterator; + typename ContainerT::size_type; + + { cc.hash_function() } -> std::same_as; + { cc.key_eq() } -> std::same_as; + { c.find(key) } -> std::same_as; + { cc.find(key) } -> std::same_as; + { c.count(key) } -> std::convertible_to; + { c.equal_range(key) }; + { cc.equal_range(key) }; + { c.bucket_count() } -> std::convertible_to; +}; + +template +concept container_compatible_range = + std::ranges::input_range && std::convertible_to, T>; + +template +concept reservable_container = + std::ranges::sized_range && + requires(ContainerT& c, std::ranges::range_size_t n) { + c.reserve(n); + { c.capacity() } -> std::same_as; + { c.max_size() } -> std::same_as; + }; + +template +concept container_appendable = requires(ContainerT& c, RefT&& ref) { + requires + requires { c.emplace_back(std::forward(ref)); } || + requires { c.push_back(std::forward(ref)); } || + requires { { c.emplace(c.end(), std::forward(ref)) } -> std::same_as; } || + requires { c.insert(c.end(), std::forward(ref)); }; +}; + +} // namespace ceph::concepts + +// Capability Queries: +namespace ceph::concepts { + +template +concept has_emplace_back = requires(ContainerT& c, ArgsT&&... args) { + c.emplace_back(std::forward(args)...); +}; + +template +concept has_push_back = requires(ContainerT& c, T&& v) { + c.push_back(std::forward(v)); +}; + +template +concept has_emplace_append = requires(ContainerT& c, ArgsT&&... args) { + { c.emplace(std::end(c), std::forward(args)...) } -> + std::same_as; +}; + +template +concept has_insert_append = requires(ContainerT& c, T&& v) { + c.insert(std::end(c), std::forward(v)); +}; + +template +concept has_emplace_after = requires(ContainerT& c, IteratorT pos, ArgsT&&... args) { + c.emplace_after(pos, std::forward(args)...); +}; + +template +concept has_insert_after = requires(ContainerT& c, IteratorT pos, T&& v) { + c.insert_after(pos, std::forward(v)); +}; + +template +concept has_append_range = requires(ContainerT& c, RangeT&& range) { + c.append_range(std::forward(range)); +}; + +template +concept has_insert_range_append = requires(ContainerT& c, RangeT&& range) { + c.insert_range(std::end(c), std::forward(range)); +}; + +template +concept has_insert_iterator_range_append = requires(ContainerT& c, RangeT&& range) { + requires std::ranges::input_range; + + c.insert(std::end(c), std::begin(range), std::end(range)); +}; + +template +concept has_insert_range = requires(ContainerT& c, IteratorT pos, RangeT&& range) { + c.insert_range(pos, std::forward(range)); +}; + +template +concept has_insert_iterator_range = requires(ContainerT& c, IteratorT pos, RangeT&& range) { + requires std::ranges::input_range; + + c.insert(pos, std::begin(range), std::end(range)); +}; + +template +concept has_emplace_front = requires(ContainerT& c, T&& v) { + c.emplace_front(std::forward(v)); +}; + +template +concept has_push_front = requires(ContainerT& c, T&& v) { + c.push_front(std::forward(v)); +}; + +template +concept has_emplace_at_begin = requires(ContainerT& c, T&& v) { + { c.emplace(std::begin(c), std::forward(v)) } -> std::same_as; +}; + +template +concept has_insert_at_begin = requires(ContainerT& c, T&& v) { + c.insert(std::begin(c), std::forward(v)); +}; + +template +concept has_pop_front = requires(ContainerT& c) { + c.pop_front(); +}; + +template +concept has_erase_begin = requires(ContainerT& c) { + c.erase(std::begin(c)); +}; + +template +concept has_erase = requires(ContainerT& c, ArgT&& v) { + c.erase(std::forward(v)); +}; + +template +concept has_erase_range = requires(ContainerT& c, IteratorT first, SentinelT last) { + c.erase(first, last); +}; + +template +concept has_remove_if = requires(ContainerT& c, PredicateT pred) { + c.remove_if(pred); +}; + +template +concept has_clear = requires(ContainerT& c) { + c.clear(); +}; + +template +concept has_reserve = requires(ContainerT& c, std::size_t n) { + c.reserve(n); +}; + +template +concept has_capacity = requires(const ContainerT& c) { + { c.capacity() } -> std::convertible_to; +}; + +template +concept has_max_size = requires(const ContainerT& c) { + { c.max_size() } -> std::convertible_to; +}; + +template +concept has_size = requires(const ContainerT& c) { + { c.size() } -> std::convertible_to; +}; + +template +concept has_empty = requires(const ContainerT& c) { + { c.empty() } -> std::convertible_to; +}; + +template +concept has_resize = requires(ContainerT& c, std::size_t n) { + c.resize(n); +}; + +} // namespace ceph::concepts + +// Helpers for composing aggregate capability checks: +namespace ceph::concepts::detail { + +template +concept can_append_with_insert = + ceph::concepts::has_insert_range_append || ceph::concepts::has_insert_iterator_range_append; + +} // namespace ceph::concepts::detail + +// Aggregate Capability Checks: +namespace ceph::concepts { + +template +concept can_append = + container_appendable; + +template +concept can_emplace_append = + has_emplace_back || + has_emplace_append || + (std::constructible_from && + can_append); + +template +concept can_insert_after = + has_emplace_after || + has_insert_after; + +template +concept can_emplace_after = + has_emplace_after || + (std::constructible_from && + can_insert_after); + +template +concept can_append_range = + std::ranges::input_range && + (has_append_range || + detail::can_append_with_insert || + can_emplace_append>); + +template +concept can_insert_any_range = + has_insert_range || has_insert_iterator_range; + +template +concept can_prepend = + has_emplace_front || + has_push_front || + has_emplace_at_begin || + has_insert_at_begin; + +template +concept can_erase_remove_if = requires(ContainerT& c, PredicateT pred) { + requires std::ranges::common_range; + requires has_erase_range, + std::ranges::sentinel_t>; + + std::remove_if(std::begin(c), std::end(c), pred); +}; + +template +concept can_erase_if = + has_remove_if || can_erase_remove_if; + +template +concept can_remove_front = + has_pop_front || has_erase_begin; + +} // namespace ceph::concepts + +// Helpers: +// - most of these try to "do the right thing" with the underlying container +// as-appropriate; use some caution in performance-sensitive situations (although +// the answer may be far from cut and dried as cache locality and other features +// of modern CPUs can produce extremely counterintuitive results vis-a-vis O(n), +// n may need to be surprisingly large before inserting at the head of a list is +// *actually* faster than shifting an array, for instance): +namespace ceph::util { + +template +requires ceph::concepts::can_append +constexpr void push_back(ContainerT& c, T&& v) +{ + if constexpr (ceph::concepts::has_emplace_back) { + c.emplace_back(std::forward(v)); + return; + } + + if constexpr (ceph::concepts::has_push_back) { + c.push_back(std::forward(v)); + return; + } + + if constexpr (ceph::concepts::has_emplace_append) { + c.emplace(std::end(c), std::forward(v)); + return; + } + + if constexpr (ceph::concepts::has_insert_append) { + c.insert(std::end(c), std::forward(v)); + return; + } +} + +// Note: emplace_back() can return a reference, hence decltype(auto): +template +constexpr decltype(auto) emplace_back(ContainerT& c, ArgsT&&... args) +{ + return c.emplace_back(std::forward(args)...); +} + +template +requires ceph::concepts::can_emplace_append +constexpr void emplace_append(ContainerT& c, ArgsT&&... args) +{ + if constexpr (ceph::concepts::has_emplace_back) { + c.emplace_back(std::forward(args)...); + return; + } + + if constexpr (ceph::concepts::has_emplace_append) { + c.emplace(std::end(c), std::forward(args)...); + return; + } + + if constexpr (std::constructible_from) { + push_back(c, typename ContainerT::value_type(std::forward(args)...)); + } +} + +template +requires ceph::concepts::can_append_range +constexpr void append_range(ContainerT& c, RangeT&& range) +{ + if constexpr (ceph::concepts::has_append_range) { + c.append_range(std::forward(range)); + return; + } + + if constexpr (ceph::concepts::has_insert_range_append) { + c.insert_range(std::end(c), std::forward(range)); + return; + } + + if constexpr (ceph::concepts::has_insert_iterator_range_append) { + c.insert(std::end(c), std::begin(range), std::end(range)); + return; + } + + if constexpr (ceph::concepts::can_emplace_append>) { + for (auto&& v : range) { + emplace_append(c, std::forward(v)); + } + return; + } +} + +template +requires ceph::concepts::can_insert_any_range +constexpr void insert_range(ContainerT& c, IteratorT pos, RangeT&& range) +{ + if constexpr (ceph::concepts::has_insert_range) { + c.insert_range(pos, std::forward(range)); + return; + } + + if constexpr (ceph::concepts::has_insert_iterator_range) { + c.insert(pos, std::begin(range), std::end(range)); + return; + } +} + +template +requires ceph::concepts::can_insert_after +constexpr auto insert_after(ContainerT& c, IteratorT pos, T&& v) +{ + if constexpr (ceph::concepts::has_emplace_after) { + return c.emplace_after(pos, std::forward(v)); + } + + if constexpr (ceph::concepts::has_insert_after) { + return c.insert_after(pos, std::forward(v)); + } +} + +template +requires ceph::concepts::can_emplace_after +constexpr auto emplace_after(ContainerT& c, IteratorT pos, ArgsT&&... args) +{ + if constexpr (ceph::concepts::has_emplace_after) { + return c.emplace_after(pos, std::forward(args)...); + } + + if constexpr (std::constructible_from) { + return insert_after(c, pos, typename ContainerT::value_type(std::forward(args)...)); + } +} + +// Fills a role similar to back_inserter(), but calling emplace_append(). +template +struct container_appender final { + ContainerT& container; + + public: + explicit constexpr container_appender(ContainerT& container) + : container(container) {} + + public: + template + requires ceph::concepts::can_emplace_append + constexpr void emplace(ArgsT&&... args) + { + emplace_append(container, std::forward(args)...); + } +}; + +template +struct emplace_after_appender final { + ContainerT& container; + std::ranges::iterator_t pos; + + public: + explicit constexpr emplace_after_appender(ContainerT& container) + : container(container), + pos(container.before_begin()) {} + + public: + template + requires ceph::concepts::can_emplace_after, + ArgsT...> + constexpr void emplace(ArgsT&&... args) + { + pos = emplace_after(container, pos, std::forward(args)...); + } +}; + +template +constexpr auto make_appender(ContainerT& c) +{ + constexpr auto inserts_after = + std::ranges::forward_range && + !std::ranges::bidirectional_range && + requires (ContainerT& container) { + container.before_begin(); + }; + + if constexpr (inserts_after) { + return emplace_after_appender { c }; + } + + if constexpr (!inserts_after) { + return container_appender { c }; + } +} + +template +constexpr auto erase(ContainerT& c, ArgT&& v) +{ + return c.erase(std::forward(v)); +} + +template +constexpr auto erase(ContainerT& c, IteratorT first, SentinelT last) +{ + return c.erase(first, last); +} + +template +requires ceph::concepts::can_prepend +constexpr void push_front(ContainerT& c, T&& v) +{ + if constexpr (ceph::concepts::has_emplace_front) { + c.emplace_front(std::forward(v)); + return; + } + + if constexpr (ceph::concepts::has_push_front) { + c.push_front(std::forward(v)); + return; + } + + if constexpr (ceph::concepts::has_emplace_at_begin) { + c.emplace(std::begin(c), std::forward(v)); + return; + } + + if constexpr (ceph::concepts::has_insert_at_begin) { + c.insert(std::begin(c), std::forward(v)); + return; + } +} + +template +requires ceph::concepts::can_remove_front +constexpr void pop_front(ContainerT& c) +{ + if constexpr (ceph::concepts::has_pop_front) { + c.pop_front(); + return; + } + + if constexpr (ceph::concepts::has_erase_begin) { + c.erase(std::begin(c)); + return; + } +} + +template +constexpr std::size_t capacity(const ContainerT& c) +{ + return c.capacity(); +} + +template +constexpr std::size_t max_size(const ContainerT& c) +{ + return c.max_size(); +} + +template +constexpr void resize(ContainerT& c, std::size_t n) +{ + c.resize(n); +} + +template +constexpr void clear(ContainerT& c) +{ + c.clear(); +} + +// It's often handy to be able to easily resize if possible, +// but continue along either way-- so, here we are: +template +constexpr void maybe_resize(ContainerT& c, std::size_t n) +{ + if constexpr (ceph::concepts::has_resize) { + c.resize(n); + } +} + +// reserve() also is frequently "nice to have" in some algorithms: +template +constexpr void maybe_reserve(ContainerT& c, std::size_t n) +{ + if constexpr (ceph::concepts::has_reserve) { + c.reserve(n); + } +} + +template +requires ceph::concepts::can_append_range +constexpr ContainerT collect_as(RangeT&& range) +{ + ContainerT out; + + if constexpr (std::ranges::sized_range) { + maybe_reserve(out, std::ranges::size(range)); + } + + append_range(out, std::forward(range)); + + return out; +} + +} // namespace ceph::util + +#endif diff --git a/src/rgw/fdb/EXAMPLES.md b/src/rgw/fdb/EXAMPLES.md index e23422f1eea4..4ef5fd6f5b7e 100644 --- a/src/rgw/fdb/EXAMPLES.md +++ b/src/rgw/fdb/EXAMPLES.md @@ -136,9 +136,7 @@ lfdb::set(dbh, std::begin(people), std::end(people)); /* Read a key range into an STL associative container. */ std::map people; -lfdb::get(dbh, - lfdb::select { "person/" }, - std::inserter(people, std::end(people))); +lfdb::get(dbh, lfdb::select { "person/" }, people); ``` ## Key Ordering diff --git a/src/rgw/fdb/base.h b/src/rgw/fdb/base.h index c67804318896..3341e3515591 100644 --- a/src/rgw/fdb/base.h +++ b/src/rgw/fdb/base.h @@ -45,6 +45,8 @@ #include #include +#include "common/container_concepts.h" + #ifdef __cpp_lib_flat_map #include template @@ -70,38 +72,6 @@ extern transaction_handle make_transaction(database_handle dbh); } // namespace ceph::libfdb -// MOAR forward declarations-- "pay no attention to that man behind the curtain": -namespace ceph::libfdb::detail { - -template -concept is_any_of = (std::is_same_v || ...); - -struct future_value; - -template -std::pair to_decoded_kv_pair(const FDBKeyValue& kv); - -inline fdb_error_t do_commit(transaction_handle& txn); - -inline void transaction_set_kv_bytes(const transaction_handle& txn, - std::span k, - std::span v); - -inline future_value block_until_ready(future_value&& fv); -inline fdb_error_t get_future_error(const future_value& fv); -inline future_value wait_for_on_error(FDBTransaction* txn, fdb_error_t original_error); -inline future_value get_range_future_from_transaction(ceph::libfdb::transaction& txn, const ceph::libfdb::select& selection, int iteration); - -// A generator that produces successive spans for a range: -inline std::generator> generate_FDB_pairs(ceph::libfdb::transaction& txn, ceph::libfdb::select key_range); - -// Stores a successively-generated of kv pair results to an iterator: -template -requires std::output_iterator> -inline bool get_value_range_from_transaction(ceph::libfdb::transaction& txn, const ceph::libfdb::select& key_range, OutIterT out_iter); - -} // namespace ceph::libfdb::detail - namespace ceph::libfdb::concepts { // Note that "stringlikes" are not all "stringview-likes", such as when they can be @@ -117,6 +87,14 @@ concept key_value_iterator = kv.second; }; +template +concept string_key_value_output_iterator = + std::output_iterator>; + +template +concept string_key_value_output_container = + ceph::concepts::can_append>; + template concept value_callback = std::invocable>; @@ -137,10 +115,39 @@ concept supported_invocation_result = // There's a high likelihood that we're going to get more sophisticated selectors, // so this is doing a more important job than it may appear to be: template -concept selector = ceph::libfdb::detail::is_any_of; +concept selector = ceph::concepts::same_as_any; } // namespace ceph::libfdb::concepts +// MOAR forward declarations-- "pay no attention to that man behind the curtain": +namespace ceph::libfdb::detail { + +struct future_value; + +template +std::pair to_decoded_kv_pair(const FDBKeyValue& kv); + +inline fdb_error_t do_commit(transaction_handle& txn); + +inline void transaction_set_kv_bytes(const transaction_handle& txn, + std::span k, + std::span v); + +inline future_value block_until_ready(future_value&& fv); +inline fdb_error_t get_future_error(const future_value& fv); +inline future_value wait_for_on_error(FDBTransaction* txn, fdb_error_t original_error); +inline future_value get_range_future_from_transaction(ceph::libfdb::transaction& txn, const ceph::libfdb::select& selection, int iteration); + +// A generator that produces successive spans for a range: +inline std::generator> generate_FDB_pairs(ceph::libfdb::transaction& txn, ceph::libfdb::select key_range); + +// Stores a successively-generated of kv pair results to an iterator: +template +requires ceph::libfdb::concepts::string_key_value_output_iterator +inline bool get_value_range_from_transaction(ceph::libfdb::transaction& txn, const ceph::libfdb::select& key_range, OutIterT out_iter); + +} // namespace ceph::libfdb::detail + // libfdb_exception: How to deal, when Bad Things(TM) happen: namespace ceph::libfdb { @@ -576,7 +583,7 @@ class transaction final // JFW: it's not as easy to wedge an output_range into here as it appears, perhaps // needs to be revisited; I'm binding it to what's actually used in practice for now: template - requires std::output_iterator> + requires concepts::string_key_value_output_iterator bool get(const ceph::libfdb::select& key_range, OutIterT out_iter) { return ceph::libfdb::detail::get_value_range_from_transaction(*this, key_range, out_iter); } @@ -623,7 +630,10 @@ class transaction final std::string_view, OutputTargetOrFnT&&, const commit_after_op); - friend inline bool get(ceph::libfdb::transaction_handle, const ceph::libfdb::select&, auto, const commit_after_op); + friend inline bool get(ceph::libfdb::transaction_handle, + const ceph::libfdb::select&, + concepts::string_key_value_output_iterator auto, + const commit_after_op); friend inline void erase(ceph::libfdb::transaction_handle, std::string_view, const commit_after_op); friend inline void erase(ceph::libfdb::transaction_handle, const ceph::libfdb::select&, const commit_after_op); @@ -868,9 +878,7 @@ inline auto decode_pairs(std::span pairs) template inline AssocT collect_pairs(std::span pairs) { - AssocT out; - std::ranges::copy(decode_pairs(pairs), std::inserter(out, std::end(out))); - return out; + return ceph::util::collect_as(decode_pairs(pairs)); } template @@ -892,7 +900,7 @@ inline query_window_result materialize_query_window(transaction& txn, se } template -requires std::output_iterator> +requires concepts::string_key_value_output_iterator inline bool get_value_range_from_transaction(transaction& txn, const select& key_range, OutIterT out_iter) { auto flattened = detail::generate_FDB_pairs(txn, key_range) | std::views::join; @@ -930,23 +938,23 @@ inline std::vector as_select_seq(std::span x } // Gather the flattened list into *overlapping* libfdb::select pairs: - return std::views::iota(std::size_t{0}, xs.size() - 1) - | std::views::transform([&parent, xs](const auto i) { - const auto& fst = xs[i]; - const auto& snd = xs[i + 1]; - const auto first_key = std::string_view((const char *)fst.key, - static_cast(fst.key_length)); - const auto second_key = std::string_view((const char *)snd.key, - static_cast(snd.key_length)); - - ceph::libfdb::select split(first_key, second_key); - - split.options = parent.options; - split.begin_inclusive = (0 == i) ? parent.begin_inclusive : true; - split.end_inclusive = (i + 2 == xs.size()) ? parent.end_inclusive : false; - return split; - }) - | std::ranges::to>(); + return ceph::util::collect_as>( + std::views::iota(std::size_t{0}, xs.size() - 1) + | std::views::transform([&parent, xs](const auto i) { + const auto& fst = xs[i]; + const auto& snd = xs[i + 1]; + const auto first_key = std::string_view((const char *)fst.key, + static_cast(fst.key_length)); + const auto second_key = std::string_view((const char *)snd.key, + static_cast(snd.key_length)); + + ceph::libfdb::select split(first_key, second_key); + + split.options = parent.options; + split.begin_inclusive = (0 == i) ? parent.begin_inclusive : true; + split.end_inclusive = (i + 2 == xs.size()) ? parent.end_inclusive : false; + return split; + })); } // Finding a clear example both in the samples and in the documentation is not very easy. The // statelessness of FDB requests bleeds into here with basically no hand-holding, but note for instance diff --git a/src/rgw/fdb/interface.h b/src/rgw/fdb/interface.h index 7ad4183b2b60..60c96560928d 100644 --- a/src/rgw/fdb/interface.h +++ b/src/rgw/fdb/interface.h @@ -272,7 +272,8 @@ namespace ceph::libfdb { // JFW: Satisfying output_iterator is not as straightforward as it appears, I need to look at this mechanism again; meanwhile, the template doesn't // /prevent/ future type narrowing, but I'm forcing it to std::string for now: inline bool get(ceph::libfdb::transaction_handle txn, - const ceph::libfdb::select& key_range, auto out_iter, + const ceph::libfdb::select& key_range, + concepts::string_key_value_output_iterator auto out_iter, const ceph::libfdb::commit_after_op commit_after) { return detail::commit_noreplay(txn, commit_after, @@ -282,13 +283,15 @@ inline bool get(ceph::libfdb::transaction_handle txn, } inline bool get(ceph::libfdb::transaction_handle txn, - const ceph::libfdb::select& key_range, auto out_iter) + const ceph::libfdb::select& key_range, + concepts::string_key_value_output_iterator auto out_iter) { return get(txn, key_range, out_iter, commit_after_op::no_commit); } inline bool get(ceph::libfdb::database_handle dbh, - const ceph::libfdb::select& key_range, auto out_iter) + const ceph::libfdb::select& key_range, + concepts::string_key_value_output_iterator auto out_iter) { return detail::maybe_retry(ceph::libfdb::make_transaction(dbh), [&key_range, out_iter](transaction_handle& txn) { @@ -434,6 +437,41 @@ inline auto pair_generator(ceph::libfdb::transaction_handle txn, ceph::libfdb::s co_yield std::ranges::elements_of(decoded_pairs); } +template +inline bool get(ceph::libfdb::transaction_handle txn, + const ceph::libfdb::select& key_range, + OutContainerT& out_container, + const ceph::libfdb::commit_after_op commit_after) +{ + return detail::commit_noreplay(txn, commit_after, + [&key_range, &out_container](const transaction_handle& txn) { + for (auto&& p : pair_generator(txn, key_range)) { + ceph::util::push_back(out_container, std::move(p)); + } + + return true; + }); +} + +template +inline bool get(ceph::libfdb::transaction_handle txn, + const ceph::libfdb::select& key_range, + OutContainerT& out_container) +{ + return get(txn, key_range, out_container, commit_after_op::no_commit); +} + +template +inline bool get(ceph::libfdb::database_handle dbh, + const ceph::libfdb::select& key_range, + OutContainerT& out_container) +{ + return detail::maybe_retry(ceph::libfdb::make_transaction(dbh), + [&key_range, &out_container](transaction_handle& txn) { + return get(txn, key_range, out_container, commit_after_op::no_commit); + }); +} + // Note: block_generator() uses split planning to tackle large sets; use pair_generator() for // direct scans. // diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 2df929008e6a..8efc08d459b7 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -1060,6 +1060,9 @@ add_ceph_unittest(unittest_weighted_shuffle) add_executable(unittest_intarith test_intarith.cc) add_ceph_unittest(unittest_intarith) + +add_catch2_test(concepts) + #make check ends here # test_nvmeof_mon_encoding @@ -1109,4 +1112,3 @@ add_executable(test_nvmeof_gw_utils target_link_libraries(test_nvmeof_gw_utils mon ceph-common global-static ) - diff --git a/src/test/rgw/test_fdb.cc b/src/test/rgw/test_fdb.cc index 66b1bd551ccd..6c71552a48af 100644 --- a/src/test/rgw/test_fdb.cc +++ b/src/test/rgw/test_fdb.cc @@ -269,6 +269,19 @@ TEMPLATE_PRODUCT_TEST_CASE("multi-key ops", "[rgw][fdb]", CHECK(std::end(out_values) != std::ranges::find(out_values, string_pair { make_key(i), make_value(i) })); } } + + SECTION("check multiple key selection into container", "[fdb]") { + TestType out_values; + + auto txn = lfdb::make_transaction(j); + + CHECK(lfdb::get(txn, + lfdb::select { make_key(0), make_key(100) }, + out_values, + lfdb::commit_after_op::no_commit)); + + CHECK(100 == out_values.size()); + } } TEST_CASE("check selectors", "[fdb][rgw]") { @@ -339,6 +352,12 @@ TEST_CASE("check selectors", "[fdb][rgw]") { &std::pair::first)); } + std::map out_map; + + CHECK(lfdb::get(dbh, select_all, out_map)); + CHECK(nentries == out_map.size()); + CHECK(make_value(0) == out_map.at(make_key(0))); + lfdb::set(dbh, test_key("keyx"), "outside"); out.clear(); lfdb::get(dbh, lfdb::select { make_key_prefix() }, std::back_inserter(out)); diff --git a/src/test/test_concepts.cc b/src/test/test_concepts.cc new file mode 100644 index 000000000000..20ae19fa4173 --- /dev/null +++ b/src/test/test_concepts.cc @@ -0,0 +1,587 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2026 International Business Machines Corp. (IBM) + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include + +#include "common/container_concepts.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using ceph::concepts::associative_container; +using ceph::concepts::can_append; +using ceph::concepts::can_append_range; +using ceph::concepts::can_emplace_append; +using ceph::concepts::can_erase_if; +using ceph::concepts::can_insert_after; +using ceph::concepts::can_prepend; +using ceph::concepts::can_remove_front; +using ceph::concepts::container_appendable; +using ceph::concepts::container_compatible_range; +using ceph::concepts::has_size; +using ceph::concepts::reservable_container; +using ceph::concepts::same_as_any; +using ceph::concepts::unordered_associative_container; + +// Helper gadgets for the unit tests: +namespace { + +struct vector_backed_range { + using value_type = int; + using iterator = std::vector::iterator; + using const_iterator = std::vector::const_iterator; + + std::vector values; + + vector_backed_range() = default; + vector_backed_range(std::initializer_list init) : values(init) {} + + iterator begin() { return values.begin(); } + iterator end() { return values.end(); } + const_iterator begin() const { return values.begin(); } + const_iterator end() const { return values.end(); } +}; + +// Expose iterator interface to prove can_append() and push_back() will +// fall back to insert() when push_back()/emplace_back() don't exist: +struct append_only final : vector_backed_range { + using vector_backed_range::vector_backed_range; + + void insert(iterator pos, int value) { + values.insert(pos, value); + } +}; + +struct append_range_only final : vector_backed_range { + using vector_backed_range::vector_backed_range; + + void append_range(const std::array& range) { + values.insert(values.end(), range.begin(), range.end()); + } +}; + +struct insert_range_only final : vector_backed_range { + using vector_backed_range::vector_backed_range; + + iterator insert_range(iterator pos, const std::array& range) { + return values.insert(pos, range.begin(), range.end()); + } +}; + +// Only exposes erase(I), to help verify can_remove_front() and pop_front() +// can work without pop_front() directly being available: +struct front_erasable final : vector_backed_range { + using vector_backed_range::vector_backed_range; + + iterator erase(iterator pos) { + return values.erase(pos); + } +}; + +// May be constructed from int, but does not convert TO one: +struct explicit_from_int final { + explicit explicit_from_int(int) {} +}; + +template +struct range_compat_case final { + using range_type = RangeT; + using value_type = ValueT; +}; + +using int_predicate = bool (*)(int); +using int_deque = std::deque; +using int_forward_list = std::forward_list; +using int_list = std::list; +using int_set = std::set; +using int_vector = std::vector; +using small_array = std::array; + +bool is_even(int value) +{ + return value % 2 == 0; +} + +template +requires can_append_range&> && + can_prepend && + can_erase_if +ContainerT collect_odd_values() +{ + const std::array input{1, 2, 3, 4}; + ContainerT values; + + ceph::util::maybe_reserve(values, input.size() + 1); + ceph::util::append_range(values, input); + std::erase_if(values, is_even); + ceph::util::push_front(values, 0); + + return values; +} + +} // namespace + +/*** Tests for library general Concepts: */ + +TEMPLATE_PRODUCT_TEST_CASE("standard sequence containers model input_range", + "[concepts]", + (std::vector, std::list, std::forward_list), + (int)) +{ + STATIC_REQUIRE(std::ranges::input_range); +} + +TEMPLATE_TEST_CASE("contiguous sequence containers model contiguous_range", + "[concepts]", std::vector, (std::array)) +{ + STATIC_REQUIRE(std::ranges::contiguous_range); +} + +TEMPLATE_TEST_CASE("compatible ranges model container_compatible_range", + "[concepts]", + (range_compat_case, int>), + (range_compat_case, int>), + (range_compat_case, int>), + (range_compat_case), + (range_compat_case, const int&>)) +{ + STATIC_REQUIRE(container_compatible_range); +} + +TEMPLATE_TEST_CASE("reservable containers model reservable_container", + "[concepts]", std::vector, std::string) +{ + STATIC_REQUIRE(reservable_container); +} + +TEMPLATE_TEST_CASE("appendable containers model container_appendable", + "[concepts]", std::vector, std::deque, + std::list, std::set, append_only) +{ + STATIC_REQUIRE(container_appendable); +} + +TEST_CASE("append capability avoids invalid hinted emplace paths", + "[concepts]") +{ + STATIC_REQUIRE(can_append, int>); + STATIC_REQUIRE(container_appendable, int>); + STATIC_REQUIRE_FALSE(ceph::concepts::has_emplace_append, int>); + STATIC_REQUIRE(ceph::concepts::has_insert_append, int>); +} + +TEMPLATE_TEST_CASE("ordered associative containers model associative_container", + "[concepts]", std::set, std::multiset, + (std::map), + (std::multimap)) +{ + STATIC_REQUIRE(associative_container); +} + +TEMPLATE_TEST_CASE("unordered associative containers model unordered_associative_container", + "[concepts]", std::unordered_set, + std::unordered_multiset, + (std::unordered_map), + (std::unordered_multimap)) +{ + STATIC_REQUIRE(unordered_associative_container); +} + +/*** Tests for aggregate capability checks: */ + +TEMPLATE_TEST_CASE("appendable containers model can_append", "[concepts]", + std::vector, std::deque, std::list, + std::set, append_only) +{ + STATIC_REQUIRE(can_append); +} + +TEMPLATE_TEST_CASE("appendable containers model can_emplace_append", + "[concepts]", + std::vector, + std::deque, + std::list, + std::set) +{ + STATIC_REQUIRE(can_emplace_append); +} + +TEMPLATE_PRODUCT_TEST_CASE("prependable containers model can_prepend", + "[concepts]", + (std::deque, std::list, std::forward_list, + std::vector), + (int)) +{ + STATIC_REQUIRE(can_prepend); +} + +TEMPLATE_PRODUCT_TEST_CASE("standard range appendable containers model " + "can_append_range", + "[concepts]", (std::vector, std::set), (int)) +{ + STATIC_REQUIRE(can_append_range>); +} + +TEST_CASE("forward-only containers model can_insert_after", + "[concepts]") +{ + STATIC_REQUIRE(can_insert_after); + STATIC_REQUIRE_FALSE(can_insert_after); +} + +TEMPLATE_PRODUCT_TEST_CASE("predicate erasable containers model can_erase_if", + "[concepts]", + (std::vector, std::list, std::forward_list), + (int)) +{ + STATIC_REQUIRE(can_erase_if); +} + +TEMPLATE_PRODUCT_TEST_CASE("standard front removable containers model " + "can_remove_front", + "[concepts]", + (std::deque, std::list, std::forward_list, + std::vector), + (int)) +{ + STATIC_REQUIRE(can_remove_front); +} + +TEST_CASE("concept edge cases", + "[concepts]") +{ + STATIC_REQUIRE(same_as_any); + STATIC_REQUIRE_FALSE(same_as_any); + STATIC_REQUIRE_FALSE(std::ranges::contiguous_range>); + STATIC_REQUIRE_FALSE(container_compatible_range, + std::string>); + STATIC_REQUIRE_FALSE(container_compatible_range, + explicit_from_int>); + STATIC_REQUIRE_FALSE(container_compatible_range); + STATIC_REQUIRE_FALSE(associative_container>); + STATIC_REQUIRE_FALSE(unordered_associative_container>); + STATIC_REQUIRE_FALSE(can_append, int>); + STATIC_REQUIRE_FALSE(can_emplace_append, int>); + STATIC_REQUIRE_FALSE(container_appendable, int>); + STATIC_REQUIRE(can_append_range>); + STATIC_REQUIRE_FALSE(can_append_range, + std::array>); + STATIC_REQUIRE_FALSE(has_size>); + STATIC_REQUIRE_FALSE(reservable_container>); + STATIC_REQUIRE(can_remove_front); +} + +/*** Tests for library helpers: */ + +TEMPLATE_PRODUCT_TEST_CASE("helpers compose into container-generic algorithms", + "[concepts][util]", + (std::deque, std::list, std::vector), + (int)) +{ + const TestType expected{0, 1, 3}; + + CHECK(collect_odd_values() == expected); +} + +TEST_CASE("push_back appends through the best available container operation", + "[concepts][util]") +{ + SECTION("vector uses back insertion") { + std::vector values; + + ceph::util::push_back(values, 1); + ceph::util::push_back(values, 2); + + CHECK(values == std::vector{1, 2}); + } + + SECTION("set falls back to hinted insertion at end") { + std::set values; + + ceph::util::push_back(values, 2); + ceph::util::push_back(values, 1); + + CHECK(values == std::set{1, 2}); + } + + SECTION("custom append-only type uses insert(end, value)") { + append_only values; + + ceph::util::push_back(values, 1); + ceph::util::push_back(values, 2); + + CHECK(values.values == std::vector{1, 2}); + } +} + +TEST_CASE("emplace_append constructs at the natural append position", + "[concepts][util]") +{ + SECTION("vector constructs from string_view") { + std::vector values; + + ceph::util::emplace_append(values, std::string_view { "alpha" }); + ceph::util::emplace_append(values, std::string_view { "beta" }); + + CHECK(values == std::vector { "alpha", "beta" }); + } + + SECTION("list constructs from string_view") { + std::list values; + + ceph::util::emplace_append(values, std::string_view { "alpha" }); + ceph::util::emplace_append(values, std::string_view { "beta" }); + + CHECK(values == std::list { "alpha", "beta" }); + } + + SECTION("set falls back to value insertion") { + std::set values; + + ceph::util::emplace_append(values, std::string_view { "beta" }); + ceph::util::emplace_append(values, std::string_view { "alpha" }); + + CHECK(values == std::set { "alpha", "beta" }); + } +} + +TEST_CASE("append_range appends ranges with container-specific fallbacks", + "[concepts][util]") +{ + SECTION("vector appends iterator ranges") { + std::vector values{1}; + const std::array more{2, 3}; + + ceph::util::append_range(values, more); + + CHECK(values == std::vector{1, 2, 3}); + } + + SECTION("set inserts an input range") { + std::set values{1}; + const std::array more{3, 2}; + + ceph::util::append_range(values, more); + + CHECK(values == std::set{1, 2, 3}); + } + + SECTION("custom append-only type falls back to element appends") { + append_only values; + const std::array more{1, 2, 3}; + + ceph::util::append_range(values, more); + + CHECK(values.values == std::vector{1, 2, 3}); + } +} + +TEST_CASE("collect_as materializes ranges with append fallbacks", + "[concepts][util]") +{ + SECTION("sequence from view") { + auto values = ceph::util::collect_as>( + std::views::iota(0, 4)); + + CHECK(values == std::vector{0, 1, 2, 3}); + } + + SECTION("set from transformed view") { + auto values = ceph::util::collect_as>( + std::views::iota(0, 4) | + std::views::transform([](int i) { return 3 - i; })); + + CHECK(values == std::set{0, 1, 2, 3}); + } + + SECTION("custom append-only type") { + auto values = ceph::util::collect_as( + std::array{1, 2, 3}); + + CHECK(values.values == std::vector{1, 2, 3}); + } +} + +TEST_CASE("insert_range inserts at the requested position", "[concepts][util]") +{ + std::vector values{1, 4}; + const std::array more{2, 3}; + + ceph::util::insert_range(values, values.begin() + 1, more); + + CHECK(values == std::vector{1, 2, 3, 4}); +} + +TEST_CASE("insert_after inserts after the requested position", + "[concepts][util]") +{ + std::forward_list values; + auto pos = values.before_begin(); + + pos = ceph::util::insert_after(values, pos, 1); + pos = ceph::util::emplace_after(values, pos, 2); + ceph::util::insert_after(values, pos, 3); + + CHECK(values == std::forward_list{1, 2, 3}); +} + +TEST_CASE("make_appender appends through container-specific operations", + "[concepts][util]") +{ + SECTION("vector appender constructs at the back") { + std::vector values; + auto append = ceph::util::make_appender(values); + + append.emplace(std::string_view { "alpha" }); + append.emplace(std::string_view { "beta" }); + + CHECK(values == std::vector { "alpha", "beta" }); + } + + SECTION("forward_list appender tracks the insertion position") { + std::forward_list values; + auto append = ceph::util::make_appender(values); + + append.emplace(std::string_view { "alpha" }); + append.emplace(std::string_view { "beta" }); + + CHECK(values == std::forward_list { "alpha", "beta" }); + } +} + +TEST_CASE("front helpers use front-specific operations or begin erasure", + "[concepts][util]") +{ + SECTION("push_front prepends values") { + std::vector values{2, 3}; + + ceph::util::push_front(values, 1); + + CHECK(values == std::vector{1, 2, 3}); + } + + SECTION("pop_front removes the first value") { + front_erasable values{{1, 2, 3}}; + + ceph::util::pop_front(values); + + CHECK(values.values == std::vector{2, 3}); + } +} + +TEST_CASE("optional sizing helpers call supported operations only", + "[concepts][util]") +{ + std::vector values; + + ceph::util::maybe_reserve(values, 4); + ceph::util::maybe_resize(values, 3); + + CHECK(ceph::util::capacity(values) >= 4); + CHECK(std::ranges::size(values) == 3); + CHECK(!std::ranges::empty(values)); + + ceph::util::clear(values); + + CHECK(std::ranges::empty(values)); +} + +/* The use of X-macros here is a bit unfortunate, however as we're not yet using C++26 I + * don't think there's another way to express this without having... well, a *LOT* of essentially + * redundant tests: */ +#define CEPH_HAS_MEMBER_CAPABILITY_CASES(X) \ + X(emplace_back, (ceph::concepts::has_emplace_back), \ + (ceph::concepts::has_emplace_back)) \ + X(push_back, (ceph::concepts::has_push_back), \ + (ceph::concepts::has_push_back)) \ + X(emplace_append, (ceph::concepts::has_emplace_append), \ + (ceph::concepts::has_emplace_append)) \ + X(insert_append, (ceph::concepts::has_insert_append), \ + (ceph::concepts::has_insert_append)) \ + X(emplace_after, (ceph::concepts::has_emplace_after), \ + (ceph::concepts::has_emplace_after)) \ + X(insert_after, (ceph::concepts::has_insert_after), \ + (ceph::concepts::has_insert_after)) \ + X(append_range, (ceph::concepts::has_append_range), \ + (ceph::concepts::has_append_range)) \ + X(insert_range_append, (ceph::concepts::has_insert_range_append), \ + (ceph::concepts::has_insert_range_append)) \ + X(insert_iterator_range_append, (ceph::concepts::has_insert_iterator_range_append), \ + (ceph::concepts::has_insert_iterator_range_append)) \ + X(insert_range, (ceph::concepts::has_insert_range), \ + (ceph::concepts::has_insert_range)) \ + X(insert_iterator_range, (ceph::concepts::has_insert_iterator_range), \ + (ceph::concepts::has_insert_iterator_range)) \ + X(emplace_front, (ceph::concepts::has_emplace_front), \ + (ceph::concepts::has_emplace_front)) \ + X(push_front, (ceph::concepts::has_push_front), \ + (ceph::concepts::has_push_front)) \ + X(emplace_at_begin, (ceph::concepts::has_emplace_at_begin), \ + (ceph::concepts::has_emplace_at_begin)) \ + X(insert_at_begin, (ceph::concepts::has_insert_at_begin), \ + (ceph::concepts::has_insert_at_begin)) \ + X(pop_front, (ceph::concepts::has_pop_front), \ + (ceph::concepts::has_pop_front)) \ + X(erase_begin, (ceph::concepts::has_erase_begin), \ + (ceph::concepts::has_erase_begin)) \ + X(erase, (ceph::concepts::has_erase), \ + (ceph::concepts::has_erase)) \ + X(erase_range, (ceph::concepts::has_erase_range), \ + (ceph::concepts::has_erase_range)) \ + X(remove_if, (ceph::concepts::has_remove_if), \ + (ceph::concepts::has_remove_if)) \ + X(clear, (ceph::concepts::has_clear), \ + (ceph::concepts::has_clear)) \ + X(reserve, (ceph::concepts::has_reserve), \ + (ceph::concepts::has_reserve)) \ + X(capacity, (ceph::concepts::has_capacity), \ + (ceph::concepts::has_capacity)) \ + X(max_size, (ceph::concepts::has_max_size), \ + (ceph::concepts::has_max_size)) \ + X(size, (ceph::concepts::has_size), \ + (ceph::concepts::has_size)) \ + X(empty, (ceph::concepts::has_empty), \ + (ceph::concepts::has_empty)) \ + X(resize, (ceph::concepts::has_resize), \ + (ceph::concepts::has_resize)) + +#define CEPH_CHECK_HAS_MEMBER_CAPABILITY(member, positive, negative) \ + STATIC_REQUIRE positive; \ + STATIC_REQUIRE_FALSE negative; + +TEST_CASE("has_* member capability predicates match member availability", + "[concepts]") +{ + CEPH_HAS_MEMBER_CAPABILITY_CASES(CEPH_CHECK_HAS_MEMBER_CAPABILITY) +} + +// Put the X-macro machinery back into the original packaging: +#undef CEPH_CHECK_HAS_MEMBER_CAPABILITY +#undef CEPH_HAS_MEMBER_CAPABILITY_CASES