From 132c77376f6b28990bdabe838192c4a650178016 Mon Sep 17 00:00:00 2001 From: "Jesse F. Williamson" Date: Fri, 12 Sep 2025 13:07:43 -0700 Subject: [PATCH] Add FoundationDB client library "libfdb". Assisted-by: Codex:GPT-5 Signed-off-by: Jesse F. Williamson --- CMakeLists.txt | 23 +- examples/libfdb/Makefile | 21 + examples/libfdb/mini_example.cc | 77 +++ src/include/buffer.h | 3 + src/rgw/CMakeLists.txt | 54 +- src/rgw/ceph_fdb.h | 96 ++++ src/rgw/fdb/EXAMPLES.md | 324 ++++++++++++ src/rgw/fdb/NOTES.txt | 55 ++ src/rgw/fdb/base.h | 881 ++++++++++++++++++++++++++++++++ src/rgw/fdb/conversion.h | 129 +++++ src/rgw/fdb/fdb.h | 23 + src/rgw/fdb/interface.h | 657 ++++++++++++++++++++++++ src/test/bufferlist.cc | 11 + src/test/rgw/CMakeLists.txt | 38 +- src/test/rgw/test_fdb.cc | 754 +++++++++++++++++++++++++++ src/test/rgw/test_fdb_ceph.cc | 496 ++++++++++++++++++ src/test/rgw/test_fdb_common.h | 138 +++++ 17 files changed, 3755 insertions(+), 25 deletions(-) create mode 100644 examples/libfdb/Makefile create mode 100644 examples/libfdb/mini_example.cc create mode 100644 src/rgw/ceph_fdb.h create mode 100644 src/rgw/fdb/EXAMPLES.md create mode 100644 src/rgw/fdb/NOTES.txt create mode 100644 src/rgw/fdb/base.h create mode 100644 src/rgw/fdb/conversion.h create mode 100644 src/rgw/fdb/fdb.h create mode 100644 src/rgw/fdb/interface.h create mode 100644 src/test/rgw/test_fdb.cc create mode 100644 src/test/rgw/test_fdb_ceph.cc create mode 100644 src/test/rgw/test_fdb_common.h diff --git a/CMakeLists.txt b/CMakeLists.txt index b173c638cd0..43ce6ef4147 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -592,7 +592,7 @@ option(WITH_RADOSGW_DBSTORE "DBStore backend for RADOS Gateway" ON) option(WITH_RADOSGW_MOTR "CORTX-Motr backend for RADOS Gateway" OFF) option(WITH_RADOSGW_DAOS "DAOS backend for RADOS Gateway" OFF) option(WITH_RADOSGW_D4N "D4N wrapper for RADOS Gateway" ON) -cmake_dependent_option(WITH_RADOSGW_POSIX "POSIX backend for RADOS Gateway" ON WITH_RADOSGW_DBSTORE OFF) # posix depends on dbstore +option(WITH_RADOSGW_FDB "FoundationDB support for RADOS Gateway (experimental)" OFF) option(WITH_RADOSGW_RADOS "RADOS backend for Rados Gateway" ON) option(WITH_RADOSGW_SELECT_PARQUET "Support for s3 select on parquet objects" ON) option(WITH_RADOSGW_ARROW_FLIGHT "Build arrow flight when not using system-provided arrow" OFF) @@ -601,6 +601,9 @@ option(WITH_RADOSGW_BACKTRACE_LOGGING "Enable backtraces in rgw logs" OFF) option(WITH_SYSTEM_ARROW "Use system-provided arrow" OFF) option(WITH_SYSTEM_UTF8PROC "Use system-provided utf8proc" OFF) +# POSIX depends on DBStore: +cmake_dependent_option(WITH_RADOSGW_POSIX "POSIX backend for RADOS Gateway" ON WITH_RADOSGW_DBSTORE OFF) + if(WITH_RADOSGW) find_package(EXPAT REQUIRED) find_package(OATH REQUIRED) @@ -654,6 +657,23 @@ if(WITH_RADOSGW) message(STATUS "crypto soname: ${LIBCRYPTO_SONAME}") endif (WITH_RADOSGW) +if(WITH_RADOSGW_FDB) + include(${CMAKE_MODULE_PATH}/CPM.cmake) + + CPMAddPackage("gh:eyalz800/zpp_bits@4.5.1") + message("-- enabled zpp_bits") + +# JFW: sadly, I'm having trouble getting FoundationDB to build, but this is VERY close +# to "just working". But, how would we install the RPMs, etc.? This will need looking at +# before this can be non-experimental: +# CPMAddPackage("gh:apple/foundationdb@7.3.63") + message("FoundationDB support is EXPERIMENTAL and requires manual setup:") + message(" wget https://github.com/apple/foundationdb/releases/download/7.3.63/foundationdb-clients-7.3.63-1.el7.x86_64.rpm") + message(" wget https://github.com/apple/foundationdb/releases/download/7.3.63/foundationdb-server-7.3.63-1.el7.x86_64.rpm") + message("-- enabled FoundationDB (whether its there or not)") + +endif() + #option for CephFS option(WITH_CEPHFS "CephFS is enabled" ON) @@ -924,4 +944,3 @@ add_tags(ctags EXCLUDE_OPTS ${CTAG_EXCLUDES} EXCLUDES "*.js" "*.css" ".tox" "python-common/build") add_custom_target(tags DEPENDS ctags) - diff --git a/examples/libfdb/Makefile b/examples/libfdb/Makefile new file mode 100644 index 00000000000..18304dfa314 --- /dev/null +++ b/examples/libfdb/Makefile @@ -0,0 +1,21 @@ +# Mini-example for libfdb + +CXX?=g++ +CXXFLAGS+=-std=c++23 -Wno-unused-parameter -Wall -Wextra -Werror -g + +CEPH_SRC?=../../src +CEPH_BUILD?=../../build + +INCLUDES=-I$(CEPH_SRC) -I$(CEPH_BUILD)/_deps/zpp_bits-src +LIBRARY_PATH=-L$(CEPH_BUILD)/lib/ -Wl,-rpath,$(CEPH_BUILD)/lib +LIBRARIES=-lfdb_c -lfmtd + +all: mini_example + +.PHONY: clean + +mini_example: mini_example.cc + $(CXX) $(CXXFLAGS) $(INCLUDES) $(LIBRARY_PATH) -o mini_example mini_example.cc $(LIBRARIES) + +clean: + rm -f mini_example diff --git a/examples/libfdb/mini_example.cc b/examples/libfdb/mini_example.cc new file mode 100644 index 00000000000..f1e14af4f67 --- /dev/null +++ b/examples/libfdb/mini_example.cc @@ -0,0 +1,77 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2026 International Business Machines Corp. (IBM) + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + * Whee! A mini-demo for libfdb! + * + * This shows how to take a user-defined data stucture and both store and + * retrieve it in FoundationDB. + * +*/ + +#include "rgw/fdb/interface.h" + +#include +#include +#include +#include + +namespace lfdb = ceph::libfdb; + +using namespace std; + +struct person +{ + string name; + + int year_born = 2000; + + vector titles; +}; + +static const vector nifty_people { + { + .name = "Albrecht Duerer", + .year_born = 1471, + .titles = { "Painter", "Printmaker", "Northern Renaissance artist" }, + }, + { + .name = "Maria Theresa", + .year_born = 1717, + .titles = { "Habsburg ruler", "Archduchess of Austria", "Queen of Hungary and Bohemia" }, + } +}; + +int main() +try +{ + auto dbh = lfdb::create_database(); + + lfdb::set(dbh, "example/people", nifty_people); + + vector people; + if(lfdb::get(dbh, "example/people", people)) { + println("read {} people, some of them are likely nifty", people.size()); + } +} +catch(const lfdb::libfdb_exception& e) { + println(stderr, "libfdb exception: {}", e.what()); + return 1; +} +catch(const exception& e) { + println(stderr, "exception: {}", e.what()); + return 1; +} +catch(...) { + println(stderr, "Well, now we're in a real jam, huh?"); + return 1; +} diff --git a/src/include/buffer.h b/src/include/buffer.h index 73b7309c15a..a182fe06aa2 100644 --- a/src/include/buffer.h +++ b/src/include/buffer.h @@ -289,6 +289,7 @@ struct error_code; const char *end_c_str() const; char *end_c_str(); unsigned length() const { return _len; } + unsigned size() const { return length(); } unsigned offset() const { return _off; } unsigned start() const { return _off; } unsigned end() const { return _off + _len; } @@ -1010,6 +1011,7 @@ struct error_code; const buffers_t& buffers() const { return _buffers; } buffers_t& mut_buffers() { return _buffers; } void swap(list& other) noexcept; + unsigned length() const { #if 0 // DEBUG: verify _len @@ -1027,6 +1029,7 @@ struct error_code; #endif return _len; } + unsigned size() const { return length(); } bool contents_equal(const buffer::list& other) const; bool contents_equal(const void* other, size_t length) const; diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index a8082f580ad..875c5b21397 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -262,19 +262,22 @@ endif() if(WITH_RADOSGW_DAOS) list(APPEND librgw_common_srcs driver/motr/rgw_sal_daos.cc) endif() + if(WITH_RADOSGW_POSIX) #add_subdirectory(driver/posix) find_package(LMDB REQUIRED) add_compile_definitions(LMDB_SAFE_NO_CPP_UTILITIES) list(APPEND librgw_common_srcs - driver/posix/rgw_sal_posix.cc driver/posix/lmdb-safe.cc - driver/posix/posixDB.cc - driver/posix/notify.cpp) + driver/posix/notify.cpp + driver/posix/posixDB.cc + driver/posix/rgw_sal_posix.cc) endif() + if(WITH_JAEGER) list(APPEND librgw_common_srcs rgw_tracer.cc) endif() + if(WITH_RADOSGW_ARROW_FLIGHT) # NOTE: eventually don't want this in common but just in radosgw daemon # list(APPEND radosgw_srcs rgw_flight.cc rgw_flight_frontend.cc) @@ -393,10 +396,6 @@ if(WITH_RADOSGW_DBSTORE) target_link_libraries(rgw_common PRIVATE global dbstore) endif() -if(WITH_RADOSGW_POSIX) - target_link_libraries(rgw_common PRIVATE global dbstore) -endif() - if(WITH_RADOSGW_MOTR) find_package(motr REQUIRED) target_link_libraries(rgw_common PRIVATE motr::motr) @@ -516,6 +515,7 @@ set(radosgw_srcs rgw_main.cc) add_executable(radosgw ${radosgw_srcs}) + if(WITH_RADOSGW_ARROW_FLIGHT) # target_compile_definitions(radosgw PUBLIC WITH_ARROW_FLIGHT) target_compile_definitions(rgw_common PUBLIC WITH_ARROW_FLIGHT) @@ -538,6 +538,46 @@ target_link_libraries(radosgw PRIVATE ${rgw_libs} ${ALLOC_LIBS}) +if(WITH_RADOSGW_FDB) + message("FoundationDB support is active (experimental)") + + add_library(rgw_fdb INTERFACE) + + target_link_libraries(rgw_fdb INTERFACE + -lfdb_c + -lm + -lrt + ) + + target_compile_options(rgw_fdb INTERFACE + -pthread) + +#JFW: this is where the FDB client library packages install +#fdb_c headers-- we need to find a smarter way to handle this +#end of it before libfdb can be non-experimental: + target_include_directories(rgw_fdb INTERFACE + /usr/include/foundationdb) + + target_include_directories(rgw_common PRIVATE + /usr/include/foundationdb + "${CMAKE_SOURCE_DIR}/src/rgw") + + target_link_libraries(rgw_common PRIVATE + rgw_fdb) + + target_include_directories(radosgw PRIVATE + /usr/include/foundationdb) + + # Use the zpp_bits back-end for serialization: + add_library(zpp_bits INTERFACE) + + # JFW: there must be a nicer way to refer to this path (or ask it to be installed somewhere?): + target_include_directories(zpp_bits INTERFACE "${CMAKE_BINARY_DIR}/_deps/zpp_bits-src/") + + target_link_libraries(rgw_fdb INTERFACE zpp_bits) + +endif(WITH_RADOSGW_FDB) + install(TARGETS radosgw DESTINATION bin) set(radosgw_admin_srcs diff --git a/src/rgw/ceph_fdb.h b/src/rgw/ceph_fdb.h new file mode 100644 index 00000000000..763828c1c19 --- /dev/null +++ b/src/rgw/ceph_fdb.h @@ -0,0 +1,96 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2025-2026 International Business Machines Corp. (IBM) + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * +*/ + +#ifndef CEPH_FDB_H + #define CEPH_FDB_H + +/* Welcome, brave adventurer! This where Ceph types are converted back and forth +between FDB's types! If you have a user type to add, this is the place! +*/ + +#include "include/buffer.h" + +#include "fdb/fdb.h" + +#include +#include +#include +#include + +/*** Conversions: */ + +namespace ceph::libfdb::detail { + +auto as_fdb_span(ceph::buffer::list& bl) +{ + // c_str() makes the buffer::list contiguous. Use length(), not C-string + // rules, because buffer::list may contain arbitrary bytes: + auto p = bl.c_str(); + + return std::span( + reinterpret_cast(p), bl.length()); +} + +} // namespace ceph::libfdb::detail + +namespace ceph::buffer { + +auto serialize(auto& archive, ceph::buffer::list& target) +{ + // This should be revisited after the library is separated from the larger + // surrounding project-- essentially, we can't write directly /into/ the buffer::list + // that I know of; yet, it would obviously be great to eliminate the copy + // here. I believe that somewhere in zpp::bits there's probably a way to get the + // archive to call a custom function-- but it is not clear to me at this time and + // I need to move on for now, unfortunately: + std::vector out; + + auto r = archive(out); + + // JFW: this is really annoying, but again buffer::list is not something I find + // easy to wrangle-- I'm not sure there's a straightforward way to just assign + // a new value... so if you know what that is, please improve this: + target.clear(); + target.append(out); + + return r; +} + +auto serialize(auto& archive, const ceph::buffer::list& src) +{ + std::vector bytes; + + auto i = std::cbegin(src); + + // buffer::list copy already resizes the vector: + i.copy(src.length(), bytes); + + return archive(std::span { bytes.data(), bytes.size() }); +} + +// This transliteration is one-way: we read from a buffer::ptr to an archive; writing +// to a ceph::buffer::ptr as output seems pointless to me, as we already support spans +// and buffer::list-- I feel in fact like we would be courting danger we don't need as it +// is a non-owning structure. +auto serialize(auto& archive, const ceph::buffer::ptr& src) +{ + std::span src_span((std::uint8_t *)src.c_str(), src.length()); + + return archive(src_span); +} + +} // namespace ceph::buffer + +#endif diff --git a/src/rgw/fdb/EXAMPLES.md b/src/rgw/fdb/EXAMPLES.md new file mode 100644 index 00000000000..f06f24d6890 --- /dev/null +++ b/src/rgw/fdb/EXAMPLES.md @@ -0,0 +1,324 @@ +# libfdb Examples + +"Take a thousand days of practice for forging, and ten thousand days of practice for refining." + -- Miyamoto Musashi, Go Rin No Sho (~1645) + +Welcome, traveller! Grab your favorite walking stick, and let us journey into +the realm of libfdb! + +While this is not proper documentation, hopefully this "cookbook-stye" set +of mini-examples will help you on your libfdb path. + +Errata: Please report errata, or contact with examples you would like to see! + +These examples use a short namespace alias for readability and also to save +typing with one's poor fingers! Yoikes! + +See examples/libfdb/ for some working, compilable simple examples. + +```cpp +namespace lfdb = ceph::libfdb; +using namespace std::string_literals; +``` + +## General Recipes + +```cpp +/* Use a database_handle when you desire a single logical operation. Behind + * the scenes, libfdb will create and complete its own transaction for you. + * Database-handle operations may retry after recoverable FoundationDB errors, + * so callbacks and output iterators may be activated more than once. */ +lfdb::set(dbh, "person/barbara-moo/name", "Barbara Moo"); +``` + +```cpp +/* Pass a transaction handle when several operations must be grouped in the + * same transaction. Do not use the transaction after commit(). */ +auto txn = lfdb::make_transaction(dbh); + +lfdb::set(txn, "person/barbara-moo/name", "Barbara Moo"); +lfdb::set(txn, "person/barbara-moo/book", "Accelerated C++"); + +if (!lfdb::commit(txn)) { + /* Retry the transaction body with a fresh or recovered transaction. */ +} +``` + +## Setup + +```cpp +/* Open the default FoundationDB database. */ +auto dbh = lfdb::create_database(); +``` + +```cpp +/* Open a database with database and network options. Flag-only options use + * lfdb::option_flag because they have no value. Network options are applied + * only during the first FoundationDB network initialization; later calls to + * create_database() cannot change them. */ +lfdb::database_options dbopts{ + { FDB_DB_OPTION_TRANSACTION_TIMEOUT, std::int64_t{5000} }, +}; + +lfdb::network_options netopts{ + { FDB_NET_OPTION_TRACE_ENABLE, lfdb::option_flag }, +}; + +auto dbh = lfdb::create_database(dbopts, netopts); +``` + +```cpp +/* Open a database with an explicit cluster file plus database/network options. */ +auto dbh = lfdb::create_database("/path/to/fdb.cluster", dbopts, netopts); +``` + +## Single-Key Operations + +```cpp +/* Store and retrieve one value by key. */ +lfdb::set(dbh, "person/konrad-zuse/name", "Konrad Zuse"); + +std::string name; +if (lfdb::get(dbh, "person/konrad-zuse/name", name)) { + /* use name */ +} +``` + +```cpp +/* Use a callback when the raw serialized bytes must be copied or decoded + * immediately. The span is only valid during the callback. */ +lfdb::get(dbh, "person/konrad-zuse/name", + [](std::span bytes) { + /* copy or decode bytes here */ + }); +``` + +## Key Existence And Erase + +```cpp +/* Check for a key and erase it if it exists. */ +if (lfdb::key_exists(dbh, "person/jose-capablanca/title")) { + lfdb::erase(dbh, "person/jose-capablanca/title"); +} +``` + +## Multi-Key Writes + +```cpp +/* Write key/value pairs from an STL associative container in one transaction. */ +std::map people{ + { "person/saladin/name", "Saladin" }, + { "person/al-khwarizmi/name", "Al-Khwarizmi" }, + { "person/albrecht-duerer/name", "Albrecht Duerer" }, +}; + +lfdb::set(dbh, std::begin(people), std::end(people)); +``` + +## Multi-Key Reads + +```cpp +/* Read a key range into an STL associative container. */ +std::map people; + +lfdb::get(dbh, + lfdb::select { "person/" }, + std::inserter(people, std::end(people))); +``` + +## Key Ordering + +```cpp +/* FoundationDB keys are ordered lexicographically by byte string. Choose key + * formats so lexical order matches the scan order you want. Numeric suffixes + * should usually be fixed-width and zero-padded. */ +lfdb::set(dbh, "person/000001/name", "Barbara Moo"); +lfdb::set(dbh, "person/000010/name", "Konrad Zuse"); +``` + +## Prefix Selection + +"select" has two constructor forms. The one-argument form is usually the one +you want: it selects every key with a shared prefix. This is a natural fit for +FoundationDB key design, where related records are commonly grouped under a +prefix such as `person/`, `bucket/index/`, or `object/metadata/`. + +```cpp +/* Select all keys beginning with "person/". */ +auto people = lfdb::select { "person/" }; +``` + +Using a key beginning with 0xFF will result in unpredictable behavior. + +## Explicit Key Ranges + +```cpp +/* Select a half-open lexicographic key range: begin is included, end is + * excluded. */ +auto medieval_people = lfdb::select { "person/charlemagne", "person/saladin/" }; +``` + +## Pair Generator + +```cpp +/* Use pair_generator() for ordinary range scans where processing one key/value + * pair at a time is natural. */ +std::map people; + +std::ranges::copy(lfdb::pair_generator(dbh, lfdb::select { "person/" }), + std::inserter(people, std::end(people))); +``` + +## Block Generator + +For very large prefix scans, use the same one-argument selector with +`block_generator()`. This lets libfdb plan and read the range in blocks while +the call site still names the logical key family directly. + +```cpp +/* Use block_generator() for large range scans where split planning and + * block-at-a-time processing are useful. */ +for (auto&& block : lfdb::block_generator(dbh, lfdb::select { "object/metadata/" })) { + /* process block */ +} +``` + +## STL Containers As Values + +```cpp +/* Store an STL container as one serialized value. */ +std::vector roles{ "compiler"s, "systems"s, "naval-officer"s }; + +lfdb::set(dbh, "person/grace-hopper/roles", roles); + +std::vector out_roles; +lfdb::get(dbh, "person/grace-hopper/roles", out_roles); +``` + +## Associative Containers As Values + +```cpp +/* Store an associative container as one serialized value. */ +std::map profile{ + { "name", "Maria Theresa" }, + { "title", "Archduchess of Austria" }, +}; + +lfdb::set(dbh, "person/maria-theresa/profile", profile); + +std::map out_profile; +lfdb::get(dbh, "person/maria-theresa/profile", out_profile); +``` + +## User Types As Values + +```cpp +/* Store a user-defined type as one serialized value. */ +struct person_profile +{ + using serialize = zpp::bits::members<3>; + + std::string name; + std::string field; + std::vector tags; +}; + +auto profile = person_profile{ + .name = "Edsger Dijkstra", + .field = "computer science", + .tags = std::vector{ "algorithms"s, "formal-methods"s }, +}; + +lfdb::set(dbh, "person/edsger-dijkstra/profile", profile); + +person_profile out_profile; +lfdb::get(dbh, "person/edsger-dijkstra/profile", out_profile); +``` + +## Manual Transactions + +```cpp +/* Group multiple operations in one explicit transaction. */ +auto txn = lfdb::make_transaction(dbh); + +lfdb::set(txn, "person/matilda-of-tuscany/name", "Matilda of Tuscany"); +lfdb::set(txn, "person/matilda-of-tuscany/title", "Margravine"); + +if (!lfdb::commit(txn)) { + /* Retry the transaction body. */ +} +``` + +## Manual Transactions With Options + +```cpp +/* Create an explicit transaction with transaction options. */ +lfdb::transaction_options opts{ + { FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE, lfdb::option_flag }, +}; + +auto txn = lfdb::make_transaction(dbh, opts); + +lfdb::set(txn, "person/hypatia/name", "Hypatia"); + +if (!lfdb::commit(txn)) { + /* Retry the transaction body. */ +} +``` + +## Transactors: Replayable Transactions + +Transactors are function objects created with `make_transactor()`. Creating a +transactor does not start a transaction; calling `operator()` creates the +transaction, invokes the body, and commits it. + +The body may be called more than once after retryable FoundationDB errors. Keep +it deterministic and free of non-idempotent external side effects. If recovery +is not possible, or if user code throws, the exception escapes to the caller. + +```cpp +/* Use a transactor when the transaction body should be replayed after retryable + * FoundationDB errors. */ +auto txr = lfdb::make_transactor(dbh); + +txr([](auto& txn) { + lfdb::set(txn, "person/eleanor-of-aquitaine/name", "Eleanor of Aquitaine"); + lfdb::set(txn, "person/eleanor-of-aquitaine/title", "Duchess of Aquitaine"); +}); +``` + +### Transactor options + +```cpp +/* Options are applied to each transaction the transactor creates. */ +lfdb::transaction_options opts{ + { FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE, lfdb::option_flag }, +}; + +auto txr = lfdb::make_transactor(dbh, opts); + +txr([](auto& txn) { + lfdb::set(txn, "person/zenobia/name", "Zenobia"); +}); +``` + +```cpp +/* Retryable FoundationDB errors are handled before control returns here. */ +auto txr = lfdb::make_transactor(dbh); + +try { + txr([](auto& txn) { + /* User exceptions propagate; the body is not committed. */ + validate_profile_update(); + + lfdb::set(txn, "person/jose-capablanca/title", + std::vector{ "Original Grandmaster"s, "World Chess Champion"s }); + }); +} +catch (const lfdb::libfdb_exception& e) { + /* FoundationDB reported an error that libfdb could not recover from. */ +} +catch (const std::exception& e) { + /* Application or system error from user code. */ +} +``` diff --git a/src/rgw/fdb/NOTES.txt b/src/rgw/fdb/NOTES.txt new file mode 100644 index 00000000000..2f01767bb9b --- /dev/null +++ b/src/rgw/fdb/NOTES.txt @@ -0,0 +1,55 @@ + +* If you're looking for some information about how to use the library, interface.h is the "user interface". + +---- +Network and Database Options: + +There are various database at network settings that can be adjusted; see the "options" test for some examples of using +the network_options and database_options structures. But what may be tweaked, you ask? Excellent question-- and when +you go to fdb_c's documentation, you will find little, because they are GENERATED! And, from whence? Well... because +this file is VERY hard to find (in the FDB source), here you are: + fdbclient/vexillographer/fdb.options + +---- +Equivalence with FDBStreamingMode: + +FDB_STREAMING_MODE_ITERATOR + +The caller is implementing an iterator (most likely in a binding to a higher level language). The amount of data returned depends on the value of the iteration parameter to fdb_transaction_get_range(). + +FDB_STREAMING_MODE_SMALL + +Data is returned in small batches (not much more expensive than reading individual key-value pairs). + +FDB_STREAMING_MODE_MEDIUM + +Data is returned in batches between _SMALL and _LARGE. + +FDB_STREAMING_MODE_LARGE + +Data is returned in batches large enough to be, in a high-concurrency environment, nearly as efficient as possible. If the caller does not need the entire range, some disk and network bandwidth may be wasted. The batch size may be still be too small to allow a single client to get high throughput from the database. + +FDB_STREAMING_MODE_SERIAL + +Data is returned in batches large enough that an individual client can get reasonable read bandwidth from the database. If the caller does not need the entire range, considerable disk and network bandwidth may be wasted. + +FDB_STREAMING_MODE_WANT_ALL + +The caller intends to consume the entire range and would like it all transferred as early as possible. + +FDB_STREAMING_MODE_EXACT + +The caller has passed a specific row limit and wants that many rows delivered in a single batch. + +enum struct streaming_mode_t : int { + iterator = FDB_STREAMING_MODE_ITERATOR, + small = FDB_STREAMING_MODE_SMALL, + medium = FDB_STREAMING_MODE_MEDIUM, + large = FDB_STREAMING_MODE_LARGE, + serial = FDB_STREAMING_MODE_SERIAL, + all = FDB_STREAMING_MODE_WANT_ALL, +3F exact = FDB_STREAMING_MODE_EXACT +}; + +...these are not defined in terms of int or enum as far as I can tell, needs more exploring. + diff --git a/src/rgw/fdb/base.h b/src/rgw/fdb/base.h new file mode 100644 index 00000000000..b62ce6e855c --- /dev/null +++ b/src/rgw/fdb/base.h @@ -0,0 +1,881 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2025-2026 International Business Machines Corp. (IBM) + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * +*/ + +#ifndef CEPH_FDB_BASE_H + #define CEPH_FDB_BASE_H + +// The API version we're writing against, which can (and probably does) differ +// from the installed version. This must be defined before the FoundationDB header +// is included (see fdb_c_apiversion.g.h): +#define FDB_API_VERSION 730 +#include + +// Ceph uses libfmt rather than : +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cpp_lib_flat_map + #include + template + using flat_map = std::flat_map; +#else + #include + template + using flat_map = boost::container::flat_map; +#endif + +// Wrangle some forward declarations: +namespace ceph::libfdb { + +struct select; + +class database; +class transaction; + +using database_handle = std::shared_ptr; +using transaction_handle = std::shared_ptr; + +extern transaction_handle make_transaction(database_handle dbh); + +} // namespace ceph::libfdb + +// MOAR forward declarations-- "pay no attention to that man behind the curtain": +namespace ceph::libfdb::detail { + +template +concept is_any_of = (std::is_same_v || ...); + +struct future_value; + +template +std::pair to_decoded_kv_pair(const FDBKeyValue& kv); + +inline fdb_error_t do_commit(transaction_handle& txn); + +inline void transaction_set_kv_bytes(const transaction_handle& txn, + std::span k, + std::span v); + +inline future_value await_ready_key_range_future(transaction_handle txn, const ceph::libfdb::select&, int); +inline future_value await_ready_keyvalue_range_future(transaction_handle txn, const ceph::libfdb::select& key_range, int& iteration); + +// A generator that produces successive spans for a range: +inline std::generator> generate_FDB_pairs(ceph::libfdb::transaction& txn, ceph::libfdb::select key_range); + +// Stores a successively-generated of kv pair results to an iterator: +template +requires std::output_iterator> +inline bool get_value_range_from_transaction(ceph::libfdb::transaction& txn, const ceph::libfdb::select& key_range, OutIterT out_iter); + +} // namespace ceph::libfdb::detail + +namespace ceph::libfdb::concepts { + +// Note that "stringlikes" are not all "stringview-likes", such as when they can be +// written to: +template +concept stringview_convertible = std::convertible_to; + +template +concept key_value_iterator = + std::input_iterator and + requires(const std::iter_value_t& kv) { + kv.first; + kv.second; + }; + +template +concept value_callback = + std::invocable>; + +template +concept value_output = + !value_callback> && + std::is_lvalue_reference_v; + +template +concept storable_invocation_result = + !std::is_void_v && !std::is_reference_v; + +template +concept supported_invocation_result = + std::is_void_v || storable_invocation_result; + +// There's a high likelihood that we're going to get more sophisticated selectors, +// so this is doing a more important job than it may appear to be: +template +concept selector = ceph::libfdb::detail::is_any_of; + +} // namespace ceph::libfdb::concepts + +// libfdb_exception: How to deal, when Bad Things(TM) happen: +namespace ceph::libfdb { + +// Should we commit after the (possibly) mutating operation? +enum struct commit_after_op { commit, no_commit }; + +struct libfdb_exception final : std::runtime_error +{ + using std::runtime_error::runtime_error; + + fdb_error_t fdb_error_value = -1; + + bool retryable() const noexcept + { + return 0 < fdb_error_value && + fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, fdb_error_value); + } + + libfdb_exception(std::string_view msg) + : std::runtime_error(make_error_string(msg)) + {} + + explicit libfdb_exception(fdb_error_t fdb_error_value) + : std::runtime_error(make_fdb_error_string(fdb_error_value)), + fdb_error_value(fdb_error_value) + {} + + static std::string make_error_string(const std::string_view msg) + { + return fmt::format("libfdb: {}", msg); + } + + static std::string make_fdb_error_string(const fdb_error_t ec) + { + return make_error_string(fmt::format("FoundationDB error {}: {}", ec, fdb_get_error(ec))); + } +}; + +// A straightforward but pretty handy predicate-- ask an exception if it's something the +// user can try replaying an operation to correct: +inline bool retryable(const libfdb_exception& e) noexcept +{ + return e.retryable(); +} + +namespace detail { + +struct future_value final +{ + std::unique_ptr future_ptr; + + public: + explicit future_value(FDBFuture *future_handle) + : future_ptr(future_handle, &fdb_future_destroy) + {} + + public: + FDBFuture *raw_handle() const noexcept { return future_ptr.get(); } + + private: + void destroy() noexcept { future_ptr.reset(nullptr); } + + private: + friend class ceph::libfdb::transaction; +}; + +constexpr auto as_fdb_span(const char *s) +{ + return std::span((const std::uint8_t *)s, std::strlen(s)); +} + +constexpr auto as_fdb_span(std::string_view sv) +{ + return std::span((const std::uint8_t *)sv.data(), sv.size()); +} + +/* Prefix selectors assume user keys do not start with 0xFF. Appending 0xFF +would exclude valid keys (beginning with prefix + 0xFF), so we build the +next-in-lexicographic-order key instead. Callers can still specify an +explicit full range if they need something special: */ +inline std::string make_range_end_key_for_prefix(std::string_view prefix) +{ + if (prefix.empty()) { + return "\xFF"; + } + + if (0xFF == static_cast(prefix.front())) { + throw libfdb_exception("requested prefix has no (finite) end key"); + } + + auto i = prefix.find_last_not_of(static_cast(0xFF)); + auto end_key = std::string(prefix.substr(0, i + 1)); + + end_key.back() = static_cast(static_cast(end_key.back()) + 1); + + return end_key; +} + +} // namespace ceph::libfdb::detail + +/* lfdb::select is for handling batch queries-- there are two constructors to consider: + - single-key select creates the range containing all keys with that prefix; + - passing a start and end key creates a half-open lexicographic key range: [begin_key, end_key). */ +struct select final +{ + std::string begin_key, end_key; + + public: + // We'll eventually need a way to get settings into the base library from the binding layer; there + // are a few ways we could do it, this is one I'm mulling over; do not use this right now. + mutable struct { + int stride = 0; // "unlimited" + + // Some parts of the documentation claim FDB_STREAMING_MODE_ITERATOR is the default, other parts + // don't... examples tend to use FDB_STREAMING_MODE_WANT_ALL, but they operate on fairly small amounts + // of data. It's pretty hard to understand what the Right Thing(TM) to do is, the sure the answer may + // evolve as I learn more, but for now this setting at least means it's "plumbed through" (this + // particular mode starts with small batches and then grows to larger increments as more data is sent), + // even though this isn't really used at the moment: + FDBStreamingMode streaming_mode = FDB_STREAMING_MODE_ITERATOR; + + } options; + + public: + select(std::string_view begin_key_, std::string_view end_key_) + : begin_key(begin_key_), end_key(end_key_) + {} + + select(std::string_view prefix) + : begin_key(prefix), + end_key(detail::make_range_end_key_for_prefix(prefix)) + {} + +}; + +// Flag-only options are indicated with an explicit option_flag, as they have no actual value: +struct option_flag_t final {}; +inline constexpr option_flag_t option_flag; + +using option_value = std::variant>; + +// i.e. option /code/ to the value of the option itself (e.g. FDB_FOO, 42): +template +using option_map = flat_map; + +using network_options = option_map; +using database_options = option_map; +using transaction_options = option_map; + +namespace detail { + +// Note that these are specific to FDB's needs (see return type, casts): +constexpr const std::uint8_t *data_of(const std::vector& xs) { return (const std::uint8_t *)xs.data(); } +constexpr const std::uint8_t *data_of(const std::string& xs) { return (const std::uint8_t *)xs.data(); } +constexpr const std::uint8_t *data_of(const std::int64_t& x) { return reinterpret_cast(&x); } +constexpr const std::uint8_t *data_of(const option_flag_t&) { return nullptr; } + +// Note that these are specific to FDB's needs (see return type, casts): +constexpr int size_of(const std::vector& xs) { return static_cast(xs.size()); } +constexpr int size_of(const std::string& xs) { return static_cast(xs.size()); } +constexpr int size_of(const std::int64_t) { return 8; } // 8b = size required by FDB +constexpr int size_of(const option_flag_t&) { return 0; } + +// ...also used: +constexpr std::uint8_t *data_of(std::string_view xs) { return (std::uint8_t *)xs.data(); } +constexpr int size_of(std::string_view xs) { return static_cast(xs.size()); } + +inline auto as_fdb_option_args(const auto& option) +{ + auto [data, size] = std::visit([](const auto& x) { + return std::tuple { data_of(x), size_of(x) }; + }, + option.second); + + return std::tuple { option.first, data, size }; +} + +inline void apply_options(const auto& option_map, auto&& set_option) +{ + std::ranges::for_each(option_map, [&set_option](const auto& option) { + auto args = as_fdb_option_args(option); + + if (auto r = std::apply(set_option, args); 0 != r) { + throw libfdb_exception(fmt::format("while setting option {}; {}", + static_cast(option.first), + libfdb_exception::make_fdb_error_string(r))); + } + }); +} + +// The global DB state and management thread: +// JFW: more user hooks that go into FDB system possible here +class database_system final +{ + database_system() = delete; + + private: + static inline bool was_initialized = false; + + static inline std::once_flag fdb_was_initialized; + static inline std::jthread fdb_network_thread; + + static inline void initialize_fdb(const network_options& options) + { + // This must be called before ANY other API function: + if (fdb_error_t r = fdb_select_api_version(FDB_API_VERSION); 0 != r) + throw libfdb_exception(r); + + // Zero or more calls to this may now be made: + apply_options(options, fdb_network_set_option); + + // This must be called before any other API function (besides >= 0 calls to fdb_network_set_option()): + if (fdb_error_t r = fdb_setup_network(); 0 != r) + throw libfdb_exception(r); + + // Launch network thread: + fdb_network_thread = std::jthread { &fdb_run_network }; + + // Okie-dokie, we're all set (distinct from fdb_was_initialized): + was_initialized = true; + } + + private: + database_system(const network_options& network_opts) + { + std::call_once(ceph::libfdb::detail::database_system::fdb_was_initialized, + ceph::libfdb::detail::database_system::initialize_fdb, + network_opts); + } + + public: + static bool& initialized() noexcept { return was_initialized; } + + public: + static inline void shutdown_fdb() + { + using namespace std::chrono_literals; + + if (not initialized()) { + return; + } + + // shut down network and database: + if (int r = fdb_stop_network(); 0 != r) + { + // In this case, we likely don't want to throw from our dtor, but we may + // have something to log. As there's no higher-level hook for that, we + // have nothing to do right now. + // fmt::println("database::shutdown_fdb() error {}", r); + } + + // This may not actually be needed, but it's a traditional courtesy: + std::this_thread::sleep_for(7ms); + + if (fdb_network_thread.joinable()) { + fdb_network_thread.join(); + } + } + + private: + friend struct ceph::libfdb::database; +}; + +} // namespace ceph::libfdb::detail + +class database final +{ + detail::database_system db_system; + + private: + std::shared_ptr db_handle; + + FDBDatabase *create_database_ptr(const std::filesystem::path cluster_file_path) { + FDBDatabase *fdbp = nullptr; + + if (fdb_error_t r = fdb_create_database(cluster_file_path.c_str(), &fdbp); 0 != r) { + throw libfdb_exception(r); + } + + return fdbp; + } + + FDBTransaction *create_transaction() { + FDBTransaction *txn_p = nullptr; + + if (fdb_error_t r = fdb_database_create_transaction(raw_handle(), &txn_p); 0 != r) { + throw libfdb_exception(r); + } + + return txn_p; + } + + public: + database(const std::filesystem::path cluster_file_path, const ceph::libfdb::database_options& db_opts, const network_options& network_opts) + : db_system(network_opts), + db_handle(create_database_ptr(cluster_file_path), &fdb_database_destroy) + { + detail::apply_options(db_opts, + [handle = raw_handle()](auto option_code, auto data, auto size) { + return fdb_database_set_option(handle, option_code, data, size); + }); + } + + database(const std::filesystem::path cluster_file_path, const ceph::libfdb::database_options& db_opts) + : database(cluster_file_path, db_opts, {}) + {} + + database(const ceph::libfdb::database_options& db_opts, const ceph::libfdb::network_options& net_opts) + : database("", db_opts, net_opts) + {} + + database(const ceph::libfdb::database_options& db_opts) + : database("", db_opts, {}) + {} + + database(const std::filesystem::path cluster_file_path) + : database(cluster_file_path, {}, {}) + {} + + database() + : database(std::filesystem::path {}, {}, {}) + {} + + public: + explicit operator bool() const noexcept { return nullptr != raw_handle(); } + + public: + FDBDatabase *raw_handle() const noexcept { return db_handle.get(); } + + private: + friend transaction; + +}; + +class transaction final +{ + database_handle dbh; + + std::unique_ptr txn_handle; + + private: + bool get_single_value_from_transaction(const std::span& key, std::invocable> auto&& write_output); + + public: + transaction(database_handle dbh_) + : dbh(dbh_), + txn_handle(dbh->create_transaction(), &fdb_transaction_destroy) + {} + + transaction(database_handle dbh_, const transaction_options& opts) + : transaction(dbh_) + { + detail::apply_options(opts, + [handle = raw_handle()](auto code, auto data, auto size) { + return fdb_transaction_set_option(handle, code, data, size); + }); + } + + public: + explicit operator bool() const noexcept { return dbh && nullptr != raw_handle(); } + + public: + FDBTransaction *raw_handle() const noexcept { return txn_handle.get(); } + + private: + void set(std::span k, std::span v) { + fdb_transaction_set(raw_handle(), + (const uint8_t*)k.data(), k.size(), + (const uint8_t*)v.data(), v.size()); + } + + // JFW: it's not as easy to wedge an output_range into here as it appears, perhaps + // needs to be revisited; I'm binding it to what's actually used in practice for now: + template + requires std::output_iterator> + bool get(const ceph::libfdb::select& key_range, OutIterT out_iter) { + return ceph::libfdb::detail::get_value_range_from_transaction(*this, key_range, out_iter); + } + + bool get(const std::span k, concepts::value_callback auto&& val_collector) { + return get_single_value_from_transaction(k, val_collector); + } + + void erase(std::span k) { + fdb_transaction_clear(raw_handle(), + (const std::uint8_t *)k.data(), k.size()); + } + + void erase(const ceph::libfdb::concepts::selector auto& key_range) { + fdb_transaction_clear_range(raw_handle(), + (const uint8_t *)key_range.begin_key.data(), key_range.begin_key.size(), + (const uint8_t *)key_range.end_key.data(), key_range.end_key.size()); + } + + bool key_exists(std::string_view k) { + return get_single_value_from_transaction(detail::as_fdb_span(k), [](auto) {}); + } + + bool commit(fdb_error_t *replay_error = nullptr); + void destroy() noexcept { txn_handle.reset(); } + + // As you can see, friends are used extensively to implement the public interface; it would be nice to come up + // with a strategy for making these "lists" a bit more managable, but for now it's what we have and it allows me ot + // keep these handles mostly-opaque (this has proven to be a good idea as I've a few times shuffled internal details + // with no disruption to the user interface surface): + private: + friend inline void set(transaction_handle, const std::string_view, const auto&, const commit_after_op); + friend inline void set(database_handle, std::string_view, const auto&); + friend inline void set(transaction_handle, std::string_view, const ceph::libfdb::concepts::stringview_convertible auto&, const commit_after_op); + friend inline void set(database_handle, std::string_view, const ceph::libfdb::concepts::stringview_convertible auto&); + + template + requires concepts::value_callback> || + concepts::value_output + friend inline bool get(ceph::libfdb::transaction_handle, + std::string_view, + OutputTargetOrFnT&&, + const commit_after_op); + friend inline bool get(ceph::libfdb::transaction_handle, const ceph::libfdb::select&, auto, const commit_after_op); + + friend inline void erase(ceph::libfdb::transaction_handle, std::string_view, const commit_after_op); + friend inline void erase(ceph::libfdb::transaction_handle, const ceph::libfdb::select&, const commit_after_op); + friend inline void erase(ceph::libfdb::database_handle, std::string_view); + friend inline void erase(ceph::libfdb::database_handle, const ceph::libfdb::select&); + + friend inline bool key_exists(transaction_handle txn, std::string_view k, const commit_after_op commit_after); + + friend inline bool commit(transaction_handle& txn); + friend inline fdb_error_t ceph::libfdb::detail::do_commit(transaction_handle& txn); + friend inline void ceph::libfdb::detail::transaction_set_kv_bytes(const transaction_handle&, + std::span, + std::span); + +}; + +namespace detail { + +// Since lambdas cannot be friend-functions, we use a named helper: +inline void transaction_set_kv_bytes(const transaction_handle& txn, + std::span k, + std::span v) +{ + txn->set(k, v); +} + +} // namespace detail + +inline bool ceph::libfdb::transaction::get_single_value_from_transaction(const std::span& key, std::invocable> auto&& write_output) +{ + const fdb_bool_t is_snapshot = false; + + detail::future_value fv(fdb_transaction_get( + raw_handle(), + (const uint8_t *)key.data(), key.size(), + is_snapshot)); + + if (fdb_error_t r = fdb_future_block_until_ready(fv.raw_handle()); 0 != r) { + throw libfdb_exception(r); + } + + fdb_bool_t key_was_found = false; + const uint8_t *out_buffer = nullptr; + int out_len = 0; + + if (fdb_error_t r = fdb_future_get_value(fv.raw_handle(), &key_was_found, &out_buffer, &out_len); 0 != r) { + throw libfdb_exception(r); + } + + // No errors, but no value was found: + if (0 == key_was_found) { + return false; + } + + write_output(std::span(out_buffer, out_len)); + + // The happy path is the simple path: + return true; +} + +[[nodiscard]] inline bool ceph::libfdb::transaction::commit(fdb_error_t *replay_error) +{ + if (nullptr != replay_error) { + *replay_error = 0; + } + + // We don't want to try to vivify for an "empty" commit: + if (!*this) + return false; + + detail::future_value fv(fdb_transaction_commit(raw_handle())); + + if (fdb_error_t r = fdb_future_block_until_ready(fv.raw_handle()); 0 != r) { + throw libfdb_exception(r); + } + + if (fdb_error_t r = fdb_future_get_error(fv.raw_handle()); 0 != r) { + detail::future_value ferror_result(fdb_transaction_on_error(raw_handle(), r)); + + if (0 != fdb_future_block_until_ready(ferror_result.raw_handle())) { + // In their example, they use the first error as the message source, so I will do that also: + throw libfdb_exception(r); + } + + if (0 != fdb_future_get_error(ferror_result.raw_handle())) { + // Destroy the futures AND be sure to invalidate the transaction-- according to the documentation, + // this must happen in the specified order (which /should/ currently match the dtor order, but I am + // not going to touch this any more right now): + fv.destroy(), ferror_result.destroy(), destroy(); + + // Again, use the original error: + throw libfdb_exception(r); + } + + // A false result means on_error() succeeded; the caller should replay the transaction: + if (nullptr != replay_error) { + *replay_error = r; + } + + return false; + } + + // Ok: + return true; +} + +} // namespace ceph::libfdb + +// Future-wrangling and tricky retry-handling: +namespace ceph::libfdb::detail { + +inline future_value block_until_ready(future_value&& fv) +{ + if (fdb_error_t r = fdb_future_block_until_ready(fv.raw_handle()); 0 != r) { + throw libfdb_exception(r); + } + + // Note that fdb_future_block_until_ready() does not by itself check for errors + // with the value; so, we need to do this separately: + fdb_error_t r = fdb_future_get_error(fv.raw_handle()); + + if (r) { + throw libfdb_exception(r); + } + + return fv; +} + +inline future_value wait_until_ready(future_value&& fv) +{ + if (fdb_error_t r = fdb_future_block_until_ready(fv.raw_handle()); 0 != r) { + throw libfdb_exception(r); + } + + return fv; +} + +template +requires std::invocable +inline future_value await_future_of(FnT&& fn, XS&& ...params) +{ + return block_until_ready( + std::invoke(std::forward(fn), std::forward(params)...)); +} + +template +requires std::output_iterator> +inline bool get_value_range_from_transaction(transaction& txn, const select& key_range, OutIterT out_iter) +{ + auto flattened = detail::generate_FDB_pairs(txn, key_range) | std::views::join; + std::ranges::transform(flattened, out_iter, to_decoded_kv_pair); + + return true; +} + +inline fdb_error_t value_from_db(future_value& fv, + std::invocable auto extract_fn) +{ + return std::invoke(extract_fn, fv); +} + +inline bool retry_after_error(ceph::libfdb::transaction_handle& txn, const fdb_error_t r) +{ + if (0 == r) { + return false; + } + + if (not fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, r)) { + // Non-retryable errors cannot be repaired by on_error(): + throw ceph::libfdb::libfdb_exception(r); + } + + future_value on_error_future(fdb_transaction_on_error(txn->raw_handle(), r)); + + if (fdb_error_t on_error_r = fdb_future_block_until_ready(on_error_future.raw_handle()); 0 != on_error_r) { + throw ceph::libfdb::libfdb_exception(r); + } + + if (fdb_error_t on_error_r = fdb_future_get_error(on_error_future.raw_handle()); 0 != on_error_r) { + throw ceph::libfdb::libfdb_exception(r); + } + + return true; +} + +// Convert FDBKey array into something useful: +inline std::vector as_select_seq(const FDBKey* const xs, const int n) +{ + std::vector out; + + if (1 < n) { + out.reserve(static_cast(n - 1)); + } + + // Gather the flattened list into *overlapping* libfdb::select pairs: + for (const auto& dyad : std::span(xs, n) | std::views::slide(2)) { + const auto& fst = std::ranges::begin(dyad)[0]; + const auto& snd = std::ranges::begin(dyad)[1]; + + out.push_back({ + { (const char *)fst.key, static_cast(fst.key_length) }, + { (const char *)snd.key, static_cast(snd.key_length) } + }); + } + + return out; +} +// Finding a clear example both in the samples and in the documentation is not very easy. The +// statelessness of FDB requests bleeds into here with basically no hand-holding, so it's fairly subtle and +// not easy to see what to do (I'm still not sure I have it right), but note for instance that the call +// parameters have to change for subsequent reads. +inline future_value get_range_future_from_transaction(ceph::libfdb::transaction& txn, const ceph::libfdb::select& selection, int iteration) +{ + const auto& begin_key = selection.begin_key; + const auto& end_key = selection.end_key; + + // Hook for getting settings into here through the selector: + const auto& streaming_mode = selection.options.streaming_mode; + + // The documentation makes this stuff about as clear as mud... read VERY carefully + // when you fiddle with these: + int begin_or_eq = (1 == iteration) ? 0 : 1; + const int begin_offset = 1; + const int end_or_eq = 0; + const int end_offset = 1; + + // See validate_and_update_parameters() in fdb_c.cpp (FDB source) if greater clarity is needed on the meaning of some of these, it can be + // hard to deduce from the documentation: + // Returns an FDBKeyValueArray in the future: + + return future_value(fdb_transaction_get_range(txn.raw_handle(), + (const uint8_t *)begin_key.data(), begin_key.size(), // the reference-point key + begin_or_eq, // begin or eq + begin_offset, // begin offset + + (const uint8_t *)end_key.data(), end_key.size(), // the end selector key + end_or_eq, // end or eq + end_offset, // end offset (a shift AFTER end is matched) + + // How should results be grouped/chunked: + 0, // limit (0 == unlimited) + 0, // target bytes (0 == unlimited) + streaming_mode, // streaming mode (e.g.: FDB_STREAMING_MODE_WANT_ALL) + iteration, // iteration # (produced side effect) + + // Other options: + 0, // 0 unless this IS a snapshot read + 0 // reverse: should items come in reverse order? + )); +} + +inline std::vector locate_split_points( + ceph::libfdb::database_handle dbh, + ceph::libfdb::select selector, + const std::int64_t remote_chunk_size) +{ + using ceph::libfdb::detail::wait_until_ready; + + auto txn = ceph::libfdb::make_transaction(dbh); + + // We can do this without side-effects by combining some lambdas, but for now I'm satisfied if it works: + FDBKey const *keys = nullptr; + int nkeys = 0; + + for (bool should_retry = true; should_retry;) { + + auto fv = wait_until_ready(future_value(fdb_transaction_get_range_split_points( + txn->raw_handle(), + (const std::uint8_t *)selector.begin_key.data(), static_cast(selector.begin_key.length()), + (const std::uint8_t *)selector.end_key.data(), static_cast(selector.end_key.length()), + remote_chunk_size))); + + should_retry = retry_after_error(txn, value_from_db(fv, + [&keys, &nkeys](future_value& fv) { + return fdb_future_get_key_array(fv.raw_handle(), &keys, &nkeys); + })); + + if (not should_retry) { + return as_select_seq(keys, nkeys); + } + } + + return {}; +} + +// Generators: +// The returned memory should be copied immediately as its lifetime will end once the Future is destroyed: +// (This is pretty much why this is in the "detail" namespace for the moment.) +inline std::generator> generate_FDB_pairs(transaction& txn, select key_range) +{ + // As FDB is stateless, we need to track state somewhere: + int more_available = 1; + int out_count = 0; + int iteration = 1; // expected to be 1 when FDB is called in iterator mode + + const FDBKeyValue *out_kvs = nullptr; + + for (; more_available; iteration++) { + + auto fv = await_future_of([&]() { return get_range_future_from_transaction(txn, key_range, iteration); }); + + if (fdb_error_t r = fdb_future_get_keyvalue_array(fv.raw_handle(), &out_kvs, &out_count, &more_available); 0 != r) { + throw libfdb_exception(r); + } + + auto result = std::span(out_kvs, out_count); + + if (more_available) { + // Make the first part of the range for the new search equal to the last one from the old search: + const auto& last_key = result.back(); + key_range.begin_key = std::string_view((const char *)last_key.key, last_key.key_length); + } + + co_yield result; + } +} + +} // namespace ceph::libfdb::detail + + +#endif diff --git a/src/rgw/fdb/conversion.h b/src/rgw/fdb/conversion.h new file mode 100644 index 00000000000..eea6882fe22 --- /dev/null +++ b/src/rgw/fdb/conversion.h @@ -0,0 +1,129 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2025-2026 International Business Machines Corp. (IBM) + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * +*/ + +#ifndef CEPH_FDB_CONVERSION_H + #define CEPH_FDB_CONVERSION_H + +#include "base.h" + +#include "zpp_bits.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* This module is for converting internal types "owned" by FoundationDB. They've initially been implemented in +a conversion namespace with overloads, which is the same way that user conversions work, but especially with Concepts +this technique doesn't have to be used-- it is likely possible to avoid default construction and possible extra copies. +Since I'm building a prototype right now, it's more important to me on this pass to make things understandable and +clear, because past experience informs me that it's better to have a clear understanding of what the goals of "to" +and "from" versions actually are in relationship to user-level types than to have every nanosecond of performance +be available on day one. + +The target of "to" conversions is not a USER type, but rather the FUNCTIONS provided inside of libfdb-- users +should NOT see the output of these or have to handle them outside of tests or edge-cases (and even then, I doubt it's +needed, though I won't work hard to stop it). + +I'm hoping that later down the line I can sit and spend more time with this-- it would be nice, for example, if we could +use memory provided by the caller. + +Additionally, this mechanism is mostly obviated by forwarding the work to zpp_bits, but keeping it here provides an additional +hook "ahead" of that library, and indeed eliminates any actual dependency on it-- we use it to smooth out a few areas where +zpp_bits is designed for serialization ONLY rather than also playing nice with some conversions, especially those where the +input and output sizes of an array may not match. +*/ +namespace ceph::libfdb::to { + +inline auto convert(const auto& from, std::vector& out_data) -> std::span +{ + out_data.clear(); + + zpp::bits::out out(out_data); + + // zpp::bits won't write a size if we start with a fixed size array: + // (see dynamic_extent): + if constexpr (std::is_array_v) { + out(std::span(from, std::size(from))).or_throw(); + + return out_data; + } + + out(from).or_throw(); + + return out_data; +} + +inline auto convert(const auto& from) -> std::vector +{ + std::vector out_data; + convert(from, out_data); + + return out_data; +} + +} // namespace ceph::libfdb::to + +/* Map from FDB inputs from FDB TYPE to CONCRETE (i.e. copyable) userland types. Do NOT add +non-FDB input sources here (or any non-matching user output sources). Do NOT add +non-owning targets, lest Antevorda be angered!: */ +namespace ceph::libfdb::from { + +inline void convert(const std::span& from, auto& to) +{ + zpp::bits::in zpp_in(from); + zpp_in(to).or_throw(); +} + +template OutputFunction> +inline void convert(const std::span& in, OutputFunction& fn) +{ + fn((const char *)in.data(), in.size()); +} + +} // namespace ceph::libfdb::from + +namespace ceph::libfdb::detail { + +template +inline std::pair to_decoded_kv_pair(const FDBKeyValue& kv) +{ + std::pair r; + + r.first.assign((const char *)kv.key, static_cast(kv.key_length)); + + try + { + ceph::libfdb::from::convert(std::span(kv.value, kv.value_length), r.second); + } + catch (const std::system_error& e) { + // Translate from underlying (e.g. zpp_bits) conversion error into the right type: + // This is a bit bound to zpp_bits for the moment, but there's not a more direct way to distinguish this + // from a different system_error. We could do that, by using zpp_bits' non-throwing modes and throwing a + // special type, but this will do for now. + throw ceph::libfdb::libfdb_exception(e.what()); + } + + return r; +} + +} // namespace ceph::libfdb::detail + +#endif diff --git a/src/rgw/fdb/fdb.h b/src/rgw/fdb/fdb.h new file mode 100644 index 00000000000..0165a6c96d1 --- /dev/null +++ b/src/rgw/fdb/fdb.h @@ -0,0 +1,23 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2025-2026 International Business Machines Corp. (IBM) + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * +*/ + +#ifndef CEPH_RGW_FDB_H + #define CEPH_RGW_FDB_H + +#include "base.h" +#include "interface.h" +#include "conversion.h" + +#endif diff --git a/src/rgw/fdb/interface.h b/src/rgw/fdb/interface.h new file mode 100644 index 00000000000..d5677e0f18f --- /dev/null +++ b/src/rgw/fdb/interface.h @@ -0,0 +1,657 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab ft=cpp +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2025-2026 International Business Machines Corp. (IBM) + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * +*/ + +#ifndef CEPH_FDB_BINDINGS_H + #define CEPH_FDB_BINDINGS_H + +#include "base.h" +#include "conversion.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ceph::libfdb { + +/* This needs to be called once and only once during application lifetime, +when no more FoundationDB calls will be made: */ +inline void shutdown_libfdb() +{ + ceph::libfdb::detail::database_system::shutdown_fdb(); +} + +// By default, libfdb will in turn use FDB's own defaults (see FDB docs): +inline database_handle create_database() +{ + return std::make_shared(); +} + +inline database_handle create_database(const std::filesystem::path dbfile) +{ + return std::make_shared(dbfile); +} + +inline database_handle create_database(const std::filesystem::path dbfile, + const database_options& dbopts, + const network_options& netopts) +{ + return std::make_shared(dbfile, dbopts, netopts); +} + +inline database_handle create_database(const database_options& dbopts, + const network_options& netopts) +{ + return std::make_shared(dbopts, netopts); +} + +inline database_handle create_database(const database_options& opts) +{ + return std::make_shared(opts, network_options{}); +} + +inline database_handle create_database(const std::filesystem::path dbfile, + const database_options& dbopts) +{ + return std::make_shared(dbfile, dbopts, network_options{}); +} + +inline transaction_handle make_transaction(database_handle dbh) +{ + return std::make_shared(dbh); +} + +inline transaction_handle make_transaction(database_handle dbh, const transaction_options& opts) +{ + return std::make_shared(dbh, opts); +} + +// Note: only rarely is a direct call to this needed. You can use transactors or pass database_handles +// to get automagic. +// Note: after a transaction is committed, it cannot be used again; but right now, that is NOT +// an error with respect to the object. So, don't do operations on the object after you've committed +// it or the behavior could be surprising. +// On false, the client should retry the transaction: +[[nodiscard]] inline bool commit(transaction_handle& txn) +{ + return txn->commit(); +} + +} // namespace ceph::libfdb + +namespace ceph::libfdb::detail { + +// Forward declarations: +template +using transaction_invocation_result_t = + std::invoke_result_t; + +template +concept supported_transaction_invocation = + concepts::supported_invocation_result>; + +template +using operation_result_t = + std::conditional_t>, + void, + std::remove_cvref_t>>; + +template +struct value_collector_t final +{ + OutValuesT& out_values; + + void operator()(std::span out_data) const; +}; + +template +auto value_collector(OutValuesT& out_values) -> value_collector_t; + +template +auto maybe_retry(transaction_handle txn, FnT&& fn) -> operation_result_t; + +template +auto maybe_commit(transaction_handle txn, const commit_after_op commit_after, FnT&& fn) + -> operation_result_t; + +template +auto commit_noreplay(transaction_handle txn, const commit_after_op commit_after, FnT&& fn) + -> operation_result_t; + +} // namespace ceph::libfdb::detail + +namespace ceph::libfdb { + +inline void set(transaction_handle txn, + std::string_view k, const auto& v, + const commit_after_op commit_after) +{ + return detail::commit_noreplay(txn, commit_after, + [key = detail::as_fdb_span(k), &v](const transaction_handle& txn) { + return txn->set(key, ceph::libfdb::to::convert(v)); + }); +} + +// If someone gives us an explicit transaction handle, they almost certainly don't want to commit +// it (though they can always specify otherwise): +inline void set(transaction_handle txn, std::string_view k, const auto& v) +{ + return set(txn, k, v, commit_after_op::no_commit); +} + +// ...conversely, with a database handle given, we can assume they DO want to auto-commit: +inline void set(database_handle dbh, + std::string_view k, const auto& v) +{ + return detail::maybe_commit(make_transaction(dbh), commit_after_op::commit, + [key = detail::as_fdb_span(k), &v](const transaction_handle& txn) { + return txn->set(key, ceph::libfdb::to::convert(v)); + }); +} + +template