From 8e92cbac3e4c1d7643b5e02735a1dabb3326658d Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Mon, 13 Nov 2023 20:05:47 -0500 Subject: [PATCH] common: add versioned encodings for std::variant adds two encoding strategies for `std::variant<>` under the namespaces `ceph::versioned_variant` and `ceph::converted_variant` these versioned encodings allow the variant to be extended with new types, provided that they're always added to the end without changing or removing existing types. because of this requirement, no default encoding is provided for `std::variant`. callers must opt in to one namespace or the other the `converted_variant` encoding requires the variant's first type T to use versioned encoding, and guarantees that the variant's encoding is backward-compatible with T's Signed-off-by: Casey Bodley --- src/common/versioned_variant.h | 215 ++++++++++++++ src/test/common/CMakeLists.txt | 4 + src/test/common/test_versioned_variant.cc | 323 ++++++++++++++++++++++ 3 files changed, 542 insertions(+) create mode 100644 src/common/versioned_variant.h create mode 100644 src/test/common/test_versioned_variant.cc diff --git a/src/common/versioned_variant.h b/src/common/versioned_variant.h new file mode 100644 index 0000000000000..9c9c5ada9b013 --- /dev/null +++ b/src/common/versioned_variant.h @@ -0,0 +1,215 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright contributors to the Ceph project + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#pragma once + +#include +#include +#include + +#include // for mp_with_index +#include "include/encoding.h" + +/// \file +/// \brief Contains binary encoding strategies for std::variant. + +namespace ceph { + +// null encoding for std::monostate +void encode(const std::monostate&, bufferlist& bl) {} +void decode(std::monostate&, bufferlist::const_iterator& p) {} + +// largest value that can be represented by `__u8 struct_v` +inline constexpr size_t max_version = std::numeric_limits<__u8>::max(); + +/// \namespace versioned_variant +/// \brief A backward-compatible binary encoding for std::variant. +/// +/// The variant index is encoded in struct_v so the correct decoder can be +/// selected. This means that existing variant types cannot be changed or +/// removed without breaking the decode of earlier ceph versions. New types +/// can only be added to the end of the variant. +/// +/// In addition to struct_v, the variant index is also encoded in compatv. As +/// the variant is extended, this means that existing decoders can continue to +/// decode the types they recognize, but reject the encodings of new types they +/// don't. +/// +/// The variant types themselves are free to change their encodings, provided +/// they manage their own versioning. The types must be default-constructible +/// so they can be constructed before decode. +/// +/// The contained encode/decode functions won't be found by argument-dependent +/// lookup, so you must either qualify the calls with `versioned_variant::` or +/// add `using namespace versioned_variant` to the calling scope. +namespace versioned_variant { + +// Requirements for the list of types for versioned std::variant encoding. +template +concept valid_types = requires { + sizeof...(Ts) > 0; // variant cannot be empty + sizeof...(Ts) <= max_version; // index must fit in u8 + requires (std::default_initializable && ...); // default-constructible + }; + +/// \brief A versioned_variant encoder. +/// +/// Example: +/// \code +/// struct example { +/// std::variant value; +/// +/// void encode(bufferlist& bl) const { +/// ENCODE_START(0, 0, bl); +/// ceph::versioned_variant::encode(value, bl); +/// ... +/// \endcode +template requires valid_types +void encode(const std::variant& v, bufferlist& bl, uint64_t features=0) +{ + // encode the variant index in struct_v and compatv + const uint8_t ver = static_cast(v.index()); + ENCODE_START(ver, ver, bl); + // use the variant type's encoder + std::visit([&bl] (const auto& value) mutable { + encode(value, bl); + }, v); + ENCODE_FINISH(bl); +} + +/// \brief A versioned_variant decoder. +/// +/// Example: +/// \code +/// struct example { +/// std::variant value; +/// +/// void decode(bufferlist::const_iterator& bl) const { +/// DECODE_START(0, bl); +/// ceph::versioned_variant::decode(value, bl); +/// ... +/// \endcode +template requires valid_types +void decode(std::variant& v, bufferlist::const_iterator& p) +{ + constexpr uint8_t max_version = sizeof...(Ts) - 1; + DECODE_START(max_version, p); + // use struct_v as an index into the variant after converting it into a + // compile-time index I + const uint8_t index = struct_v; + boost::mp11::mp_with_index(index, [&v, &p] (auto I) { + // default-construct the type at index I and call its decoder + decode(v.template emplace(), p); + }); + DECODE_FINISH(p); +} + +} // namespace versioned_variant + + +/// \namespace converted_variant +/// \brief A std::variant encoding that is backward-compatible with T. +/// +/// The encoding works the same as versioned_variant, except that a block of +/// version numbers are reserved for the first type T to allow its encoding +/// to continue evolving. T must itself use versioned encoding (ie +/// ENCODE_START/FINISH). +/// +/// This encoding strategy allows a serialized type T to be transparently +/// converted into a variant that can represent other types too. +namespace converted_variant { + +// For converted variants, reserve the first 128 versions for the original +// type. Variant types after the first use the version numbers above this. +inline constexpr uint8_t converted_max_version = 128; + +// Requirements for the list of types for converted std::variant encoding. +template +concept valid_types = requires { + sizeof...(Ts) > 0; // variant cannot be empty + sizeof...(Ts) <= (max_version - converted_max_version); // index must fit in u8 + requires (std::default_initializable && ...); // default-constructible + }; + +/// \brief A converted_variant encoder. +/// +/// Example: +/// \code +/// struct example { +/// std::variant value; // replaced `int value` +/// +/// void encode(bufferlist& bl) const { +/// ENCODE_START(1, 0, bl); +/// ceph::converted_variant::encode(value, bl); +/// ... +/// \endcode +template requires valid_types +void encode(const std::variant& v, bufferlist& bl, uint64_t features=0) +{ + const uint8_t index = static_cast(v.index()); + if (index == 0) { + // encode the first type with its own versioning scheme + encode(std::get<0>(v), bl); + return; + } + + // encode the variant index in struct_v and compatv + const uint8_t ver = converted_max_version + index; + ENCODE_START(ver, ver, bl); + // use the variant type's encoder + std::visit([&bl] (const auto& value) mutable { + encode(value, bl); + }, v); + ENCODE_FINISH(bl); +} + +/// \brief A converted_variant decoder. +/// +/// Example: +/// \code +/// struct example { +/// std::variant value; // replaced `int value` +/// +/// void decode(bufferlist::const_iterator& bl) { +/// DECODE_START(1, bl); +/// ceph::converted_variant::decode(value, bl); +/// ... +/// \endcode +template requires valid_types +void decode(std::variant& v, bufferlist::const_iterator& p) +{ + // save the iterator position so the first type can restart decode + const bufferlist::const_iterator prev = p; + + constexpr uint8_t max_version = converted_max_version + sizeof...(Ts) - 1; + DECODE_START(max_version, p); + if (struct_v <= converted_max_version) { + p = prev; // rewind and use type 0's DECODE_START/FINISH + decode(v.template emplace<0>(), p); + return; + } + + // use struct_v as an index into the variant after converting it into a + // compile-time index I + const uint8_t index = struct_v - converted_max_version; + boost::mp11::mp_with_index(index, [&v, &p] (auto I) { + // default-construct the type at index I and call its decoder + decode(v.template emplace(), p); + }); + DECODE_FINISH(p); +} + +} // namespace converted_variant + +} // namespace ceph diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt index c044daf662ab7..b2ed06ee3062c 100644 --- a/src/test/common/CMakeLists.txt +++ b/src/test/common/CMakeLists.txt @@ -390,6 +390,10 @@ target_link_libraries(unittest_blocked_completion Boost::system GTest::GTest) add_executable(unittest_allocate_unique test_allocate_unique.cc) add_ceph_unittest(unittest_allocate_unique) +add_executable(unittest_versioned_variant test_versioned_variant.cc) +add_ceph_unittest(unittest_versioned_variant) +target_link_libraries(unittest_versioned_variant common) + if(WITH_SYSTEMD) add_executable(unittest_journald_logger test_journald_logger.cc) target_link_libraries(unittest_journald_logger ceph-common) diff --git a/src/test/common/test_versioned_variant.cc b/src/test/common/test_versioned_variant.cc new file mode 100644 index 0000000000000..b91e24b6f13fe --- /dev/null +++ b/src/test/common/test_versioned_variant.cc @@ -0,0 +1,323 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright contributors to the Ceph project + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "common/versioned_variant.h" +#include +#include + +namespace { + +// type with custom encoding +struct custom_type { + void encode(bufferlist& bl) const { + ENCODE_START(0, 0, bl); + ENCODE_FINISH(bl); + } + void decode(bufferlist::const_iterator& bl) { + DECODE_START(0, bl); + DECODE_FINISH(bl); + } +}; +WRITE_CLASS_ENCODER(custom_type); + +} // anonymous namespace + +namespace ceph { + +TEST(VersionedVariant, Monostate) +{ + using Variant = std::variant; + bufferlist bl; + { + Variant in; + versioned_variant::encode(in, bl); + } + { + Variant out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(versioned_variant::decode(out, p)); + EXPECT_TRUE(std::holds_alternative(out)); + } +} + +TEST(VersionedVariant, Custom) +{ + using Variant = std::variant; + bufferlist bl; + { + Variant in = custom_type{}; + versioned_variant::encode(in, bl); + } + { + Variant out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(versioned_variant::decode(out, p)); + EXPECT_TRUE(std::holds_alternative(out)); + } +} + +TEST(VersionedVariant, DuplicateFirst) +{ + using Variant = std::variant; + bufferlist bl; + { + Variant in; + in.emplace<0>(42); + versioned_variant::encode(in, bl); + } + { + Variant out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(versioned_variant::decode(out, p)); + ASSERT_EQ(0, out.index()); + EXPECT_EQ(42, std::get<0>(out)); + } +} + +TEST(VersionedVariant, DuplicateSecond) +{ + using Variant = std::variant; + bufferlist bl; + { + Variant in; + in.emplace<1>(42); + versioned_variant::encode(in, bl); + } + { + Variant out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(versioned_variant::decode(out, p)); + ASSERT_EQ(1, out.index()); + EXPECT_EQ(42, std::get<1>(out)); + } +} + +TEST(VersionedVariant, EncodeOld) +{ + using V1 = std::variant; + using V2 = std::variant; + + bufferlist bl; + { + // use V1 to encode the initial type + V1 in = 42; + versioned_variant::encode(in, bl); + } + { + // can decode as V1 + V1 out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(versioned_variant::decode(out, p)); + ASSERT_TRUE(std::holds_alternative(out)); + EXPECT_EQ(42, std::get(out)); + } + { + // can also decode as V2 + V2 out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(versioned_variant::decode(out, p)); + ASSERT_TRUE(std::holds_alternative(out)); + EXPECT_EQ(42, std::get(out)); + } +} + +TEST(VersionedVariant, EncodeExisting) +{ + using V1 = std::variant; + using V2 = std::variant; + + bufferlist bl; + { + // use V2 to encode the type shared with V1 + V2 in = 42; + versioned_variant::encode(in, bl); + } + { + // can decode as V2 + V2 out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(versioned_variant::decode(out, p)); + ASSERT_TRUE(std::holds_alternative(out)); + EXPECT_EQ(42, std::get(out)); + } + { + // can also decode as V1 + V1 out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(versioned_variant::decode(out, p)); + ASSERT_TRUE(std::holds_alternative(out)); + EXPECT_EQ(42, std::get(out)); + } +} + +TEST(VersionedVariant, EncodeNew) +{ + using V1 = std::variant; + using V2 = std::variant; + + bufferlist bl; + { + // use V2 to encode the new string type + V2 in = "42"; + versioned_variant::encode(in, bl); + } + { + // can decode as V2 + V2 out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(versioned_variant::decode(out, p)); + ASSERT_TRUE(std::holds_alternative(out)); + EXPECT_EQ("42", std::get(out)); + } + { + // can't decode as V1 + V1 out; + auto p = bl.cbegin(); + EXPECT_THROW(versioned_variant::decode(out, p), buffer::malformed_input); + } +} + + +TEST(ConvertedVariant, Custom) +{ + using Variant = std::variant; + bufferlist bl; + { + Variant in = custom_type{}; + converted_variant::encode(in, bl); + } + { + Variant out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(converted_variant::decode(out, p)); + EXPECT_TRUE(std::holds_alternative(out)); + } +} + +TEST(ConvertedVariant, DuplicateFirst) +{ + using Variant = std::variant; + bufferlist bl; + { + Variant in; + in.emplace<1>(42); + converted_variant::encode(in, bl); + } + { + Variant out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(converted_variant::decode(out, p)); + ASSERT_EQ(1, out.index()); + EXPECT_EQ(42, std::get<1>(out)); + } +} + +TEST(ConvertedVariant, DuplicateSecond) +{ + using Variant = std::variant; + bufferlist bl; + { + Variant in; + in.emplace<2>(42); + converted_variant::encode(in, bl); + } + { + Variant out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(converted_variant::decode(out, p)); + ASSERT_EQ(2, out.index()); + EXPECT_EQ(42, std::get<2>(out)); + } +} + +TEST(ConvertedVariant, EncodeOld) +{ + using V1 = custom_type; + using V2 = std::variant; + + bufferlist bl; + { + // use V1 to encode the initial type + V1 in; + encode(in, bl); + } + { + // can decode as V1 + V1 out; + auto p = bl.cbegin(); + EXPECT_NO_THROW(decode(out, p)); + } + { + // can also decode as V2 + V2 out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(converted_variant::decode(out, p)); + EXPECT_TRUE(std::holds_alternative(out)); + } +} + +TEST(ConvertedVariant, EncodeExisting) +{ + using V1 = custom_type; + using V2 = std::variant; + + bufferlist bl; + { + // use V2 to encode the type shared with V1 + V2 in; + converted_variant::encode(in, bl); + } + { + // can decode as V2 + V2 out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(converted_variant::decode(out, p)); + EXPECT_TRUE(std::holds_alternative(out)); + } + { + // can also decode as V1 + V1 out; + auto p = bl.cbegin(); + EXPECT_NO_THROW(decode(out, p)); + } +} + +TEST(ConvertedVariant, EncodeNew) +{ + using V1 = custom_type; + using V2 = std::variant; + + bufferlist bl; + { + // use V2 to encode the new type + V2 in = 42; + converted_variant::encode(in, bl); + } + { + // can decode as V2 + V2 out; + auto p = bl.cbegin(); + ASSERT_NO_THROW(converted_variant::decode(out, p)); + ASSERT_TRUE(std::holds_alternative(out)); + EXPECT_EQ(42, std::get(out)); + } + { + // can't decode as V1 + V1 out; + auto p = bl.cbegin(); + EXPECT_THROW(decode(out, p), buffer::malformed_input); + } +} + +} // namespace ceph -- 2.39.5