From b15524076d7d7ca72eac09af9b216effa44bddd6 Mon Sep 17 00:00:00 2001 From: Chunmei Liu Date: Tue, 21 Jan 2020 23:25:02 -0800 Subject: [PATCH] crimson:add alien blue store adds an alien store wrapper for ceph blue store, which will create a thread pool, so crimson-osd can call bluestore API which is running in Posix thread from seastar thread. Signed-off-by: Chunmei Liu --- src/common/RefCountedObj.h | 10 +- src/common/WorkQueue.h | 2 +- src/common/admin_socket.h | 2 +- src/common/ceph_atomic.h | 2 +- src/common/ceph_context.cc | 2 +- src/common/ceph_context.h | 4 +- src/common/ceph_mutex.h | 4 +- src/common/config_proxy.h | 5 + src/common/dout.h | 15 +- src/common/perf_counters.cc | 2 +- src/common/perf_counters.h | 6 +- src/crimson/CMakeLists.txt | 3 - src/crimson/common/config_proxy.h | 3 + src/crimson/os/CMakeLists.txt | 6 + src/crimson/os/alienstore/CMakeLists.txt | 65 ++++ src/crimson/os/alienstore/alien_collection.h | 25 ++ src/crimson/os/alienstore/alien_store.cc | 380 +++++++++++++++++++ src/crimson/os/alienstore/alien_store.h | 89 +++++ src/crimson/os/cyanstore/cyan_store.cc | 1 + src/crimson/os/futurized_store.cc | 6 +- src/crimson/os/futurized_store.h | 3 +- src/crimson/osd/osd.cc | 3 +- src/global/global_context.cc | 4 +- src/include/Context.h | 4 +- src/include/common_fwd.h | 2 +- src/os/ObjectStore.cc | 14 + src/os/bluestore/BlockDevice.cc | 3 +- 27 files changed, 635 insertions(+), 30 deletions(-) create mode 100644 src/crimson/os/alienstore/CMakeLists.txt create mode 100644 src/crimson/os/alienstore/alien_collection.h create mode 100644 src/crimson/os/alienstore/alien_store.cc create mode 100644 src/crimson/os/alienstore/alien_store.h diff --git a/src/common/RefCountedObj.h b/src/common/RefCountedObj.h index 49f29dcf84231..b23f488cad90b 100644 --- a/src/common/RefCountedObj.h +++ b/src/common/RefCountedObj.h @@ -74,11 +74,11 @@ protected: private: void _get() const; -#ifndef WITH_SEASTAR - mutable std::atomic nref{1}; -#else +#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN) // crimson is single threaded at the moment mutable uint64_t nref{1}; +#else + mutable std::atomic nref{1}; #endif CephContext *cct{nullptr}; }; @@ -94,7 +94,7 @@ template virtual ~RefCountedObjectSafe() override {} }; -#ifndef WITH_SEASTAR +#if !defined(WITH_SEASTAR)|| defined(WITH_ALIEN) /** * RefCountedCond @@ -185,7 +185,7 @@ struct RefCountedWaitObject { } }; -#endif // WITH_SEASTAR +#endif // !defined(WITH_SEASTAR)|| defined(WITH_ALIEN) static inline void intrusive_ptr_add_ref(const RefCountedObject *p) { p->get(); diff --git a/src/common/WorkQueue.h b/src/common/WorkQueue.h index 21e9669bfb4c6..0c16cfc31f200 100644 --- a/src/common/WorkQueue.h +++ b/src/common/WorkQueue.h @@ -15,7 +15,7 @@ #ifndef CEPH_WORKQUEUE_H #define CEPH_WORKQUEUE_H -#ifdef WITH_SEASTAR +#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN) // for ObjectStore.h struct ThreadPool { struct TPHandle { diff --git a/src/common/admin_socket.h b/src/common/admin_socket.h index fad3e556fb9a5..2105097bbeeb6 100644 --- a/src/common/admin_socket.h +++ b/src/common/admin_socket.h @@ -15,7 +15,7 @@ #ifndef CEPH_COMMON_ADMIN_SOCKET_H #define CEPH_COMMON_ADMIN_SOCKET_H -#ifdef WITH_SEASTAR +#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN) #include "crimson/admin/admin_socket.h" #else diff --git a/src/common/ceph_atomic.h b/src/common/ceph_atomic.h index 26a20ee283175..320963500e7f5 100644 --- a/src/common/ceph_atomic.h +++ b/src/common/ceph_atomic.h @@ -17,7 +17,7 @@ // like a full memory barrier stalling execution till CPU's store and // load buffers are drained. -#ifdef WITH_SEASTAR +#if defined(WITH_SEASTAR) && !defined(WITH_BLUESTORE) #include diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc index 389033429418a..e7116e38875cd 100644 --- a/src/common/ceph_context.cc +++ b/src/common/ceph_context.cc @@ -55,7 +55,7 @@ using ceph::HeartbeatMap; #include #include -#ifdef WITH_SEASTAR +#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN) namespace crimson::common { CephContext::CephContext() : _conf{crimson::common::local_conf()}, diff --git a/src/common/ceph_context.h b/src/common/ceph_context.h index 88f94ff72280d..adacecebfc399 100644 --- a/src/common/ceph_context.h +++ b/src/common/ceph_context.h @@ -30,7 +30,7 @@ #include "common/cmdparse.h" #include "common/code_environment.h" -#ifdef WITH_SEASTAR +#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN) #include "crimson/common/config_proxy.h" #include "crimson/common/perf_counters_collection.h" #else @@ -60,7 +60,7 @@ namespace ceph { } } -#ifdef WITH_SEASTAR +#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN) namespace crimson::common { class CephContext { public: diff --git a/src/common/ceph_mutex.h b/src/common/ceph_mutex.h index 4e888d72bb19c..4029a5a9f159c 100644 --- a/src/common/ceph_mutex.h +++ b/src/common/ceph_mutex.h @@ -14,7 +14,7 @@ // and make_recursive_mutex() factory methods, which take a string // naming the mutex for the purposes of the lockdep debug variant. -#ifdef WITH_SEASTAR +#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN) namespace ceph { // an empty class satisfying the mutex concept @@ -58,7 +58,7 @@ namespace ceph { #define ceph_mutex_is_locked_by_me(m) true } -#else // WITH_SEASTAR +#else // defined (WITH_SEASTAR) && !defined(WITH_ALIEN) // // For legacy Mutex users that passed recursive=true, use // ceph::make_recursive_mutex. For legacy Mutex users that passed diff --git a/src/common/config_proxy.h b/src/common/config_proxy.h index 025f0a0a3bfc0..9c0850fd08e1c 100644 --- a/src/common/config_proxy.h +++ b/src/common/config_proxy.h @@ -124,6 +124,11 @@ public: ConfigValues* operator->() noexcept { return &values; } +#ifdef WITH_SEASTAR + void set_config_values(const ConfigValues& val) { + values = val; + } +#endif int get_val(const std::string_view key, char** buf, int len) const { std::lock_guard l{lock}; return config.get_val(values, key, buf, len); diff --git a/src/common/dout.h b/src/common/dout.h index 2fdc07a19a658..c7c08182e5390 100644 --- a/src/common/dout.h +++ b/src/common/dout.h @@ -19,7 +19,8 @@ #include #include "include/ceph_assert.h" -#ifdef WITH_SEASTAR +#include "include/common_fwd.h" +#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN) #include #include "crimson/common/log.h" #include "crimson/common/config_proxy.h" @@ -117,7 +118,7 @@ struct is_dynamic> : public std::true_type {}; // generic macros #define dout_prefix *_dout -#ifdef WITH_SEASTAR +#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN) #define dout_impl(cct, sub, v) \ do { \ if (crimson::common::local_conf()->subsys.should_gather(sub, v)) { \ @@ -131,6 +132,16 @@ struct is_dynamic> : public std::true_type {}; _out.str().c_str()); \ } \ } while (0) +#elif defined(WITH_SEASTAR) && defined(WITH_ALIEN) +#define dout_impl(cct, sub, v) \ + do { \ + if (0) { \ + ceph::logging::MutableEntry _dout_e(v, sub); \ + std::ostream* _dout = &_dout_e.get_ostream(); + +#define dendl_impl std::flush; \ + } \ + } while (0) #else #define dout_impl(cct, sub, v) \ do { \ diff --git a/src/common/perf_counters.cc b/src/common/perf_counters.cc index dda589e1ca96c..80c96413eb16b 100644 --- a/src/common/perf_counters.cc +++ b/src/common/perf_counters.cc @@ -476,7 +476,7 @@ PerfCounters::PerfCounters(CephContext *cct, const std::string &name, m_lower_bound(lower_bound), m_upper_bound(upper_bound), m_name(name) -#ifndef WITH_SEASTAR +#if !defined(WITH_SEASTAR) || defined(WITH_ALIEN) , m_lock_name(std::string("PerfCounters::") + name.c_str()), m_lock(ceph::make_mutex(m_lock_name)) diff --git a/src/common/perf_counters.h b/src/common/perf_counters.h index 1f9391137e876..c5f69aa7ce31a 100644 --- a/src/common/perf_counters.h +++ b/src/common/perf_counters.h @@ -113,7 +113,7 @@ public: prio_default = prio_; } - TOPNSPC::common::PerfCounters* create_perf_counters(); + PerfCounters* create_perf_counters(); private: PerfCountersBuilder(const PerfCountersBuilder &rhs); PerfCountersBuilder& operator=(const PerfCountersBuilder &rhs); @@ -121,7 +121,7 @@ private: const char *description, const char *nick, int prio, int ty, int unit=UNIT_NONE, std::unique_ptr> histogram = nullptr); - TOPNSPC::common::PerfCounters *m_perf_counters; + PerfCounters *m_perf_counters; int prio_default = 0; }; @@ -289,7 +289,7 @@ private: int prio_adjust = 0; -#ifndef WITH_SEASTAR +#if !defined(WITH_SEASTAR) || defined(WITH_ALIEN) const std::string m_lock_name; /** Protects m_data */ ceph::mutex m_lock; diff --git a/src/crimson/CMakeLists.txt b/src/crimson/CMakeLists.txt index 17550fb13280a..e51af009c5ec5 100644 --- a/src/crimson/CMakeLists.txt +++ b/src/crimson/CMakeLists.txt @@ -52,8 +52,6 @@ add_library(crimson-common STATIC ${PROJECT_SOURCE_DIR}/src/common/hobject.cc ${PROJECT_SOURCE_DIR}/src/common/hostname.cc ${PROJECT_SOURCE_DIR}/src/common/ipaddr.cc - ${PROJECT_SOURCE_DIR}/src/common/lockdep.cc - ${PROJECT_SOURCE_DIR}/src/common/mutex_debug.cc ${PROJECT_SOURCE_DIR}/src/common/mempool.cc ${PROJECT_SOURCE_DIR}/src/common/options.cc ${PROJECT_SOURCE_DIR}/src/common/perf_counters.cc @@ -80,7 +78,6 @@ add_library(crimson-common STATIC ${PROJECT_SOURCE_DIR}/src/common/SubProcess.cc ${PROJECT_SOURCE_DIR}/src/common/TextTable.cc ${PROJECT_SOURCE_DIR}/src/common/Thread.cc - ${PROJECT_SOURCE_DIR}/src/common/HeartbeatMap.cc ${PROJECT_SOURCE_DIR}/src/common/PluginRegistry.cc ${PROJECT_SOURCE_DIR}/src/common/RefCountedObj.cc ${PROJECT_SOURCE_DIR}/src/crush/builder.c diff --git a/src/crimson/common/config_proxy.h b/src/crimson/common/config_proxy.h index f0c1e59231737..d6132455eced5 100644 --- a/src/crimson/common/config_proxy.h +++ b/src/crimson/common/config_proxy.h @@ -89,6 +89,9 @@ public: const ConfigValues* operator->() const noexcept { return values.get(); } + const ConfigValues get_config_values() { + return *values.get(); + } ConfigValues* operator->() noexcept { return values.get(); } diff --git a/src/crimson/os/CMakeLists.txt b/src/crimson/os/CMakeLists.txt index 3be23677d1b0e..8936fd101a618 100644 --- a/src/crimson/os/CMakeLists.txt +++ b/src/crimson/os/CMakeLists.txt @@ -2,6 +2,12 @@ add_library(crimson-os futurized_store.cc ${PROJECT_SOURCE_DIR}/src/os/Transaction.cc) add_subdirectory(cyanstore) + +if(WITH_BLUESTORE) + add_subdirectory(alienstore) +endif() + target_link_libraries(crimson-os crimson-cyanstore + crimson-alienstore crimson) diff --git a/src/crimson/os/alienstore/CMakeLists.txt b/src/crimson/os/alienstore/CMakeLists.txt new file mode 100644 index 0000000000000..a0fbeef131fbd --- /dev/null +++ b/src/crimson/os/alienstore/CMakeLists.txt @@ -0,0 +1,65 @@ +include_directories(SYSTEM "${CMAKE_SOURCE_DIR}/src/rocksdb/include") +set(crimson_alien_srcs + alien_store.cc) + +list(APPEND crimson_alien_srcs + ${PROJECT_SOURCE_DIR}/src/common/admin_socket.cc + ${PROJECT_SOURCE_DIR}/src/common/blkdev.cc + ${PROJECT_SOURCE_DIR}/src/common/ceph_context.cc + ${PROJECT_SOURCE_DIR}/src/common/ceph_crypto.cc + ${PROJECT_SOURCE_DIR}/src/common/condition_variable_debug.cc + ${PROJECT_SOURCE_DIR}/src/common/cmdparse.cc + ${PROJECT_SOURCE_DIR}/src/common/Finisher.cc + ${PROJECT_SOURCE_DIR}/src/common/HeartbeatMap.cc + ${PROJECT_SOURCE_DIR}/src/common/PluginRegistry.cc + ${PROJECT_SOURCE_DIR}/src/common/lockdep.cc + ${PROJECT_SOURCE_DIR}/src/common/mutex_debug.cc + ${PROJECT_SOURCE_DIR}/src/common/perf_counters.cc + ${PROJECT_SOURCE_DIR}/src/common/perf_counters_collection.cc + ${PROJECT_SOURCE_DIR}/src/common/RefCountedObj.cc + ${PROJECT_SOURCE_DIR}/src/common/shared_mutex_debug.cc + ${PROJECT_SOURCE_DIR}/src/common/Throttle.cc + ${PROJECT_SOURCE_DIR}/src/common/Timer.cc + ${PROJECT_SOURCE_DIR}/src/common/TrackedOp.cc + ${PROJECT_SOURCE_DIR}/src/common/WorkQueue.cc + ${PROJECT_SOURCE_DIR}/src/common/util.cc + ${PROJECT_SOURCE_DIR}/src/global/global_context.cc + ${PROJECT_SOURCE_DIR}/src/os/ObjectStore.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/Allocator.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/AvlAllocator.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/BitmapFreelistManager.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/BlockDevice.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/BlueFS.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/bluefs_types.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/BlueRocksEnv.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/BlueStore.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/bluestore_types.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/fastbmap_allocator_impl.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/FreelistManager.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/io_uring.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/StupidAllocator.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/BitmapAllocator.cc) + +if(HAVE_LIBAIO OR HAVE_POSIXAIO) + list(APPEND crimson_alien_srcs + ${PROJECT_SOURCE_DIR}/src/os/bluestore/KernelDevice.cc + ${PROJECT_SOURCE_DIR}/src/os/bluestore/aio.cc) +endif() + +add_library(crimson-alienstore STATIC ${crimson_alien_srcs} + $ + $ + $) +if(HAVE_LIBAIO) + target_link_libraries(crimson-alienstore ${AIO_LIBRARIES}) +endif(HAVE_LIBAIO) + +target_compile_definitions(crimson-alienstore PRIVATE -DWITH_SEASTAR -DWITH_ALIEN) +target_include_directories(crimson-alienstore PRIVATE + $) +target_link_libraries(crimson-alienstore fmt::fmt) +target_link_libraries(crimson-alienstore kv) +target_link_libraries(crimson-alienstore heap_profiler) +target_link_libraries(crimson-alienstore ${BLKID_LIBRARIES}) +target_link_libraries(crimson-alienstore ${UDEV_LIBRARIES}) +target_link_libraries(crimson-alienstore crimson) diff --git a/src/crimson/os/alienstore/alien_collection.h b/src/crimson/os/alienstore/alien_collection.h new file mode 100644 index 0000000000000..30364b80ff019 --- /dev/null +++ b/src/crimson/os/alienstore/alien_collection.h @@ -0,0 +1,25 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +#include "os/ObjectStore.h" + +#include "crimson/os/futurized_collection.h" +#include "crimson/os/futurized_store.h" +#include "alien_store.h" + +namespace crimson::os { + +struct AlienCollection final : public FuturizedCollection { + + ObjectStore::CollectionHandle collection; + + AlienCollection(ObjectStore::CollectionHandle ch) + : FuturizedCollection(ch->cid), + collection(ch) {} + + ~AlienCollection() {} +}; + +} diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc new file mode 100644 index 0000000000000..e3d176c314a98 --- /dev/null +++ b/src/crimson/os/alienstore/alien_store.cc @@ -0,0 +1,380 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "alien_collection.h" +#include "alien_store.h" + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "common/ceph_context.h" +#include "global/global_context.h" +#include "include/Context.h" +#include "os/bluestore/BlueStore.h" +#include "os/ObjectStore.h" +#include "os/Transaction.h" + +#include "crimson/common/log.h" +#include "crimson/os/futurized_store.h" + +namespace { + seastar::logger& logger() + { + return crimson::get_logger(ceph_subsys_filestore); + } + +class OnCommit final: public Context +{ + int cpuid; + Context* on_commit = nullptr; +public: + seastar::promise<> alien_done; + OnCommit(int id, ceph::os::Transaction& txn): cpuid(id) { + if (txn.has_contexts()) { + on_commit = txn.get_on_commit(); + } + } + + void finish(int) final { + auto fut = seastar::alien::submit_to(cpuid, [this] { + if (on_commit) { + on_commit->complete(0); + } + alien_done.set_value(); + return seastar::make_ready_future<>(); + }); + fut.wait(); + } +}; +} + +namespace crimson::os { + +AlienStore::AlienStore(const std::string& path, const ConfigValues& values) + : path{path} +{ + cct = std::make_unique(CEPH_ENTITY_TYPE_OSD); + g_ceph_context = cct.get(); + cct->_conf.set_config_values(values); + store = std::make_unique(cct.get(), path); + tp = std::make_unique(1, 128, seastar::engine().cpu_id() + 10); +} + +seastar::future<> AlienStore::start() +{ + return tp->start(); +} + +seastar::future<> AlienStore::stop() +{ + return tp->submit([this] { + for (auto [cid, ch]: coll_map) + static_cast(ch.get())->collection.reset(); + store.reset(); + }).then([this] { + return tp->stop(); + }); +} + +AlienStore::~AlienStore() = default; + +seastar::future<> AlienStore::mount() +{ + logger().debug("{}", __func__); + return tp->submit([this] { + return store->mount(); + }).then([] (int) { + return seastar::now(); + }); +} + +seastar::future<> AlienStore::umount() +{ + logger().debug("{}", __func__); + return transaction_gate.close().then([this] { + return tp->submit([this] { + return store->umount(); + }); + }).then([] (int) { + return seastar::now(); + }); +} + +seastar::future<> AlienStore::mkfs(uuid_d new_osd_fsid) +{ + logger().debug("{}", __func__); + osd_fsid = new_osd_fsid; + return tp->submit([this] { + return store->mkfs(); + }).then([] (int) { + return seastar::now(); + }); +} + +seastar::future, ghobject_t> +AlienStore::list_objects(CollectionRef ch, + const ghobject_t& start, + const ghobject_t& end, + uint64_t limit) const +{ + logger().debug("{}", __func__); + return seastar::do_with(std::vector(), ghobject_t(), + [=] (auto &objects, auto &next) { + objects.reserve(limit); + return tp->submit([=, &objects, &next] { + auto c = static_cast(ch.get()); + return store->collection_list(c->collection, start, end, + store->get_ideal_list_max(), + &objects, &next); + }).then([&objects, &next] (int) { + return seastar::make_ready_future, ghobject_t>( + std::move(objects), std::move(next)); + }); + }); +} + +seastar::future AlienStore::create_new_collection(const coll_t& cid) +{ + logger().debug("{}", __func__); + return tp->submit([this, cid] { + return store->create_new_collection(cid); + }).then([this, cid] (ObjectStore::CollectionHandle c) { + CollectionRef ch; + auto cp = coll_map.find(c->cid); + if (cp == coll_map.end()) { + ch = new AlienCollection(c); + coll_map[c->cid] = ch; + } else { + ch = cp->second; + auto ach = static_cast(ch.get()); + if (ach->collection != c) { + ach->collection = c; + } + } + return seastar::make_ready_future(ch); + }); + +} + +seastar::future AlienStore::open_collection(const coll_t& cid) +{ + logger().debug("{}", __func__); + return tp->submit([this, cid] { + return store->open_collection(cid); + }).then([this] (ObjectStore::CollectionHandle c) { + CollectionRef ch; + auto cp = coll_map.find(c->cid); + if (cp == coll_map.end()){ + ch = new AlienCollection(c); + coll_map[c->cid] = ch; + } else { + ch = cp->second; + auto ach = static_cast(ch.get()); + if (ach->collection != c){ + ach->collection = c; + } + } + return seastar::make_ready_future(ch); + }); +} + +seastar::future> AlienStore::list_collections() +{ + logger().debug("{}", __func__); + + return seastar::do_with(std::vector{}, [=] (auto &ls) { + return tp->submit([this, &ls] { + return store->list_collections(ls); + }).then([&ls] (int) { + return seastar::make_ready_future>(std::move(ls)); + }); + }); +} + +AlienStore::read_errorator::future +AlienStore::read(CollectionRef ch, + const ghobject_t& oid, + uint64_t offset, + size_t len, + uint32_t op_flags) +{ + logger().debug("{}", __func__); + return seastar::do_with(ceph::bufferlist{}, [=] (auto &bl) { + return tp->submit([=, &bl] { + auto c = static_cast(ch.get()); + return store->read(c->collection, oid, offset, len, bl, op_flags); + }).then([&bl] (int r) -> read_errorator::future { + if (r == -ENOENT) { + return crimson::ct_error::enoent::make(); + } else if (r == -EIO) { + return crimson::ct_error::input_output_error::make(); + } else { + return read_errorator::make_ready_future(std::move(bl)); + } + }); + }); +} + +AlienStore::get_attr_errorator::future +AlienStore::get_attr(CollectionRef ch, + const ghobject_t& oid, + std::string_view name) const +{ + logger().debug("{}", __func__); + return seastar::do_with(ceph::bufferptr{}, [=] (auto &value) { + return tp->submit([=, &value] { + auto c =static_cast(ch.get()); + return store->getattr(c->collection, oid, + static_cast(name).c_str(), value); + }).then([oid, name, &value] (int r) -> get_attr_errorator::future { + if (r == -ENOENT) { + return crimson::ct_error::enoent::make(); + } else if (r == -ENODATA) { + return crimson::ct_error::enodata::make(); + } else { + return get_attr_errorator::make_ready_future( + std::move(value)); + } + }); + }); +} + +AlienStore::get_attrs_ertr::future +AlienStore::get_attrs(CollectionRef ch, + const ghobject_t& oid) +{ + logger().debug("{}", __func__); + return seastar::do_with(attrs_t{}, [=] (auto &aset) { + return tp->submit([=, &aset] { + auto c = static_cast(ch.get()); + return store->getattrs(c->collection, oid, + reinterpret_cast&>(aset)); + }).then([&aset] (int r) -> get_attrs_ertr::future { + if (r == -ENOENT) { + return crimson::ct_error::enoent::make();; + } else { + return get_attrs_ertr::make_ready_future(std::move(aset)); + } + }); + }); +} + +seastar::future +AlienStore::omap_get_values(CollectionRef ch, + const ghobject_t& oid, + const set& keys) +{ + logger().debug("{}", __func__); + return seastar::do_with(omap_values_t{}, [=] (auto &values) { + return tp->submit([=, &values] { + auto c = static_cast(ch.get()); + return store->omap_get_values(c->collection, oid, keys, + reinterpret_cast*>(&values)); + }).then([&values] (int) { + return seastar::make_ready_future(std::move(values)); + }); + }); +} + +seastar::future +AlienStore::omap_get_values(CollectionRef ch, + const ghobject_t &oid, + const std::optional &start) +{ + logger().debug("{} with_start", __func__); + return seastar::do_with(omap_values_t{}, [=] (auto &values) { + return tp->submit([=, &values] { + auto c = static_cast(ch.get()); + return store->omap_get_values(c->collection, oid, start, + reinterpret_cast*>(&values)); + }).then([&values] (int r) { + return seastar::make_ready_future(true, std::move(values)); + }); + }); +} + +seastar::future<> AlienStore::do_transaction(CollectionRef ch, + ceph::os::Transaction&& txn) +{ + logger().debug("{}", __func__); + auto callback = + std::make_unique(seastar::engine().cpu_id(), txn); + return seastar::do_with(std::move(txn), std::move(callback), + [this, ch] (ceph::os::Transaction &txn, auto &callback) { + return seastar::with_gate(transaction_gate, [this, ch, &txn, &callback] { + return tp_mutex.lock().then ([this, ch, &txn, &callback] { + return tp->submit([=, &txn, &callback] { + txn.register_on_commit(callback.get()); + auto c = static_cast(ch.get()); + return store->queue_transaction(c->collection, std::move(txn)); + }); + }).then([this, &callback] (int) { + tp_mutex.unlock(); + return callback->alien_done.get_future(); + }); + }); + }); +} + +seastar::future<> AlienStore::write_meta(const std::string& key, + const std::string& value) +{ + logger().debug("{}", __func__); + return tp->submit([=] { + return store->write_meta(key, value); + }).then([] (int) { + return seastar::make_ready_future<>(); + }); +} + +seastar::future AlienStore::read_meta(const std::string& key) +{ + logger().debug("{}", __func__); + return tp->submit([this, key] { + std::string value; + int r = store->read_meta(key, &value); + if (r > 0) { + value.resize(r); + boost::algorithm::trim_right_if(value, + [] (unsigned char c) {return isspace(c);}); + } else { + value.clear(); + } + return std::make_pair(r, value); + }).then([] (auto entry) { + return seastar::make_ready_future(entry.first, entry.second); + }); +} + +uuid_d AlienStore::get_fsid() const +{ + logger().debug("{}", __func__); + return osd_fsid; +} + +seastar::future AlienStore::stat() const +{ + logger().info("{}", __func__); + return seastar::do_with(store_statfs_t{}, [this] (store_statfs_t &st) { + return tp->submit([this, &st] { + return store->statfs(&st, nullptr); + }).then([&st] (int) { + return seastar::make_ready_future(std::move(st)); + }); + }); +} + +unsigned AlienStore::get_max_attr_name_length() const +{ + logger().info("{}", __func__); + return 256; +} + +} diff --git a/src/crimson/os/alienstore/alien_store.h b/src/crimson/os/alienstore/alien_store.h new file mode 100644 index 0000000000000..d8dd155b79926 --- /dev/null +++ b/src/crimson/os/alienstore/alien_store.h @@ -0,0 +1,89 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +#include +#include + +#include "common/ceph_context.h" +#include "os/ObjectStore.h" +#include "osd/osd_types.h" + +#include "crimson/os/futurized_collection.h" +#include "crimson/os/futurized_store.h" +#include "crimson/thread/ThreadPool.h" + +namespace ceph::os { +class Transaction; +} + +namespace crimson::os { +class AlienStore final : public FuturizedStore { +public: + mutable std::unique_ptr tp; + AlienStore(const std::string& path, const ConfigValues& values); + ~AlienStore() final; + + seastar::future<> start() final; + seastar::future<> stop() final; + seastar::future<> mount() final; + seastar::future<> umount() final; + + seastar::future<> mkfs(uuid_d new_osd_fsid) final; + read_errorator::future read(CollectionRef c, + const ghobject_t& oid, + uint64_t offset, + size_t len, + uint32_t op_flags = 0) final; + + get_attr_errorator::future get_attr(CollectionRef c, + const ghobject_t& oid, + std::string_view name) const final; + get_attrs_ertr::future get_attrs(CollectionRef c, + const ghobject_t& oid) final; + + seastar::future omap_get_values( + CollectionRef c, + const ghobject_t& oid, + const omap_keys_t& keys) final; + + seastar::future, ghobject_t> list_objects( + CollectionRef c, + const ghobject_t& start, + const ghobject_t& end, + uint64_t limit) const final; + + /// Retrieves paged set of values > start (if present) + seastar::future omap_get_values( + CollectionRef c, ///< [in] collection + const ghobject_t &oid, ///< [in] oid + const std::optional &start ///< [in] start, empty for begin + ) final; ///< @return values.empty() iff done + + seastar::future create_new_collection(const coll_t& cid) final; + seastar::future open_collection(const coll_t& cid) final; + seastar::future> list_collections() final; + + seastar::future<> do_transaction(CollectionRef c, + ceph::os::Transaction&& txn) final; + + seastar::future<> write_meta(const std::string& key, + const std::string& value) final; + seastar::future read_meta(const std::string& key) final; + uuid_d get_fsid() const final; + seastar::future stat() const final; + unsigned get_max_attr_name_length() const final; + +private: + constexpr static unsigned MAX_KEYS_PER_OMAP_GET_CALL = 32; + const std::string path; + uint64_t used_bytes = 0; + uuid_d osd_fsid; + std::unique_ptr store; + std::unique_ptr cct; + seastar::gate transaction_gate; + std::unordered_map coll_map; + seastar::shared_mutex tp_mutex; +}; +} diff --git a/src/crimson/os/cyanstore/cyan_store.cc b/src/crimson/os/cyanstore/cyan_store.cc index b08859a43196d..0269c133957e3 100644 --- a/src/crimson/os/cyanstore/cyan_store.cc +++ b/src/crimson/os/cyanstore/cyan_store.cc @@ -416,6 +416,7 @@ seastar::future<> CyanStore::do_transaction(CollectionRef ch, t.get_on_applied_sync()}) { if (i) { i->complete(0); + delete i; } } return seastar::now(); diff --git a/src/crimson/os/futurized_store.cc b/src/crimson/os/futurized_store.cc index 0e535396b426a..bb73c34787f12 100644 --- a/src/crimson/os/futurized_store.cc +++ b/src/crimson/os/futurized_store.cc @@ -1,14 +1,18 @@ #include "futurized_store.h" #include "cyanstore/cyan_store.h" +#include "alienstore/alien_store.h" namespace crimson::os { std::unique_ptr FuturizedStore::create(const std::string& type, - const std::string& data) + const std::string& data, + const ConfigValues& values) { if (type == "memstore") { return std::make_unique(data); + } else if (type == "bluestore") { + return std::make_unique(data, values); } else { ceph_abort_msgf("unsupported objectstore type: %s", type.c_str()); return {}; diff --git a/src/crimson/os/futurized_store.h b/src/crimson/os/futurized_store.h index 6bc228a00d003..5bc9d775bf3c3 100644 --- a/src/crimson/os/futurized_store.h +++ b/src/crimson/os/futurized_store.h @@ -27,7 +27,8 @@ class FuturizedStore { public: static std::unique_ptr create(const std::string& type, - const std::string& data); + const std::string& data, + const ConfigValues& values); FuturizedStore() = default; virtual ~FuturizedStore() = default; diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index 147e6b9d3d8d3..935114a730916 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -68,7 +68,8 @@ OSD::OSD(int id, uint32_t nonce, mgrc{new crimson::mgr::Client{*public_msgr, *this}}, store{crimson::os::FuturizedStore::create( local_conf().get_val("osd_objectstore"), - local_conf().get_val("osd_data"))}, + local_conf().get_val("osd_data"), + local_conf().get_config_values())}, shard_services{*this, *cluster_msgr, *public_msgr, *monc, *mgrc, *store}, heartbeat{new Heartbeat{shard_services, *monc, hb_front_msgr, hb_back_msgr}}, // do this in background diff --git a/src/global/global_context.cc b/src/global/global_context.cc index f6c2672371fc0..b1e37bfbe595e 100644 --- a/src/global/global_context.cc +++ b/src/global/global_context.cc @@ -16,7 +16,7 @@ #include #include "common/ceph_context.h" -#ifdef WITH_SEASTAR +#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN) #include "crimson/common/config_proxy.h" #endif @@ -27,7 +27,7 @@ namespace TOPNSPC::global { CephContext *g_ceph_context = NULL; ConfigProxy& g_conf() { -#ifdef WITH_SEASTAR +#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN) return crimson::common::local_conf(); #else return g_ceph_context->_conf; diff --git a/src/include/Context.h b/src/include/Context.h index 6d39be55ba1a2..26533b5449932 100644 --- a/src/include/Context.h +++ b/src/include/Context.h @@ -75,7 +75,9 @@ class Context { virtual ~Context() {} // we want a virtual destructor!!! virtual void complete(int r) { finish(r); - delete this; +#ifndef WITH_SEASTAR + delete this; //alien store need its callback fun alive to get future. +#endif } virtual bool sync_complete(int r) { if (sync_finish(r)) { diff --git a/src/include/common_fwd.h b/src/include/common_fwd.h index e7ed9cae6a352..d906aadfa4603 100644 --- a/src/include/common_fwd.h +++ b/src/include/common_fwd.h @@ -1,6 +1,6 @@ #pragma once -#ifdef WITH_SEASTAR +#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN) #define TOPNSPC crimson #else #define TOPNSPC ceph diff --git a/src/os/ObjectStore.cc b/src/os/ObjectStore.cc index 05637f2a208df..f0fb32c723175 100644 --- a/src/os/ObjectStore.cc +++ b/src/os/ObjectStore.cc @@ -17,12 +17,16 @@ #include "common/Formatter.h" #include "common/safe_io.h" +#ifndef WITH_SEASTAR #include "filestore/FileStore.h" #include "memstore/MemStore.h" +#endif #if defined(WITH_BLUESTORE) #include "bluestore/BlueStore.h" #endif +#ifndef WITH_SEASTAR #include "kstore/KStore.h" +#endif ObjectStore *ObjectStore::create(CephContext *cct, const string& type, @@ -30,16 +34,19 @@ ObjectStore *ObjectStore::create(CephContext *cct, const string& journal, osflagbits_t flags) { +#ifndef WITH_SEASTAR if (type == "filestore") { return new FileStore(cct, data, journal, flags); } if (type == "memstore") { return new MemStore(cct, data); } +#endif #if defined(WITH_BLUESTORE) if (type == "bluestore") { return new BlueStore(cct, data); } +#ifndef WITH_SEASTAR if (type == "random") { if (rand() % 2) { return new FileStore(cct, data, journal, flags); @@ -47,15 +54,20 @@ ObjectStore *ObjectStore::create(CephContext *cct, return new BlueStore(cct, data); } } +#endif #else +#ifndef WITH_SEASTAR if (type == "random") { return new FileStore(cct, data, journal, flags); } #endif +#endif +#ifndef WITH_SEASTAR if (type == "kstore" && cct->check_experimental_feature_enabled("kstore")) { return new KStore(cct, data); } +#endif return NULL; } @@ -77,6 +89,7 @@ int ObjectStore::probe_block_device_fsid( } #endif +#ifndef WITH_SEASTAR // okay, try FileStore (journal). r = FileStore::get_block_device_fsid(cct, path, fsid); if (r == 0) { @@ -84,6 +97,7 @@ int ObjectStore::probe_block_device_fsid( << *fsid << dendl; return r; } +#endif return -EINVAL; } diff --git a/src/os/bluestore/BlockDevice.cc b/src/os/bluestore/BlockDevice.cc index ea4170d786c5b..832f6b85393f8 100644 --- a/src/os/bluestore/BlockDevice.cc +++ b/src/os/bluestore/BlockDevice.cc @@ -125,12 +125,13 @@ BlockDevice *BlockDevice::create(CephContext* cct, const string& path, return new KernelDevice(cct, cb, cbpriv, d_cb, d_cbpriv); } #endif +#ifndef WITH_SEASTAR #if defined(HAVE_SPDK) if (type == "ust-nvme") { return new NVMEDevice(cct, cb, cbpriv); } #endif - +#endif derr << __func__ << " unknown backend " << type << dendl; ceph_abort(); -- 2.39.5