From b734735a0db9a5dac1b2eb015e4b166dae17fd96 Mon Sep 17 00:00:00 2001 From: Changcheng Liu Date: Tue, 21 Apr 2020 13:38:53 +0800 Subject: [PATCH] src: extract backend driver from bluestore to access device 1. Both bluestore or other component e.g. rbd could use the same driver to access storage device. It's better to use one library to integrate the driver. 2. os and crimson-alienstore are static libraries. Link blk library into them. Main changes are below: 1. move backend driver into src/blk src/$ mkdir -p blk/{aio,kernel,spdk,pmem,zns} src/$ mv os/bluestore/{BlockDevice.h,BlockDevice.cc} blk src/$ mv os/bluestore/{ceph_aio.h} aio/aio.h src/$ mv os/bluestore/{aio.cc} aio/ src/$ mv os/bluestore/{KernelDevice.h,KernelDevice.cc} kernel/ src/$ mv os/bluestore/{ceph_io_uring.h} kernel/io_uring.h src/$ mv os/bluestore/{io_uring.cc} kernel_drv/ src/$ mv os/bluestore/{NVMEDevice.h,NVMEDevice.cc} spdk/ src/$ mv os/bluestore/{PMEMDevice.h,PMEMDevice.cc} pmem/ src/$ mv os/bluestore/{HMSMRDevice.h,HMSMRDevice.cc} zns/ 2. customize macro name in header file to remove bluestore specific text 3. adjust header file patch in source code 4. create cmake rule blk/CMakeLists.txt to build blk 5. modify src/CMakeLists.txt to integrate blk 6. modify other CMakeLists.txt to adapt to new file structure. Signed-off-by: Changcheng Liu Co-authored-by: Kefu Chai --- src/CMakeLists.txt | 1 + src/{os/bluestore => blk}/BlockDevice.cc | 8 +-- src/{os/bluestore => blk}/BlockDevice.h | 8 +-- src/blk/CMakeLists.txt | 61 +++++++++++++++++++ src/{os/bluestore => blk/aio}/aio.cc | 2 +- .../bluestore/ceph_aio.h => blk/aio/aio.h} | 0 .../bluestore => blk/kernel}/KernelDevice.cc | 2 +- .../bluestore => blk/kernel}/KernelDevice.h | 6 +- src/{os/bluestore => blk/kernel}/io_uring.cc | 2 +- .../ceph_io_uring.h => blk/kernel/io_uring.h} | 2 +- src/{os/bluestore => blk/pmem}/PMEMDevice.cc | 0 src/{os/bluestore => blk/pmem}/PMEMDevice.h | 6 +- src/{os/bluestore => blk/spdk}/NVMEDevice.cc | 0 src/{os/bluestore => blk/spdk}/NVMEDevice.h | 4 +- src/{os/bluestore => blk/zns}/HMSMRDevice.cc | 2 +- src/{os/bluestore => blk/zns}/HMSMRDevice.h | 8 +-- src/crimson/os/alienstore/CMakeLists.txt | 12 +--- src/librbd/CMakeLists.txt | 3 +- src/os/CMakeLists.txt | 51 +--------------- src/os/bluestore/BlueFS.cc | 1 - src/os/bluestore/BlueFS.h | 2 +- src/os/bluestore/BlueStore.h | 1 - src/test/objectstore/test_bdev.cc | 2 +- 23 files changed, 92 insertions(+), 92 deletions(-) rename src/{os/bluestore => blk}/BlockDevice.cc (97%) rename src/{os/bluestore => blk}/BlockDevice.h (98%) create mode 100644 src/blk/CMakeLists.txt rename src/{os/bluestore => blk/aio}/aio.cc (99%) rename src/{os/bluestore/ceph_aio.h => blk/aio/aio.h} (100%) rename src/{os/bluestore => blk/kernel}/KernelDevice.cc (99%) rename src/{os/bluestore => blk/kernel}/KernelDevice.h (97%) rename src/{os/bluestore => blk/kernel}/io_uring.cc (99%) rename src/{os/bluestore/ceph_io_uring.h => blk/kernel/io_uring.h} (97%) rename src/{os/bluestore => blk/pmem}/PMEMDevice.cc (100%) rename src/{os/bluestore => blk/pmem}/PMEMDevice.h (95%) rename src/{os/bluestore => blk/spdk}/NVMEDevice.cc (100%) rename src/{os/bluestore => blk/spdk}/NVMEDevice.h (96%) rename src/{os/bluestore => blk/zns}/HMSMRDevice.cc (99%) rename src/{os/bluestore => blk/zns}/HMSMRDevice.h (97%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6d829e0542d1..0227d8750f2f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -544,6 +544,7 @@ endif() add_subdirectory(kv) add_subdirectory(os) +add_subdirectory(blk) add_subdirectory(osd) diff --git a/src/os/bluestore/BlockDevice.cc b/src/blk/BlockDevice.cc similarity index 97% rename from src/os/bluestore/BlockDevice.cc rename to src/blk/BlockDevice.cc index f431aaf190cf..c1d9d7ca9a74 100644 --- a/src/os/bluestore/BlockDevice.cc +++ b/src/blk/BlockDevice.cc @@ -20,20 +20,20 @@ #include "BlockDevice.h" #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO) -#include "KernelDevice.h" +#include "kernel/KernelDevice.h" #endif #if defined(HAVE_SPDK) -#include "NVMEDevice.h" +#include "spdk/NVMEDevice.h" #endif #if defined(HAVE_BLUESTORE_PMEM) -#include "PMEMDevice.h" +#include "pmem/PMEMDevice.h" #include "libpmem.h" #endif #if defined(HAVE_LIBZBC) -#include "HMSMRDevice.h" +#include "zns/HMSMRDevice.h" extern "C" { #include } diff --git a/src/os/bluestore/BlockDevice.h b/src/blk/BlockDevice.h similarity index 98% rename from src/os/bluestore/BlockDevice.h rename to src/blk/BlockDevice.h index 9612c32f6cd8..b0929bad12dc 100644 --- a/src/os/bluestore/BlockDevice.h +++ b/src/blk/BlockDevice.h @@ -14,8 +14,8 @@ * */ -#ifndef CEPH_OS_BLUESTORE_BLOCKDEVICE_H -#define CEPH_OS_BLUESTORE_BLOCKDEVICE_H +#ifndef CEPH_BLK_BLOCKDEVICE_H +#define CEPH_BLK_BLOCKDEVICE_H #include #include @@ -31,7 +31,7 @@ #include "include/common_fwd.h" #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO) -#include "ceph_aio.h" +#include "aio/aio.h" #endif #include "include/ceph_assert.h" #include "include/buffer.h" @@ -261,4 +261,4 @@ protected: } }; -#endif //CEPH_OS_BLUESTORE_BLOCKDEVICE_H +#endif //CEPH_BLK_BLOCKDEVICE_H diff --git a/src/blk/CMakeLists.txt b/src/blk/CMakeLists.txt new file mode 100644 index 000000000000..d42a9226b460 --- /dev/null +++ b/src/blk/CMakeLists.txt @@ -0,0 +1,61 @@ +if(WITH_BLUESTORE OR WITH_RBD_RWL) +list(APPEND libblk_srcs + BlockDevice.cc) +endif() + +if(HAVE_LIBAIO OR HAVE_POSIXAIO) + list(APPEND libblk_srcs + kernel/KernelDevice.cc + kernel/io_uring.cc + aio/aio.cc) +endif() + +if(WITH_BLUESTORE_PMEM) + list(APPEND libblk_srcs + pmem/PMEMDevice.cc) +endif() + +if(WITH_SPDK) + list(APPEND libblk_srcs + spdk/NVMEDevice.cc) +endif() + +if(HAVE_LIBZBC) + list(APPEND libblk_srcs + zns/HMSMRDevice.cc) +endif() + +add_library(blk ${libblk_srcs}) +target_include_directories(blk PRIVATE "./") + +if(HAVE_LIBAIO) + target_link_libraries(blk PUBLIC ${AIO_LIBRARIES}) +endif(HAVE_LIBAIO) + +if(WITH_SPDK) + target_link_libraries(blk PRIVATE ${SPDK_LIBRARIES}) +endif() + +if(HAVE_LIBZBC) + target_link_libraries(blk ${ZBC_LIBRARIES}) +endif() + +if(WITH_BLUESTORE_PMEM OR WITH_RBD_RWL) + target_link_libraries(blk + PUBLIC pmem::pmemobj + PRIVATE pmem::pmem) +endif() + +if(WITH_EVENTTRACE) + add_dependencies(blk eventtrace_tp) +endif() + +if(WITH_LIBURING) + if(WITH_SYSTEM_LIBURING) + find_package(uring REQUIRED) + else() + include(Builduring) + build_uring() + endif() + target_link_libraries(blk PRIVATE uring::uring) +endif() diff --git a/src/os/bluestore/aio.cc b/src/blk/aio/aio.cc similarity index 99% rename from src/os/bluestore/aio.cc rename to src/blk/aio/aio.cc index eb0c13fe9580..00a12bfd16af 100644 --- a/src/os/bluestore/aio.cc +++ b/src/blk/aio/aio.cc @@ -2,7 +2,7 @@ // vim: ts=8 sw=2 smarttab #include -#include "ceph_aio.h" +#include "aio.h" std::ostream& operator<<(std::ostream& os, const aio_t& aio) { diff --git a/src/os/bluestore/ceph_aio.h b/src/blk/aio/aio.h similarity index 100% rename from src/os/bluestore/ceph_aio.h rename to src/blk/aio/aio.h diff --git a/src/os/bluestore/KernelDevice.cc b/src/blk/kernel/KernelDevice.cc similarity index 99% rename from src/os/bluestore/KernelDevice.cc rename to src/blk/kernel/KernelDevice.cc index 3ff2f22251ea..ddbed426a2da 100644 --- a/src/os/bluestore/KernelDevice.cc +++ b/src/blk/kernel/KernelDevice.cc @@ -33,7 +33,7 @@ #include "common/numa.h" #include "global/global_context.h" -#include "ceph_io_uring.h" +#include "io_uring.h" #define dout_context cct #define dout_subsys ceph_subsys_bdev diff --git a/src/os/bluestore/KernelDevice.h b/src/blk/kernel/KernelDevice.h similarity index 97% rename from src/os/bluestore/KernelDevice.h rename to src/blk/kernel/KernelDevice.h index 57a0955d0942..7ac9b1e7e1e3 100644 --- a/src/os/bluestore/KernelDevice.h +++ b/src/blk/kernel/KernelDevice.h @@ -12,8 +12,8 @@ * */ -#ifndef CEPH_OS_BLUESTORE_KERNELDEVICE_H -#define CEPH_OS_BLUESTORE_KERNELDEVICE_H +#ifndef CEPH_BLK_KERNELDEVICE_H +#define CEPH_BLK_KERNELDEVICE_H #include @@ -22,7 +22,7 @@ #include "common/Thread.h" #include "include/utime.h" -#include "ceph_aio.h" +#include "aio/aio.h" #include "BlockDevice.h" #define RW_IO_MAX (INT_MAX & CEPH_PAGE_MASK) diff --git a/src/os/bluestore/io_uring.cc b/src/blk/kernel/io_uring.cc similarity index 99% rename from src/os/bluestore/io_uring.cc rename to src/blk/kernel/io_uring.cc index 54fa0f9535f3..e8b2140f8c8d 100644 --- a/src/os/bluestore/io_uring.cc +++ b/src/blk/kernel/io_uring.cc @@ -1,7 +1,7 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab -#include "ceph_io_uring.h" +#include "io_uring.h" #if defined(HAVE_LIBURING) && defined(__x86_64__) diff --git a/src/os/bluestore/ceph_io_uring.h b/src/blk/kernel/io_uring.h similarity index 97% rename from src/os/bluestore/ceph_io_uring.h rename to src/blk/kernel/io_uring.h index f14135ac184f..f4ac2f6e12d8 100644 --- a/src/os/bluestore/ceph_io_uring.h +++ b/src/blk/kernel/io_uring.h @@ -6,7 +6,7 @@ #include "acconfig.h" #include "include/types.h" -#include "ceph_aio.h" +#include "aio/aio.h" struct ioring_data; diff --git a/src/os/bluestore/PMEMDevice.cc b/src/blk/pmem/PMEMDevice.cc similarity index 100% rename from src/os/bluestore/PMEMDevice.cc rename to src/blk/pmem/PMEMDevice.cc diff --git a/src/os/bluestore/PMEMDevice.h b/src/blk/pmem/PMEMDevice.h similarity index 95% rename from src/os/bluestore/PMEMDevice.h rename to src/blk/pmem/PMEMDevice.h index 3077375a2ac8..8b9bbfd25ab3 100644 --- a/src/os/bluestore/PMEMDevice.h +++ b/src/blk/pmem/PMEMDevice.h @@ -14,14 +14,14 @@ * */ -#ifndef CEPH_OS_BLUESTORE_PMEMDEVICE_H -#define CEPH_OS_BLUESTORE_PMEMDEVICE_H +#ifndef CEPH_BLK_PMEMDEVICE_H +#define CEPH_BLK_PMEMDEVICE_H #include #include "os/fs/FS.h" #include "include/interval_set.h" -#include "ceph_aio.h" +#include "aio/aio.h" #include "BlockDevice.h" class PMEMDevice : public BlockDevice { diff --git a/src/os/bluestore/NVMEDevice.cc b/src/blk/spdk/NVMEDevice.cc similarity index 100% rename from src/os/bluestore/NVMEDevice.cc rename to src/blk/spdk/NVMEDevice.cc diff --git a/src/os/bluestore/NVMEDevice.h b/src/blk/spdk/NVMEDevice.h similarity index 96% rename from src/os/bluestore/NVMEDevice.h rename to src/blk/spdk/NVMEDevice.h index f44aeb59784f..9ad568549088 100644 --- a/src/os/bluestore/NVMEDevice.h +++ b/src/blk/spdk/NVMEDevice.h @@ -14,8 +14,8 @@ * */ -#ifndef CEPH_OS_BLUESTORE_NVMEDEVICE -#define CEPH_OS_BLUESTORE_NVMEDEVICE +#ifndef CEPH_BLK_NVMEDEVICE +#define CEPH_BLK_NVMEDEVICE #include #include diff --git a/src/os/bluestore/HMSMRDevice.cc b/src/blk/zns/HMSMRDevice.cc similarity index 99% rename from src/os/bluestore/HMSMRDevice.cc rename to src/blk/zns/HMSMRDevice.cc index 4fd6e8a37f02..fcf36c833c54 100644 --- a/src/os/bluestore/HMSMRDevice.cc +++ b/src/blk/zns/HMSMRDevice.cc @@ -35,7 +35,7 @@ #include "common/numa.h" #include "global/global_context.h" -#include "ceph_io_uring.h" +#include "kernel/io_uring.h" extern "C" { #include diff --git a/src/os/bluestore/HMSMRDevice.h b/src/blk/zns/HMSMRDevice.h similarity index 97% rename from src/os/bluestore/HMSMRDevice.h rename to src/blk/zns/HMSMRDevice.h index 4fd83db8032b..5ec8ee7d0ff0 100644 --- a/src/os/bluestore/HMSMRDevice.h +++ b/src/blk/zns/HMSMRDevice.h @@ -16,8 +16,8 @@ // Copied from KernelDevice with HM-SMR specific functionality added. Will be // further specialized for HM-SMR. -#ifndef CEPH_OS_BLUESTORE_HMSMRDEVICE_H -#define CEPH_OS_BLUESTORE_HMSMRDEVICE_H +#ifndef CEPH_BLK_HMSMRDEVICE_H +#define CEPH_BLK_HMSMRDEVICE_H #include @@ -26,7 +26,7 @@ #include "common/Thread.h" #include "include/utime.h" -#include "ceph_aio.h" +#include "aio/aio.h" #include "BlockDevice.h" #define RW_IO_MAX (INT_MAX & CEPH_PAGE_MASK) @@ -159,4 +159,4 @@ public: void close() final; }; -#endif +#endif //CEPH_BLK_HMSMRDEVICE_H diff --git a/src/crimson/os/alienstore/CMakeLists.txt b/src/crimson/os/alienstore/CMakeLists.txt index 1d06ce6eea6c..e3343d52d6fe 100644 --- a/src/crimson/os/alienstore/CMakeLists.txt +++ b/src/crimson/os/alienstore/CMakeLists.txt @@ -28,7 +28,6 @@ list(APPEND crimson_alien_srcs ${PROJECT_SOURCE_DIR}/src/os/bluestore/Allocator.cc ${PROJECT_SOURCE_DIR}/src/os/bluestore/AvlAllocator.cc ${PROJECT_SOURCE_DIR}/src/os/bluestore/BitmapFreelistManager.cc - ${PROJECT_SOURCE_DIR}/src/os/bluestore/BlockDevice.cc ${PROJECT_SOURCE_DIR}/src/os/bluestore/BlueFS.cc ${PROJECT_SOURCE_DIR}/src/os/bluestore/bluefs_types.cc ${PROJECT_SOURCE_DIR}/src/os/bluestore/BlueRocksEnv.cc @@ -37,23 +36,13 @@ list(APPEND crimson_alien_srcs ${PROJECT_SOURCE_DIR}/src/os/bluestore/fastbmap_allocator_impl.cc ${PROJECT_SOURCE_DIR}/src/os/bluestore/FreelistManager.cc ${PROJECT_SOURCE_DIR}/src/os/bluestore/HybridAllocator.cc - ${PROJECT_SOURCE_DIR}/src/os/bluestore/io_uring.cc ${PROJECT_SOURCE_DIR}/src/os/bluestore/StupidAllocator.cc ${PROJECT_SOURCE_DIR}/src/os/bluestore/BitmapAllocator.cc) -if(HAVE_LIBAIO OR HAVE_POSIXAIO) - list(APPEND crimson_alien_srcs - ${PROJECT_SOURCE_DIR}/src/os/bluestore/KernelDevice.cc - ${PROJECT_SOURCE_DIR}/src/os/bluestore/aio.cc) -endif() - add_library(crimson-alienstore STATIC ${crimson_alien_srcs} $ $ $) -if(HAVE_LIBAIO) - target_link_libraries(crimson-alienstore ${AIO_LIBRARIES}) -endif(HAVE_LIBAIO) target_compile_definitions(crimson-alienstore PRIVATE -DWITH_SEASTAR -DWITH_ALIEN) target_include_directories(crimson-alienstore PRIVATE @@ -64,3 +53,4 @@ target_link_libraries(crimson-alienstore heap_profiler) target_link_libraries(crimson-alienstore ${BLKID_LIBRARIES}) target_link_libraries(crimson-alienstore ${UDEV_LIBRARIES}) target_link_libraries(crimson-alienstore crimson) +target_link_libraries(crimson-alienstore blk) diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index 49952dc25cee..349941e9307d 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -217,8 +217,7 @@ target_link_libraries(rbd_internal PRIVATE if(WITH_RBD_RWL) target_link_libraries(rbd_internal - PUBLIC pmem::pmemobj - PRIVATE pmem::pmem) + PUBLIC blk) endif() add_library(librbd ${CEPH_SHARED} diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt index 1cc98a7aac9a..2611a5a7d1a6 100644 --- a/src/os/CMakeLists.txt +++ b/src/os/CMakeLists.txt @@ -23,7 +23,6 @@ if(WITH_BLUESTORE) list(APPEND libos_srcs bluestore/Allocator.cc bluestore/BitmapFreelistManager.cc - bluestore/BlockDevice.cc bluestore/BlueFS.cc bluestore/bluefs_types.cc bluestore/BlueRocksEnv.cc @@ -35,19 +34,11 @@ if(WITH_BLUESTORE) bluestore/BitmapAllocator.cc bluestore/AvlAllocator.cc bluestore/HybridAllocator.cc - bluestore/io_uring.cc ) endif(WITH_BLUESTORE) -if(HAVE_LIBAIO OR HAVE_POSIXAIO) - list(APPEND libos_srcs - bluestore/KernelDevice.cc - bluestore/aio.cc) -endif() - if(HAVE_LIBZBC) list(APPEND libos_srcs - bluestore/HMSMRDevice.cc bluestore/ZonedAllocator.cc) endif() @@ -56,11 +47,6 @@ if(WITH_FUSE) FuseStore.cc) endif(WITH_FUSE) -if(WITH_BLUESTORE_PMEM) - list(APPEND libos_srcs - bluestore/PMEMDevice.cc) -endif() - if(HAVE_LIBXFS) list(APPEND libos_srcs filestore/XfsFileStoreBackend.cc @@ -76,12 +62,8 @@ if(HAVE_LIBZFS) list(APPEND libos_srcs $) endif() -if(WITH_SPDK) - list(APPEND libos_srcs - bluestore/NVMEDevice.cc) -endif() - add_library(os STATIC ${libos_srcs}) +target_link_libraries(os blk) target_link_libraries(os heap_profiler kv) @@ -94,14 +76,6 @@ if(WITH_BLUEFS) install(TARGETS bluefs DESTINATION lib) endif(WITH_BLUEFS) -if(HAVE_LIBAIO) - target_link_libraries(os ${AIO_LIBRARIES}) -endif(HAVE_LIBAIO) - -if(HAVE_LIBZBC) - target_link_libraries(os ${ZBC_LIBRARIES}) -endif() - if(WITH_FUSE) target_link_libraries(os FUSE::FUSE) endif() @@ -110,11 +84,6 @@ if(HAVE_LIBZFS) target_link_libraries(os ${ZFS_LIBRARIES}) endif() -if(WITH_SPDK) - target_link_libraries(os - ${SPDK_LIBRARIES}) -endif() - if(WITH_LTTNG) add_dependencies(os objectstore-tp) add_dependencies(os bluestore-tp) @@ -134,21 +103,3 @@ if(WITH_BLUESTORE) install(TARGETS ceph-bluestore-tool DESTINATION bin) endif() - -if(WITH_BLUESTORE_PMEM) - target_link_libraries(os pmem::pmem) -endif() - -if(WITH_EVENTTRACE) - add_dependencies(os eventtrace_tp) -endif() - -if(WITH_LIBURING) - if(WITH_SYSTEM_LIBURING) - find_package(uring REQUIRED) - else() - include(Builduring) - build_uring() - endif() - target_link_libraries(os uring::uring) -endif() diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index 281c156a6d2c..396a7c72632f 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -8,7 +8,6 @@ #include "common/debug.h" #include "common/errno.h" #include "common/perf_counters.h" -#include "BlockDevice.h" #include "Allocator.h" #include "include/ceph_assert.h" #include "common/admin_socket.h" diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h index 9d699069d77d..33368a5c4498 100644 --- a/src/os/bluestore/BlueFS.h +++ b/src/os/bluestore/BlueFS.h @@ -7,7 +7,7 @@ #include #include "bluefs_types.h" -#include "BlockDevice.h" +#include "blk/BlockDevice.h" #include "common/RefCountedObj.h" #include "common/ceph_context.h" diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index 8685fb8782a5..95c55a04ff11 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -49,7 +49,6 @@ #include "os/ObjectStore.h" #include "bluestore_types.h" -#include "BlockDevice.h" #include "BlueFS.h" #include "common/EventTrace.h" diff --git a/src/test/objectstore/test_bdev.cc b/src/test/objectstore/test_bdev.cc index 09a5226ebcd5..b24368f0a056 100755 --- a/src/test/objectstore/test_bdev.cc +++ b/src/test/objectstore/test_bdev.cc @@ -12,7 +12,7 @@ #include "include/stringify.h" #include "common/errno.h" -#include "os/bluestore/BlockDevice.h" +#include "blk/BlockDevice.h" class TempBdev { public: -- 2.47.3