From 9d01a7cec694406e9791fc596d6c280212a9079a Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Tue, 21 Feb 2023 17:10:20 +0900 Subject: [PATCH] crimson/os/seastore/rbm: move RBMDevice related functions to rbm_device.cc Signed-off-by: Myoungwon Oh --- src/crimson/os/seastore/CMakeLists.txt | 1 + .../random_block_manager/nvme_block_device.cc | 199 ---------------- .../random_block_manager/rbm_device.cc | 216 ++++++++++++++++++ 3 files changed, 217 insertions(+), 199 deletions(-) create mode 100644 src/crimson/os/seastore/random_block_manager/rbm_device.cc diff --git a/src/crimson/os/seastore/CMakeLists.txt b/src/crimson/os/seastore/CMakeLists.txt index ad43b4c4e19..0e998674c3e 100644 --- a/src/crimson/os/seastore/CMakeLists.txt +++ b/src/crimson/os/seastore/CMakeLists.txt @@ -38,6 +38,7 @@ set(crimson_seastore_srcs seastore.cc random_block_manager.cc random_block_manager/block_rb_manager.cc + random_block_manager/rbm_device.cc random_block_manager/nvme_block_device.cc random_block_manager/avlallocator.cc journal/segmented_journal.cc diff --git a/src/crimson/os/seastore/random_block_manager/nvme_block_device.cc b/src/crimson/os/seastore/random_block_manager/nvme_block_device.cc index 96d3a677224..19d244ddf9b 100644 --- a/src/crimson/os/seastore/random_block_manager/nvme_block_device.cc +++ b/src/crimson/os/seastore/random_block_manager/nvme_block_device.cc @@ -20,123 +20,6 @@ namespace { } } -namespace crimson::os::seastore::random_block_device { -#include "crimson/os/seastore/logging.h" -SET_SUBSYS(seastore_device); - -RBMDevice::mkfs_ret RBMDevice::do_mkfs(device_config_t config) { - LOG_PREFIX(RBMDevice::mkfs); - return stat_device( - ).handle_error( - mkfs_ertr::pass_further{}, - crimson::ct_error::assert_all{ - "Invalid error stat_device in RBMDevice::mkfs"} - ).safe_then([this, FNAME, config=std::move(config)](auto st) { - super.block_size = st.block_size; - super.size = st.size; - super.feature |= RBM_BITMAP_BLOCK_CRC; - super.config = std::move(config); - assert(super.journal_size); - assert(super.size >= super.journal_size); - DEBUG("super {} ", super); - // write super block - return open(get_device_path(), - seastar::open_flags::rw | seastar::open_flags::dsync - ).handle_error( - mkfs_ertr::pass_further{}, - crimson::ct_error::assert_all{ - "Invalid error open in RBMDevice::mkfs"} - ).safe_then([this] { - return write_rbm_header( - ).safe_then([this] { - return close(); - }).handle_error( - mkfs_ertr::pass_further{}, - crimson::ct_error::assert_all{ - "Invalid error write_rbm_header in RBMDevice::mkfs" - }); - }); - }); -} - -write_ertr::future<> RBMDevice::write_rbm_header() -{ - bufferlist meta_b_header; - super.crc = 0; - encode(super, meta_b_header); - // If NVMeDevice supports data protection, CRC for checksum is not required - // NVMeDevice is expected to generate and store checksum internally. - // CPU overhead for CRC might be saved. - if (is_data_protection_enabled()) { - super.crc = -1; - } else { - super.crc = meta_b_header.crc32c(-1); - } - - bufferlist bl; - encode(super, bl); - auto iter = bl.begin(); - auto bp = bufferptr(ceph::buffer::create_page_aligned(super.block_size)); - assert(bl.length() < super.block_size); - iter.copy(bl.length(), bp.c_str()); - return write(RBM_START_ADDRESS, std::move(bp)); -} - -read_ertr::future RBMDevice::read_rbm_header( - rbm_abs_addr addr) -{ - LOG_PREFIX(RBMDevice::read_rbm_header); - assert(super.block_size > 0); - return seastar::do_with( - bufferptr(ceph::buffer::create_page_aligned(super.block_size)), - [this, addr, FNAME](auto &bptr) { - return read( - addr, - bptr - ).safe_then([length=bptr.length(), this, bptr, FNAME]() - -> read_ertr::future { - bufferlist bl; - bl.append(bptr); - auto p = bl.cbegin(); - rbm_metadata_header_t super_block; - try { - decode(super_block, p); - } - catch (ceph::buffer::error& e) { - DEBUG("read_rbm_header: unable to decode rbm super block {}", - e.what()); - return crimson::ct_error::enoent::make(); - } - checksum_t crc = super_block.crc; - bufferlist meta_b_header; - super_block.crc = 0; - encode(super_block, meta_b_header); - assert(ceph::encoded_sizeof(super_block) < - super_block.block_size); - - // Do CRC verification only if data protection is not supported. - if (is_data_protection_enabled() == false) { - if (meta_b_header.crc32c(-1) != crc) { - DEBUG("bad crc on super block, expected {} != actual {} ", - meta_b_header.crc32c(-1), crc); - return crimson::ct_error::input_output_error::make(); - } - } else { - ceph_assert_always(crc == (checksum_t)-1); - } - super_block.crc = crc; - super = super_block; - DEBUG("got {} ", super); - return read_ertr::future( - read_ertr::ready_future_marker{}, - super_block - ); - }); - }); -} - -} - namespace crimson::os::seastore::random_block_device::nvme { open_ertr::future<> NVMeBlockDevice::open( @@ -398,85 +281,3 @@ nvme_command_ertr::future NVMeBlockDevice::pass_through_io( } } - -namespace crimson::os::seastore::random_block_device { - -EphemeralRBMDeviceRef create_test_ephemeral(uint64_t journal_size, uint64_t data_size) { - return EphemeralRBMDeviceRef( - new EphemeralRBMDevice(journal_size + data_size + - random_block_device::RBMDevice::get_journal_start(), - EphemeralRBMDevice::TEST_BLOCK_SIZE)); -} - -open_ertr::future<> EphemeralRBMDevice::open( - const std::string &in_path, - seastar::open_flags mode) { - if (buf) { - return open_ertr::now(); - } - - logger().debug( - "Initializing test memory device {}", - size); - - void* addr = ::mmap( - nullptr, - size, - PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, - -1, - 0); - - buf = (char*)addr; - - ::memset(buf, 0, size); - return open_ertr::now(); -} - -write_ertr::future<> EphemeralRBMDevice::write( - uint64_t offset, - bufferptr &&bptr, - uint16_t stream) { - ceph_assert(buf); - logger().debug( - "EphemeralRBMDevice: write offset {} len {}", - offset, - bptr.length()); - - ::memcpy(buf + offset, bptr.c_str(), bptr.length()); - - return write_ertr::now(); -} - -read_ertr::future<> EphemeralRBMDevice::read( - uint64_t offset, - bufferptr &bptr) { - ceph_assert(buf); - logger().debug( - "EphemeralRBMDevice: read offset {} len {}", - offset, - bptr.length()); - - bptr.copy_in(0, bptr.length(), buf + offset); - return read_ertr::now(); -} - -Device::close_ertr::future<> EphemeralRBMDevice::close() { - logger().debug(" close "); - return close_ertr::now(); -} - -write_ertr::future<> EphemeralRBMDevice::writev( - uint64_t offset, - ceph::bufferlist bl, - uint16_t stream) { - ceph_assert(buf); - logger().debug( - "EphemeralRBMDevice: write offset {} len {}", - offset, - bl.length()); - - bl.begin().copy(bl.length(), buf + offset); - return write_ertr::now(); -} - -} diff --git a/src/crimson/os/seastore/random_block_manager/rbm_device.cc b/src/crimson/os/seastore/random_block_manager/rbm_device.cc new file mode 100644 index 00000000000..d5e8827f112 --- /dev/null +++ b/src/crimson/os/seastore/random_block_manager/rbm_device.cc @@ -0,0 +1,216 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include +#include + +#include + +#include "crimson/common/log.h" +#include "crimson/common/errorator-loop.h" + +#include "include/buffer.h" +#include "rbm_device.h" +#include "nvme_block_device.h" +#include "block_rb_manager.h" + +namespace crimson::os::seastore::random_block_device { +#include "crimson/os/seastore/logging.h" +SET_SUBSYS(seastore_device); + +RBMDevice::mkfs_ret RBMDevice::do_mkfs(device_config_t config) { + LOG_PREFIX(RBMDevice::mkfs); + return stat_device( + ).handle_error( + mkfs_ertr::pass_further{}, + crimson::ct_error::assert_all{ + "Invalid error stat_device in RBMDevice::mkfs"} + ).safe_then([this, FNAME, config=std::move(config)](auto st) { + super.block_size = st.block_size; + super.size = st.size; + super.feature |= RBM_BITMAP_BLOCK_CRC; + super.config = std::move(config); + assert(super.journal_size); + assert(super.size >= super.journal_size); + DEBUG("super {} ", super); + // write super block + return open(get_device_path(), + seastar::open_flags::rw | seastar::open_flags::dsync + ).handle_error( + mkfs_ertr::pass_further{}, + crimson::ct_error::assert_all{ + "Invalid error open in RBMDevice::mkfs"} + ).safe_then([this] { + return write_rbm_header( + ).safe_then([this] { + return close(); + }).handle_error( + mkfs_ertr::pass_further{}, + crimson::ct_error::assert_all{ + "Invalid error write_rbm_header in RBMDevice::mkfs" + }); + }); + }); +} + +write_ertr::future<> RBMDevice::write_rbm_header() +{ + bufferlist meta_b_header; + super.crc = 0; + encode(super, meta_b_header); + // If NVMeDevice supports data protection, CRC for checksum is not required + // NVMeDevice is expected to generate and store checksum internally. + // CPU overhead for CRC might be saved. + if (is_data_protection_enabled()) { + super.crc = -1; + } else { + super.crc = meta_b_header.crc32c(-1); + } + + bufferlist bl; + encode(super, bl); + auto iter = bl.begin(); + auto bp = bufferptr(ceph::buffer::create_page_aligned(super.block_size)); + assert(bl.length() < super.block_size); + iter.copy(bl.length(), bp.c_str()); + return write(RBM_START_ADDRESS, std::move(bp)); +} + +read_ertr::future RBMDevice::read_rbm_header( + rbm_abs_addr addr) +{ + LOG_PREFIX(RBMDevice::read_rbm_header); + assert(super.block_size > 0); + return seastar::do_with( + bufferptr(ceph::buffer::create_page_aligned(super.block_size)), + [this, addr, FNAME](auto &bptr) { + return read( + addr, + bptr + ).safe_then([length=bptr.length(), this, bptr, FNAME]() + -> read_ertr::future { + bufferlist bl; + bl.append(bptr); + auto p = bl.cbegin(); + rbm_metadata_header_t super_block; + try { + decode(super_block, p); + } + catch (ceph::buffer::error& e) { + DEBUG("read_rbm_header: unable to decode rbm super block {}", + e.what()); + return crimson::ct_error::enoent::make(); + } + checksum_t crc = super_block.crc; + bufferlist meta_b_header; + super_block.crc = 0; + encode(super_block, meta_b_header); + assert(ceph::encoded_sizeof(super_block) < + super_block.block_size); + + // Do CRC verification only if data protection is not supported. + if (is_data_protection_enabled() == false) { + if (meta_b_header.crc32c(-1) != crc) { + DEBUG("bad crc on super block, expected {} != actual {} ", + meta_b_header.crc32c(-1), crc); + return crimson::ct_error::input_output_error::make(); + } + } else { + ceph_assert_always(crc == (checksum_t)-1); + } + super_block.crc = crc; + super = super_block; + DEBUG("got {} ", super); + return read_ertr::future( + read_ertr::ready_future_marker{}, + super_block + ); + }); + }); +} + +EphemeralRBMDeviceRef create_test_ephemeral(uint64_t journal_size, uint64_t data_size) { + return EphemeralRBMDeviceRef( + new EphemeralRBMDevice(journal_size + data_size + + random_block_device::RBMDevice::get_journal_start(), + EphemeralRBMDevice::TEST_BLOCK_SIZE)); +} + +open_ertr::future<> EphemeralRBMDevice::open( + const std::string &in_path, + seastar::open_flags mode) { + LOG_PREFIX(EphemeralRBMDevice::open); + if (buf) { + return open_ertr::now(); + } + + DEBUG( + "Initializing test memory device {}", + size); + + void* addr = ::mmap( + nullptr, + size, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, + -1, + 0); + + buf = (char*)addr; + + ::memset(buf, 0, size); + return open_ertr::now(); +} + +write_ertr::future<> EphemeralRBMDevice::write( + uint64_t offset, + bufferptr &&bptr, + uint16_t stream) { + LOG_PREFIX(EphemeralRBMDevice::write); + ceph_assert(buf); + DEBUG( + "EphemeralRBMDevice: write offset {} len {}", + offset, + bptr.length()); + + ::memcpy(buf + offset, bptr.c_str(), bptr.length()); + + return write_ertr::now(); +} + +read_ertr::future<> EphemeralRBMDevice::read( + uint64_t offset, + bufferptr &bptr) { + LOG_PREFIX(EphemeralRBMDevice::read); + ceph_assert(buf); + DEBUG( + "EphemeralRBMDevice: read offset {} len {}", + offset, + bptr.length()); + + bptr.copy_in(0, bptr.length(), buf + offset); + return read_ertr::now(); +} + +Device::close_ertr::future<> EphemeralRBMDevice::close() { + LOG_PREFIX(EphemeralRBMDevice::close); + DEBUG(" close "); + return close_ertr::now(); +} + +write_ertr::future<> EphemeralRBMDevice::writev( + uint64_t offset, + ceph::bufferlist bl, + uint16_t stream) { + LOG_PREFIX(EphemeralRBMDevice::writev); + ceph_assert(buf); + DEBUG( + "EphemeralRBMDevice: write offset {} len {}", + offset, + bl.length()); + + bl.begin().copy(bl.length(), buf + offset); + return write_ertr::now(); +} + +} + -- 2.39.5