}
void validate() const {
- ceph_assert(shard_num == seastar::smp::count);
ceph_assert(block_size > 0);
for (unsigned int i = 0; i < seastar::smp::count; i ++) {
ceph_assert(shard_infos[i].size > block_size &&
namespace crimson::os::seastore::random_block_device::nvme {
+seastar::future<> NVMeBlockDevice::start(unsigned int shard_nums)
+{
+ device_shard_nums = shard_nums;
+ auto num_shard_services = (device_shard_nums + seastar::smp::count - 1 ) / seastar::smp::count;
+ LOG_PREFIX(NVMeBlockDevice::start);
+ DEBUG("device_shard_nums={} seastar::smp={}, num_shard_services={}", device_shard_nums, seastar::smp::count, num_shard_services);
+ return shard_devices.start(num_shard_services, device_path);
+
+}
+
+seastar::future<> NVMeBlockDevice::stop()
+{
+ return shard_devices.stop();
+}
+
+Device& NVMeBlockDevice::get_sharded_device(unsigned int store_index)
+{
+ assert(store_index < shard_devices.local().mshard_devices.size());
+ return *shard_devices.local().mshard_devices[store_index];
+}
+
NVMeBlockDevice::mkfs_ret NVMeBlockDevice::mkfs(device_config_t config) {
using crimson::common::get_conf;
- co_await shard_devices.local().do_primary_mkfs(config,
+ co_await shard_devices.local().mshard_devices[0]->do_primary_mkfs(config,
seastar::smp::count,
get_conf<Option::size_t>("seastore_cbjournal_size")
);
LOG_PREFIX(NVMeBlockDevice::mount);
DEBUG("mount");
co_await shard_devices.invoke_on_all([](auto &local_device) {
- return local_device.do_shard_mount(
- ).handle_error(
- crimson::ct_error::assert_all{
- "Invalid error in NVMeBlockDevice::do_shard_mount"
+ return seastar::do_for_each(local_device.mshard_devices, [](auto& mshard_device) {
+ return mshard_device->do_shard_mount(
+ ).handle_error(
+ crimson::ct_error::assert_all{
+ "Invalid error in NVMeBlockDevice::do_shard_mount"
+ });
});
});
* atomic_write_unit does not require fsync().
*/
- NVMeBlockDevice(std::string device_path) : device_path(device_path) {}
+ NVMeBlockDevice(std::string device_path, unsigned int store_index = 0)
+ : RBMDevice(store_index),
+ device_path(device_path) {}
~NVMeBlockDevice() = default;
open_ertr::future<> open(
return device_path;
}
- seastar::future<> start() final {
- return shard_devices.start(device_path);
- }
+ seastar::future<> start(unsigned int shard_nums) final;
- seastar::future<> stop() final {
- return shard_devices.stop();
- }
+ seastar::future<> stop() final;
- Device& get_sharded_device() final {
- return shard_devices.local();
- }
+ Device& get_sharded_device(unsigned int store_index = 0) final;
uint64_t get_preffered_write_granularity() const { return write_granularity; }
uint64_t get_preffered_write_alignment() const { return write_alignment; }
int namespace_id; // TODO: multi namespaces
std::string device_path;
- seastar::sharded<NVMeBlockDevice> shard_devices;
+
+ class MultiShardDevices {
+ public:
+ std::vector<std::unique_ptr<NVMeBlockDevice>> mshard_devices;
+
+ public:
+ MultiShardDevices(size_t count,
+ const std::string path)
+ : mshard_devices() {
+ mshard_devices.reserve(count);
+ for (size_t store_index = 0; store_index < count; ++store_index) {
+ mshard_devices.emplace_back(std::make_unique<NVMeBlockDevice>(
+ path, store_index));
+ }
+ }
+ ~MultiShardDevices() {
+ mshard_devices.clear();
+ }
+ };
+ seastar::sharded<MultiShardDevices> shard_devices;
};
}
return std::nullopt;
}));
if (!st) {
- co_await mount_ertr::future<>(
+ co_return co_await mount_ertr::future<>(
crimson::ct_error::input_output_error::make()
);
"Invalid error read_rbm_superblock in RBMDevice::do_shard_mount"}
);
LOG_PREFIX(RBMDevice::do_shard_mount);
- shard_info = s.shard_infos[seastar::this_shard_id()];
+ if(seastar::this_shard_id() + seastar::smp::count * store_index >= s.shard_num) {
+ INFO("{} shard_id {} out of range {}",
+ device_id_printer_t{get_device_id()},
+ seastar::this_shard_id() + seastar::smp::count * store_index,
+ s.shard_num);
+ shard_status = false;
+ co_return;
+ }
+ shard_info = s.shard_infos[seastar::this_shard_id() + seastar::smp::count * store_index];
INFO("{} read {}", device_id_printer_t{get_device_id()}, shard_info);
s.validate();
}
+read_ertr::future<unsigned int> RBMDevice::get_shard_nums()
+{
+ co_await open(get_device_path(),
+ seastar::open_flags::rw | seastar::open_flags::dsync
+ ).handle_error(
+ crimson::ct_error::assert_all{
+ "Invalid error open in RBMDevice::get_shard_nums"}
+ );
+
+ auto st = co_await stat_device(
+ ).handle_error(
+ crimson::ct_error::assert_all{
+ "Invalid error stat_device in RBMDevice::get_shard_nums"}
+ );
+
+ assert(st.block_size > 0);
+ super.block_size = st.block_size;
+ auto sb = co_await read_rbm_superblock(RBM_START_ADDRESS
+ ).handle_error(
+ crimson::ct_error::assert_all{
+ "Invalid error in RBMDevice::get_shard_nums"}
+ );
+
+ co_return sb.shard_num;
+}
+
EphemeralRBMDeviceRef create_test_ephemeral(uint64_t journal_size, uint64_t data_size) {
return EphemeralRBMDeviceRef(
new EphemeralRBMDevice(journal_size + data_size +
protected:
rbm_superblock_t super;
rbm_shard_info_t shard_info;
+ unsigned int device_shard_nums = 0;
+ unsigned int store_index = 0;
+ bool shard_status = true;
public:
- RBMDevice() {}
+ RBMDevice(unsigned int store_index = 0)
+ : store_index(store_index) {}
virtual ~RBMDevice() = default;
template <typename T>
std::size_t get_available_size() const { return super.size; }
extent_len_t get_block_size() const { return super.block_size; }
+ read_ertr::future<unsigned int> get_shard_nums() final;
+
virtual read_ertr::future<> read(
uint64_t offset,
bufferptr &bptr) = 0;
namespace crimson::os::seastore::segment_manager::zbd {
+seastar::future<> ZBDSegmentManager::start(unsigned int shard_nums)
+{
+ LOG_PREFIX(ZBDSegmentManager::start);
+ device_shard_nums = shard_nums;
+ auto num_shard_services = (device_shard_nums + seastar::smp::count - 1 ) / seastar::smp::count;
+ INFO("device_shard_nums={} seastar::smp={}, num_shard_services={}", device_shard_nums, seastar::smp::count, num_shard_services);
+ return shard_devices.start(num_shard_services, device_path);
+
+}
+
+seastar::future<> ZBDSegmentManager::stop()
+{
+ return shard_devices.stop();
+}
+
+Device& ZBDSegmentManager::get_sharded_device(unsigned int store_index)
+{
+ assert(store_index < shard_devices.local().mshard_devices.size());
+ return *shard_devices.local().mshard_devices[store_index];
+}
+
using open_device_ret = ZBDSegmentManager::access_ertr::future<
std::pair<seastar::file, seastar::stat_data>>;
static open_device_ret open_device(
});
}
+ZBDSegmentManager::read_ertr::future<unsigned int> ZBDSegmentManager::get_shard_nums()
+{
+ return open_device(
+ device_path, seastar::open_flags::rw
+ ).safe_then([this](auto p) {
+ device = std::move(p.first);
+ auto sd = p.second;
+ return read_metadata(device, sd);
+ }).safe_then([this](auto meta){
+ return read_ertr::make_ready_future<int>(meta.shard_num);
+ }).handle_error(
+ crimson::ct_error::assert_all{
+ "Invalid error in ZBDSegmentManager::get_shard_nums"
+ });
+}
+
ZBDSegmentManager::mount_ret ZBDSegmentManager::mount()
{
return shard_devices.invoke_on_all([](auto &local_device) {
- return local_device.shard_mount(
- ).handle_error(
- crimson::ct_error::assert_all{
- "Invalid error in ZBDSegmentManager::mount"
+ return seastar::do_for_each(local_device.mshard_devices, [](auto& mshard_device) {
+ return mshard_device->shard_mount(
+ ).handle_error(
+ crimson::ct_error::assert_all{
+ "Invalid error in ZBDSegmentManager::mount"
+ });
});
});
}
auto sd = p.second;
return read_metadata(device, sd);
}).safe_then([=, this](auto meta){
- shard_info = meta.shard_infos[seastar::this_shard_id()];
+ if(seastar::this_shard_id() + seastar::smp::count * store_index >= meta.shard_num) {
+ INFO("{} shard_id {} out of range {}",
+ device_id_printer_t{get_device_id()},
+ seastar::this_shard_id() + seastar::smp::count * store_index,
+ sb.shard_num);
+ shard_status = false;
+ return mount_ertr::now();
+ }
+ shard_info = meta.shard_infos[seastar::this_shard_id() + seastar::smp::count * store_index];
metadata = meta;
return mount_ertr::now();
});
ZBDSegmentManager::mkfs_ret ZBDSegmentManager::mkfs(
device_config_t config)
{
- return shard_devices.local().primary_mkfs(config
+ return shard_devices.local().mshard_devices[0]->primary_mkfs(config
).safe_then([this] {
return shard_devices.invoke_on_all([](auto &local_device) {
- return local_device.shard_mkfs(
- ).handle_error(
- crimson::ct_error::assert_all{
- "Invalid error in ZBDSegmentManager::mkfs"
+ return seastar::do_for_each(local_device.mshard_devices, [](auto& mshard_device) {
+ return mshard_device->shard_mkfs(
+ ).handle_error(
+ crimson::ct_error::assert_all{
+ "Invalid error in ZBDSegmentManager::mkfs"
+ });
});
});
});
}
void validate() const {
- ceph_assert_always(shard_num == seastar::smp::count);
for (unsigned int i = 0; i < seastar::smp::count; i++) {
ceph_assert_always(shard_infos[i].size > 0);
ceph_assert_always(shard_infos[i].size <= DEVICE_OFF_MAX);
class ZBDSegmentManager final : public SegmentManager{
// interfaces used by Device
public:
- seastar::future<> start() {
- return shard_devices.start(device_path);
- }
+ seastar::future<> start(int shard_nums) final;
- seastar::future<> stop() {
- return shard_devices.stop();
- }
+ seastar::future<> stop() final;
- Device& get_sharded_device() final {
- return shard_devices.local();
- }
+ Device& get_sharded_device(unsigned int store_index = 0) final;
mount_ret mount() final;
mkfs_ret mkfs(device_config_t meta) final;
- ZBDSegmentManager(const std::string &path) : device_path(path) {}
+ ZBDSegmentManager(const std::string &path, unsigned int store_index = 0)
+ : device_path(path),
+ store_index(store_index) {}
~ZBDSegmentManager() final = default;
size_t len,
ceph::bufferptr &out) final;
+ read_ertr::future<unsigned int> get_shard_nums() final;
+
device_type_t get_device_type() const final {
return device_type_t::ZBD;
}
}
} stats;
- void register_metrics();
+ void register_metrics(unsigned int store_index);
seastar::metrics::metric_group metrics;
Segment::close_ertr::future<> segment_close(
mount_ret shard_mount();
- seastar::sharded<ZBDSegmentManager> shard_devices;
+ unsigned int device_shard_nums = 0;
+ unsigned int store_index = 0;
+ bool shard_status = true;
+ class MultiShardDevices {
+ public:
+ std::vector<std::unique_ptr<ZBDSegmentManager>> mshard_devices;
+
+ public:
+ MultiShardDevices(size_t count,
+ const std::string path)
+ : mshard_devices() {
+ mshard_devices.reserve(count);
+ for (size_t store_index = 0; store_index < count; ++store_index) {
+ mshard_devices.emplace_back(std::make_unique<ZBDSegmentManager>(
+ path, store_index));
+ }
+ }
+ ~MultiShardDevices() {
+ mshard_devices.clear();
+ }
+ };
+ seastar::sharded<MultiShardDevices> shard_devices;
};
}
run_async([this] {
device.reset(new random_block_device::nvme::NVMeBlockDevice(dev_path));
local_conf().set_val("seastore_cbjournal_size", "1048576").get();
- device->start().get();
+ device->start(seastar::smp::count).get();
device->mkfs(
device_config_t{
true,
projects / ceph-ci.git / commitdiff