$ MDS=0 MON=1 OSD=1 MGR=1 taskset -ac '0-95' /ceph/src/vstart.sh --new -x \
--localhost --without-dashboard --redirect-output --seastore --osd-args \
"--seastore_max_concurrent_transactions=128 --seastore_cachepin_type=LRU \
- --seastore_main_device_type=RANDOM_BLOCK_SSD" --seastore-devs /dev/nvme0n1 \
+ --seastore_hot_device_type=RANDOM_BLOCK_SSD" --seastore-devs /dev/nvme0n1 \
--crimson --crimson-smp 1 --no-restart
Another SeaStore example::
Device Types
------------
-Configured via the ``seastore_main_device_type`` option, the device types are
+Configured via the ``seastore_hot_device_type`` option, the device types are
separated into **Segmented** and **RBM** backend types as follows:
level: advanced
desc: maximum concurrent transactions that seastore allows (per reactor)
default: 128
-- name: seastore_main_device_type
+- name: seastore_hot_device_type
type: str
level: dev
- desc: The main device type seastore uses (SSD or RANDOM_BLOCK_SSD)
+ desc: The type of the devices in the hot (or only) tier, valid values are HDD, SSD, RANDOM_BLOCK_SSD and RANDOM_BLOCK_HDD
default: SSD
+- name: seastore_hot_backend_type
+ type: str
+ level: dev
+ desc: The back end used by the devices in the hot (or only) tier, valid values are SEGMENTED and RANDOM_BLOCK
+ default: SEGMENTED
+- name: seastore_cold_device_type
+ type: str
+ level: dev
+ desc: The cold device type SeaStore uses, valid values are RANDOM_BLOCK_SSD, RANDOM_BLOCK_HDD, SSD and HDD.
+ This is only effective when there are two tiers, a hot one and a cold one.
+ default: RANDOM_BLOCK_HDD
+- name: seastore_cold_backend_type
+ type: str
+ level: dev
+ desc: The backend used by cold devices (SEGMENTED or RANDOM_BLOCK)
+ default: RANDOM_BLOCK
- name: seastore_cbjournal_size
type: size
level: dev
- desc: Total size to use for CircularBoundedJournal if created, it is valid only if seastore_main_device_type is RANDOM_BLOCK
+ desc: Total size to use for CircularBoundedJournal if created, it is valid only if seastore_hot_device_type is RANDOM_BLOCK
default: 5_G
- name: seastore_multiple_tiers_stop_evict_ratio
type: float
}
ceph_assert(block_size > 0);
ceph_assert(config.spec.magic != 0);
+ // allow HDD devices use segmented backend
+ ceph_assert(config.spec.btype != backend_type_t::NONE);
ceph_assert(config.spec.id <= DEVICE_ID_MAX_VALID);
if (!config.major_dev) {
ceph_assert(config.secondary_devices.empty());
}
seastar::future<DeviceRef>
-Device::make_device(const std::string& device, device_type_t dtype)
+Device::make_device(
+ const std::string& device,
+ device_type_t dtype,
+ backend_type_t btype)
{
- if (get_default_backend_of_device(dtype) == backend_type_t::SEGMENTED) {
+ if (btype == backend_type_t::SEGMENTED) {
return SegmentManager::get_segment_manager(device, dtype
).then([](DeviceRef ret) {
return ret;
});
- }
- assert(get_default_backend_of_device(dtype) == backend_type_t::RANDOM_BLOCK);
- return get_rb_device(device
- ).then([](DeviceRef ret) {
- return ret;
- });
+ } else {
+ ceph_assert(btype != backend_type_t::NONE);
+ return get_rb_device(device
+ ).then([](DeviceRef ret) {
+ return ret;
+ });
+ }
}
check_create_device_ret check_create_device(
struct device_spec_t {
magic_t magic = 0;
device_type_t dtype = device_type_t::NONE;
+ backend_type_t btype = backend_type_t::NONE;
device_id_t id = DEVICE_ID_NULL;
DENC(device_spec_t, v, p) {
DENC_START(1, 1, p);
denc(v.magic, p);
denc(v.dtype, p);
+ denc(v.btype, p);
denc(v.id, p);
DENC_FINISH(p);
}
uuid_d new_osd_fsid,
device_id_t id,
device_type_t d_type,
+ backend_type_t b_type,
secondary_device_set_t sds) {
return device_config_t{
true,
device_spec_t{
(magic_t)std::rand(),
d_type,
+ b_type,
id},
seastore_meta_t{new_osd_fsid},
sds};
uuid_d new_osd_fsid,
device_id_t id,
device_type_t d_type,
+ backend_type_t b_type,
magic_t magic) {
return device_config_t{
false,
device_spec_t{
magic,
d_type,
+ b_type,
id},
seastore_meta_t{new_osd_fsid},
secondary_device_set_t()};
static seastar::future<DeviceRef> make_device(
const std::string &device,
- device_type_t dtype);
+ device_type_t dtype,
+ backend_type_t btype);
// interfaces used by each device shard
public:
std::move(cold_cleaner),
hot_tier_generations,
pinboard);
+ ceph_assert(get_main_backend_type() != backend_type_t::NONE);
if (cold_segment_cleaner) {
ceph_assert(get_main_backend_type() == backend_type_t::SEGMENTED);
ceph_assert(background_process.has_cold_tier());
++secondary_index) {
device_id_t secondary_id = static_cast<device_id_t>(secondary_index);
secondary_devices.insert({
- secondary_index, device_spec_t{magic, type, secondary_id}
+ secondary_index, device_spec_t{magic, type, backend_type_t::RANDOM_BLOCK, secondary_id}
});
}
} else { // index > 0
device_id_t id = static_cast<device_id_t>(DEVICE_ID_RANDOM_BLOCK_MIN + index);
seastore_meta_t meta = {};
return {is_major_device,
- device_spec_t{magic, type, id},
+ device_spec_t{magic, type, backend_type_t::RANDOM_BLOCK, id},
meta,
secondary_devices};
}
bool is_test = false;
#endif
using crimson::common::get_conf;
- std::string type = get_conf<std::string>("seastore_main_device_type");
+ std::string type = get_conf<std::string>("seastore_hot_device_type");
device_type_t d_type = string_to_device_type(type);
assert(d_type == device_type_t::SSD ||
d_type == device_type_t::RANDOM_BLOCK_SSD);
+ type = get_conf<std::string>("seastore_hot_backend_type");
+ auto b_type = string_to_backend_type(type);
+ INFO("main device type: {}, main backend type: {}", d_type, b_type);
ceph_assert(root != "");
- DeviceRef device_obj = co_await Device::make_device(root, d_type);
+ DeviceRef device_obj = co_await Device::make_device(root, d_type, b_type);
device = std::move(device_obj);
co_await get_shard_nums();
co_await device->start(store_shard_nums);
device_id_t id = device_entry.first;
[[maybe_unused]] magic_t magic = device_entry.second.magic;
device_type_t dtype = device_entry.second.dtype;
- std::string path = fmt::format("{}/block.{}.{}", root, dtype, std::to_string(id));
- DeviceRef sec_dev = co_await Device::make_device(path, dtype);
+ backend_type_t btype = device_entry.second.btype;
+ auto btype_conf_str = get_conf<std::string>("seastore_cold_backend_type");
+ ceph_assert(string_to_backend_type(btype_conf_str) == btype);
+ std::string path = fmt::format("{}/block.{}", root, std::to_string(id));
+ DeviceRef sec_dev = co_await Device::make_device(path, dtype, btype);
co_await sec_dev->start(store_shard_nums);
co_await sec_dev->mount();
auto sec_block_size = sec_dev->get_sharded_device(0).get_block_size();
co_await write_meta("mkfs_done", "yes");
}
+std::optional<device_id_t> parse_device_id(const seastar::sstring &name) {
+ auto prefix_len = sizeof("block.") - 1;
+ if (name.starts_with("block.") && name.length() > prefix_len) {
+ int id = 0;
+ std::string id_str = name.substr(prefix_len);
+ std::istringstream iss(id_str);
+ iss >> id;
+ assert(id < std::numeric_limits<uint8_t>::max());
+ return std::make_optional<device_id_t>(id);
+ }
+ return std::nullopt;
+}
+
Device::access_ertr::future<> SeaStore::_mkfs(uuid_d new_osd_fsid)
{
LOG_PREFIX(SeaStore::_mkfs);
co_return;
}
DEBUG("mkfs_done does not exist, starting mkfs");
+ auto dtype_str = get_conf<std::string>("seastore_cold_device_type");
+ auto dtype = string_to_device_type(dtype_str);
+ auto btype_str = get_conf<std::string>("seastore_cold_backend_type");
+ auto btype = string_to_backend_type(btype_str);
+ ceph_assert(!root.empty());
+ INFO("secondary device type: {}, secondary backend type: {}", dtype, btype);
secondary_device_set_t sds;
if (!root.empty()) {
seastar::file rdir = co_await seastar::open_directory(root);
while (auto de = co_await lister()) {
auto& entry = *de;
DEBUG("found file: {}", entry.name);
- if (entry.name.find("block.") == 0 && entry.name.length() > 6 ) {
- // 6 for "block."
- std::string entry_name = entry.name;
- auto dtype_end = entry_name.find_first_of('.', 6);
- device_type_t dtype =
- string_to_device_type(
- entry_name.substr(6, dtype_end - 6));
- if (dtype == device_type_t::NONE) {
- // invalid device type
- co_return;
- }
- auto id = std::stoi(entry_name.substr(dtype_end + 1));
- std::string path = fmt::format("{}/{}", root, entry_name);
- DeviceRef sec_dev = co_await Device::make_device(path, dtype);
- auto p_sec_dev = sec_dev.get();
- secondaries.emplace_back(std::move(sec_dev));
- co_await p_sec_dev->start(store_shard_nums);
- magic_t magic = (magic_t)std::rand();
- sds.emplace((device_id_t)id, device_spec_t{magic, dtype, (device_id_t)id});
- co_await p_sec_dev->mkfs(
- device_config_t::create_secondary(new_osd_fsid, id, dtype, magic)
- ).handle_error(crimson::ct_error::assert_all("not possible"));
- co_await set_secondaries();
+ auto p = parse_device_id(entry.name);
+ if (!p) {
+ continue;
}
+ std::string path = fmt::format("{}/{}", root, entry.name);
+ DeviceRef sec_dev = co_await Device::make_device(path, dtype, btype);
+ auto p_sec_dev = sec_dev.get();
+ secondaries.emplace_back(std::move(sec_dev));
+ co_await p_sec_dev->start(store_shard_nums);
+ magic_t magic = (magic_t)std::rand();
+ auto id = *p;
+ sds.emplace((device_id_t)id,
+ device_spec_t{magic, dtype, btype, (device_id_t)id});
+ co_await p_sec_dev->mkfs(
+ device_config_t::create_secondary(new_osd_fsid, id, dtype, btype, magic)
+ ).handle_error(crimson::ct_error::assert_all("not possible"));
+ co_await set_secondaries();
}
co_await rdir.close();
}
device_id_t id = 0;
device_type_t d_type = device->get_device_type();
+ backend_type_t b_type = device->get_backend_type();
assert(d_type == device_type_t::SSD ||
d_type == device_type_t::RANDOM_BLOCK_SSD);
+ assert(b_type != backend_type_t::NONE);
if (d_type == device_type_t::RANDOM_BLOCK_SSD) {
id = static_cast<device_id_t>(DEVICE_ID_RANDOM_BLOCK_MIN);
}
DEBUG("creating primary device");
- co_await device->mkfs(device_config_t::create_primary(new_osd_fsid, id, d_type, sds));
+ co_await device->mkfs(
+ device_config_t::create_primary(
+ new_osd_fsid, id, d_type, b_type, sds));
DEBUG("mounting {} secondaries", secondaries.size());
for (auto& sec_dev : secondaries) {
co_await sec_dev->mount();
seastar::future<std::string> SeaStore::get_default_device_class()
{
using crimson::common::get_conf;
- std::string type = get_conf<std::string>("seastore_main_device_type");
+ std::string type = get_conf<std::string>("seastore_hot_device_type");
return seastar::make_ready_future<std::string>(type);
}
}
}
+backend_type_t string_to_backend_type(const std::string &str) {
+ if (str == "SEGMENTED") {
+ return backend_type_t::SEGMENTED;
+ } else if (str == "RANDOM_BLOCK") {
+ return backend_type_t::RANDOM_BLOCK;
+ } else {
+ ceph_abort("backend str not valid");
+ return backend_type_t::SEGMENTED;
+ }
+}
+
std::ostream& operator<<(std::ostream& out, backend_type_t btype) {
- if (btype == backend_type_t::SEGMENTED) {
+ switch (btype) {
+ case backend_type_t::NONE:
+ return out << "NONE";
+ case backend_type_t::SEGMENTED:
return out << "SEGMENTED";
- } else {
+ case backend_type_t::RANDOM_BLOCK:
return out << "RANDOM_BLOCK";
}
}
bool can_delay_allocation(device_type_t type);
device_type_t string_to_device_type(std::string type);
-enum class backend_type_t {
+enum class backend_type_t : uint8_t {
+ NONE,
SEGMENTED, // SegmentManager: SSD, ZBD, HDD
RANDOM_BLOCK // RBMDevice: RANDOM_BLOCK_SSD
};
constexpr backend_type_t get_default_backend_of_device(device_type_t dtype) {
assert(dtype != device_type_t::NONE &&
- dtype != device_type_t::NUM_TYPES);
+ dtype != device_type_t::NUM_TYPES);
if (dtype >= device_type_t::HDD &&
dtype <= device_type_t::EPHEMERAL_MAIN) {
return backend_type_t::SEGMENTED;
}
}
+backend_type_t string_to_backend_type(const std::string &str);
+
/**
* Monotonically increasing identifier for the location of a
* journal_record.
device_spec_t{
magic,
get_sec_dtype(secondary_index),
+ backend_type_t::SEGMENTED,
secondary_id
}
});
device_spec_t{
magic,
get_sec_dtype(index),
+ backend_type_t::SEGMENTED,
id
},
meta,
{
assert(config.path);
logger().debug("mkfs");
- return Device::make_device(*config.path, device_type_t::SSD
+ return Device::make_device(
+ *config.path,
+ device_type_t::SSD,
+ backend_type_t::SEGMENTED
).then([this](DeviceRef dev) {
device = std::move(dev);
seastore_meta_t meta;
return device->mkfs(
device_config_t{
true,
- (magic_t)std::rand(),
- device_type_t::SSD,
- 0,
+ device_spec_t{
+ (magic_t)std::rand(),
+ device_type_t::SSD,
+ backend_type_t::SEGMENTED,
+ 0
+ },
meta,
secondary_device_set_t()});
}).safe_then([this] {
{
return (config.mkfs ? mkfs() : seastar::now()
).then([this] {
- return Device::make_device(*config.path, device_type_t::SSD);
+ return Device::make_device(
+ *config.path,
+ device_type_t::SSD,
+ backend_type_t::SEGMENTED);
}).then([this](DeviceRef dev) {
device = std::move(dev);
return device->mount();
device_spec_t{
(magic_t)std::rand(),
device_type_t::RANDOM_BLOCK_SSD,
+ backend_type_t::RANDOM_BLOCK,
static_cast<device_id_t>(DEVICE_ID_RANDOM_BLOCK_MIN)},
seastore_meta_t{uuid_d()},
secondary_device_set_t()}
declare -a bluestore_wal_devs
declare -a secondary_block_devs
declare -a cpu_table
-secondary_block_devs_type="SSD"
+seastore_hot_device_type="SSD"
+seastore_hot_backend_type="SEGMENTED"
+seastore_cold_device_type="SSD"
+seastore_cold_backend_type="SEGMENTED"
VSTART_SEC="client.vstart.sh"
--seastore-device-size: set total size of seastore
--seastore-devs: comma-separated list of blockdevs to use for seastore
--seastore-secondary-devs: comma-separated list of secondary blockdevs to use for seastore
- --seastore-secondary-devs-type: device type of all secondary blockdevs. HDD, SSD(default), ZNS or RANDOM_BLOCK_SSD
+ --seastore-main-device-type: device type of main blockdevs. (SSD or RANDOM_BLOCK_SSD)
+ --seastore-main-backend-type: the driver used by main blockdevs (SEGMENTED or RANDOM_BLOCK)
+ --seastore-secondary-device-type: device type of all secondary blockdevs. HDD, SSD(default), ZNS or RANDOM_BLOCK_SSD
+ --seastore-secondary-backend-type: the driver used by secondary blockdevs (SEGMENTED or RANDOM_BLOCK)
--crimson-smp: number of cores to use for crimson
--crimson-alien-num-threads: number of alien-tp threads
--crimson-reactor-physical-only: use only one cpu per physical core for seastar reactors
parse_block_devs --seastore-devs "$2"
shift
;;
+ --seastore-main-device-type)
+ seastore_hot_device_type="$2"
+ shift
+ ;;
+ --seastore-main-backend-type)
+ seastore_hot_backend_type="$2"
+ shift
+ ;;
--seastore-secondary-devs)
parse_secondary_devs --seastore-devs "$2"
shift
;;
- --seastore-secondary-devs-type)
- secondary_block_devs_type="$2"
+ --seastore-secondary-device-type)
+ seastore_cold_device_type="$2"
+ shift
+ ;;
+ --seastore-secondary-backend-type)
+ seastore_cold_backend_type="$2"
shift
;;
--crimson-smp)
SEASTORE_OPTS="
seastore device size = $seastore_size"
fi
+ SEASTORE_OPTS+="
+ seastore_hot_device_type=$seastore_hot_device_type
+ seastore_hot_backend_type=$seastore_hot_backend_type
+ seastore_cold_device_type=$seastore_cold_device_type
+ seastore_cold_backend_type=$seastore_cold_backend_type"
fi
wconf <<EOF
fi
if [ -n "${secondary_block_devs[$osd]}" ]; then
dd if=/dev/zero of=${secondary_block_devs[$osd]} bs=1M count=1
- mkdir -p $CEPH_DEV_DIR/osd$osd/block.${secondary_block_devs_type}.1
- ln -s ${secondary_block_devs[$osd]} $CEPH_DEV_DIR/osd$osd/block.${secondary_block_devs_type}.1/block
+ mkdir -p $CEPH_DEV_DIR/osd$osd/block.1
+ ln -s ${secondary_block_devs[$osd]} $CEPH_DEV_DIR/osd$osd/block.1/block
fi
if [ "$objectstore" == "bluestore" ]; then
wconf <<EOF
projects / ceph.git / commitdiff