From 448285dcbf72fb4a80ee30458f8d1e5089d052d5 Mon Sep 17 00:00:00 2001 From: Radoslaw Zarzynski Date: Thu, 7 Jul 2022 09:59:52 +0000 Subject: [PATCH] crimson/osd: add support for OSD Service Specification This patch is a crimson's counterpart of 1b44b6709d9400102c14ad3ea40fca19ab4ed214 that brought the support for `--osdspec-affinity` to the classical OSD. `cephadm` started passing this argv which caused the problem reported by Li, Jianxin . See: * https://gist.github.com/rzarzynski/4d1225971b6c28758cb2b68fbda3bf5f?permalink_comment_id=4223998#gistcomment-4223998 * https://docs.ceph.com/en/octopus/cephadm/drivegroups/ Testing ------- ```diff diff --git a/src/vstart.sh b/src/vstart.sh index 51ebcf26cdb..2a5b485e8d7 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -1014,7 +1014,7 @@ EOF echo "{\"cephx_secret\": \"$OSD_SECRET\"}" > $CEPH_DEV_DIR/osd$osd/new.json ceph_adm osd new $uuid -i $CEPH_DEV_DIR/osd$osd/new.json rm $CEPH_DEV_DIR/osd$osd/new.json - prun $SUDO $CEPH_BIN/$ceph_osd $extra_osd_args -i $osd $ARGS --mkfs --key $OSD_SECRET --osd-uuid $uuid $extra_seastar_args \ + prun $SUDO $CEPH_BIN/$ceph_osd $extra_osd_args -i $osd $ARGS --mkfs --osdspec-affinity None --key $OSD_SECRET --osd-uuid $uuid $extra_seastar_args \ 2>&1 | tee $CEPH_OUT_DIR/osd-mkfs.$osd.log local key_fn=$CEPH_DEV_DIR/osd$osd/keyring ``` The new meta is truly set: ``` $ bin/ceph-bluestore-tool --path dev/osd0/ show-label inferring bluefs devices from bluestore path { "dev/osd0/block": { "osd_uuid": "c0534f3a-3d2b-4080-91da-cefc76387e4b", "size": 107374182400, "btime": "2022-07-07T09:56:34.306383+0000", "description": "main", "bfm_blocks": "26214400", "bfm_blocks_per_key": "128", "bfm_bytes_per_block": "4096", "bfm_size": "107374182400", "bluefs": "1", "ceph_fsid": "c4b8836d-cbf6-434f-ac4c-4ed1f869ec31", "kv_backend": "rocksdb", "magic": "ceph osd volume v026", "mkfs_done": "yes", "osd_key": "AQDRrcZitoB+HxAAsJNAHtv9S2Kyrh0HfJOpzw==", "osdspec_affinity": "None", "ready": "ready", "require_osd_release": "17", "whoami": "0" }, "dev/osd0/block.wal": { "osd_uuid": "c0534f3a-3d2b-4080-91da-cefc76387e4b", "size": 1048576000, "btime": "2022-07-07T09:56:34.311501+0000", "description": "bluefs wal" }, "dev/osd0/block.db": { "osd_uuid": "c0534f3a-3d2b-4080-91da-cefc76387e4b", "size": 1073741824, "btime": "2022-07-07T09:56:34.308145+0000", "description": "bluefs db" } } ``` Signed-off-by: Radoslaw Zarzynski --- src/crimson/osd/main.cc | 5 ++++- src/crimson/osd/osd.cc | 7 ++++++- src/crimson/osd/osd.h | 4 +++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/crimson/osd/main.cc b/src/crimson/osd/main.cc index 98d3a2e9f2d..42b9829fd48 100644 --- a/src/crimson/osd/main.cc +++ b/src/crimson/osd/main.cc @@ -202,6 +202,8 @@ int main(int argc, const char* argv[]) ("mkfs", "create a [new] data directory") ("debug", "enable debug output on all loggers") ("trace", "enable trace output on all loggers") + ("osdspec-affinity", bpo::value()->default_value(std::string{}), + "set affinity to an osdspec") ("prometheus_port", bpo::value()->default_value(0), "Prometheus port. Set to zero to disable") ("prometheus_address", bpo::value()->default_value("0.0.0.0"), @@ -327,7 +329,8 @@ int main(int argc, const char* argv[]) 0, &crimson::osd::OSD::mkfs, osd_uuid, - local_conf().get_val("fsid")).get(); + local_conf().get_val("fsid"), + config["osdspec-affinity"].as()).get(); } if (config.count("mkkey") || config.count("mkfs")) { return EXIT_SUCCESS; diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index 3c06217ef3f..d4f462972b8 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -152,7 +152,10 @@ CompatSet get_osd_initial_compat_set() } } -seastar::future<> OSD::mkfs(uuid_d osd_uuid, uuid_d cluster_fsid) +seastar::future<> OSD::mkfs( + uuid_d osd_uuid, + uuid_d cluster_fsid, + std::string osdspec_affinity) { return store.start().then([this, osd_uuid] { return store.mkfs(osd_uuid).handle_error( @@ -184,6 +187,8 @@ seastar::future<> OSD::mkfs(uuid_d osd_uuid, uuid_d cluster_fsid) return store.write_meta("whoami", std::to_string(whoami)); }).then([this] { return _write_key_meta(); + }).then([this, osdspec_affinity=std::move(osdspec_affinity)] { + return store.write_meta("osdspec_affinity", osdspec_affinity); }).then([this] { return store.write_meta("ready", "ready"); }).then([cluster_fsid, this] { diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h index b1920352791..b71a6724d55 100644 --- a/src/crimson/osd/osd.h +++ b/src/crimson/osd/osd.h @@ -128,7 +128,9 @@ public: crimson::net::MessengerRef hb_back_msgr); ~OSD() final; - seastar::future<> mkfs(uuid_d osd_uuid, uuid_d cluster_fsid); + seastar::future<> mkfs(uuid_d osd_uuid, + uuid_d cluster_fsid, + std::string osdspec_affinity); seastar::future<> start(); seastar::future<> stop(); -- 2.39.5