From 5d9ca40e9802abd7e944c4f4fa370a7cb2d70e0c Mon Sep 17 00:00:00 2001 From: "Ma, Jianpeng" Date: Tue, 24 Sep 2024 13:50:57 +0800 Subject: [PATCH] blk: Don't forget call io_uring_unregister_files. When using vstart.sh to deploy ceph cluster w/ bdev_ioring=true, met the following error messages: > /root/ceph/build/bin/ceph-osd -i 0 -c /root/ceph/build/ceph.conf --mkfs --key AQByU/JmoSTAExAApXM0Q6Et9xgfMJ3MJg7wSw== --osd-uuid e135fff2-a02c-415a-b1fe-c154a4e61c13 >2024-09-24T13:51:46.770+0800 7c71966245c0 -1 bdev(0x5c937f62f000 /root/ceph/build/dev/osd0/block) open open w/ O_EXCL > 2024-09-24T13:51:46.772+0800 7c71966245c0 -1 bluestore(/root/ceph/build/dev/osd0/block) _read_bdev_label unable to decode label /root/ceph/build/dev/osd0/block at offset 102: void bluestore_bdev_label_t::decode(ceph::buffer::v15_2_0::list::const_iterator&) decode past end of struct encoding: Malformed input [buffer:3] > 2024-09-24T13:51:46.772+0800 7c71966245c0 -1 bluestore(/root/ceph/build/dev/osd0/block) _read_multi_bdev_label label at 0x40000000 correct, but osd_uuid=b89e2e1d-dc77-45fb-9a0b-34eb0a35c9a4 need=e135fff2-a02c-415a-b1fe-c154a4e61c13 > 2024-09-24T13:51:46.772+0800 7c71966245c0 -1 bluestore(/root/ceph/build/dev/osd0/block) _read_multi_bdev_label label at 0x280000000 correct, but osd_uuid=b89e2e1d-dc77-45fb-9a0b-34eb0a35c9a4 need=e135fff2-a02c-415a-b1fe-c154a4e61c13 > 2024-09-24T13:51:46.772+0800 7c71966245c0 -1 bluestore(/root/ceph/build/dev/osd0/block) _read_multi_bdev_label label at 0x1900000000 correct, but osd_uuid=b89e2e1d-dc77-45fb-9a0b-34eb0a35c9a4 need=e135fff2-a02c-415a-b1fe-c154a4e61c13 > 2024-09-24T13:51:46.773+0800 7c71966245c0 -1 bluestore(/root/ceph/build/dev/osd0/block) _read_multi_bdev_label label at 0xfa00000000 correct, but osd_uuid=b89e2e1d-dc77-45fb-9a0b-34eb0a35c9a4 need=e135fff2-a02c-415a-b1fe-c154a4e61c13 > 2024-09-24T13:51:46.773+0800 7c71966245c0 -1 bluestore(/root/ceph/build/dev/osd0/block) No valid bdev label found > 2024-09-24T13:51:47.022+0800 7c71966245c0 -1 bluestore(/root/ceph/build/dev/osd0) _read_fsid unparsable uuid > 2024-09-24T13:51:47.022+0800 7c71966245c0 -1 bdev(0x5c937f62f000 /root/ceph/build/dev/osd0/block) open open w/ O_EXCL > 2024-09-24T13:51:47.022+0800 7c71966245c0 -1 bdev(0x5c937f62f000 /root/ceph/build/dev/osd0/block) open open got: (16) Device or resource busy0 This because we use io_uring_register_files when use ioruing. But we forget to unreister those files. Signed-off-by: Ma, Jianpeng --- src/blk/kernel/io_uring.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/blk/kernel/io_uring.cc b/src/blk/kernel/io_uring.cc index be63d63aaf264..02cf856b19f08 100644 --- a/src/blk/kernel/io_uring.cc +++ b/src/blk/kernel/io_uring.cc @@ -146,7 +146,7 @@ int ioring_queue_t::init(std::vector &fds) d->epoll_fd = epoll_create1(0); if (d->epoll_fd < 0) { ret = -errno; - goto close_ring_fd; + goto unregister_files; } struct epoll_event ev; @@ -161,6 +161,8 @@ int ioring_queue_t::init(std::vector &fds) close_epoll_fd: close(d->epoll_fd); +unregister_files: + io_uring_unregister_files(&d->io_uring); close_ring_fd: io_uring_queue_exit(&d->io_uring); @@ -172,6 +174,7 @@ void ioring_queue_t::shutdown() d->fixed_fds_map.clear(); close(d->epoll_fd); d->epoll_fd = -1; + io_uring_unregister_files(&d->io_uring); io_uring_queue_exit(&d->io_uring); } -- 2.39.5