From e0a940947885428dc53a8518fe0c335d60dea3ea Mon Sep 17 00:00:00 2001 From: Radoslaw Zarzynski Date: Sun, 27 Jun 2021 21:50:37 +0000 Subject: [PATCH] crimson/os: fix memory corruption in AlienStore::get_attrs(). `FuturizedStore` and `ObjectStore` use different memory layout for conveying object attributes: map of `bufferlists` and map of `bptrs` respectively. Unfortunately, `AlienStore` was trying to solve this mismatch with just a `reinterpret_cast`. Very likely this problem was the root cause behind the observed crashes in `PGBackend::load_matadata` like the following one: ``` 2021-06-15T09:25:07.511 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: DEBUG 2021-06-15 09:24:19,199 [shard 0] osd - peering_event(id=412, detail=PeeringEvent(from=7 pgid=5.14 sent=49 requested=49 evt=epoch_sent: 49 epoch_requested: 49 MInfoRec from 7 info: 5.14( v 45'2 (0'0,45'2] local-lis/les=48/49 n=0 ec=44/44 lis/c=48/44 les/c/f=49/45/0 sis=48) pg_lease_ack(ruub 19.176788330s))): complete 2021-06-15T09:25:07.511 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: Segmentation fault on shard 0. 2021-06-15T09:25:07.511 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: Backtrace: 2021-06-15T09:25:07.511 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 0# 0x000055C99757FFBF in /usr/bin/ceph-osd 2021-06-15T09:25:07.511 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 1# FatalSignal::signaled(int, siginfo_t const*) in /usr/bin/ceph-osd 2021-06-15T09:25:07.511 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 2# FatalSignal::install_oneshot_signal_handler<11>()::{lambda(int, siginfo_t*, void*)#1}::_FUN(int, siginfo_t*, void*) in /usr/bin/ceph-osd 2021-06-15T09:25:07.512 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 3# 0x00007F34BB632B20 in /lib64/libpthread.so.0 2021-06-15T09:25:07.512 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 4# 0x000055C99263D4D2 in /usr/bin/ceph-osd 2021-06-15T09:25:07.512 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 5# 0x000055C992740E47 in /usr/bin/ceph-osd 2021-06-15T09:25:07.512 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 6# seastar::continuation > >, seastar::noncopyable_function > >::_future > > > (seastar::future, std::allocator >, ceph::buffer::v15_2_0::list, std::less, std::allocator, std::allocator > const, ceph::buffer::v15_2_0::list> > > >&&)>, seastar::future, std::allocator >, ceph::buffer::v15_2_0::list, std::less, std::allocator, std::allocator > const, ceph::buffer::v15_2_0::list> > > >::then_wrapped_nrvo > >::_future > > >, seastar::noncopyable_function > >::_future > > > (seastar::future, std::allocator >, ceph::buffer::v15_2_0::list, std::less, std::allocator, std::allocator > const, ceph::buffer::v15_2_0::list> > > >&&)> >(seastar::noncopyable_function > >::_future > > > (seastar::future, std::allocator >, ceph::buffer::v15_2_0::list, std::less, std::allocator, std::allocator > const, ceph::buffer::v15_2_0::list> > > >&&)>&&)::{lambda(seastar::internal::promise_base_with_type > >&&, seastar::noncopyable_function > >::_future > > > (seastar::future, std::allocator >, ceph::buffer::v15_2_0::list, std::less, std::allocator, std::allocator > const, ceph::buffer::v15_2_0::list> > > >&&)>&, seastar::future_state, std::allocator >, ceph::buffer::v15_2_0::list, std::less, std::allocator, std::allocator > const, ceph::buffer::v15_2_0::list> > > >&&)#1}, std::map, std::allocator >, ceph::buffer::v15_2_0::list, std::less, std::allocator, std::allocator > const, ceph::buffer::v15_2_0::list> > > >::run_and_dispose() in /usr/bin/ceph-osd 2021-06-15T09:25:07.512 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 7# 0x000055C99CFD195F in /usr/bin/ceph-osd 2021-06-15T09:25:07.513 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 8# 0x000055C99CFD6EA0 in /usr/bin/ceph-osd 2021-06-15T09:25:07.513 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 9# 0x000055C99D188F0B in /usr/bin/ceph-osd 2021-06-15T09:25:07.513 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 10# 0x000055C99CCE698A in /usr/bin/ceph-osd 2021-06-15T09:25:07.513 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 11# 0x000055C99CCF0AAE in /usr/bin/ceph-osd 2021-06-15T09:25:07.513 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 12# main in /usr/bin/ceph-osd 2021-06-15T09:25:07.513 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 13# __libc_start_main in /lib64/libc.so.6 2021-06-15T09:25:07.514 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: 14# _start in /usr/bin/ceph-osd 2021-06-15T09:25:07.514 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:19 smithi100 conmon[54917]: Fault at location: 0x31dfff8000 2021-06-15T09:25:07.514 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jun 15 09:24:20 smithi100 podman[55356]: 2021-06-15 09:24:20.230341885 +0000 UTC m=+0.072958807 container died a3ea2a1d0a176286b93b8f5b94458982b9038e70d09128fb55f53b92976f0c42 (image=quay.ceph.io/ceph-ci/ceph@sha256:13ae953e3f83ee011d784d6eb9126fdc692f5bb688fe7d918be61ca7a7282b3c, name=ceph-43579b90-cdba-11eb-8c13-001a4aab830c-osd.3) ``` The fix deals with the issue by wrapping the `bptrs` in `bufferlists`. Signed-off-by: Radoslaw Zarzynski --- src/crimson/os/alienstore/alien_store.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc index 4f6eb4b8665cf..173463e75b396 100644 --- a/src/crimson/os/alienstore/alien_store.cc +++ b/src/crimson/os/alienstore/alien_store.cc @@ -335,8 +335,14 @@ AlienStore::get_attrs(CollectionRef ch, return seastar::do_with(attrs_t{}, [=] (auto &aset) { return tp->submit(ch->get_cid().hash_to_shard(tp->size()), [=, &aset] { auto c = static_cast(ch.get()); - return store->getattrs(c->collection, oid, - reinterpret_cast&>(aset)); + std::map blueaset; + const auto r = store->getattrs(c->collection, oid, blueaset); + for (auto& [bluekey, blueval] : blueaset) { + ceph::bufferlist bl; + bl.push_back(std::move(blueval)); + aset.emplace(std::move(bluekey), std::move(bl)); + } + return r; }).then([&aset] (int r) -> get_attrs_ertr::future { if (r == -ENOENT) { return crimson::ct_error::enoent::make(); -- 2.39.5