From ab0007299cd7cb8dd56372d4828f4984a61ad724 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Thu, 9 May 2024 13:30:54 -0400 Subject: [PATCH] librados: disable local affinity for IoCtx::exec() and aio_exec() calls These calls are read calls but can and have been used as write calls, which doesn't work with local (and balanced) affinity. Signed-off-by: Yehuda Sadeh --- src/librados/IoCtxImpl.cc | 16 +++++++++------- src/librados/IoCtxImpl.h | 2 +- src/osdc/Objecter.h | 7 ++++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/librados/IoCtxImpl.cc b/src/librados/IoCtxImpl.cc index 92990cc4035..ebac8a100cf 100644 --- a/src/librados/IoCtxImpl.cc +++ b/src/librados/IoCtxImpl.cc @@ -682,7 +682,8 @@ int librados::IoCtxImpl::operate(const object_t& oid, ::ObjectOperation *o, int librados::IoCtxImpl::operate_read(const object_t& oid, ::ObjectOperation *o, bufferlist *pbl, - int flags) + int flags, + int flags_mask) { if (!o->size()) return 0; @@ -701,6 +702,7 @@ int librados::IoCtxImpl::operate_read(const object_t& oid, oid, oloc, *o, snap_seq, pbl, flags | extra_op_flags, + flags_mask, onack, &ver); objecter->op_submit(objecter_op); @@ -741,7 +743,7 @@ int librados::IoCtxImpl::aio_operate_read(const object_t &oid, trace.event("init root span"); Objecter::Op *objecter_op = objecter->prepare_read_op( oid, oloc, - *o, snap_seq, pbl, flags | extra_op_flags, + *o, snap_seq, pbl, flags | extra_op_flags, -1, oncomplete, &c->objver, nullptr, 0, &trace); objecter->op_submit(objecter_op, &c->tid); trace.event("rados operate read submitted"); @@ -883,7 +885,7 @@ int librados::IoCtxImpl::aio_sparse_read(const object_t oid, Objecter::Op *o = objecter->prepare_read_op( oid, oloc, - onack->m_ops, snapid, NULL, extra_op_flags, + onack->m_ops, snapid, NULL, extra_op_flags, -1, onack, &c->objver); objecter->op_submit(o, &c->tid); return 0; @@ -932,7 +934,7 @@ int librados::IoCtxImpl::aio_cmpext(const object_t& oid, onack->m_ops.cmpext(off, cmp_len, cmp_buf, NULL); Objecter::Op *o = objecter->prepare_read_op( - oid, oloc, onack->m_ops, snap_seq, NULL, extra_op_flags, onack, &c->objver); + oid, oloc, onack->m_ops, snap_seq, NULL, extra_op_flags, -1, onack, &c->objver); objecter->op_submit(o, &c->tid); return 0; } @@ -1311,7 +1313,7 @@ int librados::IoCtxImpl::exec(const object_t& oid, ::ObjectOperation rd; prepare_assert_ops(&rd); rd.call(cls, method, inbl); - return operate_read(oid, &rd, &outbl); + return operate_read(oid, &rd, &outbl, 0, ~(CEPH_OSD_FLAG_LOCALIZE_READS | CEPH_OSD_FLAG_BALANCE_READS)); } int librados::IoCtxImpl::aio_exec(const object_t& oid, AioCompletionImpl *c, @@ -1331,7 +1333,7 @@ int librados::IoCtxImpl::aio_exec(const object_t& oid, AioCompletionImpl *c, prepare_assert_ops(&rd); rd.call(cls, method, inbl); Objecter::Op *o = objecter->prepare_read_op( - oid, oloc, rd, snap_seq, outbl, extra_op_flags, oncomplete, &c->objver); + oid, oloc, rd, snap_seq, outbl, extra_op_flags, ~(CEPH_OSD_FLAG_LOCALIZE_READS | CEPH_OSD_FLAG_BALANCE_READS), oncomplete, &c->objver); objecter->op_submit(o, &c->tid); return 0; } @@ -1357,7 +1359,7 @@ int librados::IoCtxImpl::aio_exec(const object_t& oid, AioCompletionImpl *c, prepare_assert_ops(&rd); rd.call(cls, method, inbl); Objecter::Op *o = objecter->prepare_read_op( - oid, oloc, rd, snap_seq, &c->bl, extra_op_flags, oncomplete, &c->objver); + oid, oloc, rd, snap_seq, &c->bl, extra_op_flags, ~(CEPH_OSD_FLAG_LOCALIZE_READS | CEPH_OSD_FLAG_BALANCE_READS), oncomplete, &c->objver); objecter->op_submit(o, &c->tid); return 0; } diff --git a/src/librados/IoCtxImpl.h b/src/librados/IoCtxImpl.h index 23c402d7b5d..c0dece81e59 100644 --- a/src/librados/IoCtxImpl.h +++ b/src/librados/IoCtxImpl.h @@ -155,7 +155,7 @@ struct librados::IoCtxImpl { int rmxattr(const object_t& oid, const char *name); int operate(const object_t& oid, ::ObjectOperation *o, ceph::real_time *pmtime, int flags=0, const jspan_context *otel_trace = nullptr); - int operate_read(const object_t& oid, ::ObjectOperation *o, bufferlist *pbl, int flags=0); + int operate_read(const object_t& oid, ::ObjectOperation *o, bufferlist *pbl, int flags=0, int flags_mask=-1); int aio_operate(const object_t& oid, ::ObjectOperation *o, AioCompletionImpl *c, const SnapContext& snap_context, const ceph::real_time *pmtime, int flags, diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h index fcddcc55b86..800d1a54e53 100644 --- a/src/osdc/Objecter.h +++ b/src/osdc/Objecter.h @@ -3092,12 +3092,13 @@ public: Op *prepare_read_op( const object_t& oid, const object_locator_t& oloc, ObjectOperation& op, - snapid_t snapid, ceph::buffer::list *pbl, int flags, + snapid_t snapid, ceph::buffer::list *pbl, + int flags, int flags_mask, Context *onack, version_t *objver = NULL, int *data_offset = NULL, uint64_t features = 0, ZTracer::Trace *parent_trace = nullptr) { - Op *o = new Op(oid, oloc, std::move(op.ops), get_read_flags(flags), onack, objver, + Op *o = new Op(oid, oloc, std::move(op.ops), get_read_flags(flags) & flags_mask, onack, objver, data_offset, parent_trace); o->priority = op.priority; o->snapid = snapid; @@ -3118,7 +3119,7 @@ public: Context *onack, version_t *objver = NULL, int *data_offset = NULL, uint64_t features = 0) { - Op *o = prepare_read_op(oid, oloc, op, snapid, pbl, flags, onack, objver, + Op *o = prepare_read_op(oid, oloc, op, snapid, pbl, flags, -1, onack, objver, data_offset); if (features) o->features = features; -- 2.39.5