From 24087ec41fb349bbab17a815cee5af7de19c0260 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Wed, 3 Jul 2024 13:26:03 -0400 Subject: [PATCH] librados: local affinity for exec calls is configurable Signed-off-by: Yehuda Sadeh --- src/common/options/global.yaml.in | 12 ++++++++++++ src/librados/IoCtxImpl.cc | 9 ++++++--- src/librados/IoCtxImpl.h | 1 + 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in index c4a63e31d09..cc6568d3909 100644 --- a/src/common/options/global.yaml.in +++ b/src/common/options/global.yaml.in @@ -6395,6 +6395,18 @@ options: - localize flags: - runtime +- name: rados_replica_read_policy_on_objclass + type: bool + level: advanced + desc: enable read policy for sending read requests to OSD on objclass ops + fmt_desc : | + This would enable objclass ops to leverage read policy that can + determine which OSD will receive read operation. The reason + we might want to disable this is because objclass operations may + not be flagged correctly as read or write ops and we don't want + write ops to be sent to the wrong OSD (and system won't function + correctly). + default: false # true if LTTng-UST tracepoints should be enabled - name: rados_tracing type: bool diff --git a/src/librados/IoCtxImpl.cc b/src/librados/IoCtxImpl.cc index ebac8a100cf..f49d1c8ec2b 100644 --- a/src/librados/IoCtxImpl.cc +++ b/src/librados/IoCtxImpl.cc @@ -241,6 +241,9 @@ librados::IoCtxImpl::IoCtxImpl(RadosClient *c, Objecter *objecter, oloc(poolid), aio_write_seq(0), objecter(objecter) { + if (!c->cct->_conf.get_val("rados_replica_read_policy_on_objclass")) { + objclass_flags_mask = ~(CEPH_OSD_FLAG_LOCALIZE_READS | CEPH_OSD_FLAG_BALANCE_READS); + } } void librados::IoCtxImpl::set_snap_read(snapid_t s) @@ -1313,7 +1316,7 @@ int librados::IoCtxImpl::exec(const object_t& oid, ::ObjectOperation rd; prepare_assert_ops(&rd); rd.call(cls, method, inbl); - return operate_read(oid, &rd, &outbl, 0, ~(CEPH_OSD_FLAG_LOCALIZE_READS | CEPH_OSD_FLAG_BALANCE_READS)); + return operate_read(oid, &rd, &outbl, 0, objclass_flags_mask); } int librados::IoCtxImpl::aio_exec(const object_t& oid, AioCompletionImpl *c, @@ -1333,7 +1336,7 @@ int librados::IoCtxImpl::aio_exec(const object_t& oid, AioCompletionImpl *c, prepare_assert_ops(&rd); rd.call(cls, method, inbl); Objecter::Op *o = objecter->prepare_read_op( - oid, oloc, rd, snap_seq, outbl, extra_op_flags, ~(CEPH_OSD_FLAG_LOCALIZE_READS | CEPH_OSD_FLAG_BALANCE_READS), oncomplete, &c->objver); + oid, oloc, rd, snap_seq, outbl, extra_op_flags, objclass_flags_mask, oncomplete, &c->objver); objecter->op_submit(o, &c->tid); return 0; } @@ -1359,7 +1362,7 @@ int librados::IoCtxImpl::aio_exec(const object_t& oid, AioCompletionImpl *c, prepare_assert_ops(&rd); rd.call(cls, method, inbl); Objecter::Op *o = objecter->prepare_read_op( - oid, oloc, rd, snap_seq, &c->bl, extra_op_flags, ~(CEPH_OSD_FLAG_LOCALIZE_READS | CEPH_OSD_FLAG_BALANCE_READS), oncomplete, &c->objver); + oid, oloc, rd, snap_seq, &c->bl, extra_op_flags, objclass_flags_mask, oncomplete, &c->objver); objecter->op_submit(o, &c->tid); return 0; } diff --git a/src/librados/IoCtxImpl.h b/src/librados/IoCtxImpl.h index c0dece81e59..6d8c1022544 100644 --- a/src/librados/IoCtxImpl.h +++ b/src/librados/IoCtxImpl.h @@ -41,6 +41,7 @@ struct librados::IoCtxImpl { uint32_t notify_timeout = 30; object_locator_t oloc; int extra_op_flags = 0; + int objclass_flags_mask = -1; ceph::mutex aio_write_list_lock = ceph::make_mutex("librados::IoCtxImpl::aio_write_list_lock"); -- 2.39.5