librbd: Improved support for balanced and localized reads

Fixes: https://tracker.ceph.com/issues/43793
Signed-off-by: Zheng Yin <zhengyin@cmss.chinamobile.com>

diff --git a/doc/rbd/rbd-config-ref.rst b/doc/rbd/rbd-config-ref.rst
index a68f446e6eeacd242eb0776d20bc7c4b78b99d4f..6fb1676c7b524044f41145a0d92e50fff4e26deb 100644 (file)
--- a/doc/rbd/rbd-config-ref.rst
+++ b/doc/rbd/rbd-config-ref.rst
@@ -15,6 +15,14 @@ Generic IO Settings
 :Default: ``none``
 :Values: ``none``, ``compressible``, ``incompressible``
 
+``rbd read from replica policy``
+
+:Description: policy for determining which OSD will receive read operations. If set to `default`, the primary OSD will always be used for read operations. If set to `balance`, read operations will be sent to a randomly selected OSD within the replica set. If set to `localize`, read operations will be sent to the closest OSD as determined by the CRUSH map. Note: this feature requires the cluster to be configured with a minimum compatible OSD release of Octopus.
+:Type: Enum
+:Required: No
+:Default: ``default``
+:Values: ``default``, ``balance``, ``localize``
+
 Cache Settings
 =======================
 

diff --git a/src/common/options.cc b/src/common/options.cc
index ce0853df50a4f025acdc4eaa6daf843a7f877709..6bf178222c91a71de3f8edb87aeea6467a27cf00 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7180,6 +7180,12 @@ static std::vector<Option> get_rbd_options() {
     .set_description("Compression hint to send to the OSDs during writes")
     .set_flag(Option::FLAG_RUNTIME),
 
+    Option("rbd_read_from_replica_policy", Option::TYPE_STR, Option::LEVEL_BASIC)
+    .set_enum_allowed({"default", "balance", "localize"})
+    .set_default("default")
+    .set_description("Read replica policy send to the OSDS during reads")
+    .set_flag(Option::FLAG_RUNTIME),
+
     Option("rbd_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(false)
     .set_description("true if LTTng-UST tracepoints should be enabled"),

diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index 28b6c795d88937fad6c4aa575f12c57efd4d0f4b..65669db5bb8296c770ae02b3f6395d5cab078d43 100644 (file)
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -309,7 +309,11 @@ public:
   }
 
   int ImageCtx::get_read_flags(snap_t snap_id) {
-    int flags = librados::OPERATION_NOFLAG | extra_read_flags;
+    int flags = librados::OPERATION_NOFLAG | read_flags;
+    if (flags != 0)
+      return flags;
+
+    flags = librados::OPERATION_NOFLAG | extra_read_flags;
     if (snap_id == LIBRADOS_SNAP_HEAD)
       return flags;
 
@@ -795,6 +799,19 @@ public:
       alloc_hint_flags |= librados::ALLOC_HINT_FLAG_INCOMPRESSIBLE;
     }
 
+    librados::Rados rados(md_ctx);
+    int8_t require_osd_release;
+    int r = rados.get_min_compatible_osd(&require_osd_release);
+    if (r == 0 && require_osd_release >= CEPH_RELEASE_OCTOPUS) {
+      read_flags = 0;
+      auto read_policy = config.get_val<std::string>("rbd_read_from_replica_policy");
+      if (read_policy == "balance") {
+        read_flags |= CEPH_OSD_FLAG_BALANCE_READS;
+      } else if (read_policy == "localize") {
+        read_flags |= CEPH_OSD_FLAG_LOCALIZE_READS;
+      }
+    }
+
     io_work_queue->apply_qos_schedule_tick_min(
       config.get_val<uint64_t>("rbd_qos_schedule_tick_min"));
 

diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index e9132025d0fae0cdf79674919b7f56a719605042..5fe8cc3b4451aae0b060e38c49d1b44f546d8997 100644 (file)
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -207,6 +207,7 @@ namespace librbd {
     bool clone_copy_on_read;
     bool enable_alloc_hint;
     uint32_t alloc_hint_flags = 0U;
+    uint32_t read_flags = 0U;
     uint32_t discard_granularity_bytes = 0;
     bool blkin_trace_all;
     uint64_t mirroring_replay_delay;