From: zhengyin <zhengyin@cmss.chinamoblie.com>
Date: Tue, 25 Feb 2020 04:00:11 +0000 (+0800)
Subject: librbd: Improved support for balanced and localized reads
X-Git-Tag: v15.1.1~216^2~1
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=d92be8b5856d7724cb8aceea292183246444969a;p=ceph.git

librbd: Improved support for balanced and localized reads

Fixes: https://tracker.ceph.com/issues/43793
Signed-off-by: Zheng Yin <zhengyin@cmss.chinamobile.com>
---

diff --git a/doc/rbd/rbd-config-ref.rst b/doc/rbd/rbd-config-ref.rst
index a68f446e6eeac..6fb1676c7b524 100644
--- a/doc/rbd/rbd-config-ref.rst
+++ b/doc/rbd/rbd-config-ref.rst
@@ -15,6 +15,14 @@ Generic IO Settings
 :Default: ``none``
 :Values: ``none``, ``compressible``, ``incompressible``
 
+``rbd read from replica policy``
+
+:Description: policy for determining which OSD will receive read operations. If set to `default`, the primary OSD will always be used for read operations. If set to `balance`, read operations will be sent to a randomly selected OSD within the replica set. If set to `localize`, read operations will be sent to the closest OSD as determined by the CRUSH map. Note: this feature requires the cluster to be configured with a minimum compatible OSD release of Octopus.
+:Type: Enum
+:Required: No
+:Default: ``default``
+:Values: ``default``, ``balance``, ``localize``
+
 Cache Settings
 =======================
 
diff --git a/src/common/options.cc b/src/common/options.cc
index ce0853df50a4f..6bf178222c91a 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7180,6 +7180,12 @@ static std::vector<Option> get_rbd_options() {
     .set_description("Compression hint to send to the OSDs during writes")
     .set_flag(Option::FLAG_RUNTIME),
 
+    Option("rbd_read_from_replica_policy", Option::TYPE_STR, Option::LEVEL_BASIC)
+    .set_enum_allowed({"default", "balance", "localize"})
+    .set_default("default")
+    .set_description("Read replica policy send to the OSDS during reads")
+    .set_flag(Option::FLAG_RUNTIME),
+
     Option("rbd_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(false)
     .set_description("true if LTTng-UST tracepoints should be enabled"),
diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index 28b6c795d8893..65669db5bb829 100644
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -309,7 +309,11 @@ public:
   }
 
   int ImageCtx::get_read_flags(snap_t snap_id) {
-    int flags = librados::OPERATION_NOFLAG | extra_read_flags;
+    int flags = librados::OPERATION_NOFLAG | read_flags;
+    if (flags != 0)
+      return flags;
+
+    flags = librados::OPERATION_NOFLAG | extra_read_flags;
     if (snap_id == LIBRADOS_SNAP_HEAD)
       return flags;
 
@@ -795,6 +799,19 @@ public:
       alloc_hint_flags |= librados::ALLOC_HINT_FLAG_INCOMPRESSIBLE;
     }
 
+    librados::Rados rados(md_ctx);
+    int8_t require_osd_release;
+    int r = rados.get_min_compatible_osd(&require_osd_release);
+    if (r == 0 && require_osd_release >= CEPH_RELEASE_OCTOPUS) {
+      read_flags = 0;
+      auto read_policy = config.get_val<std::string>("rbd_read_from_replica_policy");
+      if (read_policy == "balance") {
+        read_flags |= CEPH_OSD_FLAG_BALANCE_READS;
+      } else if (read_policy == "localize") {
+        read_flags |= CEPH_OSD_FLAG_LOCALIZE_READS;
+      }
+    }
+
     io_work_queue->apply_qos_schedule_tick_min(
       config.get_val<uint64_t>("rbd_qos_schedule_tick_min"));
 
diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index e9132025d0fae..5fe8cc3b4451a 100644
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -207,6 +207,7 @@ namespace librbd {
     bool clone_copy_on_read;
     bool enable_alloc_hint;
     uint32_t alloc_hint_flags = 0U;
+    uint32_t read_flags = 0U;
     uint32_t discard_granularity_bytes = 0;
     bool blkin_trace_all;
     uint64_t mirroring_replay_delay;