From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Wed, 29 Sep 2021 20:00:20 +0000 (+0000)
Subject: crimson/osd: write the 'osd_key' meta on OSD::mkfs().
X-Git-Tag: v17.1.0~776^2~1
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=42a1462f30a9c90008336ff65ee1aab85c4aa6f3;p=ceph.git

crimson/osd: write the 'osd_key' meta on OSD::mkfs().

This commit fixes an issue identified during the Rook-crimson effort.

Missing the `write_meta()` on `osd_key` made the CephX inoperational
because of imposibility to load the keyring. Disabling CephX in turn
caused the auth method negotation to fail when reaching out to a monitor.

```
ERROR 2021-09-28 21:19:46,598 [shard 0] none - auth: unable to find a keyring on /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory
ERROR 2021-09-28 21:19:46,598 [shard 0] none - AuthRegistry(0x7fa38c322b68) no keyring found at /var/lib/ceph/osd/ceph-0/keyring, disabling cephx
...
INFO  2021-09-28 21:19:46,601 [shard 0] monc - get_auth_request(con=[client.?(temp_mon_client) 172.17.0.1:0/2910147961@63138 >> mon.? v2:10.108.187.31:3300/0], auth_method=0)
INFO  2021-09-28 21:19:46,601 [shard 0] monc - get_auth_request no methods is supported
...
WARN  2021-09-28 21:20:06,612 [shard 0] monc - cannot establish the active_con with any mon
```

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index ea5c46bf1f407..43b67e37e4abf 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -39,6 +39,7 @@
 
 #include "crimson/admin/osd_admin.h"
 #include "crimson/admin/pg_commands.h"
+#include "crimson/common/buffer_io.h"
 #include "crimson/common/exception.h"
 #include "crimson/mon/MonClient.h"
 #include "crimson/net/Connection.h"
@@ -170,7 +171,8 @@ seastar::future<> OSD::mkfs(uuid_d osd_uuid, uuid_d cluster_fsid)
   }).then([cluster_fsid, this] {
     return when_all_succeed(
       store.write_meta("ceph_fsid", cluster_fsid.to_string()),
-      store.write_meta("whoami", std::to_string(whoami)));
+      store.write_meta("whoami", std::to_string(whoami)),
+      _write_key_meta());
   }).then_unpack([cluster_fsid, this] {
     fmt::print("created object store {} for osd.{} fsid {}\n",
                local_conf().get_val<std::string>("osd_data"),
@@ -215,6 +217,33 @@ seastar::future<> OSD::_write_superblock()
   });
 }
 
+// this `to_string` sits in the `crimson::osd` namespace, so we don't brake
+// the language rule on not overloading in `std::`.
+static std::string to_string(const seastar::temporary_buffer<char>& temp_buf)
+{
+  return {temp_buf.get(), temp_buf.size()};
+}
+
+seastar::future<> OSD::_write_key_meta()
+{
+
+  if (auto key = local_conf().get_val<std::string>("key"); !std::empty(key)) {
+    return store.write_meta("osd_key", key);
+  } else if (auto keyfile = local_conf().get_val<std::string>("keyfile");
+             !std::empty(keyfile)) {
+    return read_file(keyfile).then([this] (const auto& temp_buf) {
+      // it's on a truly cold path, so don't worry about memcpy.
+      return store.write_meta("osd_key", to_string(temp_buf));
+    }).handle_exception([keyfile] (auto ep) {
+      logger().error("_write_key_meta: failed to handle keyfile {}: {}",
+                     keyfile, ep);
+      ceph_abort();
+    });
+  } else {
+    return seastar::now();
+  }
+}
+
 namespace {
   entity_addrvec_t pick_addresses(int what) {
     entity_addrvec_t addrs;
diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h
index 731ebeb59b638..f51c89c2be615 100644
--- a/src/crimson/osd/osd.h
+++ b/src/crimson/osd/osd.h
@@ -145,6 +145,7 @@ public:
 
 private:
   seastar::future<> _write_superblock();
+  seastar::future<> _write_key_meta();
   seastar::future<> start_boot();
   seastar::future<> _preboot(version_t oldest_osdmap, version_t newest_osdmap);
   seastar::future<> _send_boot();