From: Willem Jan Withagen <wjw@digiware.nl>
Date: Sat, 26 Jun 2021 13:47:24 +0000 (+0200)
Subject: blk: use choose_fd for all filehandle references
X-Git-Tag: v17.1.0~1459^2~2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=9c13f8ad65e9436c81edeb5af7de208440930aea;p=ceph.git

blk: use choose_fd for all filehandle references

This is a leading part of the changes to implement bluestore on
FreeBSD. Without this invalid indexing in the descriptor arrays
will occur.

Creates a new enum: `blk_access_mode_t`
to describe `BUFFERED` and `DIRECT `mode access
with `choose_fd()` to get the correct file for the typed access
and adds
a pretty-printer
boolean convertor `blk_access_mode_t::buffermode(bool)`

This PR is a redo for PR #37258, since that one was lost
in rebasing errors. But the review notes are still there.

Signed-off-by: Willem Jan Withagen <wjw@digiware.nl>
---

diff --git a/src/blk/BlockDevice.cc b/src/blk/BlockDevice.cc
index d5d3169e9131..25fdc20f036e 100644
--- a/src/blk/BlockDevice.cc
+++ b/src/blk/BlockDevice.cc
@@ -47,6 +47,20 @@
 
 using std::string;
 
+
+blk_access_mode_t buffermode(bool buffered) 
+{
+  return buffered ? blk_access_mode_t::BUFFERED : blk_access_mode_t::DIRECT;
+}
+
+std::ostream& operator<<(std::ostream& os, const blk_access_mode_t buffered) 
+{
+  os << (buffered == blk_access_mode_t::BUFFERED ? "(buffered)" : "(direct)");
+  return os;
+}
+
+
+
 void IOContext::aio_wait()
 {
   std::unique_lock l(lock);
diff --git a/src/blk/BlockDevice.h b/src/blk/BlockDevice.h
index 44ed3de4d8c6..feeb6c98ead6 100644
--- a/src/blk/BlockDevice.h
+++ b/src/blk/BlockDevice.h
@@ -64,6 +64,12 @@
 #define  WRITE_LIFE_MAX  	1
 #endif
 
+enum struct blk_access_mode_t {
+  DIRECT,
+  BUFFERED
+};
+blk_access_mode_t buffermode(bool buffered);
+std::ostream& operator<<(std::ostream& os, const blk_access_mode_t buffered);
 
 /// track in-flight io
 struct IOContext {
diff --git a/src/blk/kernel/KernelDevice.cc b/src/blk/kernel/KernelDevice.cc
index 049a99de77fd..239d700e212b 100644
--- a/src/blk/kernel/KernelDevice.cc
+++ b/src/blk/kernel/KernelDevice.cc
@@ -414,7 +414,7 @@ bool KernelDevice::get_thin_utilization(uint64_t *total, uint64_t *avail) const
 
 int KernelDevice::choose_fd(bool buffered, int write_hint) const
 {
-  assert(write_hint >= WRITE_LIFE_NOT_SET && write_hint < WRITE_LIFE_MAX);
+assert(write_hint >= WRITE_LIFE_NOT_SET && write_hint < WRITE_LIFE_MAX);
   if (!enable_wrt)
     write_hint = WRITE_LIFE_NOT_SET;
   return buffered ? fd_buffereds[write_hint] : fd_directs[write_hint];
@@ -829,7 +829,7 @@ int KernelDevice::_sync_write(uint64_t off, bufferlist &bl, bool buffered, int w
 {
   uint64_t len = bl.length();
   dout(5) << __func__ << " 0x" << std::hex << off << "~" << len
-	  << std::dec << (buffered ? " (buffered)" : " (direct)") << dendl;
+	  << std::dec << " " << buffermode(buffered) << dendl;
   if (cct->_conf->bdev_inject_crash &&
       rand() % cct->_conf->bdev_inject_crash == 0) {
     derr << __func__ << " bdev_inject_crash: dropping io 0x" << std::hex
@@ -896,7 +896,7 @@ int KernelDevice::write(
 {
   uint64_t len = bl.length();
   dout(20) << __func__ << " 0x" << std::hex << off << "~" << len << std::dec
-	   << (buffered ? " (buffered)" : " (direct)")
+	   << " " << buffermode(buffered) 
 	   << dendl;
   ceph_assert(is_valid_io(off, len));
   if (cct->_conf->objectstore_blackhole) {
@@ -925,7 +925,7 @@ int KernelDevice::aio_write(
 {
   uint64_t len = bl.length();
   dout(20) << __func__ << " 0x" << std::hex << off << "~" << len << std::dec
-	   << (buffered ? " (buffered)" : " (direct)")
+	   << " " << buffermode(buffered)
 	   << dendl;
   ceph_assert(is_valid_io(off, len));
   if (cct->_conf->objectstore_blackhole) {
@@ -1032,7 +1032,7 @@ int KernelDevice::read(uint64_t off, uint64_t len, bufferlist *pbl,
 		      bool buffered)
 {
   dout(5) << __func__ << " 0x" << std::hex << off << "~" << len << std::dec
-	  << (buffered ? " (buffered)" : " (direct)")
+	  << " " << buffermode(buffered)
 	  << dendl;
   ceph_assert(is_valid_io(off, len));
 
@@ -1041,13 +1041,13 @@ int KernelDevice::read(uint64_t off, uint64_t len, bufferlist *pbl,
   auto start1 = mono_clock::now();
 
   auto p = ceph::buffer::ptr_node::create(ceph::buffer::create_small_page_aligned(len));
-  int r = ::pread(buffered ? fd_buffereds[WRITE_LIFE_NOT_SET] : fd_directs[WRITE_LIFE_NOT_SET],
+  int r = ::pread(choose_fd(buffered,  WRITE_LIFE_NOT_SET),
 		  p->c_str(), len, off);
   auto age = cct->_conf->bdev_debug_aio_log_age;
   if (mono_clock::now() - start1 >= make_timespan(age)) {
     derr << __func__ << " stalled read "
          << " 0x" << std::hex << off << "~" << len << std::dec
-         << (buffered ? " (buffered)" : " (direct)")
+         << " " << buffermode(buffered)
 	 << " since " << start1 << ", timeout is "
 	 << age
 	 << "s" << dendl;
diff --git a/src/blk/kernel/KernelDevice.h b/src/blk/kernel/KernelDevice.h
index 7ac9b1e7e1e3..9999182d0f8b 100644
--- a/src/blk/kernel/KernelDevice.h
+++ b/src/blk/kernel/KernelDevice.h
@@ -27,7 +27,6 @@
 
 #define RW_IO_MAX (INT_MAX & CEPH_PAGE_MASK)
 
-
 class KernelDevice : public BlockDevice {
   std::vector<int> fd_directs, fd_buffereds;
   bool enable_wrt = true;