blk: use choose_fd for all filehandle references

This is a leading part of the changes to implement bluestore on
FreeBSD. Without this invalid indexing in the descriptor arrays
will occur.

Creates a new enum: `blk_access_mode_t`
to describe `BUFFERED` and `DIRECT `mode access
with `choose_fd()` to get the correct file for the typed access
and adds
a pretty-printer
boolean convertor `blk_access_mode_t::buffermode(bool)`

This PR is a redo for PR #37258, since that one was lost
in rebasing errors. But the review notes are still there.

Signed-off-by: Willem Jan Withagen <wjw@digiware.nl>

diff --git a/src/blk/BlockDevice.cc b/src/blk/BlockDevice.cc
index d5d3169e9131e0f58b00d5a30c8a39b51d9a09e3..25fdc20f036e19e0e223f65911406cc76bee44a9 100644 (file)
--- a/src/blk/BlockDevice.cc
+++ b/src/blk/BlockDevice.cc
@@ -47,6 +47,20 @@
 
 using std::string;
 
+
+blk_access_mode_t buffermode(bool buffered) 
+{
+  return buffered ? blk_access_mode_t::BUFFERED : blk_access_mode_t::DIRECT;
+}
+
+std::ostream& operator<<(std::ostream& os, const blk_access_mode_t buffered) 
+{
+  os << (buffered == blk_access_mode_t::BUFFERED ? "(buffered)" : "(direct)");
+  return os;
+}
+
+
+
 void IOContext::aio_wait()
 {
   std::unique_lock l(lock);

diff --git a/src/blk/BlockDevice.h b/src/blk/BlockDevice.h
index 44ed3de4d8c610ab8219d300b063d86bf4acc054..feeb6c98ead602cbfa0b1fcea8cd523fb783a831 100644 (file)
--- a/src/blk/BlockDevice.h
+++ b/src/blk/BlockDevice.h
@@ -64,6 +64,12 @@
 #define  WRITE_LIFE_MAX        1
 #endif
 
+enum struct blk_access_mode_t {
+  DIRECT,
+  BUFFERED
+};
+blk_access_mode_t buffermode(bool buffered);
+std::ostream& operator<<(std::ostream& os, const blk_access_mode_t buffered);
 
 /// track in-flight io
 struct IOContext {

diff --git a/src/blk/kernel/KernelDevice.cc b/src/blk/kernel/KernelDevice.cc
index 049a99de77fdb3ac614278efa46ce8887ce2aeff..239d700e212be9ea7080b7dcda68817802f35b4a 100644 (file)
--- a/src/blk/kernel/KernelDevice.cc
+++ b/src/blk/kernel/KernelDevice.cc
@@ -414,7 +414,7 @@ bool KernelDevice::get_thin_utilization(uint64_t *total, uint64_t *avail) const
 
 int KernelDevice::choose_fd(bool buffered, int write_hint) const
 {
-  assert(write_hint >= WRITE_LIFE_NOT_SET && write_hint < WRITE_LIFE_MAX);
+assert(write_hint >= WRITE_LIFE_NOT_SET && write_hint < WRITE_LIFE_MAX);
   if (!enable_wrt)
     write_hint = WRITE_LIFE_NOT_SET;
   return buffered ? fd_buffereds[write_hint] : fd_directs[write_hint];
@@ -829,7 +829,7 @@ int KernelDevice::_sync_write(uint64_t off, bufferlist &bl, bool buffered, int w
 {
   uint64_t len = bl.length();
   dout(5) << __func__ << " 0x" << std::hex << off << "~" << len
-         << std::dec << (buffered ? " (buffered)" : " (direct)") << dendl;
+         << std::dec << " " << buffermode(buffered) << dendl;
   if (cct->_conf->bdev_inject_crash &&
       rand() % cct->_conf->bdev_inject_crash == 0) {
     derr << __func__ << " bdev_inject_crash: dropping io 0x" << std::hex
@@ -896,7 +896,7 @@ int KernelDevice::write(
 {
   uint64_t len = bl.length();
   dout(20) << __func__ << " 0x" << std::hex << off << "~" << len << std::dec
-          << (buffered ? " (buffered)" : " (direct)")
+          << " " << buffermode(buffered) 
           << dendl;
   ceph_assert(is_valid_io(off, len));
   if (cct->_conf->objectstore_blackhole) {
@@ -925,7 +925,7 @@ int KernelDevice::aio_write(
 {
   uint64_t len = bl.length();
   dout(20) << __func__ << " 0x" << std::hex << off << "~" << len << std::dec
-          << (buffered ? " (buffered)" : " (direct)")
+          << " " << buffermode(buffered)
           << dendl;
   ceph_assert(is_valid_io(off, len));
   if (cct->_conf->objectstore_blackhole) {
@@ -1032,7 +1032,7 @@ int KernelDevice::read(uint64_t off, uint64_t len, bufferlist *pbl,
                      bool buffered)
 {
   dout(5) << __func__ << " 0x" << std::hex << off << "~" << len << std::dec
-         << (buffered ? " (buffered)" : " (direct)")
+         << " " << buffermode(buffered)
          << dendl;
   ceph_assert(is_valid_io(off, len));
 
@@ -1041,13 +1041,13 @@ int KernelDevice::read(uint64_t off, uint64_t len, bufferlist *pbl,
   auto start1 = mono_clock::now();
 
   auto p = ceph::buffer::ptr_node::create(ceph::buffer::create_small_page_aligned(len));
-  int r = ::pread(buffered ? fd_buffereds[WRITE_LIFE_NOT_SET] : fd_directs[WRITE_LIFE_NOT_SET],
+  int r = ::pread(choose_fd(buffered,  WRITE_LIFE_NOT_SET),
                  p->c_str(), len, off);
   auto age = cct->_conf->bdev_debug_aio_log_age;
   if (mono_clock::now() - start1 >= make_timespan(age)) {
     derr << __func__ << " stalled read "
          << " 0x" << std::hex << off << "~" << len << std::dec
-         << (buffered ? " (buffered)" : " (direct)")
+         << " " << buffermode(buffered)
         << " since " << start1 << ", timeout is "
         << age
         << "s" << dendl;

diff --git a/src/blk/kernel/KernelDevice.h b/src/blk/kernel/KernelDevice.h
index 7ac9b1e7e1e3ab3a980c445982489dd9df919ff6..9999182d0f8b3eeaca5f95b480183760ae06f662 100644 (file)
--- a/src/blk/kernel/KernelDevice.h
+++ b/src/blk/kernel/KernelDevice.h
@@ -27,7 +27,6 @@
 
 #define RW_IO_MAX (INT_MAX & CEPH_PAGE_MASK)
 
-
 class KernelDevice : public BlockDevice {
   std::vector<int> fd_directs, fd_buffereds;
   bool enable_wrt = true;