From 23fd140d6afcff4a1c7536dc3ff2104ed0cb8b68 Mon Sep 17 00:00:00 2001 From: Haomai Wang Date: Sat, 9 Jan 2016 22:12:46 +0800 Subject: [PATCH] NVMEDevice: impl read_buffered method Signed-off-by: Haomai Wang --- src/common/Makefile.am | 3 +- src/common/align.h | 30 +++++++++++++++++++ src/os/bluestore/BlockDevice.h | 1 + src/os/bluestore/NVMEDevice.cc | 54 +++++++++++++++++++++++++++++++++- src/os/bluestore/NVMEDevice.h | 1 + 5 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 src/common/align.h diff --git a/src/common/Makefile.am b/src/common/Makefile.am index c94232210e2..3401bcb87f5 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -269,7 +269,8 @@ noinst_HEADERS += \ common/event_socket.h \ common/PluginRegistry.h \ common/ceph_time.h \ - common/ceph_timer.h + common/ceph_timer.h \ + common/align.h if ENABLE_XIO noinst_HEADERS += \ diff --git a/src/common/align.h b/src/common/align.h new file mode 100644 index 00000000000..b5c25b9904a --- /dev/null +++ b/src/common/align.h @@ -0,0 +1,30 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 XSky + * + * Author: Haomai Wang + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_COMMON_ALIGN_H +#define CEPH_COMMON_ALIGN_H + +template +inline constexpr T align_up(T v, T align) { + return (v + align - 1) & ~(align - 1); +} + +template +inline constexpr T align_down(T v, T align) { + return v & ~(align - 1); +} + +#endif /* CEPH_COMMON_ALIGN_H */ diff --git a/src/os/bluestore/BlockDevice.h b/src/os/bluestore/BlockDevice.h index a5057260c57..123c8461d7c 100644 --- a/src/os/bluestore/BlockDevice.h +++ b/src/os/bluestore/BlockDevice.h @@ -76,6 +76,7 @@ public: virtual int read(uint64_t off, uint64_t len, bufferlist *pbl, IOContext *ioc, bool buffered) = 0; + virtual int read_buffered(uint64_t off, uint64_t len, char *buf) = 0; virtual int aio_write(uint64_t off, bufferlist& bl, IOContext *ioc, bool buffered) = 0; diff --git a/src/os/bluestore/NVMEDevice.cc b/src/os/bluestore/NVMEDevice.cc index a9897bdec12..fbf5721a848 100644 --- a/src/os/bluestore/NVMEDevice.cc +++ b/src/os/bluestore/NVMEDevice.cc @@ -34,6 +34,7 @@ #include "include/stringify.h" #include "include/types.h" #include "include/compat.h" +#include "common/align.h" #include "common/errno.h" #include "common/debug.h" #include "common/Initialize.h" @@ -320,7 +321,6 @@ NVMEDevice::NVMEDevice(aio_callback_t cb, void *cbpriv) { zeros = buffer::create_page_aligned(1048576); zeros.zero(); - } @@ -682,6 +682,58 @@ int NVMEDevice::read(uint64_t off, uint64_t len, bufferlist *pbl, return r; } +int NVMEDevice::read_buffered(uint64_t off, uint64_t len, char *buf) +{ + dout(5) << __func__ << " " << off << "~" << len << dendl; + assert(len > 0); + assert(off < size); + assert(off + len <= size); + + uint64_t aligned_off = align_down(off, block_size); + uint64_t aligned_len = align_up(len, block_size); + IOContext ioc(nullptr); + Task *t; + int r = rte_mempool_get(task_pool, (void **)&t); + if (r < 0) { + derr << __func__ << " task_pool rte_mempool_get failed" << dendl; + return r; + } + t->start = ceph_clock_now(g_ceph_context); + t->buf = rte_malloc(NULL, aligned_len, block_size); + if (t->buf == NULL) { + derr << __func__ << " task->buf rte_malloc failed" << dendl; + r = -ENOMEM; + goto out; + } + t->ctx = ioc; + t->command = IOCommand::READ_COMMAND; + t->offset = aligned_off; + t->len = aligned_len; + t->device = this; + t->return_code = 1; + t->next = t->prev = nullptr; + ioc->num_reading.inc();; + { + Mutex::Locker l(queue_lock); + task_queue.push(t); + if (queue_empty.read()) { + queue_empty.dec(); + queue_cond.Signal(); + } + } + + { + Mutex::Locker l(ioc->lock); + while (t->return_code > 0) + ioc->cond.Wait(ioc->lock); + } + memcpy(buf, t->buf+off-aligned_off, len); + r = t->return_code; + rte_free(t->buf); + + return r; +} + int NVMEDevice::invalidate_cache(uint64_t off, uint64_t len) { dout(5) << __func__ << " " << off << "~" << len << dendl; diff --git a/src/os/bluestore/NVMEDevice.h b/src/os/bluestore/NVMEDevice.h index b7e36527768..59d8f14a821 100644 --- a/src/os/bluestore/NVMEDevice.h +++ b/src/os/bluestore/NVMEDevice.h @@ -121,6 +121,7 @@ class NVMEDevice : public BlockDevice { int aio_zero(uint64_t off, uint64_t len, IOContext *ioc) override; int flush() override; + int read_buffered(uint64_t off, uint64_t len, char *buf) override; // for managing buffered readers/writers int invalidate_cache(uint64_t off, uint64_t len) override; -- 2.39.5