From: Jason Dillaman Date: Thu, 8 Dec 2016 03:38:47 +0000 (-0500) Subject: librbd: new block guard helper to prevent concurrent IO to blocks X-Git-Tag: v11.1.1~66^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b1d624b43ec159d4a07616e86557ea48f089b7a1;p=ceph.git librbd: new block guard helper to prevent concurrent IO to blocks Signed-off-by: Jason Dillaman --- diff --git a/src/librbd/BlockGuard.h b/src/librbd/BlockGuard.h new file mode 100644 index 000000000000..4c371580c1f1 --- /dev/null +++ b/src/librbd/BlockGuard.h @@ -0,0 +1,174 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IO_BLOCK_GUARD_H +#define CEPH_LIBRBD_IO_BLOCK_GUARD_H + +#include "include/int_types.h" +#include "common/dout.h" +#include "common/Mutex.h" +#include +#include +#include +#include +#include "include/assert.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::BlockGuard: " << this << " " \ + << __func__ << ": " + +namespace librbd { + +struct BlockExtent { + uint64_t block_start = 0; + uint64_t block_end = 0; + + BlockExtent() { + } + BlockExtent(uint64_t block_start, uint64_t block_end) + : block_start(block_start), block_end(block_end) { + } +}; + +struct BlockGuardCell { +}; + +/** + * Helper class to restrict and order concurrent IO to the same block. The + * definition of a block is dependent upon the user of this class. It might + * represent a backing object, 512 byte sectors, etc. + */ +template +class BlockGuard { +private: + struct DetainedBlockExtent; + +public: + typedef std::list BlockOperations; + + BlockGuard(CephContext *cct) + : m_cct(cct), m_lock("librbd::BlockGuard::m_lock") { + } + + BlockGuard(const BlockGuard&) = delete; + BlockGuard &operator=(const BlockGuard&) = delete; + + /** + * Detain future IO for a range of blocks. the guard will assume + * ownership of the provided operation if the operation is blocked. + * @return 0 upon success and IO can be issued + * >0 if the IO is blocked, + * <0 upon error + */ + int detain(const BlockExtent &block_extent, BlockOperation *block_operation, + BlockGuardCell **cell) { + Mutex::Locker locker(m_lock); + ldout(m_cct, 20) << "block_start=" << block_extent.block_start << ", " + << "block_end=" << block_extent.block_end << ", " + << "free_slots=" << m_free_detained_block_extents.size() + << dendl; + + DetainedBlockExtent *detained_block_extent; + auto it = m_detained_block_extents.find(block_extent); + if (it != m_detained_block_extents.end()) { + // request against an already detained block + detained_block_extent = &(*it); + if (block_operation != nullptr) { + detained_block_extent->block_operations.emplace_back( + std::move(*block_operation)); + } + + // alert the caller that the IO was detained + *cell = nullptr; + return detained_block_extent->block_operations.size(); + } else { + if (!m_free_detained_block_extents.empty()) { + detained_block_extent = &m_free_detained_block_extents.front(); + detained_block_extent->block_operations.clear(); + m_free_detained_block_extents.pop_front(); + } else { + ldout(m_cct, 20) << "no free detained block cells" << dendl; + m_detained_block_extent_pool.emplace_back(); + detained_block_extent = &m_detained_block_extent_pool.back(); + } + + detained_block_extent->block_extent = block_extent; + m_detained_block_extents.insert(*detained_block_extent); + *cell = reinterpret_cast(detained_block_extent); + return 0; + } + } + + /** + * Release any detained IO operations from the provided cell. + */ + void release(BlockGuardCell *cell, BlockOperations *block_operations) { + Mutex::Locker locker(m_lock); + + assert(cell != nullptr); + auto &detained_block_extent = reinterpret_cast( + *cell); + ldout(m_cct, 20) << "block_start=" + << detained_block_extent.block_extent.block_start << ", " + << "block_end=" + << detained_block_extent.block_extent.block_end << ", " + << "pending_ops=" + << (detained_block_extent.block_operations.empty() ? + 0 : detained_block_extent.block_operations.size() - 1) + << dendl; + + *block_operations = std::move(detained_block_extent.block_operations); + m_detained_block_extents.erase(detained_block_extent.block_extent); + m_free_detained_block_extents.push_back(detained_block_extent); + } + +private: + struct DetainedBlockExtent : public boost::intrusive::list_base_hook<>, + public boost::intrusive::set_base_hook<> { + BlockExtent block_extent; + BlockOperations block_operations; + }; + + struct DetainedBlockExtentKey { + typedef BlockExtent type; + const BlockExtent &operator()(const DetainedBlockExtent &value) { + return value.block_extent; + } + }; + + struct DetainedBlockExtentCompare { + bool operator()(const BlockExtent &lhs, + const BlockExtent &rhs) const { + // check for range overlap (lhs < rhs) + if (lhs.block_end <= rhs.block_start) { + return true; + } + return false; + } + }; + + typedef std::deque DetainedBlockExtentsPool; + typedef boost::intrusive::list DetainedBlockExtents; + typedef boost::intrusive::set< + DetainedBlockExtent, + boost::intrusive::compare, + boost::intrusive::key_of_value > + BlockExtentToDetainedBlockExtents; + + CephContext *m_cct; + + Mutex m_lock; + DetainedBlockExtentsPool m_detained_block_extent_pool; + DetainedBlockExtents m_free_detained_block_extents; + BlockExtentToDetainedBlockExtents m_detained_block_extents; + +}; + +} // namespace librbd + +#undef dout_subsys +#undef dout_prefix +#define dout_prefix *_dout + +#endif // CEPH_LIBRBD_IO_BLOCK_GUARD_H diff --git a/src/test/librbd/CMakeLists.txt b/src/test/librbd/CMakeLists.txt index df7fd41cce47..1906da7cf69e 100644 --- a/src/test/librbd/CMakeLists.txt +++ b/src/test/librbd/CMakeLists.txt @@ -23,6 +23,7 @@ set_target_properties(rbd_test_mock PROPERTIES COMPILE_FLAGS # unittest_librbd # doesn't use add_ceph_test because it is called by run-rbd-unit-tests.sh set(unittest_librbd_srcs + test_BlockGuard.cc test_ConsistencyGroups.cc test_main.cc test_mock_fixture.cc diff --git a/src/test/librbd/test_BlockGuard.cc b/src/test/librbd/test_BlockGuard.cc new file mode 100644 index 000000000000..b16188b04a47 --- /dev/null +++ b/src/test/librbd/test_BlockGuard.cc @@ -0,0 +1,98 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/librbd/test_fixture.h" +#include "test/librbd/test_support.h" +#include "librbd/BlockGuard.h" + +namespace librbd { + +class TestIOBlockGuard : public TestFixture { +public: + static uint32_t s_index; + + struct Operation { + uint32_t index; + Operation() : index(++s_index) { + } + Operation(Operation &&rhs) : index(rhs.index) { + } + Operation(const Operation &) = delete; + + Operation& operator=(Operation &&rhs) { + index = rhs.index; + return *this; + } + + bool operator==(const Operation &rhs) const { + return index == rhs.index; + } + }; + + typedef std::list Operations; + + typedef BlockGuard OpBlockGuard; + + virtual void SetUp() override { + TestFixture::SetUp(); + m_cct = reinterpret_cast(m_ioctx.cct()); + } + + CephContext *m_cct; +}; + +TEST_F(TestIOBlockGuard, NonDetainedOps) { + OpBlockGuard op_block_guard(m_cct); + + Operation op1; + BlockGuardCell *cell1; + ASSERT_EQ(0, op_block_guard.detain({1, 3}, &op1, &cell1)); + + Operation op2; + BlockGuardCell *cell2; + ASSERT_EQ(0, op_block_guard.detain({0, 1}, &op2, &cell2)); + + Operation op3; + BlockGuardCell *cell3; + ASSERT_EQ(0, op_block_guard.detain({3, 6}, &op3, &cell3)); + + Operations released_ops; + op_block_guard.release(cell1, &released_ops); + ASSERT_TRUE(released_ops.empty()); + + op_block_guard.release(cell2, &released_ops); + ASSERT_TRUE(released_ops.empty()); + + op_block_guard.release(cell3, &released_ops); + ASSERT_TRUE(released_ops.empty()); +} + +TEST_F(TestIOBlockGuard, DetainedOps) { + OpBlockGuard op_block_guard(m_cct); + + Operation op1; + BlockGuardCell *cell1; + ASSERT_EQ(0, op_block_guard.detain({1, 3}, &op1, &cell1)); + + Operation op2; + BlockGuardCell *cell2; + ASSERT_EQ(1, op_block_guard.detain({2, 6}, &op2, &cell2)); + ASSERT_EQ(nullptr, cell2); + + Operation op3; + BlockGuardCell *cell3; + ASSERT_EQ(2, op_block_guard.detain({0, 2}, &op3, &cell3)); + ASSERT_EQ(nullptr, cell3); + + Operations expected_ops; + expected_ops.push_back(std::move(op2)); + expected_ops.push_back(std::move(op3)); + Operations released_ops; + op_block_guard.release(cell1, &released_ops); + ASSERT_EQ(expected_ops, released_ops); +} + +uint32_t TestIOBlockGuard::s_index = 0; + +} // namespace librbd +