elseif(FREEBSD)
set(HAVE_UDEV OFF)
set(HAVE_LIBAIO OFF)
- set(HAVE_LIBDML OFF)
set(HAVE_BLKID OFF)
set(HAVE_KEYUTILS OFF)
else()
endif()
endif()
-CMAKE_DEPENDENT_OPTION(WITH_BLUESTORE_PMEM "Enable PMDK libraries" OFF
- "WITH_BLUESTORE" OFF)
-if(WITH_BLUESTORE_PMEM)
- find_package(dml)
- set(HAVE_LIBDML ${DML_FOUND})
-endif()
-
CMAKE_DEPENDENT_OPTION(WITH_RBD_MIRROR "Enable build for rbd-mirror daemon executable" OFF
"WITH_RBD" OFF)
"WITH_RBD" OFF)
CMAKE_DEPENDENT_OPTION(WITH_SYSTEM_PMDK "Require and build with system PMDK" OFF
- "WITH_RBD_RWL OR WITH_BLUESTORE_PMEM" OFF)
+ "WITH_RBD_RWL" OFF)
CMAKE_DEPENDENT_OPTION(WITH_RBD_UBBD "Enable ubbd support for rbd device utility" OFF
"WITH_RBD" OFF)
build_ubbd()
endif()
-if(WITH_BLUESTORE_PMEM)
- set(HAVE_BLUESTORE_PMEM ON)
-endif()
-
CMAKE_DEPENDENT_OPTION(WITH_SPDK "Enable SPDK" OFF
"CMAKE_SYSTEM_PROCESSOR MATCHES i386|i686|amd64|x86_64|AMD64|aarch64|riscv64" OFF)
option(WITH_SYSTEM_SPDK "Require a system SPDK instead of building bundled src/spdk" OFF)
endif(WITH_SPDK)
if(WITH_BLUESTORE)
- if(NOT AIO_FOUND AND NOT HAVE_POSIXAIO AND NOT WITH_SPDK AND NOT WITH_BLUESTORE_PMEM)
+ if(NOT AIO_FOUND AND NOT HAVE_POSIXAIO AND NOT WITH_SPDK)
message(SEND_ERROR "WITH_BLUESTORE is ON, "
"but none of the bluestore backends is enabled. "
- "Please install libaio, or enable WITH_SPDK or WITH_BLUESTORE_PMEM (experimental)")
+ "Please install libaio, or enable WITH_SPDK")
endif()
endif()
rather rigid and also challenging to disseminate in some environments. The
key can be embedded in the migration spec or just referenced from there while
stored in the MON config-key store.
+* BlueStore: The experimental PMEM device backend has been removed, together
+ with the ``pmem`` value of the ``bdev_type`` option and the DML/DSA offload
+ path. The hardware it targeted, Intel Optane DC persistent memory, was
+ discontinued in 2022. The RBD persistent write-back cache, which also uses
+ PMDK, is unaffected.
+
+ Existing OSDs backed by persistent memory in fsdax mode (a block device such
+ as ``/dev/pmem0``) keep working on the default ``aio`` backend over the same
+ data. Remove any explicit ``bdev_type = pmem`` from their configuration. OSDs
+ on a devdax namespace (a character device such as ``/dev/dax0.0``) will no
+ longer start, because no remaining backend can open one; reconfigure the
+ namespace to fsdax with ``ndctl`` or re-provision them before upgrading.
>=20.0.0
BuildRequires: pkgconfig(libevent)
%endif
%if 0%{with system_pmdk}
-%if 0%{?suse_version}
-BuildRequires: pkgconfig(libndctl) >= 63
-%else
-BuildRequires: pkgconfig(libndctl) >= 63
-BuildRequires: pkgconfig(libdaxctl) >= 63
-%endif
-BuildRequires: pkgconfig(libpmem)
BuildRequires: pkgconfig(libpmemobj) >= 1.8
%endif
%if 0%{with system_arrow}
-function(build_pmdk enable_ndctl)
+function(build_pmdk)
include(FindMake)
find_make("MAKE_EXECUTABLE" "make_cmd")
endif()
set(LIBPMEM_INTERFACE_LINK_LIBRARIES Threads::Threads)
- if(${enable_ndctl})
- set(ndctl "y")
- list(APPEND LIBPMEM_INTERFACE_LINK_LIBRARIES ndctl::ndctl daxctl::daxctl)
- else()
- set(ndctl "n")
- endif()
# Use debug PMDK libs in debug lib/rbd builds
if(CMAKE_BUILD_TYPE STREQUAL Debug)
ExternalProject_Add(pmdk_ext
${source_dir_args}
CONFIGURE_COMMAND ""
- BUILD_COMMAND ${make_cmd} CC=${CMAKE_C_COMPILER} "EXTRA_CFLAGS=${pmdk_cflags}" NDCTL_ENABLE=${ndctl} BUILD_EXAMPLES=n BUILD_BENCHMARKS=n DOC=n
+ BUILD_COMMAND ${make_cmd} CC=${CMAKE_C_COMPILER} "EXTRA_CFLAGS=${pmdk_cflags}" NDCTL_ENABLE=n BUILD_EXAMPLES=n BUILD_BENCHMARKS=n DOC=n
BUILD_IN_SOURCE 1
BUILD_BYPRODUCTS "<SOURCE_DIR>/src/${PMDK_LIB_DIR}/libpmem.a" "<SOURCE_DIR>/src/${PMDK_LIB_DIR}/libpmemobj.a"
INSTALL_COMMAND "")
+++ /dev/null
-# - Find libdaxctl
-# Find the daxctl libraries and includes
-#
-# daxctl_INCLUDE_DIR - where to find libdaxctl.h etc.
-# daxctl_LIBRARIES - List of libraries when using daxctl.
-# daxctl_FOUND - True if daxctl found.
-
-find_path(daxctl_INCLUDE_DIR daxctl/libdaxctl.h)
-
-if(daxctl_INCLUDE_DIR AND EXISTS "${daxctl_INCLUDE_DIR}/libdaxctl.h")
- foreach(ver "MAJOR" "MINOR" "RELEASE")
- file(STRINGS "${daxctl_INCLUDE_DIR}/libdaxctl.h" daxctl_VER_${ver}_LINE
- REGEX "^#define[ \t]+daxctl_VERSION_${ver}[ \t]+[0-9]+[ \t]+.*$")
- string(REGEX REPLACE "^#define[ \t]+daxctl_VERSION_${ver}[ \t]+([0-9]+)[ \t]+.*$"
- "\\1" daxctl_VERSION_${ver} "${daxctl_VER_${ver}_LINE}")
- unset(${daxctl_VER_${ver}_LINE})
- endforeach()
- set(daxctl_VERSION_STRING
- "${daxctl_VERSION_MAJOR}.${daxctl_VERSION_MINOR}.${daxctl_VERSION_RELEASE}")
-endif()
-
-find_library(daxctl_LIBRARY daxctl)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(daxctl
- REQUIRED_VARS daxctl_LIBRARY daxctl_INCLUDE_DIR
- VERSION_VAR daxctl_VERSION_STRING)
-
-mark_as_advanced(daxctl_INCLUDE_DIR daxctl_LIBRARY)
-
-if(daxctl_FOUND)
- set(daxctl_INCLUDE_DIRS ${daxctl_INCLUDE_DIR})
- set(daxctl_LIBRARIES ${daxctl_LIBRARY})
- if(NOT (TARGET daxctl::daxctl))
- add_library(daxctl::daxctl UNKNOWN IMPORTED)
- set_target_properties(daxctl::daxctl PROPERTIES
- INTERFACE_INCLUDE_DIRECTORIES "${daxctl_INCLUDE_DIRS}"
- IMPORTED_LINK_INTERFACE_LANGUAGES "C"
- IMPORTED_LOCATION "${daxctl_LIBRARIES}"
- VERSION "${daxctl_VERSION_STRING}")
- endif()
-endif()
+++ /dev/null
-# - Find libdml
-# Find the dml and dmlhl libraries and includes
-#
-# DML_INCLUDE_DIR - where to find dml.hpp etc.
-# DML_LIBRARIES - List of libraries when using dml.
-# DML_HL_LIBRARIES - List of libraries when using dmlhl.
-# DML_FOUND - True if DML found.
-
-
-find_path(DML_INCLUDE_DIR
- dml/dml.hpp
- PATHS
- /usr/include
- /usr/local/include)
-
-find_library(DML_LIBRARIES NAMES dml libdml PATHS
- /usr/local/
- /usr/local/lib64
- /usr/lib64
- /usr/lib)
-
-find_library(DML_HL_LIBRARIES NAMES dmlhl libdmlhl PATHS
- /usr/local/
- /usr/local/lib64
- /usr/lib64
- /usr/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(dml DEFAULT_MSG
- DML_LIBRARIES
- DML_INCLUDE_DIR
- DML_HL_LIBRARIES)
-
-mark_as_advanced(
- DML_LIBRARIES
- DML_INCLUDE_DIR
- DML_HL_LIBRARIES)
-
-if(DML_FOUND)
- if(NOT (TARGET dml::dml))
- add_library(dml::dml UNKNOWN IMPORTED)
- set_target_properties(dml::dml PROPERTIES
- INTERFACE_INCLUDE_DIRECTORIES "${DML_INCLUDE_DIR}"
- IMPORTED_LINK_INTERFACE_LANGUAGES "C"
- IMPORTED_LOCATION "${DML_LIBRARIES}")
- endif()
-
- if(NOT (TARGET dml::dmlhl))
- add_library(dml::dmlhl UNKNOWN IMPORTED)
- set_target_properties(dml::dmlhl PROPERTIES
- INTERFACE_INCLUDE_DIRECTORIES "${DML_INCLUDE_DIR}"
- INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS}
- INTERFACE_COMPILE_FEATURES cxx_std_17
- INTERFACE_COMPILE_DEFINITIONS "DML_HW"
- IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
- IMPORTED_LOCATION "${DML_HL_LIBRARIES}")
- endif()
-endif()
+++ /dev/null
-# - Find libndctl
-# Find the ndctl libraries and includes
-#
-# ndctl_INCLUDE_DIR - where to find libndctl.h etc.
-# ndctl_LIBRARIES - List of libraries when using ndctl.
-# ndctl_FOUND - True if ndctl found.
-
-find_path(ndctl_INCLUDE_DIR ndctl/libndctl.h)
-
-if(ndctl_INCLUDE_DIR AND EXISTS "${ndctl_INCLUDE_DIR}/libndctl.h")
- foreach(ver "MAJOR" "MINOR" "RELEASE")
- file(STRINGS "${ndctl_INCLUDE_DIR}/libndctl.h" ndctl_VER_${ver}_LINE
- REGEX "^#define[ \t]+ndctl_VERSION_${ver}[ \t]+[0-9]+[ \t]+.*$")
- string(REGEX REPLACE "^#define[ \t]+ndctl_VERSION_${ver}[ \t]+([0-9]+)[ \t]+.*$"
- "\\1" ndctl_VERSION_${ver} "${ndctl_VER_${ver}_LINE}")
- unset(${ndctl_VER_${ver}_LINE})
- endforeach()
- set(ndctl_VERSION_STRING
- "${ndctl_VERSION_MAJOR}.${ndctl_VERSION_MINOR}.${ndctl_VERSION_RELEASE}")
-endif()
-
-find_library(ndctl_LIBRARY ndctl)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(ndctl
- REQUIRED_VARS ndctl_LIBRARY ndctl_INCLUDE_DIR
- VERSION_VAR ndctl_VERSION_STRING)
-
-mark_as_advanced(ndctl_INCLUDE_DIR ndctl_LIBRARY)
-
-if(ndctl_FOUND)
- set(ndctl_INCLUDE_DIRS ${ndctl_INCLUDE_DIR})
- set(ndctl_LIBRARIES ${ndctl_LIBRARY})
- if(NOT (TARGET ndctl::ndctl))
- add_library(ndctl::ndctl UNKNOWN IMPORTED)
- set_target_properties(ndctl::ndctl PROPERTIES
- INTERFACE_INCLUDE_DIRECTORIES "${ndctl_INCLUDE_DIRS}"
- IMPORTED_LINK_INTERFACE_LANGUAGES "C"
- IMPORTED_LOCATION "${ndctl_LIBRARIES}"
- VERSION "${ndctl_VERSION_STRING}")
- endif()
-endif()
libthrift-dev (>= 0.13.0),
libyaml-cpp-dev (>= 0.6),
libzstd-dev <pkg.ceph.check>,
- libdaxctl-dev (>= 63) <pkg.ceph.pmdk>,
- libndctl-dev (>= 63) <pkg.ceph.pmdk>,
- libpmem-dev <pkg.ceph.pmdk>,
libpmemobj-dev (>= 1.8) <pkg.ceph.pmdk>,
libprotobuf-dev,
libxsimd-dev <!pkg.ceph.arrow>,
packages `libzbd`_ . The upstream libzbd includes debian packaging already.
libpmem
- packages `pmdk`_ . Please note, ``ndctl`` is one of the build dependencies of
- pmdk, for an updated debian packaging, please see
- https://github.com/ceph/ceph-ndctl .
+ packages `pmdk`_ .
.. note::
.. confval:: bluestore_min_alloc_size_hdd
.. confval:: bluestore_min_alloc_size_ssd
.. confval:: bluestore_use_optimal_io_size_for_min_alloc_size
-
-DSA (Data Streaming Accelerator) Usage
-======================================
-
-If you want to use the DML library to drive the DSA device for offloading
-read/write operations on persistent memory (PMEM) in BlueStore, you need to
-install `DML`_ and the `idxd-config`_ library. This will work only on machines
-that have a SPR (Sapphire Rapids) CPU.
-
-.. _DML: https://github.com/intel/DML
-.. _idxd-config: https://github.com/intel/idxd-config
-
-After installing the DML software, configure the shared work queues (WQs) with
-reference to the following WQ configuration example:
-
-.. prompt:: bash $
-
- accel-config config-wq --group-id=1 --mode=shared --wq-size=16 --threshold=15 --type=user --name="MyApp1" --priority=10 --block-on-fault=1 dsa0/wq0.1
- accel-config config-engine dsa0/engine0.1 --group-id=1
- accel-config enable-device dsa0
- accel-config enable-wq dsa0/wq0.1
list(APPEND ceph_common_deps dlfcn_win32)
endif()
-if(WITH_BLUESTORE_PMEM OR WITH_RBD_RWL)
+if(WITH_RBD_RWL)
if(WITH_SYSTEM_PMDK)
- set(pmdk_COMPONENTS)
- if(WITH_BLUESTORE_PMEM)
- list(APPEND pmdk_COMPONENTS pmem)
- endif()
- if(WITH_RBD_RWL)
- list(APPEND pmdk_COMPONENTS pmemobj)
- endif()
- find_package(pmdk 1.8 REQUIRED COMPONENTS ${pmdk_COMPONENTS})
+ find_package(pmdk 1.8 REQUIRED COMPONENTS pmemobj)
else()
include(Buildpmdk)
- if(WITH_BLUESTORE_PMEM)
- set(enable_ndctl ON)
- find_package(ndctl 63 REQUIRED)
- find_package(daxctl 63 REQUIRED)
- else()
- set(enable_ndctl OFF)
- endif()
- build_pmdk(${enable_ndctl})
+ build_pmdk()
endif()
endif()
#include "spdk/NVMEDevice.h"
#endif
-#if defined(HAVE_BLUESTORE_PMEM)
-#include "pmem/PMEMDevice.h"
-#endif
-
#include "common/debug.h"
#include "common/EventTrace.h"
#include "common/errno.h"
return block_device_t::spdk;
}
#endif
-#if defined(HAVE_BLUESTORE_PMEM)
- if (PMEMDevice::support(path)) {
- return block_device_t::pmem;
- }
-#endif
#if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
return block_device_t::aio;
#else
if (blk_dev_name == "spdk") {
return block_device_t::spdk;
}
-#endif
-#if defined(HAVE_BLUESTORE_PMEM)
- if (blk_dev_name == "pmem") {
- return block_device_t::pmem;
- }
#endif
return block_device_t::unknown;
}
#if defined(HAVE_SPDK)
case block_device_t::spdk:
return new NVMEDevice(cct, cb, cbpriv);
-#endif
-#if defined(HAVE_BLUESTORE_PMEM)
- case block_device_t::pmem:
- return new PMEMDevice(cct, cb, cbpriv);
#endif
default:
ceph_abort_msg("unsupported device");
#endif
#if defined(HAVE_SPDK)
spdk,
-#endif
-#if defined(HAVE_BLUESTORE_PMEM)
- pmem,
#endif
};
std::queue <ceph::mono_clock::time_point> stalled_read_event_queue;
aio/aio.cc)
endif()
-if(WITH_BLUESTORE_PMEM)
- list(APPEND libblk_srcs
- pmem/PMEMDevice.cc)
-endif()
-
if(WITH_SPDK)
list(APPEND libblk_srcs
spdk/NVMEDevice.cc)
PRIVATE spdk::spdk)
endif()
-if(WITH_BLUESTORE_PMEM)
- if(HAVE_LIBDML)
- target_link_libraries(blk PRIVATE dml::dml dml::dmlhl)
- endif()
-
- target_link_libraries(blk
- PRIVATE pmdk::pmem)
-endif()
-
if(WITH_EVENTTRACE)
add_dependencies(blk eventtrace_tp)
endif()
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
-// vim: ts=8 sw=2 sts=2 expandtab
-
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2015 Intel <jianpeng.ma@intel.com>
- *
- * Author: Jianpeng Ma <jianpeng.ma@intel.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation. See file COPYING.
- *
- */
-
-#include <unistd.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/sysmacros.h>
-#include <stdio.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <string.h>
-#include <filesystem>
-#include <fstream>
-
-#include <fmt/format.h>
-
-#include "PMEMDevice.h"
-#include "libpmem.h"
-#include "include/types.h"
-#include "include/compat.h"
-#include "include/stringify.h"
-#include "common/errno.h"
-#include "common/debug.h"
-#include "common/blkdev.h"
-
-#if defined(HAVE_LIBDML)
-#include <dml/dml.hpp>
-using execution_path = dml::automatic;
-#endif
-
-#define dout_context cct
-#define dout_subsys ceph_subsys_bdev
-#undef dout_prefix
-#define dout_prefix *_dout << "bdev-PMEM(" << path << ") "
-
-PMEMDevice::PMEMDevice(CephContext *cct, aio_callback_t cb, void *cbpriv)
- : BlockDevice(cct, cb, cbpriv),
- fd(-1), addr(0),
- injecting_crash(0)
-{
-}
-
-int PMEMDevice::_lock()
-{
- struct flock l;
- memset(&l, 0, sizeof(l));
- l.l_type = F_WRLCK;
- l.l_whence = SEEK_SET;
- l.l_start = 0;
- l.l_len = 0;
- int r = ::fcntl(fd, F_SETLK, &l);
- if (r < 0)
- return -errno;
- return 0;
-}
-
-static int pmem_check_file_type(int fd, const char *pmem_file, uint64_t *total_size)
-{
- namespace fs = std::filesystem;
- if (!fs::is_character_file(pmem_file)) {
- return -EINVAL;
- }
- struct stat file_stat;
- if (::fstat(fd, &file_stat)) {
- return -EINVAL;
- }
- fs::path char_dir = fmt::format("/sys/dev/char/{}:{}",
- major(file_stat.st_rdev),
- minor(file_stat.st_rdev));
- // Need to check if it is a DAX device
- if (auto subsys_path = char_dir / "subsystem";
- fs::read_symlink(subsys_path).filename().string() != "dax") {
- return -EINVAL;
- }
- if (total_size == nullptr) {
- return 0;
- }
- if (std::ifstream size_file(char_dir / "size"); size_file) {
- size_file >> *total_size;
- return size_file ? 0 : -EINVAL;
- } else {
- return -EINVAL;
- }
-}
-
-int PMEMDevice::open(const std::string& p)
-{
- path = p;
- int r = 0;
- dout(1) << __func__ << " path " << path << dendl;
-
- fd = ::open(path.c_str(), O_RDWR | O_CLOEXEC);
- if (fd < 0) {
- r = -errno;
- derr << __func__ << " open got: " << cpp_strerror(r) << dendl;
- return r;
- }
-
- r = pmem_check_file_type(fd, path.c_str(), &size);
- if (!r) {
- dout(1) << __func__ << " This path " << path << " is a devdax dev " << dendl;
- devdax_device = true;
- // If using devdax char device, set it to not rotational device.
- rotational = false;
- }
-
- r = _lock();
- if (r < 0) {
- derr << __func__ << " failed to lock " << path << ": " << cpp_strerror(r)
- << dendl;
- goto out_fail;
- }
-
- struct stat st;
- r = ::fstat(fd, &st);
- if (r < 0) {
- r = -errno;
- derr << __func__ << " fstat got " << cpp_strerror(r) << dendl;
- goto out_fail;
- }
-
- size_t map_len;
- addr = (char *)pmem_map_file(path.c_str(), 0,
- devdax_device ? 0: PMEM_FILE_EXCL, O_RDWR,
- &map_len, NULL);
- if (addr == NULL) {
- derr << __func__ << " pmem_map_file failed: " << pmem_errormsg() << dendl;
- goto out_fail;
- }
- size = map_len;
-
- // Operate as though the block size is 4 KB. The backing file
- // blksize doesn't strictly matter except that some file systems may
- // require a read/modify/write if we write something smaller than
- // it.
- block_size = g_conf()->bdev_block_size;
- if (block_size != (unsigned)st.st_blksize) {
- dout(1) << __func__ << " backing device/file reports st_blksize "
- << st.st_blksize << ", using bdev_block_size "
- << block_size << " anyway" << dendl;
- }
-
- dout(1) << __func__
- << " size " << size
- << " (" << byte_u_t(size) << ")"
- << " block_size " << block_size
- << " (" << byte_u_t(block_size) << ")"
- << dendl;
- return 0;
-
- out_fail:
- VOID_TEMP_FAILURE_RETRY(::close(fd));
- fd = -1;
- return r;
-}
-
-void PMEMDevice::close()
-{
- dout(1) << __func__ << dendl;
-
- ceph_assert(addr != NULL);
- if (devdax_device) {
- devdax_device = false;
- }
- pmem_unmap(addr, size);
-
- ceph_assert(fd >= 0);
- VOID_TEMP_FAILURE_RETRY(::close(fd));
- fd = -1;
-
- path.clear();
-}
-
-int PMEMDevice::collect_metadata(const std::string& prefix, std::map<std::string,std::string> *pm) const
-{
- (*pm)[prefix + "rotational"] = stringify((int)(bool)rotational);
- (*pm)[prefix + "size"] = stringify(get_size());
- (*pm)[prefix + "block_size"] = stringify(get_block_size());
- (*pm)[prefix + "driver"] = "PMEMDevice";
- (*pm)[prefix + "type"] = "ssd";
-
- struct stat st;
- int r = ::fstat(fd, &st);
- if (r < 0)
- return -errno;
- if (S_ISBLK(st.st_mode)) {
- (*pm)[prefix + "access_mode"] = "blk";
- char buffer[1024] = {0};
- BlkDev blkdev(fd);
-
- blkdev.model(buffer, sizeof(buffer));
- (*pm)[prefix + "model"] = buffer;
-
- buffer[0] = '\0';
- blkdev.dev(buffer, sizeof(buffer));
- (*pm)[prefix + "dev"] = buffer;
-
- // nvme exposes a serial number
- buffer[0] = '\0';
- blkdev.serial(buffer, sizeof(buffer));
- (*pm)[prefix + "serial"] = buffer;
-
- } else if (S_ISCHR(st.st_mode)) {
- (*pm)[prefix + "access_mode"] = "chardevice";
- (*pm)[prefix + "path"] = path;
-
- } else {
- (*pm)[prefix + "access_mode"] = "file";
- (*pm)[prefix + "path"] = path;
- }
- return 0;
-}
-
-bool PMEMDevice::support(const std::string &path)
-{
- int is_pmem = 0;
- size_t map_len = 0;
- int r = 0;
- int local_fd;
-
- local_fd = ::open(path.c_str(), O_RDWR);
- if (local_fd < 0) {
- return false;
- }
-
- r = pmem_check_file_type(local_fd, path.c_str(), NULL);
- VOID_TEMP_FAILURE_RETRY(::close(local_fd));
- int flags = PMEM_FILE_EXCL;
- if (r == 0) {
- flags = 0;
- }
-
- void *addr = pmem_map_file(path.c_str(), 0, flags, O_RDONLY, &map_len, &is_pmem);
- if (addr != NULL) {
- pmem_unmap(addr, map_len);
- if (is_pmem) {
- return true;
- }
- }
-
- return false;
-}
-
-int PMEMDevice::flush()
-{
- //Because all write is persist. So no need
- return 0;
-}
-
-
-void PMEMDevice::aio_submit(IOContext *ioc)
-{
- if (ioc->priv) {
- ceph_assert(ioc->num_running == 0);
- aio_callback(aio_callback_priv, ioc->priv);
- } else {
- ioc->try_aio_wake();
- }
- return;
-}
-
-int PMEMDevice::write(uint64_t off, bufferlist& bl, bool buffered, int write_hint)
-{
- uint64_t len = bl.length();
- dout(20) << __func__ << " " << off << "~" << len << dendl;
- ceph_assert(is_valid_io(off, len));
-
- dout(40) << "data:\n";
- bl.hexdump(*_dout);
- *_dout << dendl;
-
- if (g_conf()->bdev_inject_crash &&
- rand() % g_conf()->bdev_inject_crash == 0) {
- derr << __func__ << " bdev_inject_crash: dropping io " << off << "~" << len
- << dendl;
- ++injecting_crash;
- return 0;
- }
-
- bufferlist::iterator p = bl.begin();
- uint64_t off1 = off;
- while (len) {
- const char *data;
- uint32_t l = p.get_ptr_and_advance(len, &data);
-
-#if defined(HAVE_LIBDML)
- // Take care of the persistency issue
- auto result = dml::execute<execution_path>(dml::mem_move, dml::make_view(data, l), dml::make_view(addr + off1, l));
- ceph_assert(result.status == dml::status_code::ok);
-#else
- pmem_memcpy_persist(addr + off1, data, l);
-#endif
- len -= l;
- off1 += l;
- }
- return 0;
-}
-
-int PMEMDevice::aio_write(
- uint64_t off,
- bufferlist &bl,
- IOContext *ioc,
- bool buffered,
- int write_hint)
-{
- return write(off, bl, buffered);
-}
-
-
-int PMEMDevice::read(uint64_t off, uint64_t len, bufferlist *pbl,
- IOContext *ioc,
- bool buffered)
-{
- dout(5) << __func__ << " " << off << "~" << len << dendl;
- ceph_assert(is_valid_io(off, len));
-
- bufferptr p = buffer::create_small_page_aligned(len);
-
-#if defined(HAVE_LIBDML)
- auto result = dml::execute<execution_path>(dml::mem_move, dml::make_view(addr + off, len), dml::make_view(p.c_str(), len));
- ceph_assert(result.status == dml::status_code::ok);
-#else
- memcpy(p.c_str(), addr + off, len);
-#endif
-
- pbl->clear();
- pbl->push_back(std::move(p));
-
- dout(40) << "data:\n";
- pbl->hexdump(*_dout);
- *_dout << dendl;
-
- return 0;
-}
-
-int PMEMDevice::aio_read(uint64_t off, uint64_t len, bufferlist *pbl,
- IOContext *ioc)
-{
- return read(off, len, pbl, ioc, false);
-}
-
-int PMEMDevice::read_random(uint64_t off, uint64_t len, char *buf, bool buffered)
-{
- dout(5) << __func__ << " " << off << "~" << len << dendl;
- ceph_assert(is_valid_io(off, len));
-
-
-#if defined(HAVE_LIBDML)
- auto result = dml::execute<execution_path>(dml::mem_move, dml::make_view(addr + off, len), dml::make_view(buf, len));
- ceph_assert(result.status == dml::status_code::ok);
-#else
- memcpy(buf, addr + off, len);
-#endif
- return 0;
-}
-
-
-int PMEMDevice::invalidate_cache(uint64_t off, uint64_t len)
-{
- dout(5) << __func__ << " " << off << "~" << len << dendl;
- return 0;
-}
-
-
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
-// vim: ts=8 sw=2 sts=2 expandtab
-
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2015 Intel <jianpeng.ma@intel.com>
- *
- * Author: Jianpeng Ma <jianpeng.ma@intel.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation. See file COPYING.
- *
- */
-
-#ifndef CEPH_BLK_PMEMDEVICE_H
-#define CEPH_BLK_PMEMDEVICE_H
-
-#include <atomic>
-#include <map>
-#include <string>
-
-#include "os/fs/FS.h"
-#include "include/interval_set.h"
-#include "aio/aio.h"
-#include "BlockDevice.h"
-
-class PMEMDevice : public BlockDevice {
- int fd;
- char *addr; //the address of mmap
- std::string path;
- bool devdax_device = false;
-
- ceph::mutex debug_lock = ceph::make_mutex("PMEMDevice::debug_lock");
- interval_set<uint64_t> debug_inflight;
-
- std::atomic_int injecting_crash;
- int _lock();
-
-public:
- PMEMDevice(CephContext *cct, aio_callback_t cb, void *cbpriv);
-
- bool supported_bdev_label() override { return !devdax_device; }
- void aio_submit(IOContext *ioc) override;
-
- int collect_metadata(const std::string& prefix, std::map<std::string,std::string> *pm) const override;
-
- static bool support(const std::string& path);
-
- int read(uint64_t off, uint64_t len, bufferlist *pbl,
- IOContext *ioc,
- bool buffered) override;
- int aio_read(uint64_t off, uint64_t len, bufferlist *pbl,
- IOContext *ioc) override;
-
- int read_random(uint64_t off, uint64_t len, char *buf, bool buffered) override;
- int write(uint64_t off, bufferlist& bl, bool buffered, int write_hint = WRITE_LIFE_NOT_SET) override;
- int aio_write(uint64_t off, bufferlist& bl,
- IOContext *ioc,
- bool buffered,
- int write_hint = WRITE_LIFE_NOT_SET) override;
- int flush() override;
-
- // for managing buffered readers/writers
- int invalidate_cache(uint64_t off, uint64_t len) override;
- int open(const std::string &path) override;
- void close() override;
-
-private:
- bool is_valid_io(uint64_t off, uint64_t len) const {
- return (len > 0 &&
- off < size &&
- off + len <= size);
- }
-};
-
-#endif
enum_values:
- aio
- spdk
- - pmem
- name: bdev_stalled_read_warn_lifetime
type: uint
level: advanced
/* Defined if you have libaio */
#cmakedefine HAVE_LIBAIO
-/* Defined if you have libdml */
-#cmakedefine HAVE_LIBDML
-
/* Defined if you have liburing */
#cmakedefine HAVE_LIBURING
/* DPDK conditional compilation */
#cmakedefine HAVE_DPDK
-/* PMEM_DEVICE (OSD) conditional compilation */
-#cmakedefine HAVE_BLUESTORE_PMEM
-
/* Define if you have tcmalloc */
#cmakedefine HAVE_LIBTCMALLOC
#cmakedefine LIBTCMALLOC_MISSING_ALIGNED_ALLOC
rgw_store="rados"
lockdep=${LOCKDEP:-1}
spdk_enabled=0 # disable SPDK by default
-pmem_enabled=0
io_uring_enabled=0
with_jaeger=0
force_addr=0
--multimds <count> allow multimds with maximum active count
--without-dashboard: do not run using mgr dashboard
--bluestore-spdk: enable SPDK and with a comma-delimited list of PCI-IDs of NVME device (e.g, 0000:81:00.0)
- --bluestore-pmem: enable PMEM and with path to a file mapped to PMEM
--msgr1: use msgr1 only
--msgr2: use msgr2 only
--msgr21: use msgr2 and msgr1
spdk_enabled=1
shift
;;
- --bluestore-pmem)
- [ -z "$2" ] && usage_exit
- bluestore_pmem_file="$2"
- pmem_enabled=1
- shift
- ;;
--bluestore-devs)
parse_block_devs --bluestore-devs "$2"
shift
osd max object namespace len = 64"
fi
if [ "$objectstore" == "bluestore" ]; then
- if [ "$spdk_enabled" -eq 1 ] || [ "$pmem_enabled" -eq 1 ]; then
+ if [ "$spdk_enabled" -eq 1 ]; then
BLUESTORE_OPTS=" bluestore_block_db_path = \"\"
bluestore_block_db_size = 0
bluestore_block_db_create = false
if [ "$spdk_enabled" -eq 1 ]; then
wconf <<EOF
bluestore_block_path = spdk:${bluestore_spdk_dev[$osd]}
-EOF
- elif [ "$pmem_enabled" -eq 1 ]; then
- wconf <<EOF
- bluestore_block_path = ${bluestore_pmem_file}
EOF
fi
rm -rf $CEPH_DEV_DIR/osd$osd || true
projects / ceph.git / commitdiff