From: Lucian Petrut Date: Fri, 4 Nov 2022 10:30:43 +0000 (+0200) Subject: test/librbd/fsx: add rbd-wnbd tests X-Git-Tag: v19.0.0~672^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=9a0a715cb1fceed4121d82d723cd374d8fd85d6d;p=ceph.git test/librbd/fsx: add rbd-wnbd tests rbd-wnbd can be used to map rbd images as Windows block devices. This change will allow exercising rbd-wnbd and the wnbd driver as opposed to using librbd directly. One thing to note here is that read and write operations must be 512B aligned. Also, wnbd does not currently support "compare and write". For this reason, we'll stick to the basic operations for the time being. Note that we're renaming krbd_read/krbd_write to generic_pread and generic_pwrite, reusing those functions for wnbd operations. We're also removing the duplicated ggate read/write operations, using the generic functions instead. Signed-off-by: Lucian Petrut --- diff --git a/src/include/win32/winioctl_compat.h b/src/include/win32/winioctl_compat.h new file mode 100644 index 0000000000000..5e87287b1176b --- /dev/null +++ b/src/include/win32/winioctl_compat.h @@ -0,0 +1,43 @@ +// Mingw provides a minimal version of this header and doesn't include all the +// definitions that we need. + +#pragma once + +#ifdef __MINGW32__ + +#include + +#define IOCTL_DISK_GET_DISK_ATTRIBUTES \ + CTL_CODE(IOCTL_DISK_BASE, 0x003c, METHOD_BUFFERED, FILE_ANY_ACCESS) +#define IOCTL_DISK_SET_DISK_ATTRIBUTES \ + CTL_CODE(IOCTL_DISK_BASE, 0x003d, METHOD_BUFFERED, \ + FILE_READ_ACCESS | FILE_WRITE_ACCESS) + +#define DISK_ATTRIBUTE_OFFLINE 0x0000000000000001 +#define DISK_ATTRIBUTE_READ_ONLY 0x0000000000000002 + +// +// IOCTL_DISK_SET_DISK_ATTRIBUTES +// +// Input Buffer: +// Structure of type SET_DISK_ATTRIBUTES +// +// Output Buffer: +// None +// +typedef struct _SET_DISK_ATTRIBUTES { + // Specifies the size of the structure for versioning. + DWORD Version; + // Indicates whether to remember these settings across reboots or not. + BOOLEAN Persist; + // Reserved. Must set to zero. + BYTE Reserved1[3]; + // Specifies the new attributes. + DWORDLONG Attributes; + // Specifies the attributes that are being modified. + DWORDLONG AttributesMask; + // Reserved. Must set to zero. + DWORD Reserved2[4]; +} SET_DISK_ATTRIBUTES, *PSET_DISK_ATTRIBUTES; + +#endif // __MINGW32__ diff --git a/src/test/librbd/fsx.cc b/src/test/librbd/fsx.cc index fa4314d69a3db..a727bde1ce488 100644 --- a/src/test/librbd/fsx.cc +++ b/src/test/librbd/fsx.cc @@ -46,12 +46,16 @@ #include #include #include +#include #include "include/compat.h" #include "include/intarith.h" #if defined(WITH_KRBD) #include "include/krbd.h" #endif +#ifdef _WIN32 +#include "include/win32/winioctl_compat.h" +#endif #include "include/rados/librados.h" #include "include/rados/librados.hpp" #include "include/rbd/librbd.h" @@ -1021,9 +1025,8 @@ krbd_close(struct rbd_ctx *ctx) } #endif // WITH_KRBD -#if defined(__linux__) ssize_t -krbd_read(struct rbd_ctx *ctx, uint64_t off, size_t len, char *buf) +generic_pread(struct rbd_ctx *ctx, uint64_t off, size_t len, char *buf) { ssize_t n; @@ -1038,7 +1041,7 @@ krbd_read(struct rbd_ctx *ctx, uint64_t off, size_t len, char *buf) } ssize_t -krbd_write(struct rbd_ctx *ctx, uint64_t off, size_t len, const char *buf) +generic_pwrite(struct rbd_ctx *ctx, uint64_t off, size_t len, const char *buf) { ssize_t n; @@ -1052,6 +1055,7 @@ krbd_write(struct rbd_ctx *ctx, uint64_t off, size_t len, const char *buf) return n; } +#if defined(__linux__) int __krbd_flush(struct rbd_ctx *ctx, bool invalidate) { @@ -1212,8 +1216,8 @@ krbd_flatten(struct rbd_ctx *ctx) const struct rbd_operations krbd_operations = { krbd_open, krbd_close, - krbd_read, - krbd_write, + generic_pread, + generic_pwrite, krbd_flush, krbd_discard, krbd_get_size, @@ -1338,8 +1342,8 @@ nbd_clone(struct rbd_ctx *ctx, const char *src_snapname, const struct rbd_operations nbd_operations = { nbd_open, nbd_close, - krbd_read, - krbd_write, + generic_pread, + generic_pwrite, krbd_flush, krbd_discard, krbd_get_size, @@ -1350,6 +1354,256 @@ const struct rbd_operations nbd_operations = { }; #endif // __linux__ +#ifdef _WIN32 +int +wnbd_set_writable(struct rbd_ctx *ctx) +{ + DWORD bytesReturned = 0; + + SET_DISK_ATTRIBUTES attributes = {0}; + attributes.Version = sizeof(attributes); + attributes.Attributes = 0; // clear read-only flag + attributes.AttributesMask = DISK_ATTRIBUTE_READ_ONLY; + + BOOL succeeded = DeviceIoControl( + (HANDLE) _get_osfhandle(ctx->krbd_fd), + IOCTL_DISK_SET_DISK_ATTRIBUTES, + (LPVOID) &attributes, + (DWORD) sizeof(attributes), + NULL, + 0, + &bytesReturned, + NULL); + if (!succeeded) { + DWORD err = GetLastError(); + prt("couldn't clear disk read-only flag, error: %d\n", err); + return -EINVAL; + } + return 0; +} + +int +wnbd_open(const char *name, struct rbd_ctx *ctx) +{ + int r; + int fd; + + SubProcess process( + "rbd-wnbd", + SubProcess::KEEP, SubProcess::PIPE, SubProcess::KEEP); + process.add_cmd_arg("map"); + std::string img; + img.append(pool); + img.append("/"); + img.append(name); + process.add_cmd_arg(img.c_str()); + + r = __librbd_open(name, ctx); + if (r < 0) + return r; + + r = process.spawn(); + if (r < 0) { + prt("failed to run rbd-wnbd map, error: %s\n", process.err().c_str()); + return r; + } + r = process.join(); + if (r) { + prt("rbd-wnbd map failed, error: %s\n", process.err().c_str()); + return -EINVAL; + } + + // Retrieve the device number + SubProcess process2( + "rbd-wnbd", + SubProcess::KEEP, SubProcess::PIPE, SubProcess::KEEP); + process2.add_cmd_arg("show"); + process2.add_cmd_arg(img.c_str()); + + r = process2.spawn(); + if (r < 0) { + prt("failed to run rbd-wnbd show, error: %s\n", process2.err().c_str()); + return r; + } + + // This should be more than enough for the "rbd-wnbd show" output + char map_info[4096]; + r = safe_read(process2.get_stdout(), map_info, sizeof(map_info)); + if (r < 0) { + prt("unable to retrieve rbd-wnbd map info, error: %d\n", r); + return r; + } + + r = process2.join(); + if (r) { + prt("rbd-wnbd show failed, error: %s\n", process2.err().c_str()); + return -EINVAL; + } + + std::string dev_path; + std::regex dev_num_re("\"disk_number\": ([0-9]+)"); + std::smatch cm; + std::string map_info_str(map_info); + if (std::regex_search(map_info_str, cm, dev_num_re)) { + dev_path = std::string("\\\\.\\PhysicalDrive") + cm.str(1); + } else { + prt("unable to retrieve rbd-wnbd device number, device info: %s\n", + map_info); + return -EINVAL; + } + + // TODO: consider applying o_direct (e.g. using CreateFile). + fd = open(dev_path.c_str(), O_RDWR | O_BINARY); + if (fd < 0) { + r = -errno; + prt("open(%s) failed\n", dev_path.c_str()); + return r; + } + + ctx->krbd_name = strdup(img.c_str()); + ctx->krbd_fd = fd; + + return wnbd_set_writable(ctx); +} + +int +wnbd_close(struct rbd_ctx *ctx) +{ + int r; + + ceph_assert(ctx->krbd_name && ctx->krbd_fd >= 0); + + if (close(ctx->krbd_fd) < 0) { + r = -errno; + prt("close(%s) failed\n", ctx->krbd_name); + return r; + } + + SubProcess process("rbd-wnbd"); + process.add_cmd_arg("unmap"); + process.add_cmd_arg(ctx->krbd_name); + + r = process.spawn(); + if (r < 0) { + prt("failed to run rbd-wnbd unmap, error: %s\n", process.err().c_str()); + return r; + } + r = process.join(); + if (r) { + prt("rbd-wnbd unmap failed, error: %s", process.err().c_str()); + return -EINVAL; + } + + free((void *)ctx->krbd_name); + + ctx->krbd_name = NULL; + ctx->krbd_fd = -1; + + return __librbd_close(ctx); +} + +int +wnbd_flush(struct rbd_ctx *ctx) +{ + return _commit(ctx->krbd_fd); +} + +int +wnbd_discard(struct rbd_ctx *ctx, uint64_t off, uint64_t len) +{ + // TODO: use one of the following: + // * IOCTL_SCSI_PASS_THROUGH + // * IOCTL_SCSI_PASS_THROUGH_DIRECT + // * FSCTL_FILE_LEVEL_TRIM + // * sg3_utils + return -ENOTSUP; +} + +int +wnbd_get_size(struct rbd_ctx *ctx, uint64_t *size) +{ + DWORD bytesReturned = 0; + + GET_LENGTH_INFORMATION length = {0}; + + BOOL succeeded = DeviceIoControl( + (HANDLE) _get_osfhandle(ctx->krbd_fd), + IOCTL_DISK_GET_LENGTH_INFO, + NULL, + 0, + (LPVOID) &length, + (DWORD) sizeof(length), + &bytesReturned, + NULL); + if (!succeeded) { + DWORD err = GetLastError(); + prt("unable to retrieve wnbd disk size, error: %d\n", err); + return -EINVAL; + } + + *size = length.Length.QuadPart; + return 0; +} + +int +wnbd_resize(struct rbd_ctx *ctx, uint64_t size) +{ + int ret; + + ceph_assert(size % truncbdy == 0); + + ret = wnbd_flush(ctx); + if (ret < 0) { + return ret; + } + + return __librbd_resize(ctx, size); +} + +int +wnbd_clone(struct rbd_ctx *ctx, const char *src_snapname, + const char *dst_imagename, int *order, int stripe_unit, + int stripe_count) +{ + int ret; + + ret = wnbd_flush(ctx); + if (ret < 0) { + return ret; + } + + return __librbd_clone(ctx, src_snapname, dst_imagename, order, + stripe_unit, stripe_count); +} + +int +wnbd_flatten(struct rbd_ctx *ctx) +{ + int ret; + + ret = wnbd_flush(ctx); + if (ret < 0) { + return ret; + } + + return __librbd_flatten(ctx); +} + +const struct rbd_operations wnbd_operations = { + wnbd_open, + wnbd_close, + generic_pread, + generic_pwrite, + wnbd_flush, + wnbd_discard, + wnbd_get_size, + wnbd_resize, + wnbd_clone, + wnbd_flatten, + NULL, +}; +#endif // _WIN32 + #if defined(__FreeBSD__) int ggate_open(const char *name, struct rbd_ctx *ctx) @@ -1457,36 +1711,6 @@ ggate_close(struct rbd_ctx *ctx) return __librbd_close(ctx); } -ssize_t -ggate_read(struct rbd_ctx *ctx, uint64_t off, size_t len, char *buf) -{ - ssize_t n; - - n = pread(ctx->krbd_fd, buf, len, off); - if (n < 0) { - n = -errno; - prt("pread(%llu, %zu) failed\n", off, len); - return n; - } - - return n; -} - -ssize_t -ggate_write(struct rbd_ctx *ctx, uint64_t off, size_t len, const char *buf) -{ - ssize_t n; - - n = pwrite(ctx->krbd_fd, buf, len, off); - if (n < 0) { - n = -errno; - prt("pwrite(%llu, %zu) failed\n", off, len); - return n; - } - - return n; -} - int __ggate_flush(struct rbd_ctx *ctx, bool invalidate) { @@ -1599,8 +1823,8 @@ ggate_flatten(struct rbd_ctx *ctx) const struct rbd_operations ggate_operations = { ggate_open, ggate_close, - ggate_read, - ggate_write, + generic_pread, + generic_pwrite, ggate_flush, ggate_discard, ggate_get_size, @@ -2045,7 +2269,7 @@ doread(unsigned offset, unsigned size) int ret; offset -= offset % readbdy; - if (o_direct) + if (o_direct || ops == &wnbd_operations) size -= size % readbdy; if (size == 0) { if (!quiet && testcalls > simulatedopcount && !o_direct) @@ -2135,7 +2359,7 @@ dowrite(unsigned offset, unsigned size) off_t newsize; offset -= offset % writebdy; - if (o_direct) + if (o_direct || ops == &wnbd_operations) size -= size % writebdy; if (size == 0) { if (!quiet && testcalls > simulatedopcount && !o_direct) @@ -2948,6 +3172,9 @@ usage(void) #if defined(__linux__) " -M: enable rbd-nbd mode (use -t and -h too)\n" #endif +#if defined(_WIN32) +" -M: enable rbd-wnbd mode (use -L, -r and -w too)\n" +#endif " -L: fsxLite - no file creations & no file size changes\n\ -N numops: total # operations to do (default infinity)\n\ -O: use oplen (see -o flag) for every op (default random)\n\ @@ -3225,6 +3452,12 @@ main(int argc, char **argv) prt("rbd-nbd mode enabled\n"); ops = &nbd_operations; break; +#endif +#if defined(_WIN32) + case 'M': + prt("rbd-wnbd mode enabled\n"); + ops = &wnbd_operations; + break; #endif case 'L': lite = 1;