]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
test/librbd/fsx: add rbd-wnbd tests 50141/head
authorLucian Petrut <lpetrut@cloudbasesolutions.com>
Fri, 4 Nov 2022 10:30:43 +0000 (12:30 +0200)
committerLucian Petrut <lpetrut@cloudbasesolutions.com>
Wed, 16 Aug 2023 07:51:44 +0000 (07:51 +0000)
rbd-wnbd can be used to map rbd images as Windows block devices.

This change will allow exercising rbd-wnbd and the wnbd driver
as opposed to using librbd directly.

One thing to note here is that read and write operations must be
512B aligned. Also, wnbd does not currently support "compare
and write".

For this reason, we'll stick to the basic operations for the time
being.

Note that we're renaming krbd_read/krbd_write to generic_pread
and generic_pwrite, reusing those functions for wnbd operations.
We're also removing the duplicated ggate read/write operations,
using the generic functions instead.

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
src/include/win32/winioctl_compat.h [new file with mode: 0644]
src/test/librbd/fsx.cc

diff --git a/src/include/win32/winioctl_compat.h b/src/include/win32/winioctl_compat.h
new file mode 100644 (file)
index 0000000..5e87287
--- /dev/null
@@ -0,0 +1,43 @@
+// Mingw provides a minimal version of this header and doesn't include all the
+// definitions that we need.
+
+#pragma once
+
+#ifdef __MINGW32__
+
+#include <winioctl.h>
+
+#define IOCTL_DISK_GET_DISK_ATTRIBUTES \
+       CTL_CODE(IOCTL_DISK_BASE, 0x003c, METHOD_BUFFERED, FILE_ANY_ACCESS)
+#define IOCTL_DISK_SET_DISK_ATTRIBUTES \
+       CTL_CODE(IOCTL_DISK_BASE, 0x003d, METHOD_BUFFERED, \
+               FILE_READ_ACCESS | FILE_WRITE_ACCESS)
+
+#define DISK_ATTRIBUTE_OFFLINE   0x0000000000000001
+#define DISK_ATTRIBUTE_READ_ONLY 0x0000000000000002
+
+//
+// IOCTL_DISK_SET_DISK_ATTRIBUTES
+//
+// Input Buffer:
+//     Structure of type SET_DISK_ATTRIBUTES
+//
+// Output Buffer:
+//     None
+//
+typedef struct _SET_DISK_ATTRIBUTES {
+    // Specifies the size of the structure for versioning.
+    DWORD Version;
+    // Indicates whether to remember these settings across reboots or not.
+    BOOLEAN Persist;
+    // Reserved. Must set to zero.
+    BYTE  Reserved1[3];
+    // Specifies the new attributes.
+    DWORDLONG Attributes;
+    // Specifies the attributes that are being modified.
+    DWORDLONG AttributesMask;
+    // Reserved. Must set to zero.
+    DWORD Reserved2[4];
+} SET_DISK_ATTRIBUTES, *PSET_DISK_ATTRIBUTES;
+
+#endif // __MINGW32__
index fa4314d69a3db7d88f28ed34b278bb7f649a6c6f..a727bde1ce4882163b0a78cd9f41c37e040df25c 100644 (file)
 #include <math.h>
 #include <fcntl.h>
 #include <random>
+#include <regex>
 
 #include "include/compat.h"
 #include "include/intarith.h"
 #if defined(WITH_KRBD)
 #include "include/krbd.h"
 #endif
+#ifdef _WIN32
+#include "include/win32/winioctl_compat.h"
+#endif
 #include "include/rados/librados.h"
 #include "include/rados/librados.hpp"
 #include "include/rbd/librbd.h"
@@ -1021,9 +1025,8 @@ krbd_close(struct rbd_ctx *ctx)
 }
 #endif // WITH_KRBD
 
-#if defined(__linux__)
 ssize_t
-krbd_read(struct rbd_ctx *ctx, uint64_t off, size_t len, char *buf)
+generic_pread(struct rbd_ctx *ctx, uint64_t off, size_t len, char *buf)
 {
        ssize_t n;
 
@@ -1038,7 +1041,7 @@ krbd_read(struct rbd_ctx *ctx, uint64_t off, size_t len, char *buf)
 }
 
 ssize_t
-krbd_write(struct rbd_ctx *ctx, uint64_t off, size_t len, const char *buf)
+generic_pwrite(struct rbd_ctx *ctx, uint64_t off, size_t len, const char *buf)
 {
        ssize_t n;
 
@@ -1052,6 +1055,7 @@ krbd_write(struct rbd_ctx *ctx, uint64_t off, size_t len, const char *buf)
        return n;
 }
 
+#if defined(__linux__)
 int
 __krbd_flush(struct rbd_ctx *ctx, bool invalidate)
 {
@@ -1212,8 +1216,8 @@ krbd_flatten(struct rbd_ctx *ctx)
 const struct rbd_operations krbd_operations = {
        krbd_open,
        krbd_close,
-       krbd_read,
-       krbd_write,
+       generic_pread,
+       generic_pwrite,
        krbd_flush,
        krbd_discard,
        krbd_get_size,
@@ -1338,8 +1342,8 @@ nbd_clone(struct rbd_ctx *ctx, const char *src_snapname,
 const struct rbd_operations nbd_operations = {
        nbd_open,
        nbd_close,
-       krbd_read,
-       krbd_write,
+       generic_pread,
+       generic_pwrite,
        krbd_flush,
        krbd_discard,
        krbd_get_size,
@@ -1350,6 +1354,256 @@ const struct rbd_operations nbd_operations = {
 };
 #endif // __linux__
 
+#ifdef _WIN32
+int
+wnbd_set_writable(struct rbd_ctx *ctx)
+{
+       DWORD bytesReturned = 0;
+
+       SET_DISK_ATTRIBUTES attributes = {0};
+       attributes.Version = sizeof(attributes);
+       attributes.Attributes = 0; // clear read-only flag
+       attributes.AttributesMask = DISK_ATTRIBUTE_READ_ONLY;
+
+       BOOL succeeded = DeviceIoControl(
+               (HANDLE) _get_osfhandle(ctx->krbd_fd),
+               IOCTL_DISK_SET_DISK_ATTRIBUTES,
+               (LPVOID) &attributes,
+               (DWORD) sizeof(attributes),
+               NULL,
+               0,
+               &bytesReturned,
+               NULL);
+       if (!succeeded) {
+               DWORD err = GetLastError();
+               prt("couldn't clear disk read-only flag, error: %d\n", err);
+               return -EINVAL;
+       }
+       return 0;
+}
+
+int
+wnbd_open(const char *name, struct rbd_ctx *ctx)
+{
+       int r;
+       int fd;
+
+       SubProcess process(
+               "rbd-wnbd",
+               SubProcess::KEEP, SubProcess::PIPE, SubProcess::KEEP);
+       process.add_cmd_arg("map");
+       std::string img;
+       img.append(pool);
+       img.append("/");
+       img.append(name);
+       process.add_cmd_arg(img.c_str());
+
+       r = __librbd_open(name, ctx);
+       if (r < 0)
+               return r;
+
+       r = process.spawn();
+       if (r < 0) {
+               prt("failed to run rbd-wnbd map, error: %s\n", process.err().c_str());
+               return r;
+       }
+       r = process.join();
+       if (r) {
+               prt("rbd-wnbd map failed, error: %s\n", process.err().c_str());
+               return -EINVAL;
+       }
+
+       // Retrieve the device number
+       SubProcess process2(
+               "rbd-wnbd",
+               SubProcess::KEEP, SubProcess::PIPE, SubProcess::KEEP);
+       process2.add_cmd_arg("show");
+       process2.add_cmd_arg(img.c_str());
+
+       r = process2.spawn();
+       if (r < 0) {
+               prt("failed to run rbd-wnbd show, error: %s\n", process2.err().c_str());
+               return r;
+       }
+
+       // This should be more than enough for the "rbd-wnbd show" output
+       char map_info[4096];
+       r = safe_read(process2.get_stdout(), map_info, sizeof(map_info));
+       if (r < 0) {
+               prt("unable to retrieve rbd-wnbd map info, error: %d\n", r);
+               return r;
+       }
+
+       r = process2.join();
+       if (r) {
+               prt("rbd-wnbd show failed, error: %s\n", process2.err().c_str());
+               return -EINVAL;
+       }
+
+       std::string dev_path;
+       std::regex dev_num_re("\"disk_number\": ([0-9]+)");
+       std::smatch cm;
+       std::string map_info_str(map_info);
+       if (std::regex_search(map_info_str, cm, dev_num_re)) {
+               dev_path = std::string("\\\\.\\PhysicalDrive") + cm.str(1);
+       } else {
+               prt("unable to retrieve rbd-wnbd device number, device info: %s\n",
+                   map_info);
+               return -EINVAL;
+       }
+
+       // TODO: consider applying o_direct (e.g. using CreateFile).
+       fd = open(dev_path.c_str(), O_RDWR | O_BINARY);
+       if (fd < 0) {
+               r = -errno;
+               prt("open(%s) failed\n", dev_path.c_str());
+               return r;
+       }
+
+       ctx->krbd_name = strdup(img.c_str());
+       ctx->krbd_fd = fd;
+
+       return wnbd_set_writable(ctx);
+}
+
+int
+wnbd_close(struct rbd_ctx *ctx)
+{
+       int r;
+
+       ceph_assert(ctx->krbd_name && ctx->krbd_fd >= 0);
+
+       if (close(ctx->krbd_fd) < 0) {
+               r = -errno;
+               prt("close(%s) failed\n", ctx->krbd_name);
+               return r;
+       }
+
+       SubProcess process("rbd-wnbd");
+       process.add_cmd_arg("unmap");
+       process.add_cmd_arg(ctx->krbd_name);
+
+       r = process.spawn();
+       if (r < 0) {
+               prt("failed to run rbd-wnbd unmap, error: %s\n", process.err().c_str());
+               return r;
+       }
+       r = process.join();
+       if (r) {
+               prt("rbd-wnbd unmap failed, error: %s", process.err().c_str());
+               return -EINVAL;
+       }
+
+       free((void *)ctx->krbd_name);
+
+       ctx->krbd_name = NULL;
+       ctx->krbd_fd = -1;
+
+       return __librbd_close(ctx);
+}
+
+int
+wnbd_flush(struct rbd_ctx *ctx)
+{
+       return _commit(ctx->krbd_fd);
+}
+
+int
+wnbd_discard(struct rbd_ctx *ctx, uint64_t off, uint64_t len)
+{
+       // TODO: use one of the following:
+       // * IOCTL_SCSI_PASS_THROUGH
+       // * IOCTL_SCSI_PASS_THROUGH_DIRECT
+       // * FSCTL_FILE_LEVEL_TRIM
+       // * sg3_utils
+       return -ENOTSUP;
+}
+
+int
+wnbd_get_size(struct rbd_ctx *ctx, uint64_t *size)
+{
+       DWORD bytesReturned = 0;
+
+       GET_LENGTH_INFORMATION length = {0};
+
+       BOOL succeeded = DeviceIoControl(
+               (HANDLE) _get_osfhandle(ctx->krbd_fd),
+               IOCTL_DISK_GET_LENGTH_INFO,
+               NULL,
+               0,
+               (LPVOID) &length,
+               (DWORD) sizeof(length),
+               &bytesReturned,
+               NULL);
+       if (!succeeded) {
+               DWORD err = GetLastError();
+               prt("unable to retrieve wnbd disk size, error: %d\n", err);
+               return -EINVAL;
+       }
+
+       *size = length.Length.QuadPart;
+       return 0;
+}
+
+int
+wnbd_resize(struct rbd_ctx *ctx, uint64_t size)
+{
+       int ret;
+
+       ceph_assert(size % truncbdy == 0);
+
+       ret = wnbd_flush(ctx);
+       if (ret < 0) {
+               return ret;
+       }
+
+       return __librbd_resize(ctx, size);
+}
+
+int
+wnbd_clone(struct rbd_ctx *ctx, const char *src_snapname,
+          const char *dst_imagename, int *order, int stripe_unit,
+          int stripe_count)
+{
+       int ret;
+
+       ret = wnbd_flush(ctx);
+       if (ret < 0) {
+               return ret;
+       }
+
+       return __librbd_clone(ctx, src_snapname, dst_imagename, order,
+                             stripe_unit, stripe_count);
+}
+
+int
+wnbd_flatten(struct rbd_ctx *ctx)
+{
+       int ret;
+
+       ret = wnbd_flush(ctx);
+       if (ret < 0) {
+               return ret;
+       }
+
+       return __librbd_flatten(ctx);
+}
+
+const struct rbd_operations wnbd_operations = {
+       wnbd_open,
+       wnbd_close,
+       generic_pread,
+       generic_pwrite,
+       wnbd_flush,
+       wnbd_discard,
+       wnbd_get_size,
+       wnbd_resize,
+       wnbd_clone,
+       wnbd_flatten,
+       NULL,
+};
+#endif // _WIN32
+
 #if defined(__FreeBSD__)
 int
 ggate_open(const char *name, struct rbd_ctx *ctx)
@@ -1457,36 +1711,6 @@ ggate_close(struct rbd_ctx *ctx)
        return __librbd_close(ctx);
 }
 
-ssize_t
-ggate_read(struct rbd_ctx *ctx, uint64_t off, size_t len, char *buf)
-{
-       ssize_t n;
-
-       n = pread(ctx->krbd_fd, buf, len, off);
-       if (n < 0) {
-               n = -errno;
-               prt("pread(%llu, %zu) failed\n", off, len);
-               return n;
-       }
-
-       return n;
-}
-
-ssize_t
-ggate_write(struct rbd_ctx *ctx, uint64_t off, size_t len, const char *buf)
-{
-       ssize_t n;
-
-       n = pwrite(ctx->krbd_fd, buf, len, off);
-       if (n < 0) {
-               n = -errno;
-               prt("pwrite(%llu, %zu) failed\n", off, len);
-               return n;
-       }
-
-       return n;
-}
-
 int
 __ggate_flush(struct rbd_ctx *ctx, bool invalidate)
 {
@@ -1599,8 +1823,8 @@ ggate_flatten(struct rbd_ctx *ctx)
 const struct rbd_operations ggate_operations = {
        ggate_open,
        ggate_close,
-       ggate_read,
-       ggate_write,
+       generic_pread,
+       generic_pwrite,
        ggate_flush,
        ggate_discard,
        ggate_get_size,
@@ -2045,7 +2269,7 @@ doread(unsigned offset, unsigned size)
        int ret;
 
        offset -= offset % readbdy;
-       if (o_direct)
+       if (o_direct || ops == &wnbd_operations)
                size -= size % readbdy;
        if (size == 0) {
                if (!quiet && testcalls > simulatedopcount && !o_direct)
@@ -2135,7 +2359,7 @@ dowrite(unsigned offset, unsigned size)
        off_t newsize;
 
        offset -= offset % writebdy;
-       if (o_direct)
+       if (o_direct || ops == &wnbd_operations)
                size -= size % writebdy;
        if (size == 0) {
                if (!quiet && testcalls > simulatedopcount && !o_direct)
@@ -2948,6 +3172,9 @@ usage(void)
 #if defined(__linux__)
 "      -M: enable rbd-nbd mode (use -t and -h too)\n"
 #endif
+#if defined(_WIN32)
+"      -M: enable rbd-wnbd mode (use -L, -r and -w too)\n"
+#endif
 "      -L: fsxLite - no file creations & no file size changes\n\
        -N numops: total # operations to do (default infinity)\n\
        -O: use oplen (see -o flag) for every op (default random)\n\
@@ -3225,6 +3452,12 @@ main(int argc, char **argv)
                        prt("rbd-nbd mode enabled\n");
                        ops = &nbd_operations;
                        break;
+#endif
+#if defined(_WIN32)
+               case 'M':
+                       prt("rbd-wnbd mode enabled\n");
+                       ops = &wnbd_operations;
+                       break;
 #endif
                case 'L':
                        lite = 1;