*
*/
+#include <objidl.h>
+// LOCK_WRITE is also defined by objidl.h, we have to avoid
+// a collision.
+#undef LOCK_WRITE
+
#include "include/int_types.h"
+#include <atomic>
#include <stdio.h>
#include <stdlib.h>
#include <stddef.h>
#include <unistd.h>
#include "wnbd_handler.h"
+#include "wnbd_wmi.h"
#include "rbd_wnbd.h"
#include <fstream>
using namespace std;
+// Wait 2s before recreating the wmi subscription in case of errors
+#define WMI_SUBSCRIPTION_RETRY_INTERVAL 2
+// SCSI adapter modification events aren't received until the entire polling
+// interval has elapsed (unlike other WMI classes, such as Msvm_ComputerSystem).
+// With longer intervals, it even seems to miss events. For this reason,
+// we're using a relatively short interval but have adapter state monitoring
+// as an optional feature, mainly used for dev / driver certification purposes.
+#define WNBD_ADAPTER_WMI_POLL_INTERVAL 2
+// Wait for wmi events up to two seconds
+#define WMI_EVENT_TIMEOUT 2
+
bool is_process_running(DWORD pid)
{
HANDLE process = OpenProcess(SYNCHRONIZE, FALSE, pid);
WNBDActiveDiskIterator::WNBDActiveDiskIterator()
{
DWORD status = WNBDActiveDiskIterator::fetch_list(&conn_list);
- if (status) {
+ switch (status) {
+ case 0:
+ // no error
+ break;
+ case ERROR_OPEN_FAILED:
+ error = ENOENT;
+ break;
+ default:
error = EINVAL;
+ break;
}
}
{
Config cfg;
WNBDDiskIterator iterator;
- int err = 0, r;
+ int r;
+ std::atomic<int> err = 0;
+
+ dout(0) << "remounting persistent disks" << dendl;
int total_timeout_ms = max(total_timeout, total_timeout * 1000);
int image_map_timeout_ms = max(image_map_timeout, image_map_timeout * 1000);
}
boost::asio::post(pool,
- [&, cfg]() mutable
+ [cfg, start_t, counter_freq, total_timeout_ms,
+ image_map_timeout_ms, &err]()
{
LARGE_INTEGER curr_t, elapsed_ms;
QueryPerformanceCounter(&curr_t);
// We'll try to map all devices and return a non-zero value
// if any of them fails.
- r = map_device_using_suprocess(cfg.command_line, time_left_ms);
+ int r = map_device_using_suprocess(cfg.command_line, time_left_ms);
if (r) {
err = r;
derr << "Could not create mapping: "
Config cfg;
WNBDActiveDiskIterator iterator;
- int err = 0, r;
+ int r;
+ std::atomic<int> err = 0;
boost::asio::thread_pool pool(worker_count);
LARGE_INTEGER start_t, counter_freq;
QueryPerformanceCounter(&start_t);
while (iterator.get(&cfg)) {
boost::asio::post(pool,
- [&, cfg]() mutable
+ [cfg, start_t, counter_freq, timeout_ms,
+ hard_disconnect, unregister, &err]() mutable
{
LARGE_INTEGER curr_t, elapsed_ms;
QueryPerformanceCounter(&curr_t);
<< "s. Hard disconnect: " << cfg.hard_disconnect
<< dendl;
- r = do_unmap(&cfg, unregister);
+ int r = do_unmap(&cfg, unregister);
if (r) {
err = r;
derr << "Could not remove mapping: " << cfg.devpath
pool.join();
r = iterator.get_error();
- if (r) {
+ if (r == ENOENT) {
+ dout(0) << __func__ << ": wnbd adapter unavailable, "
+ << "assuming that no wnbd mappings exist." << dendl;
+ err = 0;
+ } else if (r) {
derr << "Could not fetch all mappings. Error: " << r << dendl;
err = r;
}
int service_start_timeout;
int image_map_timeout;
bool remap_failure_fatal;
+ bool adapter_monitoring_enabled;
+
+ std::thread adapter_monitor_thread;
+
+ ceph::mutex start_lock = ceph::make_mutex("RBDService::StartLocker");
+ ceph::mutex shutdown_lock = ceph::make_mutex("RBDService::ShutdownLocker");
+ bool started = false;
+ std::atomic<bool> stop_requsted = false;
public:
RBDService(bool _hard_disconnect,
int _thread_count,
int _service_start_timeout,
int _image_map_timeout,
- bool _remap_failure_fatal)
+ bool _remap_failure_fatal,
+ bool _adapter_monitoring_enabled)
: ServiceBase(g_ceph_context)
, hard_disconnect(_hard_disconnect)
, soft_disconnect_timeout(_soft_disconnect_timeout)
, service_start_timeout(_service_start_timeout)
, image_map_timeout(_image_map_timeout)
, remap_failure_fatal(_remap_failure_fatal)
+ , adapter_monitoring_enabled(_adapter_monitoring_enabled)
{
}
return err;
}
+ void monitor_wnbd_adapter()
+ {
+ dout(5) << __func__ << ": initializing COM" << dendl;
+ // Initialize the Windows COM library for this thread.
+ COMBootstrapper com_bootstrapper;
+ HRESULT hres = com_bootstrapper.initialize();
+ if (FAILED(hres)) {
+ return;
+ }
+
+ WmiSubscription subscription = subscribe_wnbd_adapter_events(
+ WNBD_ADAPTER_WMI_POLL_INTERVAL);
+ dout(5) << __func__ << ": initializing wmi subscription" << dendl;
+ hres = subscription.initialize();
+
+ dout(0) << "monitoring wnbd adapter state changes" << dendl;
+ // The event watcher will wait at most WMI_EVENT_TIMEOUT (2s)
+ // and exit the loop if the service is being stopped.
+ while (!stop_requsted) {
+ IWbemClassObject* object;
+ ULONG returned = 0;
+
+ if (FAILED(hres)) {
+ derr << "couldn't retrieve wnbd adapter events, wmi hresult: "
+ << hres << ". Reestablishing wmi listener in "
+ << WMI_SUBSCRIPTION_RETRY_INTERVAL << " seconds." << dendl;
+ subscription.close();
+ Sleep(WMI_SUBSCRIPTION_RETRY_INTERVAL * 1000);
+
+ dout(20) << "recreating wnbd adapter wmi subscription" << dendl;
+ subscription = subscribe_wnbd_adapter_events(
+ WNBD_ADAPTER_WMI_POLL_INTERVAL);
+ hres = subscription.initialize();
+ continue;
+ }
+
+ dout(20) << "fetching wnbd adapter events" << dendl;
+ hres = subscription.next(
+ WMI_EVENT_TIMEOUT * 1000,
+ 1, // we'll process one event at a time
+ &object,
+ &returned);
+
+ if (!FAILED(hres) && returned) {
+ if (WBEM_S_NO_ERROR == object->InheritsFrom(L"__InstanceCreationEvent")) {
+ dout(0) << "wnbd adapter (re)created, remounting disks" << dendl;
+ restart_registered_mappings(
+ thread_count, service_start_timeout, image_map_timeout);
+ } else if (WBEM_S_NO_ERROR == object->InheritsFrom(L"__InstanceDeletionEvent")) {
+ dout(0) << "wnbd adapter removed" << dendl;
+ // nothing to do here
+ } else if (WBEM_S_NO_ERROR == object->InheritsFrom(L"__InstanceModificationEvent")) {
+ dout(0) << "wnbd adapter changed" << dendl;
+ // TODO: look for state changes and log the availability/status
+ }
+
+ object->Release();
+ }
+ }
+
+ dout(10) << "service stop requested, wnbd event monitor exited" << dendl;
+ }
+
int run_hook() override {
+ std::unique_lock l{start_lock};
+ if (started) {
+ // The run hook is only supposed to be called once per process,
+ // however we're staying cautious.
+ derr << "Service already running." << dendl;
+ return -EALREADY;
+ }
+
+ started = true;
// Restart registered mappings before accepting new ones.
int r = restart_registered_mappings(
thread_count, service_start_timeout, image_map_timeout);
}
}
+ if (adapter_monitoring_enabled) {
+ adapter_monitor_thread = std::thread(&monitor_wnbd_adapter, this);
+ } else {
+ dout(0) << "WNBD adapter monitoring disabled." << dendl;
+ }
+
return create_pipe_server();
}
// Invoked when the service is requested to stop.
int stop_hook() override {
- return disconnect_all_mappings(
+ std::unique_lock l{shutdown_lock};
+
+ stop_requsted = true;
+
+ int r = disconnect_all_mappings(
false, hard_disconnect, soft_disconnect_timeout, thread_count);
+
+ if (adapter_monitor_thread.joinable()) {
+ dout(10) << "waiting for wnbd event monitor thread" << dendl;
+ adapter_monitor_thread.join();
+ dout(10) << "wnbd event monitor stopped" << dendl;
+ }
+
+ return r;
}
+
// Invoked when the system is shutting down.
int shutdown_hook() override {
return stop_hook();
unflushed caches or open handles. Default: 15
Service options:
- --hard-disconnect Skip attempting a soft disconnect
- --soft-disconnect-timeout Cummulative soft disconnect timeout in seconds,
- used when disconnecting existing mappings. A hard
- disconnect will be issued when hitting the timeout
- --service-thread-count The number of workers used when mapping or
- unmapping images. Default: 8
- --start-timeout The service start timeout in seconds. Default: 120
- --map-timeout Individual image map timeout in seconds. Default: 20
- --remap-failure-fatal If set, the service will stop when failing to remap
- an image at start time, unmapping images that have
- been mapped so far.
+ --hard-disconnect Skip attempting a soft disconnect
+ --soft-disconnect-timeout Cummulative soft disconnect timeout in seconds,
+ used when disconnecting existing mappings. A hard
+ disconnect will be issued when hitting the timeout
+ --service-thread-count The number of workers used when mapping or
+ unmapping images. Default: 8
+ --start-timeout The service start timeout in seconds. Default: 120
+ --map-timeout Individual image map timeout in seconds. Default: 20
+ --remap-failure-fatal If set, the service will stop when failing to remap
+ an image at start time, unmapping images that have
+ been mapped so far.
+ --adapter-monitoring-enabled If set, the service will monitor WNBD adapter WMI
+ events and remount the images when the adapter gets
+ recreated. Mainly used for development and driver
+ certification purposes.
Show|List options:
--format plain|json|xml Output format (default: plain)
cfg->pretty_format = true;
} else if (ceph_argparse_flag(args, i, "--remap-failure-fatal", (char *)NULL)) {
cfg->remap_failure_fatal = true;
+ } else if (ceph_argparse_flag(args, i, "--adapter-monitoring-enabled", (char *)NULL)) {
+ cfg->adapter_monitoring_enabled = true;
} else if (ceph_argparse_witharg(args, i, &cfg->parent_pipe, err,
"--pipe-name", (char *)NULL)) {
if (!err.str().empty()) {
cfg.service_thread_count,
cfg.service_start_timeout,
cfg.image_map_timeout,
- cfg.remap_failure_fatal);
+ cfg.remap_failure_fatal,
+ cfg.adapter_monitoring_enabled);
// This call will block until the service stops.
r = RBDService::initialize(&service);
if (r < 0)
--- /dev/null
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (c) 2019 SUSE LLC
+ * Copyright (C) 2022 Cloudbase Solutions
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "wnbd_wmi.h"
+
+#include "common/debug.h"
+#include "common/win32/wstring.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd-wnbd: "
+
+// Initializes the COM library for use by the calling thread using
+// COINIT_MULTITHREADED.
+static HRESULT co_initialize_basic()
+{
+ dout(10) << "initializing COM library" << dendl;
+
+ HRESULT hres = CoInitializeEx(0, COINIT_MULTITHREADED);
+ if (FAILED(hres)) {
+ derr << "CoInitializeEx failed. HRESULT: " << hres << dendl;
+ return hres;
+ }
+
+ // CoInitializeSecurity must be called once per process.
+ static bool com_security_flags_set = false;
+
+ if (!com_security_flags_set) {
+ hres = CoInitializeSecurity(
+ NULL, -1, NULL, NULL,
+ RPC_C_AUTHN_LEVEL_DEFAULT,
+ RPC_C_IMP_LEVEL_IMPERSONATE,
+ NULL,
+ EOAC_NONE,
+ NULL);
+ if (FAILED(hres)) {
+ derr << "CoInitializeSecurity failed. HRESULT: " << hres << dendl;
+ CoUninitialize();
+ return hres;
+ }
+
+ com_security_flags_set = true;
+ }
+
+ return 0;
+}
+
+// co_uninitialize must be called once for every successful
+// co_initialize_basic call. Any WMI objects (including connections,
+// event subscriptions, etc) must be released beforehand.
+static void co_uninitialize()
+{
+ dout(10) << "closing COM library" << dendl;
+ CoUninitialize();
+}
+
+HRESULT COMBootstrapper::initialize()
+{
+ std::unique_lock l{init_lock};
+
+ HRESULT hres = co_initialize_basic();
+ if (!FAILED(hres)) {
+ initialized = true;
+ }
+ return hres;
+}
+
+void COMBootstrapper::cleanup()
+{
+ if (initialized) {
+ co_uninitialize();
+ initialized = false;
+ }
+}
+
+void WmiConnection::close()
+{
+ dout(20) << "closing wmi conn: " << this
+ << ", svc: " << wbem_svc
+ << ", loc: " << wbem_loc << dendl;
+ if (wbem_svc != NULL) {
+ wbem_svc->Release();
+ wbem_svc = NULL;
+ }
+ if (wbem_loc != NULL) {
+ wbem_loc->Release();
+ wbem_loc = NULL;
+ }
+}
+
+HRESULT WmiConnection::initialize()
+{
+ HRESULT hres = CoCreateInstance(
+ CLSID_WbemLocator, 0, CLSCTX_INPROC_SERVER,
+ IID_IWbemLocator, (LPVOID*)&wbem_loc);
+ if (FAILED(hres)) {
+ derr << "CoCreateInstance failed. HRESULT: " << hres << dendl;
+ return hres;
+ }
+
+ hres = wbem_loc->ConnectServer(
+ _bstr_t(ns.c_str()).GetBSTR(), NULL, NULL, NULL,
+ WBEM_FLAG_CONNECT_USE_MAX_WAIT, NULL, NULL,
+ &wbem_svc);
+ if (FAILED(hres)) {
+ derr << "Could not connect to WMI service. HRESULT: " << hres << dendl;
+ return hres;
+ }
+
+ if (!wbem_svc) {
+ hres = MAKE_HRESULT(SEVERITY_ERROR, FACILITY_WIN32,
+ ERROR_INVALID_HANDLE);
+ derr << "WMI connection failed, no WMI service object received." << dendl;
+ return hres;
+ }
+
+ hres = CoSetProxyBlanket(
+ wbem_svc, RPC_C_AUTHN_WINNT, RPC_C_AUTHZ_NONE, NULL,
+ RPC_C_AUTHN_LEVEL_CALL, RPC_C_IMP_LEVEL_IMPERSONATE, NULL, EOAC_NONE);
+ if (FAILED(hres)) {
+ derr << "CoSetProxyBlanket failed. HRESULT:" << hres << dendl;
+ }
+
+ return hres;
+}
+
+HRESULT get_property_str(
+ IWbemClassObject* cls_obj,
+ const std::wstring& property,
+ std::wstring& value)
+{
+ VARIANT vt_prop;
+ VariantInit(&vt_prop);
+ HRESULT hres = cls_obj->Get(property.c_str(), 0, &vt_prop, 0, 0);
+ if (!FAILED(hres)) {
+ VARIANT vt_bstr_prop;
+ VariantInit(&vt_bstr_prop);
+ hres = VariantChangeType(&vt_bstr_prop, &vt_prop, 0, VT_BSTR);
+ if (!FAILED(hres)) {
+ value = vt_bstr_prop.bstrVal;
+ }
+ VariantClear(&vt_bstr_prop);
+ }
+ VariantClear(&vt_prop);
+
+ if (FAILED(hres)) {
+ derr << "Could not get WMI property: " << to_string(property)
+ << ". HRESULT: " << hres << dendl;
+ }
+ return hres;
+}
+
+HRESULT get_property_int(
+ IWbemClassObject* cls_obj,
+ const std::wstring& property,
+ uint32_t& value)
+{
+ VARIANT vt_prop;
+ VariantInit(&vt_prop);
+ HRESULT hres = cls_obj->Get(property.c_str(), 0, &vt_prop, 0, 0);
+ if (!FAILED(hres)) {
+ VARIANT vt_uint_prop;
+ VariantInit(&vt_uint_prop);
+ hres = VariantChangeType(&vt_uint_prop, &vt_prop, 0, VT_UINT);
+ if (!FAILED(hres)) {
+ value = vt_uint_prop.intVal;
+ }
+ VariantClear(&vt_uint_prop);
+ }
+ VariantClear(&vt_prop);
+
+ if (FAILED(hres)) {
+ derr << "Could not get WMI property: " << to_string(property)
+ << ". HRESULT: " << hres << dendl;
+ }
+ return hres;
+}
+
+HRESULT WmiSubscription::initialize()
+{
+ HRESULT hres = conn.initialize();
+ if (FAILED(hres)) {
+ derr << "Could not create WMI connection" << dendl;
+ return hres;
+ }
+
+ hres = conn.wbem_svc->ExecNotificationQuery(
+ _bstr_t(L"WQL").GetBSTR(),
+ _bstr_t(query.c_str()).GetBSTR(),
+ WBEM_FLAG_FORWARD_ONLY | WBEM_FLAG_RETURN_IMMEDIATELY,
+ NULL,
+ &event_enum);
+
+ if (FAILED(hres)) {
+ derr << "Notification query failed, unable to subscribe to "
+ << "WMI events. HRESULT: " << hres << dendl;
+ } else {
+ dout(20) << "wmi subscription initialized: " << this
+ << ", event enum: " << event_enum
+ << ", conn: " << &conn << ", conn svc: " << conn.wbem_svc << dendl;
+ }
+
+ return hres;
+}
+
+void WmiSubscription::close()
+{
+ dout(20) << "closing wmi subscription: " << this
+ << ", event enum: " << event_enum << dendl;
+ if (event_enum != NULL) {
+ event_enum->Release();
+ event_enum = NULL;
+ }
+}
+
+HRESULT WmiSubscription::next(
+ long timeout,
+ ULONG count,
+ IWbemClassObject **objects,
+ ULONG *returned)
+{
+ if (!event_enum) {
+ HRESULT hres = MAKE_HRESULT(
+ SEVERITY_ERROR, FACILITY_WIN32,
+ ERROR_INVALID_HANDLE);
+ derr << "WMI subscription uninitialized." << dendl;
+ return hres;
+ }
+
+ HRESULT hres = event_enum->Next(timeout, count, objects, returned);
+ if (FAILED(hres)) {
+ derr << "Unable to retrieve WMI events. HRESULT: "
+ << hres << dendl;
+ }
+ return hres;
+}
+
+WmiSubscription subscribe_wnbd_adapter_events(
+ uint32_t interval)
+{
+ std::wostringstream query_stream;
+ query_stream
+ << L"SELECT * FROM __InstanceOperationEvent "
+ << L"WITHIN " << interval
+ << L"WHERE TargetInstance ISA 'Win32_ScsiController' "
+ << L"AND TargetInstance.Description="
+ << L"'WNBD SCSI Virtual Adapter'";
+
+ return WmiSubscription(L"root\\cimv2", query_stream.str());
+}
projects / ceph.git / commitdiff