ceph-volume: use 'no workqueue' options with dmcrypt

CloudFlare engineers made some testing and realized that using
workqueues with encryption on flash devices has a bad effect.

See [1] for details.

With this patch it will make ceph-volume call crypsetup with
`--perf-no_read_workqueue` and `--perf-no_write_workqueue` options
when the device is not a rotational.

[1] https://blog.cloudflare.com/speeding-up-linux-disk-encryption/

Fixes: https://tracker.ceph.com/issues/64195
Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
Co-Authored-by: Stefan Kooman <stefan@kooman.org>
(cherry picked from commit 0985e201342fa53c014a811156aed661b4b8f994)
(cherry picked from commit 956305eb5caf323cfadb772a9a1f910a90aa7740)

diff --git a/ceph.spec.in b/ceph.spec.in
index bb7d69de6e2027ef0ead7428b1aaeddd28f97bbd..77ec17499919e71d26f3bd4b0311234e16cca707 100644 (file)
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -889,6 +889,7 @@ Requires: parted
 Requires: util-linux
 Requires: xfsprogs
 Requires: python%{python3_pkgversion}-setuptools
+Requires: python%{python3_pkgversion}-packaging
 Requires: python%{python3_pkgversion}-ceph-common = %{_epoch_prefix}%{version}-%{release}
 %description volume
 This package contains a tool to deploy OSD with different devices like

diff --git a/src/ceph-volume/ceph_volume/__init__.py b/src/ceph-volume/ceph_volume/__init__.py
index b36b9f3193ab3eabea733ce43291d2f5a06759ed..addddd031af622ab2e8ff650c6e9cdb8bc266e16 100644 (file)
--- a/src/ceph-volume/ceph_volume/__init__.py
+++ b/src/ceph-volume/ceph_volume/__init__.py
@@ -14,8 +14,9 @@ class UnloadedConfig(object):
     def __getattr__(self, *a):
         raise RuntimeError("No valid ceph configuration file was loaded.")
 
-conf = namedtuple('config', ['ceph', 'cluster', 'verbosity', 'path', 'log_path'])
+conf = namedtuple('config', ['ceph', 'cluster', 'verbosity', 'path', 'log_path', 'dmcrypt_no_workqueue'])
 conf.ceph = UnloadedConfig()
+conf.dmcrypt_no_workqueue = None
 
 __version__ = "1.0.0"
 

diff --git a/src/ceph-volume/ceph_volume/devices/lvm/activate.py b/src/ceph-volume/ceph_volume/devices/lvm/activate.py
index 53ed6aa47918e786ae6109b3299cf26f60cec444..d586765d063a8fc43d2622c501ac919aac1343a8 100644 (file)
--- a/src/ceph-volume/ceph_volume/devices/lvm/activate.py
+++ b/src/ceph-volume/ceph_volume/devices/lvm/activate.py
@@ -149,6 +149,8 @@ def activate_bluestore(osd_lvs, no_systemd=False, no_tmpfs=False):
         raise RuntimeError('could not find a bluestore OSD to activate')
 
     is_encrypted = osd_block_lv.tags.get('ceph.encrypted', '0') == '1'
+    if is_encrypted and conf.dmcrypt_no_workqueue is None:
+        encryption_utils.set_dmcrypt_no_workqueue()
     dmcrypt_secret = None
     osd_id = osd_block_lv.tags['ceph.osd_id']
     conf.cluster = osd_block_lv.tags['ceph.cluster_name']

diff --git a/src/ceph-volume/ceph_volume/devices/lvm/batch.py b/src/ceph-volume/ceph_volume/devices/lvm/batch.py
index d867fe2d87e3bb33c30bbce277433095449d80f1..e0be775a02501fffbc2df2763575f8d9a7dffde4 100644 (file)
--- a/src/ceph-volume/ceph_volume/devices/lvm/batch.py
+++ b/src/ceph-volume/ceph_volume/devices/lvm/batch.py
@@ -269,7 +269,7 @@ class Batch(object):
         )
         parser.add_argument(
             '--dmcrypt',
-            action='store_true',
+            action=arg_validators.DmcryptAction,
             help='Enable device encryption via dm-crypt',
         )
         parser.add_argument(

diff --git a/src/ceph-volume/ceph_volume/devices/lvm/common.py b/src/ceph-volume/ceph_volume/devices/lvm/common.py
index edc8e1cbce117c377b60bf36d97b04090b9072e5..d2522b3d0c1250b195b1063f937f98a63b432717 100644 (file)
--- a/src/ceph-volume/ceph_volume/devices/lvm/common.py
+++ b/src/ceph-volume/ceph_volume/devices/lvm/common.py
@@ -73,7 +73,7 @@ common_args = {
         'default': "",
     },
     '--dmcrypt': {
-        'action': 'store_true',
+        'action': arg_validators.DmcryptAction,
         'help': 'Enable device encryption via dm-crypt',
     },
     '--no-systemd': {

diff --git a/src/ceph-volume/ceph_volume/devices/raw/common.py b/src/ceph-volume/ceph_volume/devices/raw/common.py
index 89ee285be5b423e6a1d3b3d1a32efbdbf6a735b1..4863b9e18e05ae763e6041009c77449215948ffd 100644 (file)
--- a/src/ceph-volume/ceph_volume/devices/raw/common.py
+++ b/src/ceph-volume/ceph_volume/devices/raw/common.py
@@ -46,7 +46,7 @@ def create_parser(prog, description):
     )
     parser.add_argument(
         '--dmcrypt',
-        action='store_true',
+        action=arg_validators.DmcryptAction,
         help='Enable device encryption via dm-crypt',
     )
     parser.add_argument(

diff --git a/src/ceph-volume/ceph_volume/util/arg_validators.py b/src/ceph-volume/ceph_volume/util/arg_validators.py
index 655f7cd55ed0f9d882e72a255a267da0524930ee..a6e7633d1d10664bc7f276f98bf77be754ff5b9c 100644 (file)
--- a/src/ceph-volume/ceph_volume/util/arg_validators.py
+++ b/src/ceph-volume/ceph_volume/util/arg_validators.py
@@ -4,11 +4,20 @@ import math
 from ceph_volume import terminal, decorators, process
 from ceph_volume.util.device import Device
 from ceph_volume.util import disk
-
+from ceph_volume.util.encryption import set_dmcrypt_no_workqueue
+from ceph_volume import process, conf
 
 def valid_osd_id(val):
     return str(int(val))
 
+class DmcryptAction(argparse._StoreTrueAction):
+    def __init__(self, *args, **kwargs):
+        super(DmcryptAction, self).__init__(*args, **kwargs)
+
+    def __call__(self, *args, **kwargs):
+        set_dmcrypt_no_workqueue()
+        super(DmcryptAction, self).__call__(*args, **kwargs)
+
 class ValidDevice(object):
 
     def __init__(self, as_string=False, gpt_ok=False):

diff --git a/src/ceph-volume/ceph_volume/util/device.py b/src/ceph-volume/ceph_volume/util/device.py
index 151ddcfcce78f4222d0bfbab2b8770518b62033b..4911b3f9a46b14501d9aa4afbc9fbc41428be243 100644 (file)
--- a/src/ceph-volume/ceph_volume/util/device.py
+++ b/src/ceph-volume/ceph_volume/util/device.py
@@ -120,13 +120,8 @@ class Device(object):
             # check if we are not a device mapper
             if "dm-" not in real_path:
                 self.path = real_path
-        if not sys_info.devices:
-            if self.path:
-                sys_info.devices = disk.get_devices(device=self.path)
-            else:
-                sys_info.devices = disk.get_devices()
-        if sys_info.devices.get(self.path, {}):
-            self.device_nodes = sys_info.devices[self.path]['device_nodes']
+        if not sys_info.devices.get(self.path):
+            sys_info.devices = disk.get_devices()
         self.sys_api = sys_info.devices.get(self.path, {})
         self.partitions = self._get_partitions()
         self.lv_api = None
@@ -142,6 +137,7 @@ class Device(object):
         self._is_lvm_member = None
         self.ceph_device = False
         self._parse()
+        self.device_nodes = sys_info.devices[self.path]['device_nodes']
         self.lsm_data = self.fetch_lsm(with_lsm)
 
         self.available_lvm, self.rejected_reasons_lvm = self._check_lvm_reject_reasons()

diff --git a/src/ceph-volume/ceph_volume/util/encryption.py b/src/ceph-volume/ceph_volume/util/encryption.py
index 0831d859700580f1f21149553810ca23e22f5849..732e3310920722c0e20a101285d071d6791103e5 100644 (file)
--- a/src/ceph-volume/ceph_volume/util/encryption.py
+++ b/src/ceph-volume/ceph_volume/util/encryption.py
@@ -6,10 +6,28 @@ from ceph_volume.util import constants, system
 from ceph_volume.util.device import Device
 from .prepare import write_keyring
 from .disk import lsblk, device_family, get_part_entry_type
+from packaging import version
 
 logger = logging.getLogger(__name__)
 mlogger = terminal.MultiLogger(__name__)
 
+def set_dmcrypt_no_workqueue(target_version: str = '2.3.4') -> None:
+    """
+    set `conf.dmcrypt_no_workqueue` to `True` if the available
+    version of `cryptsetup` is greater or equal to `version`
+    """
+    command = ["cryptsetup", "--version"]
+    out, err, rc = process.call(command)
+    try:
+        if version.parse(out[0]) >= version.parse(f'cryptsetup {target_version}'):
+            conf.dmcrypt_no_workqueue = True
+    except IndexError:
+        mlogger.debug(f'cryptsetup version check: rc={rc} out={out} err={err}')
+        raise RuntimeError("Couldn't check the cryptsetup version.")
+
+def bypass_workqueue(device: str) -> bool:
+    return not Device(device).rotational and conf.dmcrypt_no_workqueue
+
 def get_key_size_from_conf():
     """
     Return the osd dmcrypt key size from config file.
@@ -79,6 +97,10 @@ def plain_open(key, device, mapping):
         '--key-size', '256',
     ]
 
+    if bypass_workqueue(device):
+        command.extend(['--perf-no_read_workqueue',
+                        '--perf-no_write_workqueue'])
+
     process.call(command, stdin=key, terminal_verbose=True, show_command=True)
 
 
@@ -103,6 +125,11 @@ def luks_open(key, device, mapping):
         device,
         mapping,
     ]
+
+    if bypass_workqueue(device):
+        command.extend(['--perf-no_read_workqueue',
+                        '--perf-no_write_workqueue'])
+
     process.call(command, stdin=key, terminal_verbose=True, show_command=True)