ceph-volume: use 'no workqueue' options with dmcrypt

CloudFlare engineers made some testing and realized that using
workqueues with encryption on flash devices has a bad effect.

See [1] for details.

With this patch it will make ceph-volume call crypsetup with
`--perf-no_read_workqueue` and `--perf-no_write_workqueue` options
when the device is not a rotational.

[1] https://blog.cloudflare.com/speeding-up-linux-disk-encryption/

Fixes: https://tracker.ceph.com/issues/64195
Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
Co-Authored-by: Stefan Kooman <stefan@kooman.org>
(cherry picked from commit 0985e201342fa53c014a811156aed661b4b8f994)

diff --git a/ceph.spec.in b/ceph.spec.in
index 414963e2e4ca8f4df21976329b7bbbf1036237cb..11c093dcbe4427e8fe35ef769b5a04985bca67a4 100644 (file)
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -894,6 +894,7 @@ Requires: parted
 Requires: util-linux
 Requires: xfsprogs
 Requires: python%{python3_pkgversion}-setuptools
+Requires: python%{python3_pkgversion}-packaging
 Requires: python%{python3_pkgversion}-ceph-common = %{_epoch_prefix}%{version}-%{release}
 %description volume
 This package contains a tool to deploy OSD with different devices like

diff --git a/src/ceph-volume/ceph_volume/__init__.py b/src/ceph-volume/ceph_volume/__init__.py
index dad83c95bb7a81d5dae6ad2a27581f54ac2b476e..d0014284d5de60a9f5d35399f545d2001affa2e8 100644 (file)
--- a/src/ceph-volume/ceph_volume/__init__.py
+++ b/src/ceph-volume/ceph_volume/__init__.py
@@ -14,8 +14,9 @@ class UnloadedConfig(object):
     def __getattr__(self, *a):
         raise RuntimeError("No valid ceph configuration file was loaded.")
 
-conf = namedtuple('config', ['ceph', 'cluster', 'verbosity', 'path', 'log_path'])
+conf = namedtuple('config', ['ceph', 'cluster', 'verbosity', 'path', 'log_path', 'dmcrypt_no_workqueue'])
 conf.ceph = UnloadedConfig()
+conf.dmcrypt_no_workqueue = None
 
 __version__ = "1.0.0"
 

diff --git a/src/ceph-volume/ceph_volume/devices/lvm/activate.py b/src/ceph-volume/ceph_volume/devices/lvm/activate.py
index feb91053b4478982c4f973c436aca3ba1b11d7aa..17c66194c67e36f9562aa9145f1922810e03f7a0 100644 (file)
--- a/src/ceph-volume/ceph_volume/devices/lvm/activate.py
+++ b/src/ceph-volume/ceph_volume/devices/lvm/activate.py
@@ -69,6 +69,8 @@ def activate_bluestore(osd_lvs, no_systemd=False, no_tmpfs=False):
         raise RuntimeError('could not find a bluestore OSD to activate')
 
     is_encrypted = osd_block_lv.tags.get('ceph.encrypted', '0') == '1'
+    if is_encrypted and conf.dmcrypt_no_workqueue is None:
+        encryption_utils.set_dmcrypt_no_workqueue()
     dmcrypt_secret = None
     osd_id = osd_block_lv.tags['ceph.osd_id']
     conf.cluster = osd_block_lv.tags['ceph.cluster_name']

diff --git a/src/ceph-volume/ceph_volume/devices/lvm/batch.py b/src/ceph-volume/ceph_volume/devices/lvm/batch.py
index 69a3f672b4825a18b0a45378df2e596615598ab3..2118ce47aeeab5d271a12b7a2d107b2deb5db4ba 100644 (file)
--- a/src/ceph-volume/ceph_volume/devices/lvm/batch.py
+++ b/src/ceph-volume/ceph_volume/devices/lvm/batch.py
@@ -256,7 +256,7 @@ class Batch(object):
         )
         parser.add_argument(
             '--dmcrypt',
-            action='store_true',
+            action=arg_validators.DmcryptAction,
             help='Enable device encryption via dm-crypt',
         )
         parser.add_argument(

diff --git a/src/ceph-volume/ceph_volume/devices/lvm/common.py b/src/ceph-volume/ceph_volume/devices/lvm/common.py
index 35e53181aff07021d83df92eb31d02ef5e4ed4ad..198ba9417a1b5eef0dd1cc0c822a008d88511468 100644 (file)
--- a/src/ceph-volume/ceph_volume/devices/lvm/common.py
+++ b/src/ceph-volume/ceph_volume/devices/lvm/common.py
@@ -73,7 +73,7 @@ common_args = {
         'default': "",
     },
     '--dmcrypt': {
-        'action': 'store_true',
+        'action': arg_validators.DmcryptAction,
         'help': 'Enable device encryption via dm-crypt',
     },
     '--no-systemd': {

diff --git a/src/ceph-volume/ceph_volume/devices/raw/common.py b/src/ceph-volume/ceph_volume/devices/raw/common.py
index 89ee285be5b423e6a1d3b3d1a32efbdbf6a735b1..4863b9e18e05ae763e6041009c77449215948ffd 100644 (file)
--- a/src/ceph-volume/ceph_volume/devices/raw/common.py
+++ b/src/ceph-volume/ceph_volume/devices/raw/common.py
@@ -46,7 +46,7 @@ def create_parser(prog, description):
     )
     parser.add_argument(
         '--dmcrypt',
-        action='store_true',
+        action=arg_validators.DmcryptAction,
         help='Enable device encryption via dm-crypt',
     )
     parser.add_argument(

diff --git a/src/ceph-volume/ceph_volume/util/arg_validators.py b/src/ceph-volume/ceph_volume/util/arg_validators.py
index 1abb5165ec004349531c5c38847dab8ecdd9695f..e936cab895e389dae4975f95adbd25c0c6ca5934 100644 (file)
--- a/src/ceph-volume/ceph_volume/util/arg_validators.py
+++ b/src/ceph-volume/ceph_volume/util/arg_validators.py
@@ -4,11 +4,20 @@ import math
 from ceph_volume import terminal, decorators, process
 from ceph_volume.util.device import Device
 from ceph_volume.util import disk
-
+from ceph_volume.util.encryption import set_dmcrypt_no_workqueue
+from ceph_volume import process, conf
 
 def valid_osd_id(val):
     return str(int(val))
 
+class DmcryptAction(argparse._StoreTrueAction):
+    def __init__(self, *args, **kwargs):
+        super(DmcryptAction, self).__init__(*args, **kwargs)
+
+    def __call__(self, *args, **kwargs):
+        set_dmcrypt_no_workqueue()
+        super(DmcryptAction, self).__call__(*args, **kwargs)
+
 class ValidDevice(object):
 
     def __init__(self, as_string=False, gpt_ok=False):

diff --git a/src/ceph-volume/ceph_volume/util/device.py b/src/ceph-volume/ceph_volume/util/device.py
index bb806292f2c48aab02cfbf9f770187cc17af97ba..f6a4878a3fa30854afb63ad15bd9e858d7a7d509 100644 (file)
--- a/src/ceph-volume/ceph_volume/util/device.py
+++ b/src/ceph-volume/ceph_volume/util/device.py
@@ -121,13 +121,8 @@ class Device(object):
             # check if we are not a device mapper
             if "dm-" not in real_path:
                 self.path = real_path
-        if not sys_info.devices:
-            if self.path:
-                sys_info.devices = disk.get_devices(device=self.path)
-            else:
-                sys_info.devices = disk.get_devices()
-        if sys_info.devices.get(self.path, {}):
-            self.device_nodes = sys_info.devices[self.path]['device_nodes']
+        if not sys_info.devices.get(self.path):
+            sys_info.devices = disk.get_devices()
         self.sys_api = sys_info.devices.get(self.path, {})
         self.partitions = self._get_partitions()
         self.lv_api = None
@@ -143,6 +138,7 @@ class Device(object):
         self._is_lvm_member = None
         self.ceph_device = False
         self._parse()
+        self.device_nodes = sys_info.devices[self.path]['device_nodes']
         self.lsm_data = self.fetch_lsm(with_lsm)
 
         self.available_lvm, self.rejected_reasons_lvm = self._check_lvm_reject_reasons()

diff --git a/src/ceph-volume/ceph_volume/util/encryption.py b/src/ceph-volume/ceph_volume/util/encryption.py
index f8aea80b493539fe618ee239c1e15c097c864657..844a81620d22a0e132bf8346ac72e9a13d080cdf 100644 (file)
--- a/src/ceph-volume/ceph_volume/util/encryption.py
+++ b/src/ceph-volume/ceph_volume/util/encryption.py
@@ -6,10 +6,28 @@ from ceph_volume.util import constants, system
 from ceph_volume.util.device import Device
 from .prepare import write_keyring
 from .disk import lsblk, device_family, get_part_entry_type
+from packaging import version
 
 logger = logging.getLogger(__name__)
 mlogger = terminal.MultiLogger(__name__)
 
+def set_dmcrypt_no_workqueue(target_version: str = '2.3.4') -> None:
+    """
+    set `conf.dmcrypt_no_workqueue` to `True` if the available
+    version of `cryptsetup` is greater or equal to `version`
+    """
+    command = ["cryptsetup", "--version"]
+    out, err, rc = process.call(command)
+    try:
+        if version.parse(out[0]) >= version.parse(f'cryptsetup {target_version}'):
+            conf.dmcrypt_no_workqueue = True
+    except IndexError:
+        mlogger.debug(f'cryptsetup version check: rc={rc} out={out} err={err}')
+        raise RuntimeError("Couldn't check the cryptsetup version.")
+
+def bypass_workqueue(device: str) -> bool:
+    return not Device(device).rotational and conf.dmcrypt_no_workqueue
+
 def get_key_size_from_conf():
     """
     Return the osd dmcrypt key size from config file.
@@ -79,6 +97,10 @@ def plain_open(key, device, mapping):
         '--key-size', '256',
     ]
 
+    if bypass_workqueue(device):
+        command.extend(['--perf-no_read_workqueue',
+                        '--perf-no_write_workqueue'])
+
     process.call(command, stdin=key, terminal_verbose=True, show_command=True)
 
 
@@ -103,6 +125,11 @@ def luks_open(key, device, mapping):
         device,
         mapping,
     ]
+
+    if bypass_workqueue(device):
+        command.extend(['--perf-no_read_workqueue',
+                        '--perf-no_write_workqueue'])
+
     process.call(command, stdin=key, terminal_verbose=True, show_command=True)