ceph-volume: retry lvs after empty result and "devices file is missing" stderr

When LVM's devices file is out of sync with the runtime device view (common
in teuthology/container namespaces with multipath), `lvs` can exit 0 with
empty stdout and only stderr warnings about missing mapper entries.
It can leave get_lvs() empty and cause Device() to fall through to lsblk on a
vg/lv path which can produce a misleading "not a block device" error.

With this fix, ceph-volume retries once with 'use_devicesfile=0' when it
detects this specific pattern.

Fixes: https://tracker.ceph.com/issues/76959
Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>

diff --git a/src/ceph-volume/ceph_volume/api/lvm.py b/src/ceph-volume/ceph_volume/api/lvm.py
index c530265e9cda4d79447442e45295004803b1be48..ecab3a239a0f4dac627f0d7e7360d1c950ee6f3e 100644 (file)
--- a/src/ceph-volume/ceph_volume/api/lvm.py
+++ b/src/ceph-volume/ceph_volume/api/lvm.py
@@ -70,6 +70,18 @@ def make_filters_lvmcmd_ready(filters: Dict[str, Any], tags: Dict[str, Any]) ->
         return ''
 
 
+def should_retry_lvs_without_devices_file(stdout: List[str],
+                                           stderr: List[str],
+                                           returncode: int) -> bool:
+    """
+    Detect an LVM 'devices file' mismatch scenario where `lvs` can return no data
+    and only emits warnings on stderr.
+    """
+    if returncode != 0 or stdout:
+        return False
+    return any('devices file is missing' in line.lower() for line in stderr)
+
+
 def _output_parser(output: List[str], fields: str) -> List[Dict[str, Any]]:
     """
     Newer versions of LVM allow ``--reportformat=json``, but older versions,
@@ -1177,6 +1189,21 @@ def get_lvs(fields: str = LV_FIELDS, filters: Optional[Dict[str, Any]] = None, t
     args = ['lvs'] + LV_CMD_OPTIONS + ['-S', filters_str, '-o', fields]
 
     stdout, stderr, returncode = process.call(args, run_on_host=True, verbose_on_failure=False)
+    if should_retry_lvs_without_devices_file(stdout, stderr, returncode):
+        logger.warning(
+            'lvs returned no output with devices-file warnings; retrying with use_devicesfile=0'
+        )
+        retry_args = ['lvs'] + LV_CMD_OPTIONS + [
+            '--config',
+            'devices { use_devicesfile=0 }',
+            '-S',
+            filters_str,
+            '-o',
+            fields,
+        ]
+        stdout, stderr, returncode = process.call(
+            retry_args, run_on_host=True, verbose_on_failure=False
+        )
     lvs_report = _output_parser(stdout, fields)
     return [Volume(**lv_report) for lv_report in lvs_report]
 

diff --git a/src/ceph-volume/ceph_volume/tests/api/test_api.py b/src/ceph-volume/ceph_volume/tests/api/test_api.py
index 93743226dca153a35d06f4f3c0827bbbfbfbea24..61815221a42624a50ce4865425039023925b1850 100644 (file)
--- a/src/ceph-volume/ceph_volume/tests/api/test_api.py
+++ b/src/ceph-volume/ceph_volume/tests/api/test_api.py
@@ -837,6 +837,31 @@ class TestGetLVs(object):
         monkeypatch.setattr(api.process, 'call', lambda x,**kw: ('', '', 0))
         assert api.get_lvs() == []
 
+    def test_get_lvs_retries_without_devices_file_on_warning(self, monkeypatch):
+        retry_stdout = ['ceph.type=data;/dev/vg/lv;lv;vg;lv-uuid;1024']
+        calls = []
+
+        def fake_call(cmd, **kw):
+            calls.append(cmd)
+            if len(calls) == 1:
+                return (
+                    [],
+                    ['WARNING: devices file is missing /dev/mapper/eui.foo using multipath component /dev/nvme0n1.'],
+                    0,
+                )
+            return (retry_stdout, [], 0)
+
+        monkeypatch.setattr(api.process, 'call', fake_call)
+
+        lvs_ = api.get_lvs()
+
+        assert len(calls) == 2
+        assert '--config' in calls[1]
+        assert 'devices { use_devicesfile=0 }' in calls[1]
+        assert len(lvs_) == 1
+        assert lvs_[0].lv_name == 'lv'
+        assert lvs_[0].vg_name == 'vg'
+
 
 class TestGetSinglePV(object):