From: Tomer Haskalovitch <tomer.haska@ibm.com>
Date: Sun, 23 Nov 2025 07:46:30 +0000 (+0200)
Subject: mgr/nvmeof: introduce nvmeof module and create .nvmeof rbd pool on orch nvmeof apply
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=755cfbf2645f00cc47a74272aa35a1de8e9e6b6d;p=ceph-ci.git

mgr/nvmeof: introduce nvmeof module and create .nvmeof rbd pool on orch nvmeof apply

Introduce a new NVMe-oF mgr module and which create the pool
used for storing NVMe-related metadata ceph orch nvmeof apply command.
This removes the need for users to manually create and configure the
metadata pool before using the NVMe-oF functionality, simplifying
setup and reducing the chance of misconfiguration.
Signed-off-by: Tomer Haskalovitch <tomer.haska@ibm.com>
---

diff --git a/ceph.spec.in b/ceph.spec.in
index 18cd57b2683..f0f6a50c56d 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -2015,6 +2015,7 @@ fi
 %{_datadir}/ceph/mgr/telemetry
 %{_datadir}/ceph/mgr/test_orchestrator
 %{_datadir}/ceph/mgr/volumes
+%{_datadir}/ceph/mgr/nvmeof
 
 %files mgr-rook
 %{_datadir}/ceph/mgr/rook
diff --git a/debian/ceph-mgr-modules-core.install b/debian/ceph-mgr-modules-core.install
index 90359a8e3e7..17ed984b02f 100644
--- a/debian/ceph-mgr-modules-core.install
+++ b/debian/ceph-mgr-modules-core.install
@@ -25,3 +25,4 @@ usr/share/ceph/mgr/telegraf
 usr/share/ceph/mgr/telemetry
 usr/share/ceph/mgr/test_orchestrator
 usr/share/ceph/mgr/volumes
+usr/share/ceph/mgr/nvmeof
diff --git a/src/common/options/mgr.yaml.in b/src/common/options/mgr.yaml.in
index c6bdee1d156..4a21104c471 100644
--- a/src/common/options/mgr.yaml.in
+++ b/src/common/options/mgr.yaml.in
@@ -153,7 +153,7 @@ options:
     first started after installation, to populate the list of enabled manager modules.  Subsequent
     updates are done using the 'mgr module [enable|disable]' commands.  List may be
     comma or space separated.
-  default: iostat nfs
+  default: iostat nfs nvmeof
   services:
   - mon
   - common
diff --git a/src/pybind/mgr/CMakeLists.txt b/src/pybind/mgr/CMakeLists.txt
index 9e900f859d7..d4032eb1dd7 100644
--- a/src/pybind/mgr/CMakeLists.txt
+++ b/src/pybind/mgr/CMakeLists.txt
@@ -27,6 +27,7 @@ set(mgr_modules
   devicehealth
   diskprediction_local
   # hello is an example for developers, not for user
+  nvmeof
   influx
   insights
   iostat
diff --git a/src/pybind/mgr/nvmeof/__init__.py b/src/pybind/mgr/nvmeof/__init__.py
new file mode 100644
index 00000000000..33b8e63a31c
--- /dev/null
+++ b/src/pybind/mgr/nvmeof/__init__.py
@@ -0,0 +1,2 @@
+# flake8: noqa
+from .module import NVMeoF
diff --git a/src/pybind/mgr/nvmeof/module.py b/src/pybind/mgr/nvmeof/module.py
new file mode 100644
index 00000000000..cc62d70006b
--- /dev/null
+++ b/src/pybind/mgr/nvmeof/module.py
@@ -0,0 +1,76 @@
+import logging
+from typing import Any, Tuple
+
+from mgr_module import MgrModule, Option
+import rbd
+
+logger = logging.getLogger(__name__)
+
+POOL_NAME = ".nvmeof"
+PG_NUM = 1
+
+class NVMeoF(MgrModule):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super(NVMeoF, self).__init__(*args, **kwargs)
+
+    def _print_log(self, ret: int, out: str, err: str, cmd: str) -> None:
+        logger.info(f"logging command: {cmd} *** ret:{str(ret)}, out:{out}, err: {err}")
+
+    def _mon_cmd(self, cmd: dict) -> Tuple[int, str, str]:
+        ret, out, err = self.mon_command(cmd)
+        self._print_log(ret, out, err, cmd)
+        if ret != 0:
+            raise RuntimeError(f"mon_command failed: {cmd}, ret={ret}, out={out}, err={err}")
+        return ret, out, err
+
+    def _pool_exists(self, pool_name: str) -> bool:
+        logger.info(f"checking if pool {pool_name} exists")
+        pool_exists = self.rados.pool_exists(pool_name)
+        if pool_exists:
+            logger.info(f"pool {pool_name} already exists")
+        else:
+            logger.info(f"pool {pool_name} doesn't exist")
+        return pool_exists
+
+    def _create_pool(self, pool_name: str, pg_num: int) -> None:
+        create_cmd = {
+            'prefix': 'osd pool create',
+            'pool': pool_name,
+            'pg_num': pg_num,
+            'pool_type': 'replicated',
+            'yes_i_really_mean_it': True
+        }
+        try:
+            self._mon_cmd(create_cmd)
+            logger.info(f"Pool '{pool_name}' created.")
+        except RuntimeError as e:
+            logger.error(f"Error creating pool '{pool_name}", exc_info=True)
+            raise
+
+    def _enable_rbd_application(self, pool_name: str) -> None:
+        cmd = {
+            'prefix': 'osd pool application enable',
+            'pool': pool_name,
+            'app': 'rbd',
+        }
+        try:
+            self._mon_cmd(cmd)
+            logger.info(f"'rbd' application enabled on pool '{pool_name}'.")
+        except RuntimeError as e:
+            logger.error(
+                f"Failed to enable 'rbd' application on '{pool_name}'",
+                exc_info=True
+            )
+            raise
+
+    def _rbd_pool_init(self, pool_name: str) -> None:
+        with self.rados.open_ioctx(pool_name) as ioctx:
+            rbd.RBD().pool_init(ioctx, False)
+        logger.info(f"RBD pool_init completed on '{pool_name}'.")
+
+    def create_pool_if_not_exists(self) -> None:
+        if not self._pool_exists(POOL_NAME):
+            self._create_pool(POOL_NAME, PG_NUM)
+        self._enable_rbd_application(POOL_NAME)
+        self._rbd_pool_init(POOL_NAME)
+
diff --git a/src/pybind/mgr/nvmeof/tests/__init__.py b/src/pybind/mgr/nvmeof/tests/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/src/pybind/mgr/nvmeof/tests/test_nvmeof_module.py b/src/pybind/mgr/nvmeof/tests/test_nvmeof_module.py
new file mode 100644
index 00000000000..28eaf4bc751
--- /dev/null
+++ b/src/pybind/mgr/nvmeof/tests/test_nvmeof_module.py
@@ -0,0 +1,84 @@
+
+from contextlib import contextmanager
+from unittest.mock import MagicMock
+
+import nvmeof.module as nvmeof_mod
+
+
+class FakeRados:
+    def __init__(self, exists: bool):
+        self._exists = exists
+        self.opened_pools = []
+
+    def pool_exists(self, pool_name: str) -> bool:
+        return self._exists
+
+    @contextmanager
+    def open_ioctx(self, pool_name: str):
+        self.opened_pools.append(pool_name)
+        yield object()
+
+
+def patch_rbd_pool_init(monkeypatch):
+    rbd_instance = MagicMock()
+    monkeypatch.setattr(nvmeof_mod.rbd, "RBD", lambda: rbd_instance)
+    return rbd_instance
+
+
+def make_mgr(mon_handler, exists: bool, monkeypatch):
+    mgr = nvmeof_mod.NVMeoF.__new__(nvmeof_mod.NVMeoF)
+    mgr.mon_command = mon_handler
+    mgr._print_log = lambda *args, **kwargs: None
+    mgr.run = False
+
+    mgr._fake_rados = FakeRados(exists)
+
+    def _pool_exists(self, pool_name: str) -> bool:
+        return self._fake_rados.pool_exists(pool_name)
+
+    def _rbd_pool_init(self, pool_name: str):
+        with self._fake_rados.open_ioctx(pool_name) as ioctx:
+            nvmeof_mod.rbd.RBD().pool_init(ioctx, False)
+
+    monkeypatch.setattr(nvmeof_mod.NVMeoF, "_pool_exists", _pool_exists, raising=True)
+    monkeypatch.setattr(nvmeof_mod.NVMeoF, "_rbd_pool_init", _rbd_pool_init, raising=True)
+
+    return mgr
+
+
+def test_pool_exists_skips_create_calls_enable_and_pool_init(monkeypatch):
+    calls = []
+
+    def mon_command(cmd):
+        calls.append(cmd)
+        return 0, "", ""
+
+    rbd_instance = patch_rbd_pool_init(monkeypatch)
+    mgr = make_mgr(mon_command, exists=True, monkeypatch=monkeypatch)
+
+    mgr.create_pool_if_not_exists()
+
+    assert not any(c.get("prefix") == "osd pool create" for c in calls)
+    assert any(c.get("prefix") == "osd pool application enable" for c in calls)
+
+    assert mgr._fake_rados.opened_pools == [".nvmeof"]
+    rbd_instance.pool_init.assert_called_once()
+
+
+def test_pool_missing_creates_then_enables_then_pool_init(monkeypatch):
+    calls = []
+
+    def mon_command(cmd):
+        calls.append(cmd)
+        return 0, "", ""
+
+    rbd_instance = patch_rbd_pool_init(monkeypatch)
+    mgr = make_mgr(mon_command, exists=False, monkeypatch=monkeypatch)
+
+    mgr.create_pool_if_not_exists()
+
+    assert any(c.get("prefix") == "osd pool create" for c in calls)
+    assert any(c.get("prefix") == "osd pool application enable" for c in calls)
+
+    assert mgr._fake_rados.opened_pools == [".nvmeof"]
+    rbd_instance.pool_init.assert_called_once()
diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index 5fc2fce63fa..0194d54abea 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -2109,10 +2109,14 @@ Usage:
 
         return self._apply_misc([spec], dry_run, format, no_overwrite)
 
+    def _create_nvmeof_metadata_pool_if_needed(self) -> None:
+        self.remote('nvmeof', 'create_pool_if_not_exists')
+
     @OrchestratorCLICommand.Write('orch apply nvmeof')
     def _apply_nvmeof(self,
-                      pool: str,
-                      group: str,
+                      _end_positional_: int = 0,
+                      pool: str = ".nvmeof",
+                      group: str = '',
                       placement: Optional[str] = None,
                       unmanaged: bool = False,
                       dry_run: bool = False,
@@ -2120,11 +2124,18 @@ Usage:
                       no_overwrite: bool = False,
                       inbuf: Optional[str] = None) -> HandleCommandResult:
         """Scale an nvmeof service"""
+        if group == '':
+            raise OrchestratorValidationError('The --group argument is required')
+
         if inbuf:
             raise OrchestratorValidationError('unrecognized command -i; -h or --help for usage')
 
+        if pool == ".nvmeof":
+            self._create_nvmeof_metadata_pool_if_needed()
+
+        cleanpool = pool.lstrip('.')
         spec = NvmeofServiceSpec(
-            service_id=f'{pool}.{group}' if group else pool,
+            service_id=f'{cleanpool}.{group}' if group else cleanpool,
             pool=pool,
             group=group,
             placement=PlacementSpec.from_string(placement),
diff --git a/src/pybind/mgr/tox.ini b/src/pybind/mgr/tox.ini
index c2deb627261..35d7f972d00 100644
--- a/src/pybind/mgr/tox.ini
+++ b/src/pybind/mgr/tox.ini
@@ -93,6 +93,7 @@ commands =
            -m devicehealth \
            -m diskprediction_local \
            -m hello \
+           -m nvmeof \
            -m influx \
            -m iostat \
            -m localpool \
@@ -146,6 +147,7 @@ modules =
     devicehealth \
     diskprediction_local \
     hello \
+    nvmeof \
     iostat \
     localpool \
     mgr_module.py \