cephadm/nvmeof: add DSA acceleration support with conditional mounts and privileges

Signed-off-by: Alexander Indenbaum <aindenba@redhat.com>
(cherry picked from commit 3c94c1a4c3f77c2683001264c759779bab1d486d)

diff --git a/src/cephadm/cephadmlib/daemons/nvmeof.py b/src/cephadm/cephadmlib/daemons/nvmeof.py
index 75a05019e064d10e5461fddd2a91691823bba3ca..db337a2be9816254e5f3282fa30f6ee1ad9d03ce 100644 (file)
--- a/src/cephadm/cephadmlib/daemons/nvmeof.py
+++ b/src/cephadm/cephadmlib/daemons/nvmeof.py
@@ -90,6 +90,13 @@ class CephNvmeof(ContainerDaemonForm):
             mounts['/dev/vfio/vfio'] = '/dev/vfio/vfio'
         return mounts
 
+    def _get_dsa_mounts(self, files: Dict[str, str]) -> Dict[str, str]:
+        mounts = dict()
+        if 'enable_dsa_acceleration' in files:
+            mounts['/dev/dsa'] = '/dev/dsa'
+            mounts['/dev/char'] = '/dev/char'
+        return mounts
+
     def _get_tls_cert_key_mounts(
         self, data_dir: str, files: Dict[str, str]
     ) -> Dict[str, str]:
@@ -123,6 +130,7 @@ class CephNvmeof(ContainerDaemonForm):
         else:
             mounts.update(self._get_container_mounts(data_dir, log_dir))
         mounts.update(self._get_huge_pages_mounts(self.files))
+        mounts.update(self._get_dsa_mounts(self.files))
         mounts.update(self._get_tls_cert_key_mounts(data_dir, self.files))
 
     def customize_container_binds(
@@ -254,6 +262,10 @@ class CephNvmeof(ContainerDaemonForm):
         args.extend(['--ulimit', 'memlock=-1:-1'])
         args.extend(['--ulimit', 'nofile=10240'])
         args.extend(['--cap-add=CAP_SYS_NICE'])
+        # idxd/dsa
+        if 'enable_dsa_acceleration' in self.files:
+            args.extend(['--privileged'])
+            args.extend(['--cap-add=SYS_RAWIO'])
         if 'spdk_mem_size' not in self.files:
             args.extend(['--cap-add=SYS_ADMIN'])
             if 'spdk_huge_pages' in self.files:

diff --git a/src/pybind/mgr/cephadm/services/nvmeof.py b/src/pybind/mgr/cephadm/services/nvmeof.py
index 1f83aad726107c3991575b60b6fdc2c665f24a02..f34f107adf37255f1adf86e84662655e0d386956 100644 (file)
--- a/src/pybind/mgr/cephadm/services/nvmeof.py
+++ b/src/pybind/mgr/cephadm/services/nvmeof.py
@@ -93,6 +93,10 @@ class NvmeofService(CephService):
             except ValueError:
                 logger.error(f"Invalid value for SPDK huge pages: {spec.spdk_huge_pages}")
 
+        # Enable DSA probing
+        if spec.enable_dsa_acceleration:
+            daemon_spec.extra_files['enable_dsa_acceleration'] = str(spec.enable_dsa_acceleration)
+
         if spec.enable_auth:
             if (
                 not spec.client_cert

diff --git a/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2 b/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
index 735979a548f091bddcf89891200f527db1fe2cc2..b63ae15b95754ec0bf909153a1f5f34df4501c66 100644 (file)
--- a/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
+++ b/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
@@ -105,6 +105,7 @@ transports = {{ spec.transports }}
 {% if transport_tcp_options %}
 transport_tcp_options = {{ transport_tcp_options }}
 {% endif %}
+enable_dsa_acceleration = {{ spec.enable_dsa_acceleration }}
 {% if spec.tgt_cmd_extra_args %}
 tgt_cmd_extra_args = {{ spec.tgt_cmd_extra_args }}
 {% endif %}

diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
index 1792aee31e6979ceba65851417b07038c39918ac..40cabeb7be5a8bab1257919d1666cf99537a57aa 100644 (file)
--- a/src/pybind/mgr/cephadm/tests/test_services.py
+++ b/src/pybind/mgr/cephadm/tests/test_services.py
@@ -429,6 +429,7 @@ protocol_log_level = WARNING
 conn_retries = 10
 transports = tcp
 transport_tcp_options = {{"in_capsule_data_size": 8192, "max_io_qpairs_per_ctrlr": 7}}
+enable_dsa_acceleration = True
 tgt_cmd_extra_args = {tgt_cmd_extra_args}
 qos_timeslice_in_usecs = 0
 notifications_interval = 60
@@ -468,7 +469,8 @@ timeout = 1.0\n"""
                             "config": "",
                             "keyring": "[client.nvmeof.testpool.mygroup.test.qwert]\nkey = None\n",
                             "files": {
-                                "ceph-nvmeof.conf": nvmeof_gateway_conf
+                                "ceph-nvmeof.conf": nvmeof_gateway_conf,
+                                "enable_dsa_acceleration": "True"
                             }
                         }
                     }),

diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index 381f12bfb6a7235ad49bb169f5f4c8da80363c9e..c49df25f43a4de54d480249901d8dbe9b4855f0e 100644 (file)
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -1430,6 +1430,7 @@ class NvmeofServiceSpec(ServiceSpec):
                  transports: Optional[str] = 'tcp',
                  transport_tcp_options: Optional[Dict[str, int]] =
                  {"in_capsule_data_size": 8192, "max_io_qpairs_per_ctrlr": 7},
+                 enable_dsa_acceleration: bool = True,
                  tgt_cmd_extra_args: Optional[str] = None,
                  iobuf_options: Optional[Dict[str, int]] = None,
                  qos_timeslice_in_usecs: Optional[int] = 0,
@@ -1595,6 +1596,8 @@ class NvmeofServiceSpec(ServiceSpec):
         self.transports = transports
         #: List of extra arguments for transports in the form opt=value
         self.transport_tcp_options: Optional[Dict[str, int]] = transport_tcp_options
+        #: ``enable_dsa_acceleration`` enable  dsa acceleration
+        self.enable_dsa_acceleration = enable_dsa_acceleration
         #: ``tgt_cmd_extra_args`` extra arguments for the nvmf_tgt process
         self.tgt_cmd_extra_args = tgt_cmd_extra_args
         #: List of extra arguments for SPDK iobuf in the form opt=value