mgr/cephadm: Add extra-ceph-conf

Due to limitations of `config generate-minimal-conf`,
we need to provide a way for users to modify the
ceph.conf for the daemons.

This now provides a way to append config to
the daemons config. Like. e.g. `cluster_network`.

Signed-off-by: Sebastian Wagner <sebastian.wagner@suse.com>
(cherry picked from commit ff7e76348e5457fa6acb23545fcef56d6640c50a)

diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index 30b2f5ea2250c2d5f0b714478fbe6d3c36ef234d..9e9345fe2e313d752fdc7b6732812e1beb8c71a5 100644 (file)
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -410,7 +410,7 @@ class HostCache():
                 r.append(name)
         return r
 
-    def get_daemon_last_config_deps(self, host, name):
+    def get_daemon_last_config_deps(self, host, name) -> Tuple[Optional[List[str]], Optional[datetime.datetime]]:
         if host in self.daemon_config_deps:
             if name in self.daemon_config_deps[host]:
                 return self.daemon_config_deps[host][name].get('deps', []), \
@@ -485,13 +485,15 @@ class HostCache():
             return True
         if self.mgr.last_monmap > self.last_etc_ceph_ceph_conf[host]:
             return True
+        if self.mgr.extra_ceph_conf_is_newer(self.last_etc_ceph_ceph_conf[host]):
+            return True
         # already up to date:
         return False
 
     def update_last_etc_ceph_ceph_conf(self, host: str):
         if not self.mgr.last_monmap:
             return
-        self.last_etc_ceph_ceph_conf[host] = self.mgr.last_monmap
+        self.last_etc_ceph_ceph_conf[host] = datetime.datetime.utcnow()
 
     def host_needs_registry_login(self, host: str) -> bool:
         if host in self.mgr.offline_hosts:

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 42ee122bda06d144748d8739c27e43c2c9ec1edb..d9fdaf00cc7c602a6b7927805458ebdad067a1c2 100644 (file)
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -3,6 +3,7 @@ import errno
 import logging
 import shlex
 from collections import defaultdict
+from configparser import ConfigParser
 from contextlib import contextmanager
 from functools import wraps
 from tempfile import TemporaryDirectory
@@ -49,7 +50,8 @@ from .schedule import HostAssignment, HostPlacementSpec
 from .inventory import Inventory, SpecStore, HostCache, EventStore
 from .upgrade import CEPH_UPGRADE_ORDER, CephadmUpgrade
 from .template import TemplateMgr
-from .utils import forall_hosts, CephadmNoImage, cephadmNoImage, str_to_datetime, is_repo_digest
+from .utils import forall_hosts, CephadmNoImage, cephadmNoImage, \
+    str_to_datetime, datetime_to_str, is_repo_digest
 
 try:
     import remoto
@@ -1028,6 +1030,53 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule):
                     self.event.set()
         return 0, '%s (%s) ok' % (host, addr), err
 
+    @orchestrator._cli_write_command(
+        prefix='cephadm set-extra-ceph-conf',
+        desc="Text that is appended to all daemon's ceph.conf.\n"
+             "Mainly a workaround, till `config generate-minimal-conf` generates\n"
+             "a complete ceph.conf.\n\n"
+             "Warning: this is a dangerous operation.")
+    def _set_extra_ceph_conf(self, inbuf=None) -> HandleCommandResult:
+        if inbuf:
+            # sanity check.
+            cp = ConfigParser()
+            cp.read_string(inbuf, source='<infile>')
+
+        self.set_store("extra_ceph_conf", json.dumps({
+            'conf': inbuf,
+            'last_modified': datetime_to_str(datetime.datetime.utcnow())
+        }))
+        self.log.info('Set extra_ceph_conf')
+        self._kick_serve_loop()
+        return HandleCommandResult()
+
+    @orchestrator._cli_read_command(
+        'cephadm get-extra-ceph-conf',
+        desc='Get extra ceph conf that is appended')
+    def _get_extra_ceph_conf(self) -> HandleCommandResult:
+        return HandleCommandResult(stdout=self.extra_ceph_conf().conf)
+
+    class ExtraCephConf(NamedTuple):
+        conf: str
+        last_modified: Optional[datetime.datetime]
+
+    def extra_ceph_conf(self) -> 'CephadmOrchestrator.ExtraCephConf':
+        data = self.get_store('extra_ceph_conf')
+        if not data:
+            return CephadmOrchestrator.ExtraCephConf('', None)
+        try:
+            j = json.loads(data)
+        except ValueError:
+            self.log.exception('unable to laod extra_ceph_conf')
+            return CephadmOrchestrator.ExtraCephConf('', None)
+        return CephadmOrchestrator.ExtraCephConf(j['conf'], str_to_datetime(j['last_modified']))
+
+    def extra_ceph_conf_is_newer(self, dt: datetime.datetime) -> bool:
+        conf = self.extra_ceph_conf()
+        if not conf.last_modified:
+            return False
+        return conf.last_modified > dt
+
     def _get_connection(self, host: str):
         """
         Setup a connection for running commands on remote host.
@@ -1548,6 +1597,9 @@ To check that the host is reachable:
         _, config, _ = self.check_mon_command({
             "prefix": "config generate-minimal-conf",
         })
+        extra = self.extra_ceph_conf().conf
+        if extra:
+            config += '\n\n' + extra.strip() + '\n'
         return config
 
     def _invalidate_daemons_and_kick_serve(self, filter_host=None):
@@ -2301,6 +2353,10 @@ To check that the host is reachable:
                 dd.daemon_type in CEPH_TYPES:
                 self.log.info('Reconfiguring %s (monmap changed)...' % dd.name())
                 action = 'reconfig'
+            elif self.extra_ceph_conf_is_newer(last_config) and \
+                    dd.daemon_type in CEPH_TYPES:
+                self.log.info('Reconfiguring %s (extra config changed)...' % dd.name())
+                action = 'reconfig'
             if action:
                 if self.cache.get_scheduled_daemon_action(dd.hostname, dd.name()) == 'redeploy' \
                         and action == 'reconfig':

diff --git a/src/pybind/mgr/cephadm/schedule.py b/src/pybind/mgr/cephadm/schedule.py
index 2797ed1b86783920e2351531433b0100968f1ec1..4b42099b293a22baf13f7aad32d8264bcedcc1eb 100644 (file)
--- a/src/pybind/mgr/cephadm/schedule.py
+++ b/src/pybind/mgr/cephadm/schedule.py
@@ -144,7 +144,7 @@ class HostAssignment(object):
             if self.filter_new_host:
                 old = others
                 others = [h for h in others if self.filter_new_host(h.hostname)]
-                logger.debug('filtered %s down to %s' % (old, candidates))
+                logger.debug('filtered %s down to %s' % (old, others))
 
             # ask the scheduler to return a set of hosts with a up to the value of <count>
             others = self.scheduler.place(others, need)

diff --git a/src/pybind/mgr/cephadm/tests/test_cephadm.py b/src/pybind/mgr/cephadm/tests/test_cephadm.py
index 4464fa0ba5efe0112aa4fb51138450c695f349a4..f1958281b590221285f177894b8ef9b88e65eb01 100644 (file)
--- a/src/pybind/mgr/cephadm/tests/test_cephadm.py
+++ b/src/pybind/mgr/cephadm/tests/test_cephadm.py
@@ -250,6 +250,40 @@ class TestCephadm(object):
 
                 assert cephadm_module.cache.get_scheduled_daemon_action('test', daemon_name) is None
 
+    @mock.patch("cephadm.module.CephadmOrchestrator._run_cephadm")
+    def test_daemon_check_extra_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
+        _run_cephadm.return_value = ('{}', '', 0)
+
+        with with_host(cephadm_module, 'test'):
+
+            # Also testing deploying mons without explicit network placement
+            cephadm_module.check_mon_command({
+                'prefix': 'config set',
+                'who': 'mon',
+                'name': 'public_network',
+                'value': '127.0.0.0/8'
+            })
+
+            cephadm_module.cache.update_host_devices_networks(
+                'test',
+                [],
+                {
+                    "127.0.0.0/8": [
+                        "127.0.0.1"
+                    ],
+                }
+            )
+
+            with with_service(cephadm_module, ServiceSpec(service_type='mon'), CephadmOrchestrator.apply_mon, 'test') as d_names:
+                [daemon_name] = d_names
+
+                cephadm_module._set_extra_ceph_conf('[mon]\nk=v')
+
+                cephadm_module._check_daemons()
+
+                _run_cephadm.assert_called_with('test', 'mon.test', 'deploy', [
+                                                '--name', 'mon.test', '--config-json', '-', '--reconfig'], stdin='{"config": "\\n\\n[mon]\\nk=v\\n", "keyring": ""}')
+
     @mock.patch("cephadm.module.CephadmOrchestrator._run_cephadm", _run_cephadm('{}'))
     def test_daemon_check_post(self, cephadm_module: CephadmOrchestrator):
         with with_host(cephadm_module, 'test'):
@@ -801,6 +835,13 @@ class TestCephadm(object):
 
             assert not cephadm_module.cache.host_needs_new_etc_ceph_ceph_conf('test')
 
+            # set extra config and expect that we deploy another ceph.conf
+            cephadm_module._set_extra_ceph_conf('[mon]\nk=v')
+            cephadm_module._refresh_hosts_and_daemons()
+            _check.assert_called_with(
+                ANY, ['dd', 'of=/etc/ceph/ceph.conf'], stdin=b'\n\n[mon]\nk=v\n')
+
+            # reload
             cephadm_module.cache.last_etc_ceph_ceph_conf = {}
             cephadm_module.cache.load()
 

diff --git a/src/pybind/mgr/tests/__init__.py b/src/pybind/mgr/tests/__init__.py
index 9010cdc710867911f7b9dfa81ba3df92719e52c3..5e5a541fbdf8270611610ce8517fe30e50622e75 100644 (file)
--- a/src/pybind/mgr/tests/__init__.py
+++ b/src/pybind/mgr/tests/__init__.py
@@ -73,15 +73,18 @@ if 'UNITTEST' in os.environ:
             except FileNotFoundError:
                 val = None
             mo = [o for o in self.MODULE_OPTIONS if o['name'] == key]
-            if len(mo) == 1 and val is not None:
-                cls = {
-                    'str': str,
-                    'secs': int,
-                    'bool': lambda s: bool(s) and s != 'false' and s != 'False',
-                    'int': int,
-                }[mo[0].get('type', 'str')]
-                return cls(val)
-            return val
+            if len(mo) == 1:
+                if val is not None:
+                    cls = {
+                        'str': str,
+                        'secs': int,
+                        'bool': lambda s: bool(s) and s != 'false' and s != 'False',
+                        'int': int,
+                    }[mo[0].get('type', 'str')]
+                    return cls(val)
+                return val
+            else:
+                return val if val is not None else ''
 
         def _ceph_set_module_option(self, module, key, val):
             _, _, _ = self.check_mon_command({