qa/tasks/ceph: provide configuration for setting configs via mon

These configs may be set using:

ceph:
  cluster-config:
    entity:
      foo: bar

same as the current:

ceph:
  config:
    entity:
      foo: bar

The configs will be set in parallel using the `ceph config set` command.

The main benefit here is to avoid using the ceph.conf to set configs which
cannot be overriden using subsequent `ceph config` command. The only way to
override is to change the ceph.conf in the test (yuck) or the admin socket
(which gets reset when the daemon restarts).

Finally, we can now exploit the `ceph config reset` command will let us
trivially rollback config changes after a test completes. That is exposed
as the `ctx.config_epoch` variable.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

diff --git a/qa/tasks/ceph.py b/qa/tasks/ceph.py
index a7b3ad4c5eec6b75a3bd1a1672e9fa6dc1ae6ff6..6314183c3b3742ebecc78f0d792e3f47d1060c9a 100644 (file)
--- a/qa/tasks/ceph.py
+++ b/qa/tasks/ceph.py
@@ -360,6 +360,68 @@ def crush_setup(ctx, config):
     yield
 
 
+@contextlib.contextmanager
+def module_setup(ctx, config):
+    cluster_name = config['cluster']
+    first_mon = teuthology.get_first_mon(ctx, config, cluster_name)
+    (mon_remote,) = ctx.cluster.only(first_mon).remotes.keys()
+
+    modules = config.get('mgr-modules', [])
+    for m in modules:
+        m = str(m)
+        cmd = [
+           'sudo',
+           'ceph',
+           '--cluster',
+           cluster_name,
+           'mgr',
+           'module',
+           'emable',
+           m,
+        ]
+        log.info("enabling module %s", m)
+        mon_remote.run(args=cmd)
+    yield
+
+
+@contextlib.contextmanager
+def conf_setup(ctx, config):
+    cluster_name = config['cluster']
+    first_mon = teuthology.get_first_mon(ctx, config, cluster_name)
+    (mon_remote,) = ctx.cluster.only(first_mon).remotes.keys()
+
+    configs = config.get('cluster-conf', {})
+    procs = []
+    for section, confs in configs.items():
+        section = str(section)
+        for k, v in confs.items():
+            k = str(k).replace(' ', '_') # pre-pacific compatibility
+            v = str(v)
+            cmd = [
+                'sudo',
+                'ceph',
+                '--cluster',
+                cluster_name,
+                'config',
+                'set',
+                section,
+                k,
+                v,
+            ]
+            log.info("setting config [%s] %s = %s", section, k, v)
+            procs.append(mon_remote.run(args=cmd, wait=False))
+    log.debug("set %d configs", len(procs))
+    for p in procs:
+        log.debug("waiting for %s", p)
+        p.wait()
+    yield
+
+@contextlib.contextmanager
+def conf_epoch(ctx, config):
+    cm = ctx.managers[config['cluster']]
+    cm.save_conf_epoch()
+    yield
+
 @contextlib.contextmanager
 def check_enable_crimson(ctx, config):
     # enable crimson-osds if crimson
@@ -1899,7 +1961,9 @@ def task(ctx, config):
             mon_bind_addrvec=config.get('mon_bind_addrvec', True),
         )),
         lambda: run_daemon(ctx=ctx, config=config, type_='mon'),
+        lambda: module_setup(ctx=ctx, config=config),
         lambda: run_daemon(ctx=ctx, config=config, type_='mgr'),
+        lambda: conf_setup(ctx=ctx, config=config),
         lambda: crush_setup(ctx=ctx, config=config),
         lambda: check_enable_crimson(ctx=ctx, config=config),
         lambda: run_daemon(ctx=ctx, config=config, type_='osd'),
@@ -1908,6 +1972,7 @@ def task(ctx, config):
         lambda: run_daemon(ctx=ctx, config=config, type_='mds'),
         lambda: cephfs_setup(ctx=ctx, config=config),
         lambda: watchdog_setup(ctx=ctx, config=config),
+        lambda: conf_epoch(ctx=ctx, config=config),
     ]
 
     with contextutil.nested(*subtasks):

diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
index b81fb9bb213dc47de4cdd4ffea329eec388f448b..4486e554ed587cb8d561d975c14a0cebc45f3d01 100644 (file)
--- a/qa/tasks/ceph_manager.py
+++ b/qa/tasks/ceph_manager.py
@@ -1599,6 +1599,11 @@ class CephManager:
         timeout = kwargs.pop('timeout', 120)
         return ['sudo'] + self.pre + ['timeout', f'{timeout}', 'ceph',
                                       '--cluster', self.cluster]
+    def save_conf_epoch(self):
+        p = self.ceph("config log 1 --format=json")
+        J = json.loads(p.stdout.getvalue())
+        self.ctx.conf_epoch = J[0]["version"]
+        log.info("config epoch is %d", self.ctx.conf_epoch)
 
     def ceph(self, cmd, **kwargs):
         """

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index 75402752b3da7ad0d5996be6cacc029f66d57868..23a3e2cf9d7bea131ee562b6ae388c759fccea3e 100644 (file)
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -1747,6 +1747,63 @@ def crush_setup(ctx, config):
     yield
 
 
+@contextlib.contextmanager
+def module_setup(ctx, config):
+    cluster_name = config['cluster']
+    remote = ctx.ceph[cluster_name].bootstrap_remote
+
+    modules = config.get('mgr-modules', [])
+    for m in modules:
+        m = str(m)
+        cmd = [
+           'sudo',
+           'ceph',
+           '--cluster',
+           cluster_name,
+           'mgr',
+           'module',
+           'enable',
+           m,
+        ]
+        log.info("enabling module %s", m)
+        _shell(ctx, cluster_name, remote, args=cmd)
+    yield
+
+
+@contextlib.contextmanager
+def conf_setup(ctx, config):
+    cluster_name = config['cluster']
+    remote = ctx.ceph[cluster_name].bootstrap_remote
+
+    configs = config.get('cluster-conf', {})
+    procs = []
+    for section, confs in configs.items():
+        section = str(section)
+        for k, v in confs.items():
+            k = str(k).replace(' ', '_') # pre-pacific compatibility
+            v = str(v)
+            cmd = [
+                'ceph',
+                'config',
+                'set',
+                section,
+                k,
+                v,
+            ]
+            log.info("setting config [%s] %s = %s", section, k, v)
+            procs.append(_shell(ctx, cluster_name, remote, args=cmd, wait=False))
+    log.debug("set %d configs", len(procs))
+    for p in procs:
+        log.debug("waiting for %s", p)
+        p.wait()
+    yield
+
+@contextlib.contextmanager
+def conf_epoch(ctx, config):
+    cm = ctx.managers[config['cluster']]
+    cm.save_conf_epoch()
+    yield
+
 @contextlib.contextmanager
 def create_rbd_pool(ctx, config):
     if config.get('create_rbd_pool', False):
@@ -2249,7 +2306,9 @@ def task(ctx, config):
             lambda: crush_setup(ctx=ctx, config=config),
             lambda: ceph_mons(ctx=ctx, config=config),
             lambda: distribute_config_and_admin_keyring(ctx=ctx, config=config),
+            lambda: module_setup(ctx=ctx, config=config),
             lambda: ceph_mgrs(ctx=ctx, config=config),
+            lambda: conf_setup(ctx=ctx, config=config),
             lambda: ceph_osds(ctx=ctx, config=config),
             lambda: ceph_mdss(ctx=ctx, config=config),
             lambda: cephfs_setup(ctx=ctx, config=config),
@@ -2261,6 +2320,7 @@ def task(ctx, config):
             lambda: ceph_monitoring('grafana', ctx=ctx, config=config),
             lambda: ceph_clients(ctx=ctx, config=config),
             lambda: create_rbd_pool(ctx=ctx, config=config),
+            lambda: conf_epoch(ctx=ctx, config=config),
     ):
         try:
             if config.get('wait-for-healthy', True):

diff --git a/qa/tasks/vstart_runner.py b/qa/tasks/vstart_runner.py
index 1c881808d06518a3d9a8767020c1ed17a8903053..49465a825c75e9fc731804f93055da151d84a908 100644 (file)
--- a/qa/tasks/vstart_runner.py
+++ b/qa/tasks/vstart_runner.py
@@ -820,6 +820,8 @@ class LocalCephManager(CephManager):
         self.testdir = None
         self.RADOS_CMD = [RADOS_CMD]
 
+        self.save_conf_epoch()
+
     def get_ceph_cmd(self, **kwargs):
         return [CEPH_CMD]