qa/tasks/ceph: provide configuration for setting configs via mon

These configs may be set using:

ceph:
  cluster-config:
    entity:
      foo: bar

same as the current:

ceph:
  config:
    entity:
      foo: bar

The configs will be set in parallel using the `ceph config set` command.

The main benefit here is to avoid using the ceph.conf to set configs which
cannot be overriden using subsequent `ceph config` command. The only way to
override is to change the ceph.conf in the test (yuck) or the admin socket
(which gets reset when the daemon restarts).

Finally, we can now exploit the `ceph config reset` command will let us
trivially rollback config changes after a test completes. That is exposed
as the `ctx.config_epoch` variable.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 9d485ae1f450f7ce582f3d1e92045314532d5fe7)

diff --git a/qa/tasks/ceph.py b/qa/tasks/ceph.py
index 9177974ed08deba044a37a9664ea1e1e01dba673..24f0c3831e8afd2d8e683cb65ffc072f960f8501 100644 (file)
--- a/qa/tasks/ceph.py
+++ b/qa/tasks/ceph.py
@@ -360,6 +360,68 @@ def crush_setup(ctx, config):
     yield
 
 
+@contextlib.contextmanager
+def module_setup(ctx, config):
+    cluster_name = config['cluster']
+    first_mon = teuthology.get_first_mon(ctx, config, cluster_name)
+    (mon_remote,) = ctx.cluster.only(first_mon).remotes.keys()
+
+    modules = config.get('mgr-modules', [])
+    for m in modules:
+        m = str(m)
+        cmd = [
+           'sudo',
+           'ceph',
+           '--cluster',
+           cluster_name,
+           'mgr',
+           'module',
+           'emable',
+           m,
+        ]
+        log.info("enabling module %s", m)
+        mon_remote.run(args=cmd)
+    yield
+
+
+@contextlib.contextmanager
+def conf_setup(ctx, config):
+    cluster_name = config['cluster']
+    first_mon = teuthology.get_first_mon(ctx, config, cluster_name)
+    (mon_remote,) = ctx.cluster.only(first_mon).remotes.keys()
+
+    configs = config.get('cluster-conf', {})
+    procs = []
+    for section, confs in configs.items():
+        section = str(section)
+        for k, v in confs.items():
+            k = str(k).replace(' ', '_') # pre-pacific compatibility
+            v = str(v)
+            cmd = [
+                'sudo',
+                'ceph',
+                '--cluster',
+                cluster_name,
+                'config',
+                'set',
+                section,
+                k,
+                v,
+            ]
+            log.info("setting config [%s] %s = %s", section, k, v)
+            procs.append(mon_remote.run(args=cmd, wait=False))
+    log.debug("set %d configs", len(procs))
+    for p in procs:
+        log.debug("waiting for %s", p)
+        p.wait()
+    yield
+
+@contextlib.contextmanager
+def conf_epoch(ctx, config):
+    cm = ctx.managers[config['cluster']]
+    cm.save_conf_epoch()
+    yield
+
 @contextlib.contextmanager
 def check_enable_crimson(ctx, config):
     # enable crimson-osds if crimson
@@ -1894,7 +1956,9 @@ def task(ctx, config):
             mon_bind_addrvec=config.get('mon_bind_addrvec', True),
         )),
         lambda: run_daemon(ctx=ctx, config=config, type_='mon'),
+        lambda: module_setup(ctx=ctx, config=config),
         lambda: run_daemon(ctx=ctx, config=config, type_='mgr'),
+        lambda: conf_setup(ctx=ctx, config=config),
         lambda: crush_setup(ctx=ctx, config=config),
         lambda: check_enable_crimson(ctx=ctx, config=config),
         lambda: run_daemon(ctx=ctx, config=config, type_='osd'),
@@ -1903,6 +1967,7 @@ def task(ctx, config):
         lambda: run_daemon(ctx=ctx, config=config, type_='mds'),
         lambda: cephfs_setup(ctx=ctx, config=config),
         lambda: watchdog_setup(ctx=ctx, config=config),
+        lambda: conf_epoch(ctx=ctx, config=config),
     ]
 
     with contextutil.nested(*subtasks):

diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
index ac164691fa2db6771c9d45c626efc85894df8cea..657f396fa9332a4bff17b27643e293ca66afacf5 100644 (file)
--- a/qa/tasks/ceph_manager.py
+++ b/qa/tasks/ceph_manager.py
@@ -1575,6 +1575,11 @@ class CephManager:
         timeout = kwargs.pop('timeout', 120)
         return ['sudo'] + self.pre + ['timeout', f'{timeout}', 'ceph',
                                       '--cluster', self.cluster]
+    def save_conf_epoch(self):
+        p = self.ceph("config log 1 --format=json")
+        J = json.loads(p.stdout.getvalue())
+        self.ctx.conf_epoch = J[0]["version"]
+        log.info("config epoch is %d", self.ctx.conf_epoch)
 
     def ceph(self, cmd, **kwargs):
         """

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index a4da67e6ffdcc5536dc2e1c112c5eef49ca6ea2c..64889cb15efad96e5a9e37fa85cc11c076159806 100644 (file)
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -1735,6 +1735,63 @@ def crush_setup(ctx, config):
     yield
 
 
+@contextlib.contextmanager
+def module_setup(ctx, config):
+    cluster_name = config['cluster']
+    remote = ctx.ceph[cluster_name].bootstrap_remote
+
+    modules = config.get('mgr-modules', [])
+    for m in modules:
+        m = str(m)
+        cmd = [
+           'sudo',
+           'ceph',
+           '--cluster',
+           cluster_name,
+           'mgr',
+           'module',
+           'enable',
+           m,
+        ]
+        log.info("enabling module %s", m)
+        _shell(ctx, cluster_name, remote, args=cmd)
+    yield
+
+
+@contextlib.contextmanager
+def conf_setup(ctx, config):
+    cluster_name = config['cluster']
+    remote = ctx.ceph[cluster_name].bootstrap_remote
+
+    configs = config.get('cluster-conf', {})
+    procs = []
+    for section, confs in configs.items():
+        section = str(section)
+        for k, v in confs.items():
+            k = str(k).replace(' ', '_') # pre-pacific compatibility
+            v = str(v)
+            cmd = [
+                'ceph',
+                'config',
+                'set',
+                section,
+                k,
+                v,
+            ]
+            log.info("setting config [%s] %s = %s", section, k, v)
+            procs.append(_shell(ctx, cluster_name, remote, args=cmd, wait=False))
+    log.debug("set %d configs", len(procs))
+    for p in procs:
+        log.debug("waiting for %s", p)
+        p.wait()
+    yield
+
+@contextlib.contextmanager
+def conf_epoch(ctx, config):
+    cm = ctx.managers[config['cluster']]
+    cm.save_conf_epoch()
+    yield
+
 @contextlib.contextmanager
 def create_rbd_pool(ctx, config):
     if config.get('create_rbd_pool', False):
@@ -2227,7 +2284,9 @@ def task(ctx, config):
             lambda: crush_setup(ctx=ctx, config=config),
             lambda: ceph_mons(ctx=ctx, config=config),
             lambda: distribute_config_and_admin_keyring(ctx=ctx, config=config),
+            lambda: module_setup(ctx=ctx, config=config),
             lambda: ceph_mgrs(ctx=ctx, config=config),
+            lambda: conf_setup(ctx=ctx, config=config),
             lambda: ceph_osds(ctx=ctx, config=config),
             lambda: ceph_mdss(ctx=ctx, config=config),
             lambda: cephfs_setup(ctx=ctx, config=config),

diff --git a/qa/tasks/vstart_runner.py b/qa/tasks/vstart_runner.py
index ecb31015bd201609a92256229ff3b8dda19fe7f9..e2384f2bf5d6f64b112aa6719e12720ef5247e14 100644 (file)
--- a/qa/tasks/vstart_runner.py
+++ b/qa/tasks/vstart_runner.py
@@ -810,6 +810,8 @@ class LocalCephManager(CephManager):
         self.testdir = None
         self.RADOS_CMD = [RADOS_CMD]
 
+        self.save_conf_epoch()
+
     def get_ceph_cmd(self, **kwargs):
         return [CEPH_CMD]