From: Patrick Donnelly Date: Tue, 8 Aug 2023 01:09:21 +0000 (-0400) Subject: qa/tasks/ceph: provide configuration for setting configs via mon X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=aba2b1df605000d3ecbb54af27c77a6a11548afb;p=ceph-ci.git qa/tasks/ceph: provide configuration for setting configs via mon These configs may be set using: ceph: cluster-config: entity: foo: bar same as the current: ceph: config: entity: foo: bar The configs will be set in parallel using the `ceph config set` command. The main benefit here is to avoid using the ceph.conf to set configs which cannot be overriden using subsequent `ceph config` command. The only way to override is to change the ceph.conf in the test (yuck) or the admin socket (which gets reset when the daemon restarts). Finally, we can now exploit the `ceph config reset` command will let us trivially rollback config changes after a test completes. That is exposed as the `ctx.config_epoch` variable. Signed-off-by: Patrick Donnelly (cherry picked from commit 9d485ae1f450f7ce582f3d1e92045314532d5fe7) --- diff --git a/qa/tasks/ceph.py b/qa/tasks/ceph.py index 58442202ce7..4d47fe994fe 100644 --- a/qa/tasks/ceph.py +++ b/qa/tasks/ceph.py @@ -360,6 +360,68 @@ def crush_setup(ctx, config): yield +@contextlib.contextmanager +def module_setup(ctx, config): + cluster_name = config['cluster'] + first_mon = teuthology.get_first_mon(ctx, config, cluster_name) + (mon_remote,) = ctx.cluster.only(first_mon).remotes.keys() + + modules = config.get('mgr-modules', []) + for m in modules: + m = str(m) + cmd = [ + 'sudo', + 'ceph', + '--cluster', + cluster_name, + 'mgr', + 'module', + 'emable', + m, + ] + log.info("enabling module %s", m) + mon_remote.run(args=cmd) + yield + + +@contextlib.contextmanager +def conf_setup(ctx, config): + cluster_name = config['cluster'] + first_mon = teuthology.get_first_mon(ctx, config, cluster_name) + (mon_remote,) = ctx.cluster.only(first_mon).remotes.keys() + + configs = config.get('cluster-conf', {}) + procs = [] + for section, confs in configs.items(): + section = str(section) + for k, v in confs.items(): + k = str(k).replace(' ', '_') # pre-pacific compatibility + v = str(v) + cmd = [ + 'sudo', + 'ceph', + '--cluster', + cluster_name, + 'config', + 'set', + section, + k, + v, + ] + log.info("setting config [%s] %s = %s", section, k, v) + procs.append(mon_remote.run(args=cmd, wait=False)) + log.debug("set %d configs", len(procs)) + for p in procs: + log.debug("waiting for %s", p) + p.wait() + yield + +@contextlib.contextmanager +def conf_epoch(ctx, config): + cm = ctx.managers[config['cluster']] + cm.save_conf_epoch() + yield + @contextlib.contextmanager def check_enable_crimson(ctx, config): # enable crimson-osds if crimson @@ -1904,7 +1966,9 @@ def task(ctx, config): mon_bind_addrvec=config.get('mon_bind_addrvec', True), )), lambda: run_daemon(ctx=ctx, config=config, type_='mon'), + lambda: module_setup(ctx=ctx, config=config), lambda: run_daemon(ctx=ctx, config=config, type_='mgr'), + lambda: conf_setup(ctx=ctx, config=config), lambda: crush_setup(ctx=ctx, config=config), lambda: check_enable_crimson(ctx=ctx, config=config), lambda: run_daemon(ctx=ctx, config=config, type_='osd'), @@ -1913,6 +1977,7 @@ def task(ctx, config): lambda: run_daemon(ctx=ctx, config=config, type_='mds'), lambda: cephfs_setup(ctx=ctx, config=config), lambda: watchdog_setup(ctx=ctx, config=config), + lambda: conf_epoch(ctx=ctx, config=config), ] with contextutil.nested(*subtasks): diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py index 098a001e2e0..9a9dc87a0bc 100644 --- a/qa/tasks/ceph_manager.py +++ b/qa/tasks/ceph_manager.py @@ -1587,6 +1587,11 @@ class CephManager: timeout = kwargs.pop('timeout', 120) return ['sudo'] + self.pre + ['timeout', f'{timeout}', 'ceph', '--cluster', self.cluster] + def save_conf_epoch(self): + p = self.ceph("config log 1 --format=json") + J = json.loads(p.stdout.getvalue()) + self.ctx.conf_epoch = J[0]["version"] + log.info("config epoch is %d", self.ctx.conf_epoch) def ceph(self, cmd, **kwargs): """ diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py index 2f5f5c3fec3..55f46b19fda 100644 --- a/qa/tasks/cephadm.py +++ b/qa/tasks/cephadm.py @@ -1751,6 +1751,63 @@ def crush_setup(ctx, config): yield +@contextlib.contextmanager +def module_setup(ctx, config): + cluster_name = config['cluster'] + remote = ctx.ceph[cluster_name].bootstrap_remote + + modules = config.get('mgr-modules', []) + for m in modules: + m = str(m) + cmd = [ + 'sudo', + 'ceph', + '--cluster', + cluster_name, + 'mgr', + 'module', + 'enable', + m, + ] + log.info("enabling module %s", m) + _shell(ctx, cluster_name, remote, args=cmd) + yield + + +@contextlib.contextmanager +def conf_setup(ctx, config): + cluster_name = config['cluster'] + remote = ctx.ceph[cluster_name].bootstrap_remote + + configs = config.get('cluster-conf', {}) + procs = [] + for section, confs in configs.items(): + section = str(section) + for k, v in confs.items(): + k = str(k).replace(' ', '_') # pre-pacific compatibility + v = str(v) + cmd = [ + 'ceph', + 'config', + 'set', + section, + k, + v, + ] + log.info("setting config [%s] %s = %s", section, k, v) + procs.append(_shell(ctx, cluster_name, remote, args=cmd, wait=False)) + log.debug("set %d configs", len(procs)) + for p in procs: + log.debug("waiting for %s", p) + p.wait() + yield + +@contextlib.contextmanager +def conf_epoch(ctx, config): + cm = ctx.managers[config['cluster']] + cm.save_conf_epoch() + yield + @contextlib.contextmanager def create_rbd_pool(ctx, config): if config.get('create_rbd_pool', False): @@ -2243,7 +2300,9 @@ def task(ctx, config): lambda: crush_setup(ctx=ctx, config=config), lambda: ceph_mons(ctx=ctx, config=config), lambda: distribute_config_and_admin_keyring(ctx=ctx, config=config), + lambda: module_setup(ctx=ctx, config=config), lambda: ceph_mgrs(ctx=ctx, config=config), + lambda: conf_setup(ctx=ctx, config=config), lambda: ceph_osds(ctx=ctx, config=config), lambda: ceph_mdss(ctx=ctx, config=config), lambda: cephfs_setup(ctx=ctx, config=config), diff --git a/qa/tasks/vstart_runner.py b/qa/tasks/vstart_runner.py index d53e5f1eb07..be0e9098e52 100644 --- a/qa/tasks/vstart_runner.py +++ b/qa/tasks/vstart_runner.py @@ -820,6 +820,8 @@ class LocalCephManager(CephManager): self.testdir = None self.RADOS_CMD = [RADOS_CMD] + self.save_conf_epoch() + def get_ceph_cmd(self, **kwargs): return [CEPH_CMD]