From f87e6222657e6dbbbeb070c908ac5736fd18940a Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 6 Mar 2024 12:00:31 -0500 Subject: [PATCH] qa/tasks: remove generic template stuff from cephadm.py Remove the generic template stuff from cephadm.py now that the template.py file is usable and various yamls have been updated to use the new module now. Signed-off-by: John Mulligan --- qa/tasks/cephadm.py | 118 ++------------------------------------------ 1 file changed, 4 insertions(+), 114 deletions(-) diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py index 7c5d03b2da6c1..a312f7fe7bc06 100644 --- a/qa/tasks/cephadm.py +++ b/qa/tasks/cephadm.py @@ -4,7 +4,6 @@ Ceph cluster task, deployed via cephadm orchestrator import argparse import configobj import contextlib -import functools import json import logging import os @@ -13,8 +12,6 @@ import time import uuid import yaml -import jinja2 - from copy import deepcopy from io import BytesIO, StringIO from tarfile import ReadError @@ -30,6 +27,7 @@ from textwrap import dedent from tasks.cephfs.filesystem import MDSCluster, Filesystem from tasks.daemonwatchdog import DaemonWatchdog from tasks.util import chacra +from tasks import template # these items we use from ceph.py should probably eventually move elsewhere from tasks.ceph import get_mons, healthy @@ -39,73 +37,6 @@ CEPH_ROLE_TYPES = ['mon', 'mgr', 'osd', 'mds', 'rgw', 'prometheus'] log = logging.getLogger(__name__) -def _convert_strs_in(o, conv): - """A function to walk the contents of a dict/list and recurisvely apply - a conversion function (`conv`) to the strings within. - """ - if isinstance(o, str): - return conv(o) - if isinstance(o, dict): - for k in o: - o[k] = _convert_strs_in(o[k], conv) - if isinstance(o, list): - o[:] = [_convert_strs_in(v, conv) for v in o] - return o - - -def _apply_template(jinja_env, rctx, template): - """Apply jinja2 templating to the template string `template` via the jinja - environment `jinja_env`, passing a dictionary containing top-level context - to render into the template. - """ - if '{{' in template or '{%' in template: - return jinja_env.from_string(template).render(**rctx) - return template - - -def _template_transform(ctx, config, target): - """Apply jinja2 based templates to strings within the target object, - returning a transformed target. Target objects may be a list or dict or - str. - - Note that only string values in the list or dict objects are modified. - Therefore one can read & parse yaml or json that contain templates in - string values without the risk of changing the structure of the yaml/json. - """ - jenv = getattr(ctx, '_jinja_env', None) - if jenv is None: - loader = jinja2.BaseLoader() - jenv = jinja2.Environment(loader=loader) - jenv.filters['role_to_remote'] = _role_to_remote - setattr(ctx, '_jinja_env', jenv) - rctx = dict(ctx=ctx, config=config, cluster_name=config.get('cluster', '')) - _vip_vars(rctx) - conv = functools.partial(_apply_template, jenv, rctx) - return _convert_strs_in(target, conv) - - -def _vip_vars(rctx): - """For backwards compat with the previous subst_vip function.""" - ctx = rctx['ctx'] - if 'vnet' in getattr(ctx, 'vip', {}): - rctx['VIPPREFIXLEN'] = str(ctx.vip["vnet"].prefixlen) - rctx['VIPSUBNET'] = str(ctx.vip["vnet"].network_address) - if 'vips' in getattr(ctx, 'vip', {}): - vips = ctx.vip['vips'] - for idx, vip in enumerate(vips): - rctx[f'VIP{idx}'] = str(vip) - - -@jinja2.pass_context -def _role_to_remote(rctx, role): - """Return the first remote matching the given role.""" - ctx = rctx['ctx'] - for remote, roles in ctx.cluster.remotes.items(): - if role in roles: - return remote - return None - - def _shell(ctx, cluster_name, remote, args, extra_cephadm_args=[], **kwargs): teuthology.get_testdir(ctx) return remote.run( @@ -1513,22 +1444,6 @@ def stop(ctx, config): yield -def _expand_roles(ctx, config): - if 'all-roles' in config and len(config) == 1: - a = config['all-roles'] - roles = teuthology.all_roles(ctx.cluster) - config = dict((id_, a) for id_ in roles if not id_.startswith('host.')) - elif 'all-hosts' in config and len(config) == 1: - a = config['all-hosts'] - roles = teuthology.all_roles(ctx.cluster) - config = dict((id_, a) for id_ in roles if id_.startswith('host.')) - elif 'all-roles' in config or 'all-hosts' in config: - raise ValueError( - 'all-roles/all-hosts may not be combined with any other roles' - ) - return config - - def shell(ctx, config): """ Execute (shell) commands @@ -1541,8 +1456,8 @@ def shell(ctx, config): for k in config.pop('volumes', []): args.extend(['-v', k]) - config = _expand_roles(ctx, config) - config = _template_transform(ctx, config, config) + config = template.expand_roles(ctx, config) + config = template.transform(ctx, config, config) for role, cmd in config.items(): (remote,) = ctx.cluster.only(role).remotes.keys() log.info('Running commands on role %s host %s', role, remote.name) @@ -1575,31 +1490,6 @@ def _shell_command(obj): raise ValueError(f'invalid command item: {obj!r}') -def exec(ctx, config): - """ - This is similar to the standard 'exec' task, but does template substitutions. - - TODO: this should probably be moved out of cephadm.py as it's pretty generic. - """ - assert isinstance(config, dict), "task exec got invalid config" - testdir = teuthology.get_testdir(ctx) - config = _expand_roles(ctx, config) - for role, ls in config.items(): - (remote,) = ctx.cluster.only(role).remotes.keys() - log.info('Running commands on role %s host %s', role, remote.name) - for c in ls: - c.replace('$TESTDIR', testdir) - remote.run( - args=[ - 'sudo', - 'TESTDIR={tdir}'.format(tdir=testdir), - 'bash', - '-ex', - '-c', - _template_transform(ctx, config, c)], - ) - - def apply(ctx, config): """ Apply spec @@ -1622,7 +1512,7 @@ def apply(ctx, config): cluster_name = config.get('cluster', 'ceph') specs = config.get('specs', []) - specs = _template_transform(ctx, config, specs) + specs = template.transform(ctx, config, specs) y = yaml.dump_all(specs) log.info(f'Applying spec(s):\n{y}') -- 2.39.5