From 8b8546a46b86e8f1fcac9e4e75c558a1ebe68520 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 22 May 2019 13:08:10 -0500 Subject: [PATCH] Sync config_template from upstream This change pulls in the most recent release of the config_template module into the ceph_ansible action plugins. Signed-off-by: Kevin Carter (cherry picked from commit 789cef7621a3869fb42d4b2749f22d11ff08f6e0) --- plugins/actions/config_template.py | 190 +++++++++++++++++++++++++---- 1 file changed, 169 insertions(+), 21 deletions(-) diff --git a/plugins/actions/config_template.py b/plugins/actions/config_template.py index e88488fb3..fbf65081c 100644 --- a/plugins/actions/config_template.py +++ b/plugins/actions/config_template.py @@ -26,6 +26,7 @@ try: from StringIO import StringIO except ImportError: from io import StringIO +import base64 import json import os import pwd @@ -215,8 +216,8 @@ class ConfigTemplateParser(ConfigParser.RawConfigParser): comments.append('') continue - if line[0] in '#;': - comments.append(line) + if line.lstrip()[0] in '#;': + comments.append(line.lstrip()) continue if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR": @@ -297,6 +298,98 @@ class ConfigTemplateParser(ConfigParser.RawConfigParser): options[name] = _temp_item +class DictCompare(object): + """ + Calculate the difference between two dictionaries. + + Example Usage: + >>> base_dict = {'test1': 'val1', 'test2': 'val2', 'test3': 'val3'} + >>> new_dict = {'test1': 'val2', 'test3': 'val3', 'test4': 'val3'} + >>> dc = DictCompare(base_dict, new_dict) + >>> dc.added() + ... ['test4'] + >>> dc.removed() + ... ['test2'] + >>> dc.changed() + ... ['test1'] + >>> dc.get_changes() + ... {'added': + ... {'test4': 'val3'}, + ... 'removed': + ... {'test2': 'val2'}, + ... 'changed': + ... {'test1': {'current_val': 'vol1', 'new_val': 'val2'} + ... } + """ + def __init__(self, base_dict, new_dict): + self.new_dict, self.base_dict = new_dict, base_dict + self.base_items, self.new_items = set( + self.base_dict.keys()), set(self.new_dict.keys()) + self.intersect = self.new_items.intersection(self.base_items) + + def added(self): + return self.new_items - self.intersect + + def removed(self): + return self.base_items - self.intersect + + def changed(self): + return set( + x for x in self.intersect if self.base_dict[x] != self.new_dict[x]) + + def get_changes(self): + """Returns dict of differences between 2 dicts and bool indicating if + there are differences + + :param base_dict: ``dict`` + :param new_dict: ``dict`` + :returns: ``dict``, ``bool`` + """ + changed = False + mods = {'added': {}, 'removed': {}, 'changed': {}} + + for s in self.changed(): + changed = True + if type(self.base_dict[s]) is not dict: + mods['changed'] = { + s: {'current_val': self.base_dict[s], + 'new_val': self.new_dict[s]}} + continue + + diff = DictCompare(self.base_dict[s], self.new_dict[s]) + for a in diff.added(): + if s not in mods['added']: + mods['added'][s] = {a: self.new_dict[s][a]} + else: + mods['added'][s][a] = self.new_dict[s][a] + + for r in diff.removed(): + if s not in mods['removed']: + mods['removed'][s] = {r: self.base_dict[s][r]} + else: + mods['removed'][s][r] = self.base_dict[s][r] + + for c in diff.changed(): + if s not in mods['changed']: + mods['changed'][s] = { + c: {'current_val': self.base_dict[s][c], + 'new_val': self.new_dict[s][c]}} + else: + mods['changed'][s][c] = { + 'current_val': self.base_dict[s][c], + 'new_val': self.new_dict[s][c]} + + for s in self.added(): + changed = True + mods['added'][s] = self.new_dict[s] + + for s in self.removed(): + changed = True + mods['removed'][s] = self.base_dict[s] + + return mods, changed + + class ActionModule(ActionBase): TRANSFERS_FILES = True @@ -306,11 +399,12 @@ class ActionModule(ActionBase): list_extend=True, ignore_none_type=True, default_section='DEFAULT'): - """Returns string value from a modified config file. + """Returns string value from a modified config file and dict of + merged config :param config_overrides: ``dict`` :param resultant: ``str`` || ``unicode`` - :returns: ``str`` + :returns: ``str``, ``dict`` """ # If there is an exception loading the RawConfigParser The config obj # is loaded again without the extra option. This is being done to @@ -328,6 +422,7 @@ class ActionModule(ActionBase): config_object = StringIO(resultant) config.readfp(config_object) + for section, items in config_overrides.items(): # If the items value is not a dictionary it is assumed that the # value is a default item for this config type. @@ -361,10 +456,23 @@ class ActionModule(ActionBase): else: config_object.close() + config_dict_new = {} + config_defaults = config.defaults() + for s in config.sections(): + config_dict_new[s] = {} + for k, v in config.items(s): + if k not in config_defaults or config_defaults[k] != v: + config_dict_new[s][k] = v + else: + if default_section in config_dict_new: + config_dict_new[default_section][k] = v + else: + config_dict_new[default_section] = {k: v} + resultant_stringio = StringIO() try: config.write(resultant_stringio) - return resultant_stringio.getvalue() + return resultant_stringio.getvalue(), config_dict_new finally: resultant_stringio.close() @@ -391,27 +499,26 @@ class ActionModule(ActionBase): list_extend=True, ignore_none_type=True, default_section='DEFAULT'): - """Returns config json + """Returns config json and dict of merged config Its important to note that file ordering will not be preserved as the information within the json file will be sorted by keys. :param config_overrides: ``dict`` :param resultant: ``str`` || ``unicode`` - :returns: ``str`` + :returns: ``str``, ``dict`` """ original_resultant = json.loads(resultant) merged_resultant = self._merge_dict( base_items=original_resultant, new_items=config_overrides, - list_extend=list_extend, - default_section=default_section + list_extend=list_extend ) return json.dumps( merged_resultant, indent=4, sort_keys=True - ) + ), merged_resultant def return_config_overrides_yaml(self, config_overrides, @@ -419,11 +526,11 @@ class ActionModule(ActionBase): list_extend=True, ignore_none_type=True, default_section='DEFAULT'): - """Return config yaml. + """Return config yaml and dict of merged config :param config_overrides: ``dict`` :param resultant: ``str`` || ``unicode`` - :returns: ``str`` + :returns: ``str``, ``dict`` """ original_resultant = yaml.safe_load(resultant) merged_resultant = self._merge_dict( @@ -436,7 +543,7 @@ class ActionModule(ActionBase): Dumper=IDumper, default_flow_style=False, width=1000, - ) + ), merged_resultant def _merge_dict(self, base_items, new_items, list_extend=True): """Recursively merge new_items into base_items. @@ -631,16 +738,49 @@ class ActionModule(ActionBase): self._templar._available_variables ) - if _vars['config_overrides']: - type_merger = getattr(self, CONFIG_TYPES.get(_vars['config_type'])) - resultant = type_merger( - config_overrides=_vars['config_overrides'], - resultant=resultant, - list_extend=_vars.get('list_extend', True), - ignore_none_type=_vars.get('ignore_none_type', True), - default_section=_vars.get('default_section', 'DEFAULT') + config_dict_base = {} + type_merger = getattr(self, CONFIG_TYPES.get(_vars['config_type'])) + resultant, config_dict_base = type_merger( + config_overrides=_vars['config_overrides'], + resultant=resultant, + list_extend=_vars.get('list_extend', True), + ignore_none_type=_vars.get('ignore_none_type', True), + default_section=_vars.get('default_section', 'DEFAULT') + ) + + changed = False + if self._play_context.diff: + slurpee = self._execute_module( + module_name='slurp', + module_args=dict(src=_vars['dest']), + task_vars=task_vars ) + config_dict_new = {} + if 'content' in slurpee: + dest_data = base64.b64decode( + slurpee['content']).decode('utf-8') + resultant_dest = self._templar.template( + dest_data, + preserve_trailing_newlines=True, + escape_backslashes=False, + convert_data=False + ) + type_merger = getattr(self, + CONFIG_TYPES.get(_vars['config_type'])) + resultant_new, config_dict_new = type_merger( + config_overrides={}, + resultant=resultant_dest, + list_extend=_vars.get('list_extend', True), + ignore_none_type=_vars.get('ignore_none_type', True), + default_section=_vars.get('default_section', 'DEFAULT') + ) + + # Compare source+overrides with dest to look for changes and + # build diff + cmp_dicts = DictCompare(config_dict_new, config_dict_base) + mods, changed = cmp_dicts.get_changes() + # Re-template the resultant object as it may have new data within it # as provided by an override variable. resultant = self._templar.template( @@ -691,6 +831,14 @@ class ActionModule(ActionBase): module_args=new_module_args, task_vars=task_vars ) + copy_changed = rc.get('changed') + if not copy_changed: + rc['changed'] = changed + + if self._play_context.diff: + rc['diff'] = [] + rc['diff'].append( + {'prepared': json.dumps(mods, indent=4, sort_keys=True)}) if self._task.args.get('content'): os.remove(_vars['source']) return rc -- 2.39.5