From: Adam King Date: Fri, 14 Oct 2022 18:53:27 +0000 (-0400) Subject: cephadm: only pull host info from applied spec X-Git-Tag: v17.2.6~137^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=20e588be9b1b423e150977d4ddef5f956155cf7c;p=ceph.git cephadm: only pull host info from applied spec We don't need to actually try to properly parse the yaml spec for our purposes, jsut pull the host info out from it. Fixes: https://tracker.ceph.com/issues/57870 Signed-off-by: Adam King (cherry picked from commit 5db03a684bc0e33d3a2e843e9fab7723af186312) Conflicts: src/cephadm/tests/test_cephadm.py --- diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index 4e48b261e2d4..54da689338b2 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -5348,62 +5348,105 @@ def finish_bootstrap_config( pass -# funcs to process spec file for apply spec -def _parse_yaml_docs(f: Iterable[str]) -> List[List[str]]: - docs = [] - current_doc = [] # type: List[str] +def _extract_host_info_from_applied_spec(f: Iterable[str]) -> List[Dict[str, str]]: + # overall goal of this function is to go through an applied spec and find + # the hostname (and addr is provided) for each host spec in the applied spec. + # Generally, we should be able to just pass the spec to the mgr module where + # proper yaml parsing can happen, but for host specs in particular we want to + # be able to distribute ssh keys, which requires finding the hostname (and addr + # if possible) for each potential host spec in the applied spec. + + specs: List[List[str]] = [] + current_spec: List[str] = [] for line in f: if re.search(r'^---\s+', line): - if current_doc: - docs.append(current_doc) - current_doc = [] + if current_spec: + specs.append(current_spec) + current_spec = [] else: - current_doc.append(line.rstrip()) - if current_doc: - docs.append(current_doc) - return docs - - -def _parse_yaml_obj(doc: List[str]) -> Dict[str, str]: - # note: this only parses the first layer of yaml - obj = {} # type: Dict[str, str] - current_key = '' - for line in doc: - if line.startswith(' '): - obj[current_key] += line.strip() - elif line.endswith(':'): - current_key = line.strip(':') - obj[current_key] = '' - else: - current_key, val = line.split(':') - obj[current_key] = val.strip() - return obj - + line = line.strip() + if line: + current_spec.append(line) + if current_spec: + specs.append(current_spec) + + host_specs: List[List[str]] = [] + for spec in specs: + for line in spec: + if 'service_type' in line: + try: + _, type = line.split(':') + type = type.strip() + if type == 'host': + host_specs.append(spec) + except ValueError as e: + spec_str = '\n'.join(spec) + logger.error(f'Failed to pull service_type from spec:\n{spec_str}. Got error: {e}') + break + spec_str = '\n'.join(spec) + logger.error(f'Failed to find service_type within spec:\n{spec_str}') + + host_dicts = [] + for s in host_specs: + host_dict = _extract_host_info_from_spec(s) + # if host_dict is empty here, we failed to pull the hostname + # for the host from the spec. This should have already been logged + # so at this point we just don't want to include it in our output + if host_dict: + host_dicts.append(host_dict) + + return host_dicts + + +def _extract_host_info_from_spec(host_spec: List[str]) -> Dict[str, str]: + # note:for our purposes here, we only really want the hostname + # and address of the host from each of these specs in order to + # be able to distribute ssh keys. We will later apply the spec + # through the mgr module where proper yaml parsing can be done + # The returned dicts from this function should only contain + # one or two entries, one (required) for hostname, one (optional) for addr + # { + # hostname: + # addr: + # } + # if we fail to find the hostname, an empty dict is returned + + host_dict = {} # type: Dict[str, str] + for line in host_spec: + for field in ['hostname', 'addr']: + if field in line: + try: + _, field_value = line.split(':') + field_value = field_value.strip() + host_dict[field] = field_value + except ValueError as e: + spec_str = '\n'.join(host_spec) + logger.error(f'Error trying to pull {field} from host spec:\n{spec_str}. Got error: {e}') -def parse_yaml_objs(f: Iterable[str]) -> List[Dict[str, str]]: - objs = [] - for d in _parse_yaml_docs(f): - objs.append(_parse_yaml_obj(d)) - return objs + if 'hostname' not in host_dict: + spec_str = '\n'.join(host_spec) + logger.error(f'Could not find hostname in host spec:\n{spec_str}') + return {} + return host_dict -def _distribute_ssh_keys(ctx: CephadmContext, host_spec: Dict[str, str], bootstrap_hostname: str) -> int: +def _distribute_ssh_keys(ctx: CephadmContext, host_info: Dict[str, str], bootstrap_hostname: str) -> int: # copy ssh key to hosts in host spec (used for apply spec) ssh_key = CEPH_DEFAULT_PUBKEY if ctx.ssh_public_key: ssh_key = ctx.ssh_public_key.name - if bootstrap_hostname != host_spec['hostname']: - if 'addr' in host_spec: - addr = host_spec['addr'] + if bootstrap_hostname != host_info['hostname']: + if 'addr' in host_info: + addr = host_info['addr'] else: - addr = host_spec['hostname'] + addr = host_info['hostname'] out, err, code = call(ctx, ['sudo', '-u', ctx.ssh_user, 'ssh-copy-id', '-f', '-i', ssh_key, '-o StrictHostKeyChecking=no', '%s@%s' % (ctx.ssh_user, addr)]) if code: - logger.info('\nCopying ssh key to host %s at address %s failed!\n' % (host_spec['hostname'], addr)) + logger.error('\nCopying ssh key to host %s at address %s failed!\n' % (host_info['hostname'], addr)) return 1 else: - logger.info('Added ssh key to host %s at address %s\n' % (host_spec['hostname'], addr)) + logger.info('Added ssh key to host %s at address %s' % (host_info['hostname'], addr)) return 0 @@ -5637,12 +5680,9 @@ def command_bootstrap(ctx): logger.info('Applying %s to cluster' % ctx.apply_spec) # copy ssh key to hosts in spec file with open(ctx.apply_spec) as f: - try: - for spec in parse_yaml_objs(f): - if spec.get('service_type') == 'host': - _distribute_ssh_keys(ctx, spec, hostname) - except ValueError: - logger.info('Unable to parse %s succesfully' % ctx.apply_spec) + host_dicts = _extract_host_info_from_applied_spec(f) + for h in host_dicts: + _distribute_ssh_keys(ctx, h, hostname) mounts = {} mounts[pathify(ctx.apply_spec)] = '/tmp/spec.yml:ro' diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py index b259ce55eca0..c5504f93b0bd 100644 --- a/src/cephadm/tests/test_cephadm.py +++ b/src/cephadm/tests/test_cephadm.py @@ -2098,7 +2098,7 @@ class TestPull: class TestApplySpec: - def test_parse_yaml(self, cephadm_fs): + def test_extract_host_info_from_applied_spec(self, cephadm_fs): yaml = '''--- service_type: host hostname: vm-00 @@ -2115,7 +2115,6 @@ labels: --- service_type: host hostname: vm-02 -addr: 192.168.122.165 --- --- service_type: rgw @@ -2141,20 +2140,12 @@ spec: ''' cephadm_fs.create_file('spec.yml', contents=yaml) - retdic = [{'service_type': 'host', 'hostname': 'vm-00', 'addr': '192.168.122.44', 'labels': '- example1- example2'}, - {'service_type': 'host', 'hostname': 'vm-01', 'addr': '192.168.122.247', 'labels': '- grafana'}, - {'service_type': 'host', 'hostname': 'vm-02', 'addr': '192.168.122.165'}, - {'service_id': 'myrgw', - 'service_type': 'rgw', - 'spec': - 'rgw_frontend_ssl_certificate: |-----BEGIN PRIVATE ' - 'KEY-----V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3MgZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=-----END ' - 'PRIVATE KEY----------BEGIN ' - 'CERTIFICATE-----V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3MgZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=-----END ' - 'CERTIFICATE-----ssl: true'}] + retdic = [{'hostname': 'vm-00', 'addr': '192.168.122.44'}, + {'hostname': 'vm-01', 'addr': '192.168.122.247'}, + {'hostname': 'vm-02',}] with open('spec.yml') as f: - dic = cd.parse_yaml_objs(f) + dic = cd._extract_host_info_from_applied_spec(f) assert dic == retdic @mock.patch('cephadm.call', return_value=('', '', 0))