pass
-# funcs to process spec file for apply spec
-def _parse_yaml_docs(f: Iterable[str]) -> List[List[str]]:
- docs = []
- current_doc = [] # type: List[str]
+def _extract_host_info_from_applied_spec(f: Iterable[str]) -> List[Dict[str, str]]:
+ # overall goal of this function is to go through an applied spec and find
+ # the hostname (and addr is provided) for each host spec in the applied spec.
+ # Generally, we should be able to just pass the spec to the mgr module where
+ # proper yaml parsing can happen, but for host specs in particular we want to
+ # be able to distribute ssh keys, which requires finding the hostname (and addr
+ # if possible) for each potential host spec in the applied spec.
+
+ specs: List[List[str]] = []
+ current_spec: List[str] = []
for line in f:
if re.search(r'^---\s+', line):
- if current_doc:
- docs.append(current_doc)
- current_doc = []
+ if current_spec:
+ specs.append(current_spec)
+ current_spec = []
else:
- current_doc.append(line.rstrip())
- if current_doc:
- docs.append(current_doc)
- return docs
-
-
-def _parse_yaml_obj(doc: List[str]) -> Dict[str, str]:
- # note: this only parses the first layer of yaml
- obj = {} # type: Dict[str, str]
- current_key = ''
- for line in doc:
- if line.startswith(' '):
- obj[current_key] += line.strip()
- elif line.endswith(':'):
- current_key = line.strip(':')
- obj[current_key] = ''
- else:
- current_key, val = line.split(':')
- obj[current_key] = val.strip()
- return obj
-
+ line = line.strip()
+ if line:
+ current_spec.append(line)
+ if current_spec:
+ specs.append(current_spec)
+
+ host_specs: List[List[str]] = []
+ for spec in specs:
+ for line in spec:
+ if 'service_type' in line:
+ try:
+ _, type = line.split(':')
+ type = type.strip()
+ if type == 'host':
+ host_specs.append(spec)
+ except ValueError as e:
+ spec_str = '\n'.join(spec)
+ logger.error(f'Failed to pull service_type from spec:\n{spec_str}. Got error: {e}')
+ break
+ spec_str = '\n'.join(spec)
+ logger.error(f'Failed to find service_type within spec:\n{spec_str}')
+
+ host_dicts = []
+ for s in host_specs:
+ host_dict = _extract_host_info_from_spec(s)
+ # if host_dict is empty here, we failed to pull the hostname
+ # for the host from the spec. This should have already been logged
+ # so at this point we just don't want to include it in our output
+ if host_dict:
+ host_dicts.append(host_dict)
+
+ return host_dicts
+
+
+def _extract_host_info_from_spec(host_spec: List[str]) -> Dict[str, str]:
+ # note:for our purposes here, we only really want the hostname
+ # and address of the host from each of these specs in order to
+ # be able to distribute ssh keys. We will later apply the spec
+ # through the mgr module where proper yaml parsing can be done
+ # The returned dicts from this function should only contain
+ # one or two entries, one (required) for hostname, one (optional) for addr
+ # {
+ # hostname: <hostname>
+ # addr: <ip-addr>
+ # }
+ # if we fail to find the hostname, an empty dict is returned
+
+ host_dict = {} # type: Dict[str, str]
+ for line in host_spec:
+ for field in ['hostname', 'addr']:
+ if field in line:
+ try:
+ _, field_value = line.split(':')
+ field_value = field_value.strip()
+ host_dict[field] = field_value
+ except ValueError as e:
+ spec_str = '\n'.join(host_spec)
+ logger.error(f'Error trying to pull {field} from host spec:\n{spec_str}. Got error: {e}')
-def parse_yaml_objs(f: Iterable[str]) -> List[Dict[str, str]]:
- objs = []
- for d in _parse_yaml_docs(f):
- objs.append(_parse_yaml_obj(d))
- return objs
+ if 'hostname' not in host_dict:
+ spec_str = '\n'.join(host_spec)
+ logger.error(f'Could not find hostname in host spec:\n{spec_str}')
+ return {}
+ return host_dict
-def _distribute_ssh_keys(ctx: CephadmContext, host_spec: Dict[str, str], bootstrap_hostname: str) -> int:
+def _distribute_ssh_keys(ctx: CephadmContext, host_info: Dict[str, str], bootstrap_hostname: str) -> int:
# copy ssh key to hosts in host spec (used for apply spec)
ssh_key = CEPH_DEFAULT_PUBKEY
if ctx.ssh_public_key:
ssh_key = ctx.ssh_public_key.name
- if bootstrap_hostname != host_spec['hostname']:
- if 'addr' in host_spec:
- addr = host_spec['addr']
+ if bootstrap_hostname != host_info['hostname']:
+ if 'addr' in host_info:
+ addr = host_info['addr']
else:
- addr = host_spec['hostname']
+ addr = host_info['hostname']
out, err, code = call(ctx, ['sudo', '-u', ctx.ssh_user, 'ssh-copy-id', '-f', '-i', ssh_key, '-o StrictHostKeyChecking=no', '%s@%s' % (ctx.ssh_user, addr)])
if code:
- logger.info('\nCopying ssh key to host %s at address %s failed!\n' % (host_spec['hostname'], addr))
+ logger.error('\nCopying ssh key to host %s at address %s failed!\n' % (host_info['hostname'], addr))
return 1
else:
- logger.info('Added ssh key to host %s at address %s\n' % (host_spec['hostname'], addr))
+ logger.info('Added ssh key to host %s at address %s' % (host_info['hostname'], addr))
return 0
logger.info('Applying %s to cluster' % ctx.apply_spec)
# copy ssh key to hosts in spec file
with open(ctx.apply_spec) as f:
- try:
- for spec in parse_yaml_objs(f):
- if spec.get('service_type') == 'host':
- _distribute_ssh_keys(ctx, spec, hostname)
- except ValueError:
- logger.info('Unable to parse %s succesfully' % ctx.apply_spec)
+ host_dicts = _extract_host_info_from_applied_spec(f)
+ for h in host_dicts:
+ _distribute_ssh_keys(ctx, h, hostname)
mounts = {}
mounts[pathify(ctx.apply_spec)] = '/tmp/spec.yml:ro'
class TestApplySpec:
- def test_parse_yaml(self, cephadm_fs):
+ def test_extract_host_info_from_applied_spec(self, cephadm_fs):
yaml = '''---
service_type: host
hostname: vm-00
---
service_type: host
hostname: vm-02
-addr: 192.168.122.165
---
---
service_type: rgw
'''
cephadm_fs.create_file('spec.yml', contents=yaml)
- retdic = [{'service_type': 'host', 'hostname': 'vm-00', 'addr': '192.168.122.44', 'labels': '- example1- example2'},
- {'service_type': 'host', 'hostname': 'vm-01', 'addr': '192.168.122.247', 'labels': '- grafana'},
- {'service_type': 'host', 'hostname': 'vm-02', 'addr': '192.168.122.165'},
- {'service_id': 'myrgw',
- 'service_type': 'rgw',
- 'spec':
- 'rgw_frontend_ssl_certificate: |-----BEGIN PRIVATE '
- 'KEY-----V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3MgZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=-----END '
- 'PRIVATE KEY----------BEGIN '
- 'CERTIFICATE-----V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3MgZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=-----END '
- 'CERTIFICATE-----ssl: true'}]
+ retdic = [{'hostname': 'vm-00', 'addr': '192.168.122.44'},
+ {'hostname': 'vm-01', 'addr': '192.168.122.247'},
+ {'hostname': 'vm-02',}]
with open('spec.yml') as f:
- dic = cd.parse_yaml_objs(f)
+ dic = cd._extract_host_info_from_applied_spec(f)
assert dic == retdic
@mock.patch('cephadm.call', return_value=('', '', 0))
projects / ceph.git / commitdiff