From: Pere Diaz Bou Date: Thu, 14 Oct 2021 12:56:06 +0000 (+0200) Subject: cephadm/box: add cluster ssh key to hosts X-Git-Tag: v17.1.0~577^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=87aeb1c8ae8b236fa8570356981076c63ad519a5;p=ceph.git cephadm/box: add cluster ssh key to hosts Signed-off-by: Pere Diaz Bou --- diff --git a/ceph.spec.in b/ceph.spec.in index 2a0ccbd21c1..2c61b7e8bc4 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -466,6 +466,7 @@ BuildArch: noarch Requires: lvm2 Requires: python%{python3_pkgversion} Requires: openssh-server +Requires: which %if 0%{?weak_deps} Recommends: podman >= 2.0.2 %endif diff --git a/src/ceph-volume/ceph_volume/process.py b/src/ceph-volume/ceph_volume/process.py index 08791c276c0..400063ae085 100644 --- a/src/ceph-volume/ceph_volume/process.py +++ b/src/ceph-volume/ceph_volume/process.py @@ -5,6 +5,11 @@ from select import select from ceph_volume import terminal from ceph_volume.util import as_bytes +import locale + +if locale.getpreferredencoding().upper() != 'UTF-8': + locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') + import logging logger = logging.getLogger(__name__) @@ -31,7 +36,7 @@ def log_output(descriptor, message, terminal_logging, logfile_logging): if terminal_logging: getattr(terminal, descriptor)(message) if logfile_logging: - logger.info(line.encode('ascii', 'ignore').decode('ascii')) + logger.info(line) def log_descriptors(reads, process, terminal_logging): diff --git a/src/cephadm/box/Dockerfile b/src/cephadm/box/Dockerfile index 5b7a48045ec..3b685a22e9c 100644 --- a/src/cephadm/box/Dockerfile +++ b/src/cephadm/box/Dockerfile @@ -1,6 +1,7 @@ # https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/ FROM centos:8 as centos-systemd ENV container docker +ENV CEPHADM_PATH=/usr/local/sbin/cephadm #RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ #rm -f /lib/systemd/system/multi-user.target.wants/*;\ #rm -f /etc/systemd/system/*.wants/*;\ @@ -10,7 +11,9 @@ ENV container docker #rm -f /lib/systemd/system/basic.target.wants/*;\ #rm -f /lib/systemd/system/anaconda.target.wants/*; RUN dnf -y install chrony firewalld lvm2 \ - openssh-server openssh-clients python3 yum-utils && dnf clean all + openssh-server openssh-clients python3 \ + yum-utils sudo which && dnf clean all + RUN systemctl enable chronyd firewalld sshd @@ -20,10 +23,11 @@ RUN yum-config-manager --add-repo https://download.docker.com/linux/centos/docke RUN dnf -y install docker-ce && \ dnf clean all && systemctl enable docker +# ssh utilities +RUN dnf install epel-release -y && dnf makecache && dnf install sshpass -y EXPOSE 8443 - -RUN dnf install sudo which -y +EXPOSE 22 FROM centos-systemd-docker WORKDIR /root diff --git a/src/cephadm/box/bootstrap.sh b/src/cephadm/box/bootstrap.sh index 4f689afe164..f1baecb56ca 100755 --- a/src/cephadm/box/bootstrap.sh +++ b/src/cephadm/box/bootstrap.sh @@ -9,7 +9,11 @@ function print_usage() { echo "options:" echo " --hosts n: number of hosts to add" echo " --osds n: number of osds to add" - echo " --update-image: create/update ceph image" + echo " --update-ceph-image: create/update ceph image" + echo " --update-box-image: create/update cephadm box image" + echo " --skip-create-loop: skip creating loopback device" + echo " -l | --list-hosts: list available cephad-box hosts/seed" + echo " -h | --help: this help :)" } function docker-ips() { @@ -18,15 +22,18 @@ function docker-ips() { while [ $# -ge 1 ]; do case $1 in - --help) + -h | --help) print_usage exit ;; - --list-hosts) # TODO remove when ceph-ci updated + -l | --list-hosts) # TODO remove when ceph-ci updated docker-ips | grep box exit ;; - --update-image) # TODO remove when ceph-ci updated + --update-box-image) + docker build -t cephadm-box -f Dockerfile . + ;; + --update-ceph-image) # TODO remove when ceph-ci updated source ./get_ceph_image.sh ;; --hosts) @@ -53,12 +60,17 @@ then exit fi -if [[ SKIP_LOOP -eq 0 ]] +if [[ $OSDS -eq 0 ]] +then + SKIP_LOOP=1 +fi + +if [[ $SKIP_LOOP -eq 0 ]] then source setup_loop.sh + create_loops $OSDS fi -create_loops $OSDS # loops should be created before starting docker-compose or else docker could # not find lvs @@ -66,6 +78,18 @@ docker-compose down docker-compose up --scale hosts=$HOSTS -d sleep 3 -# setup ssh in hosts -docker-compose exec hosts /cephadm/box/setup_ssh.sh +IPS=$(docker-ips | grep "box_hosts" | awk '{ print $1 }') +echo "IPS: " +echo $IPS + +sudo sysctl net.ipv4.conf.all.forwarding=1 +sudo iptables -P FORWARD ACCEPT + +for ((i=1;i<=$HOSTS;i++)) +do + docker-compose exec --index=$i hosts /cephadm/box/setup_ssh.sh run-sshd +done + docker-compose exec -e NUM_OSDS=${OSDS} seed /cephadm/box/start + +docker-compose exec -e HOST_IPS="${IPS}" seed /cephadm/box/setup_ssh.sh copy-cluster-ssh-key diff --git a/src/cephadm/box/get_ceph_image.sh b/src/cephadm/box/get_ceph_image.sh index f34d208d418..3c431c86695 100755 --- a/src/cephadm/box/get_ceph_image.sh +++ b/src/cephadm/box/get_ceph_image.sh @@ -7,5 +7,6 @@ docker pull $IMAGE # update image with deps docker build -t $IMAGE docker/ceph # store to later load within docker -rm docker/ceph/image/quay.ceph.image.tar +mkdir -p docker/ceph/image +rm -f docker/ceph/image/quay.ceph.image.tar docker save quay.ceph.io/ceph-ci/ceph:master -o docker/ceph/image/quay.ceph.image.tar diff --git a/src/cephadm/box/setup_loop.sh b/src/cephadm/box/setup_loop.sh index 4bfb126b1fa..9ff6b814c4d 100755 --- a/src/cephadm/box/setup_loop.sh +++ b/src/cephadm/box/setup_loop.sh @@ -1,8 +1,7 @@ #!/bin/bash function clean_vg() { - # sudo lvm lvremove -y "/dev/vg1/lv${i}" - sudo lvm vgremove -y vg1 + sudo lvm vgremove -f -y vg1 sudo rm loop-images/* } diff --git a/src/cephadm/box/setup_ssh.sh b/src/cephadm/box/setup_ssh.sh index 3eef28bd572..5b81c2cbf68 100755 --- a/src/cephadm/box/setup_ssh.sh +++ b/src/cephadm/box/setup_ssh.sh @@ -1,19 +1,50 @@ #!/usr/bin/env bash -# SSH -if [[ ! -f "/root/.ssh/id_rsa" ]]; then - mkdir -p ~/.ssh - chmod 700 ~/.ssh - ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N "" -fi +set -e -cat ~/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys -if [[ ! -f "/root/.ssh/known_hosts" ]]; then - ssh-keygen -A -fi -dnf install -y openssh-server -/usr/sbin/sshd +function run-sshd() { + echo "Creating sshd server on $(hostname):$(hostname -i)" + # SSH + if [[ ! -f "/root/.ssh/id_rsa" ]]; then + mkdir -p ~/.ssh + chmod 700 ~/.ssh + ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N "" + fi -# no password -echo "root:" | chpasswd -echo "PermitRootLogin yes" >> /etc/ssh/sshd_config + cat ~/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys + if [[ ! -f "/root/.ssh/known_hosts" ]]; then + ssh-keygen -A + fi + + # change password + echo "root:root" | chpasswd + echo "PermitRootLogin yes" >> /etc/ssh/sshd_config + echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config + + /usr/sbin/sshd + echo "sshd finished" +} + +function copy-cluster-ssh-key() { + echo "Adding cluster ssh key to all hosts: ${HOST_IPS}" + HOST_IPS=$(echo $HOST_IPS) + for ip in $(echo $HOST_IPS) + do + if [[ ! $ip == $(hostname -i) ]] + then + echo $ip + # copy cluster key + sshpass -p "root" ssh-copy-id -f -o StrictHostKeyChecking=no -i /etc/ceph/ceph.pub "root@${ip}" + fi + done + echo "Finished adding keys, you can now add existing hosts containers to the cluster!" +} + +case $1 in + run-sshd) + run-sshd + ;; + copy-cluster-ssh-key) + copy-cluster-ssh-key + ;; +esac diff --git a/src/cephadm/box/start b/src/cephadm/box/start index 07e6d3dd989..7d12e41db41 100755 --- a/src/cephadm/box/start +++ b/src/cephadm/box/start @@ -1,8 +1,5 @@ #!/usr/bin/env bash set -euxo pipefail -CEPHADM_PATH=/usr/local/sbin/cephadm - -dnf install which sudo -y # link so we can debug cephadm ln -s -f /cephadm/cephadm $CEPHADM_PATH diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index afc4331a557..2db102d722d 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -2615,6 +2615,7 @@ def extract_uid_gid(ctx, img='', file_path='/var/lib/ceph'): def validate_osd_data_dir(data_dir): + # type: (str) -> None required_files = ['keyring', 'block', 'type', 'config'] current_files = os.listdir(data_dir) error_msg = '' @@ -2626,6 +2627,7 @@ def validate_osd_data_dir(data_dir): def configure_osd_data_dir(ctx, fsid, daemon_id, uid, gid): + # type: (CephadmContext, str, Union[int, str], int, int) -> None daemon_type = 'osd' data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) @@ -4789,8 +4791,8 @@ def extract_uid_gid_monitoring(ctx, daemon_type): @default_image @infer_fsid def command_deploy(ctx): - assert ctx.fsid # type: (CephadmContext) -> None + assert ctx.fsid daemon_type, daemon_id = ctx.name.split('.', 1) lock = FileLock(ctx, ctx.fsid) @@ -5086,7 +5088,7 @@ def command_ceph_volume(ctx): # Ceph-volume uses the bootstrap-osd key in order to do its operations. # This function retrieves the keyring so it can be provided. - def get_bootstrap_osd_keyring() -> str: + def get_bootstrap_osd_keyring() -> Optional[str]: if not ctx.keyring and os.path.exists(SHELL_DEFAULT_KEYRING): ctx.keyring = SHELL_DEFAULT_KEYRING (config, keyring) = get_config_and_keyring(ctx) @@ -5121,34 +5123,34 @@ def command_ceph_volume(ctx): # If ceph-volume creates osd data directories which won't be persisted # so we use a tmp dir for that. # FIXME: probably we can use /var/lib/ceph/{fsid}? - tmp_osd_dir = tempfile.TemporaryDirectory() - # match /var/lib/ceph/osd/ dir permissions - os.chown(tmp_osd_dir.name, uid, gid) - os.chmod(tmp_osd_dir.name, 0o755) - # store newly created osds here - mounts[tmp_osd_dir.name] = '/var/lib/ceph/osd/:z' + with tempfile.TemporaryDirectory() as tmp_osd_dir: + # match /var/lib/ceph/osd/ dir permissions + os.chown(tmp_osd_dir, uid, gid) + os.chmod(tmp_osd_dir, 0o755) + # store newly created osds here + mounts[tmp_osd_dir] = '/var/lib/ceph/osd/:z' - c = CephContainer( - ctx, - image=ctx.image, - entrypoint='/usr/sbin/ceph-volume', - envs=ctx.env, - args=ctx.command, - privileged=True, - volume_mounts=mounts, - ) + c = CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/sbin/ceph-volume', + envs=ctx.env, + args=ctx.command, + privileged=True, + volume_mounts=mounts, + ) - out, err, code = call_throws(ctx, c.run_cmd()) - if not code: - print(out) - else: - print(err) - # If osds were created move osd's data directories - for osd_folder_name in os.listdir(tmp_osd_dir.name): - if 'ceph-' in osd_folder_name[:5]: - osd_id = osd_folder_name[5:] - osd_data_dir = os.path.join(tmp_osd_dir.name, osd_folder_name) - copy_tree(ctx, [osd_data_dir], f'/var/lib/ceph/{ctx.fsid}/osd.{osd_id}', uid=uid, gid=gid) + out, err, code = call_throws(ctx, c.run_cmd()) + if not code: + print(out) + else: + print(err) + # If osds were created move osd's data directories + for osd_folder_name in os.listdir(tmp_osd_dir): + if 'ceph-' in osd_folder_name[:5]: + osd_id = osd_folder_name[5:] + osd_data_dir = os.path.join(tmp_osd_dir, osd_folder_name) + copy_tree(ctx, [osd_data_dir], f'/var/lib/ceph/{ctx.fsid}/osd.{osd_id}', uid=uid, gid=gid) ##################################