From: Pere Diaz Bou Date: Tue, 5 Oct 2021 08:13:33 +0000 (+0200) Subject: cephadm/box: create osds with cephadm and cleanups X-Git-Tag: v17.1.0~577^2~4 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=772db8c1b68f3fa943898066bcf07710321b1730;p=ceph.git cephadm/box: create osds with cephadm and cleanups Signed-off-by: Pere Diaz Bou --- diff --git a/ceph.spec.in b/ceph.spec.in index 3579d7e628e..2a0ccbd21c1 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -465,6 +465,7 @@ Summary: Utility to bootstrap Ceph clusters BuildArch: noarch Requires: lvm2 Requires: python%{python3_pkgversion} +Requires: openssh-server %if 0%{?weak_deps} Recommends: podman >= 2.0.2 %endif diff --git a/src/ceph-volume/ceph_volume/devices/lvm/prepare.py b/src/ceph-volume/ceph_volume/devices/lvm/prepare.py index 2f715fdba12..3121ede3bf5 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/prepare.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/prepare.py @@ -104,7 +104,7 @@ def prepare_bluestore(block, wal, db, secrets, tags, osd_id, fsid): db = prepare_dmcrypt(key, db, 'db', tags) # create the directory - prepare_utils.create_osd_path(osd_id, tmpfs=True) + prepare_utils.create_osd_path(osd_id, tmpfs=False) # symlink the block prepare_utils.link_block(block, osd_id) # get the latest monmap diff --git a/src/ceph-volume/ceph_volume/process.py b/src/ceph-volume/ceph_volume/process.py index e70986892b7..08791c276c0 100644 --- a/src/ceph-volume/ceph_volume/process.py +++ b/src/ceph-volume/ceph_volume/process.py @@ -31,7 +31,7 @@ def log_output(descriptor, message, terminal_logging, logfile_logging): if terminal_logging: getattr(terminal, descriptor)(message) if logfile_logging: - logger.info(line) + logger.info(line.encode('ascii', 'ignore').decode('ascii')) def log_descriptors(reads, process, terminal_logging): diff --git a/src/cephadm/box/bootstrap.sh b/src/cephadm/box/bootstrap.sh new file mode 100755 index 00000000000..4f689afe164 --- /dev/null +++ b/src/cephadm/box/bootstrap.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +OSDS=1 +HOSTS=0 +SKIP_LOOP=0 + +function print_usage() { + echo "./bootstrap.sh [OPTIONS]" + echo "options:" + echo " --hosts n: number of hosts to add" + echo " --osds n: number of osds to add" + echo " --update-image: create/update ceph image" +} + +function docker-ips() { + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}} %tab% {{.Name}}' $(docker ps -aq) | sed 's#%tab%#\t#g' | sed 's#/##g' | sort -t . -k 1,1n -k 2,2n -k 3,3n -k 4,4n +} + +while [ $# -ge 1 ]; do +case $1 in + --help) + print_usage + exit + ;; + --list-hosts) # TODO remove when ceph-ci updated + docker-ips | grep box + exit + ;; + --update-image) # TODO remove when ceph-ci updated + source ./get_ceph_image.sh + ;; + --hosts) + HOSTS="$2" + echo "number of hosts: $HOSTS" + shift + ;; + --osds) + OSDS="$2" + echo "number of osds: $OSDS" + shift + ;; + --skip-create-loop) + SKIP_LOOP=1 + ;; +esac +shift +done + +# TODO: remove when ceph-ci image has required deps +if [[ ! -a docker/ceph/image/quay.ceph.image.tar ]] +then + echo -e "\033[33mWARNING:\033[0m run ./get_ceph_image.sh to get an updated ceph-ci/ceph image with correct deps." + exit +fi + +if [[ SKIP_LOOP -eq 0 ]] +then + source setup_loop.sh +fi + +create_loops $OSDS + +# loops should be created before starting docker-compose or else docker could +# not find lvs +docker-compose down +docker-compose up --scale hosts=$HOSTS -d +sleep 3 + +# setup ssh in hosts +docker-compose exec hosts /cephadm/box/setup_ssh.sh +docker-compose exec -e NUM_OSDS=${OSDS} seed /cephadm/box/start diff --git a/src/cephadm/box/docker-compose.yml b/src/cephadm/box/docker-compose.yml index f55d2d286da..79271761f2d 100644 --- a/src/cephadm/box/docker-compose.yml +++ b/src/cephadm/box/docker-compose.yml @@ -16,6 +16,8 @@ services: stop_signal: RTMIN+3 volumes: - "/sys/fs/cgroup:/sys/fs/cgroup:ro" + - ../../../:/ceph + - ..:/cephadm networks: - public mem_limit: "20g" @@ -25,9 +27,6 @@ services: seed: extends: service: cephadm-host-base - volumes: - - ../../../:/ceph - - ..:/cephadm ports: - "3000:3000" - "8443:8443" diff --git a/src/cephadm/box/docker/ceph/.bashrc b/src/cephadm/box/docker/ceph/.bashrc new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/cephadm/box/docker/ceph/Dockerfile b/src/cephadm/box/docker/ceph/Dockerfile index 5aafd22d401..354728db2d1 100644 --- a/src/cephadm/box/docker/ceph/Dockerfile +++ b/src/cephadm/box/docker/ceph/Dockerfile @@ -1,4 +1,9 @@ FROM quay.ceph.io/ceph-ci/ceph:master -RUN dnf install which sudo -y + +RUN dnf install which sudo glibc-all-langpacks langpacks-en -y +RUN yum -y install glibc-locale-source glibc-langpack-en + +RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8 +COPY locale.conf /etc/locale.conf EXPOSE 8443 diff --git a/src/cephadm/box/docker/ceph/locale.conf b/src/cephadm/box/docker/ceph/locale.conf new file mode 100644 index 00000000000..00d76c8cdb6 --- /dev/null +++ b/src/cephadm/box/docker/ceph/locale.conf @@ -0,0 +1,2 @@ +LANG="en_US.UTF-8" +LC_ALL="en_US.UTF-8" diff --git a/src/cephadm/box/get_ceph_image.sh b/src/cephadm/box/get_ceph_image.sh new file mode 100755 index 00000000000..f34d208d418 --- /dev/null +++ b/src/cephadm/box/get_ceph_image.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -ex + +IMAGE=quay.ceph.io/ceph-ci/ceph:master +docker pull $IMAGE +# update image with deps +docker build -t $IMAGE docker/ceph +# store to later load within docker +rm docker/ceph/image/quay.ceph.image.tar +docker save quay.ceph.io/ceph-ci/ceph:master -o docker/ceph/image/quay.ceph.image.tar diff --git a/src/cephadm/box/setup_loop.sh b/src/cephadm/box/setup_loop.sh index 1e8ef9580b6..4bfb126b1fa 100755 --- a/src/cephadm/box/setup_loop.sh +++ b/src/cephadm/box/setup_loop.sh @@ -1,33 +1,50 @@ #!/bin/bash -set -x - -# This script works best outside docker right now. - -# TODO: remove this file in the future or extend with something more extensible. -# For now let's just use this. - -# look for an available loop device -avail_loop=$(sudo losetup -f) -loop_name=$(basename -- $avail_loop) - -# in case we have to create the loop, find the minor device number. -num_loops=$(lsmod | grep loop | awk '{print $3}') -num_loops=$((num_loops + 1)) -echo creating loop $avail_loop minor: $num_loops -mknod $avail_loop b 7 $num_loops -sudo umount $avail_loop -sudo losetup -d $avail_loop -mkdir -p loop-images -sudo fallocate -l 10G "loop-images/disk${loop_name}.img" -sudo losetup $avail_loop "loop-images/disk${loop_name}.img" -sudo wipefs -a $avail_loop - - -# TODO: We will need more than one LVs -sudo lvm lvremove /dev/vg1/lv1 -sudo lvm vgremove vg1 -sudo pvcreate $avail_loop -sudo vgcreate vg1 $avail_loop -# 6G is arbitrary, osds need 5 I think. Just in case. -sudo lvcreate --size 6G --name lv1 vg1 +function clean_vg() { + # sudo lvm lvremove -y "/dev/vg1/lv${i}" + sudo lvm vgremove -y vg1 + sudo rm loop-images/* +} + + +function create_loops() { + clean_vg + + NUM_OSDS=$1 + if [[ -z $NUM_OSDS ]]; then + echo "Call setup_loop to setup with more osds" + echo "Using default number of osds: 1." + NUM_OSDS=1 + fi + + # minimum 5 GB for each osd + SIZE=$(expr $NUM_OSDS \* 5) + # extra space just in case + SIZE=$(expr $SIZE + 2) + + echo "Using ${SIZE} GB of space" + + # look for an available loop device + avail_loop=$(sudo losetup -f) + loop_name=$(basename -- $avail_loop) + + # in case we have to create the loop, find the minor device number. + num_loops=$(lsmod | grep loop | awk '{print $3}') + num_loops=$((num_loops + 1)) + echo creating loop $avail_loop minor: $num_loops + mknod $avail_loop b 7 $num_loops + sudo umount $avail_loop + sudo losetup -d $avail_loop + mkdir -p loop-images + # sudo fallocate -l 10G "loop-images/disk${loop_name}.img" + sudo dd if=/dev/zero of="loop-images/disk${loop_name}.img" bs=1G count=$SIZE + sudo losetup $avail_loop "loop-images/disk${loop_name}.img" + + sudo vgcreate vg1 $avail_loop + sudo pvcreate $avail_loop + + for ((i=0;i<$NUM_OSDS;i++)); do + sudo vgchange --refresh + sudo lvcreate --size 5G --name "lv${i}" "vg1" + done; +} diff --git a/src/cephadm/box/setup_ssh.sh b/src/cephadm/box/setup_ssh.sh index f680feb032f..3eef28bd572 100755 --- a/src/cephadm/box/setup_ssh.sh +++ b/src/cephadm/box/setup_ssh.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # SSH if [[ ! -f "/root/.ssh/id_rsa" ]]; then diff --git a/src/cephadm/box/start b/src/cephadm/box/start index 32d15113504..07e6d3dd989 100755 --- a/src/cephadm/box/start +++ b/src/cephadm/box/start @@ -4,7 +4,8 @@ CEPHADM_PATH=/usr/local/sbin/cephadm dnf install which sudo -y -cp /cephadm/cephadm $CEPHADM_PATH +# link so we can debug cephadm +ln -s -f /cephadm/cephadm $CEPHADM_PATH chmod +x $CEPHADM_PATH tail -f /var/log/ceph/cephadm.log 1>&2 & @@ -14,20 +15,34 @@ if [[ -n "${SHARED_CEPH_FOLDER-}" ]]; then EXTRA_ARGS+=(--shared_ceph_folder "$SHARED_CEPH_FOLDER") fi -# TODO: remove docker build image and skill pull when cephadm's dependencies -# use which or it is removed. -# If we use a ceph image cephadm won't skip pulling the image. If it's a -# local image, it will fail. -docker build -t quay.ceph.io/ceph-ci/ceph:master /cephadm/box/docker/ceph -CEPHADM_IMAGE=quay.ceph.io/ceph-ci/ceph:master +docker load < /cephadm/box/docker/ceph/image/quay.ceph.image.tar + +# cephadm guid error because it sometimes tries to use quay.ceph.io/ceph-ci/ceph: +# instead of master's tag +export CEPHADM_IMAGE=quay.ceph.io/ceph-ci/ceph:master +echo "export CEPHADM_IMAGE=quay.ceph.io/ceph-ci/ceph:master" >> ~/.bashrc + if [[ -n "$CEPHADM_IMAGE" ]]; then EXTRA_ARGS+=--skip-pull fi -$CEPHADM_PATH bootstrap \ +export CEPH_SOURCE_FOLDER=/ceph +$CEPHADM_PATH --verbose bootstrap \ --mon-ip "$(hostname -i)" \ --allow-fqdn-hostname \ --initial-dashboard-password admin \ --dashboard-password-noupdate \ - --allow-overwrite \ + --shared_ceph_folder /ceph \ "${EXTRA_ARGS[@]}" + +# make sure vg and lvs are visible +vgchange --refresh +for((i=0;i<$NUM_OSDS;i++)); do + echo "Creating osd.${i}" + # create osd folder + $CEPHADM_PATH ceph-volume --shared_ceph_folder /ceph lvm create --bluestore --data "/dev/vg1/lv${i}" --no-systemd + echo "Deploying osd.${i}..." + # deploy osd with osd data folder + $CEPHADM_PATH deploy --name "osd.${i}" + echo "osd.${i} deployed!" +done; diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index ed392bd187d..afc4331a557 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -1785,7 +1785,6 @@ def default_image(func: FuncT) -> FuncT: ctx.image = _get_default_image(ctx) return func(ctx) - return cast(FuncT, _default_image) @@ -2217,7 +2216,6 @@ def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid, # type: (CephadmContext, str, str, Union[int, str], int, int, Optional[str], Optional[str]) -> None data_dir = make_data_dir(ctx, fsid, daemon_type, daemon_id, uid=uid, gid=gid) make_log_dir(ctx, fsid, uid=uid, gid=gid) - if config: config_path = os.path.join(data_dir, 'config') with open(config_path, 'w') as f: @@ -2616,6 +2614,77 @@ def extract_uid_gid(ctx, img='', file_path='/var/lib/ceph'): raise RuntimeError('uid/gid not found') +def validate_osd_data_dir(data_dir): + required_files = ['keyring', 'block', 'type', 'config'] + current_files = os.listdir(data_dir) + error_msg = '' + for file in required_files: + if file not in current_files: + error_msg += f'File {file} not found in {data_dir}\n' + if error_msg: + raise RuntimeError(error_msg) + + +def configure_osd_data_dir(ctx, fsid, daemon_id, uid, gid): + daemon_type = 'osd' + data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) + + # Ensure user:group is the expected + for f in os.listdir(data_dir): + os.chown(os.path.join(data_dir, f), uid, gid) + + # Create minimal config + touch(os.path.join(data_dir, 'config'), uid, gid) + mounts = get_container_mounts(ctx, fsid, daemon_type, daemon_id, no_config=True) + mounts[data_dir] = '/var/lib/ceph/osd/ceph-%s' % daemon_id + mounts['/etc/ceph/ceph.conf'] = '/etc/ceph/ceph.conf:z' + mounts['/etc/ceph/ceph.client.admin.keyring'] = '/etc/ceph/ceph.keyring:z' + + CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/bin/ceph', + args=[ + 'config', 'generate-minimal-conf', + '-o', '/var/lib/ceph/osd/ceph-%s/config' % daemon_id + ], + privileged=True, + volume_mounts=mounts + ).run() + + # Create keyring and then import + key = CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/bin/ceph-authtool', + args=['--gen-print-key'], + ).run().strip() + + keyring = ('[%s.%s]\n' + '\tkey = %s\n' + '\tcaps osd = allow *\n' + '\tcaps mon = allow *\n' + % (daemon_type, daemon_id, key)) + with open(os.path.join(data_dir, 'keyring'), 'w+') as f: + os.fchmod(f.fileno(), 0o600) + os.fchown(f.fileno(), uid, gid) + f.write(keyring) + CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/bin/ceph', + args=[ + 'auth', 'import', + '-i', '/var/lib/ceph/osd/ceph-%s/keyring' % daemon_id + ], + privileged=True, + volume_mounts=mounts + ).run() + + # Validate we have needed files + validate_osd_data_dir(data_dir) + + def deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid, config=None, keyring=None, osd_fsid=None, @@ -2682,6 +2751,9 @@ def deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid, uid, gid, config, keyring) + if daemon_type == 'osd': + configure_osd_data_dir(ctx, fsid, daemon_id, uid, gid) + if not reconfig: if daemon_type == CephadmAgent.daemon_type: if ctx.config_json == '-': @@ -2793,6 +2865,14 @@ def deploy_daemon_units( ) -> None: # cmd data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) + + # if osd then try to read parameters if not provided + if daemon_type == 'osd': + osd_fsid_path = os.path.join(data_dir, 'fsid') + if 'fsid' in os.listdir(data_dir) and not osd_fsid: + with open(osd_fsid_path, 'r') as f: + osd_fsid = f.read() + with open(data_dir + '/unit.run.new', 'w') as f, \ open(data_dir + '/unit.meta.new', 'w') as metaf: f.write('set -e\n') @@ -2922,7 +3002,9 @@ def deploy_daemon_units( verbosity=CallVerbosity.DEBUG) if enable: call_throws(ctx, ['systemctl', 'enable', unit_name]) + if start: + clean_cgroup(ctx, fsid, unit_name) call_throws(ctx, ['systemctl', 'start', unit_name]) @@ -4705,7 +4787,9 @@ def extract_uid_gid_monitoring(ctx, daemon_type): @default_image +@infer_fsid def command_deploy(ctx): + assert ctx.fsid # type: (CephadmContext) -> None daemon_type, daemon_id = ctx.name.split('.', 1) @@ -4887,6 +4971,7 @@ def command_shell(ctx): mounts[pathify(ctx.config)] = '/etc/ceph/ceph.conf:z' if ctx.keyring: mounts[pathify(ctx.keyring)] = '/etc/ceph/ceph.keyring:z' + if ctx.mount: for _mount in ctx.mount: split_src_dst = _mount.split(':') @@ -4988,7 +5073,6 @@ def command_ceph_volume(ctx): (uid, gid) = (0, 0) # ceph-volume runs as root mounts = get_container_mounts(ctx, ctx.fsid, 'osd', None) - tmp_config = None tmp_keyring = None @@ -4999,11 +5083,51 @@ def command_ceph_volume(ctx): tmp_config = write_tmp(config, uid, gid) mounts[tmp_config.name] = '/etc/ceph/ceph.conf:z' + # Ceph-volume uses the bootstrap-osd key in order to do its operations. + # This function retrieves the keyring so it can be provided. + + def get_bootstrap_osd_keyring() -> str: + if not ctx.keyring and os.path.exists(SHELL_DEFAULT_KEYRING): + ctx.keyring = SHELL_DEFAULT_KEYRING + (config, keyring) = get_config_and_keyring(ctx) + + mounts = get_container_mounts(ctx, ctx.fsid, 'osd', None, + no_config=True if ctx.config else False) + if ctx.config: + mounts[pathify(ctx.config)] = '/etc/ceph/ceph.conf:z' + if ctx.keyring: + mounts[pathify(ctx.keyring)] = '/etc/ceph/ceph.keyring:z' + c = CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/bin/ceph', + args='auth get client.bootstrap-osd'.split(), + volume_mounts=mounts, + ) + out, err, code = call_throws(ctx, c.run_cmd()) + if not code: + return out + else: + return None + + if not keyring: + keyring = get_bootstrap_osd_keyring() + if keyring: # tmp keyring file tmp_keyring = write_tmp(keyring, uid, gid) mounts[tmp_keyring.name] = '/var/lib/ceph/bootstrap-osd/ceph.keyring:z' + # If ceph-volume creates osd data directories which won't be persisted + # so we use a tmp dir for that. + # FIXME: probably we can use /var/lib/ceph/{fsid}? + tmp_osd_dir = tempfile.TemporaryDirectory() + # match /var/lib/ceph/osd/ dir permissions + os.chown(tmp_osd_dir.name, uid, gid) + os.chmod(tmp_osd_dir.name, 0o755) + # store newly created osds here + mounts[tmp_osd_dir.name] = '/var/lib/ceph/osd/:z' + c = CephContainer( ctx, image=ctx.image, @@ -5017,6 +5141,14 @@ def command_ceph_volume(ctx): out, err, code = call_throws(ctx, c.run_cmd()) if not code: print(out) + else: + print(err) + # If osds were created move osd's data directories + for osd_folder_name in os.listdir(tmp_osd_dir.name): + if 'ceph-' in osd_folder_name[:5]: + osd_id = osd_folder_name[5:] + osd_data_dir = os.path.join(tmp_osd_dir.name, osd_folder_name) + copy_tree(ctx, [osd_data_dir], f'/var/lib/ceph/{ctx.fsid}/osd.{osd_id}', uid=uid, gid=gid) ################################## @@ -7577,7 +7709,7 @@ def _get_parser(): help='cluster FSID') parser_ceph_volume.add_argument( '--config-json', - help='JSON file with config and (client.bootrap-osd) key') + help='JSON file with config and (client.bootstrap-osd) key') parser_ceph_volume.add_argument( '--config', '-c', help='ceph conf file') @@ -7587,6 +7719,10 @@ def _get_parser(): parser_ceph_volume.add_argument( 'command', nargs=argparse.REMAINDER, help='command') + parser_ceph_volume.add_argument( + '--shared_ceph_folder', + metavar='CEPH_SOURCE_FOLDER', + help='Development mode. Several folders in containers are volumes mapped to different sub-folders in the ceph source folder') parser_zap_osds = subparsers.add_parser( 'zap-osds', help='zap all OSDs associated with a particular fsid') @@ -7808,7 +7944,6 @@ def _get_parser(): help='daemon name (type.id)') parser_deploy.add_argument( '--fsid', - required=True, help='cluster FSID') parser_deploy.add_argument( '--config', '-c',