ContainerEngine is the baseclass of PodmanEngine and DockerEngine.
Furthermore, a HostContainer class was added as struct of info related
to a container.
Signed-off-by: Pere Diaz Bou <pdiazbou@redhat.com>
run_cephadm_shell_command,
run_dc_shell_command,
run_dc_shell_commands,
+ get_container_engine,
run_shell_command,
run_shell_commands,
+ ContainerEngine,
+ DockerEngine,
+ PodmanEngine,
colored,
engine,
engine_compose,
# extract_tag
assert image_name.find(':')
image_name, tag = image_name.split(':')
- images = run_shell_command(f'{engine()} image ls').split('\n')
+ engine = get_container_engine()
+ images = engine.run('image ls').split('\n')
IMAGE_NAME = 0
TAG = 1
for image in images:
def get_ceph_image():
print('Getting ceph image')
- run_shell_command(f'{engine()} pull {CEPH_IMAGE}')
+ engine = get_container_engine()
+ engine.run('pull {CEPH_IMAGE}')
# update
- run_shell_command(f'{engine()} build -t {CEPH_IMAGE} docker/ceph')
+ engine.run('build -t {CEPH_IMAGE} docker/ceph')
if not os.path.exists('docker/ceph/image'):
os.mkdir('docker/ceph/image')
remove_ceph_image_tar()
- run_shell_command(f'{engine()} save {CEPH_IMAGE} -o {CEPH_IMAGE_TAR}')
+ engine.run('save {CEPH_IMAGE} -o {CEPH_IMAGE_TAR}')
run_shell_command(f'chmod 777 {CEPH_IMAGE_TAR}')
print('Ceph image added')
def get_box_image():
print('Getting box image')
- if engine() == 'docker':
- run_shell_command(f'{engine()} build -t cephadm-box -f DockerfileDocker .')
- else:
- run_shell_command(f'{engine()} build -t cephadm-box -f DockerfilePodman .')
+ engine = get_container_engine()
+ engine.run(f'build -t cephadm-box -f {engine.dockerfile} .')
print('Box image added')
def check_dashboard():
print(colored('selinux should be disabled, please disable it if you '
'don\'t want unexpected behaviour.', Colors.WARNING))
-def setup_podman_env(hosts: int = 1, osd_devs={}):
- network_name = 'box_network'
- networks = run_shell_command('podman network ls')
- if network_name not in networks:
- run_shell_command(f'podman network create -d bridge {network_name}')
-
- run_default_options = """--group-add keep-groups --device /dev/fuse -it -d \\
- --cap-add SYS_ADMIN \\
- --cap-add NET_ADMIN \\
- --cap-add SYS_TIME \\
- --cap-add SYS_RAWIO \\
- --cap-add MKNOD \\
- --cap-add NET_RAW \\
- --cap-add SETUID \\
- --cap-add SETGID \\
- --cap-add CHOWN \\
- --cap-add SYS_PTRACE \\
- --cap-add SYS_TTY_CONFIG \\
- --cap-add CAP_AUDIT_WRITE \\
- --cap-add CAP_AUDIT_CONTROL \\
- -e CEPH_BRANCH=main \\
- -v ../../../:/ceph:z \\
- -v ../:/cephadm:z \\
- -v /run/udev:/run/udev \\
- --tmpfs /run \\
- --tmpfs /tmp \\
- -v /sys/dev/block:/sys/dev/block \\
- -v /sys/fs/cgroup:/sys/fs/cgroup:ro \\
- -v /dev/fuse:/dev/fuse \\
- -v /dev/disk:/dev/disk \\
- -v /sys/devices/virtual/block:/sys/devices/virtual/block \\
- -v /sys/block:/dev/block \\
- -v /dev/mapper:/dev/mapper \\
- -v /dev/mapper/control:/dev/mapper/control \\
- --stop-signal RTMIN+3 -m 20g cephadm-box \\
- """
- def add_option(dest, src):
- dest = f'{src} {dest}'
- return dest
- for osd_dev in osd_devs.values():
- device = osd_dev["device"]
- run_default_options = add_option(run_default_options, f'--device {device}:{device}')
-
-
- for host in range(hosts+1): # 0 will be the seed
- options = run_default_options
- options = add_option(options, f'--name box_hosts_{host}')
- options = add_option(options, f'--network {network_name}')
- if host == 0:
- options = add_option(options, f'-p 8443:8443') # dashboard
- options = add_option(options, f'-p 3000:3000') # grafana
- options = add_option(options, f'-p 9093:9093') # alertmanager
- options = add_option(options, f'-p 9095:9095') # prometheus
-
- run_shell_command(f'podman run {options}')
-
class Cluster(Target):
_help = 'Manage docker cephadm boxes'
- actions = ['bootstrap', 'start', 'down', 'list', 'sh', 'setup', 'cleanup', 'doctor']
+ actions = ['bootstrap', 'start', 'down', 'list', 'sh', 'setup', 'cleanup']
def set_args(self):
self.parser.add_argument(
print('Running bootstrap on seed')
cephadm_path = str(os.environ.get('CEPHADM_PATH'))
- if engine() == 'docker':
- # restart to ensure docker is using daemon.json
- run_shell_command(
- 'systemctl restart docker'
- )
-
+ engine = get_container_engine()
+ if isinstance(engine, DockerEngine):
+ engine.restart()
st = os.stat(cephadm_path)
os.chmod(cephadm_path, st.st_mode | stat.S_IEXEC)
- run_shell_command(f'{engine()} load < /cephadm/box/docker/ceph/image/quay.ceph.image.tar')
+ engine.run('load < /cephadm/box/docker/ceph/image/quay.ceph.image.tar')
# cephadm guid error because it sometimes tries to use quay.ceph.io/ceph-ci/ceph:<none>
# instead of main branch's tag
run_shell_command('export CEPH_SOURCE_FOLDER=/ceph')
)
print('Running cephadm bootstrap...')
- run_shell_command(cephadm_bootstrap_command)
+ run_shell_command(cephadm_bootstrap_command, expect_exit_code=120)
print('Cephadm bootstrap complete')
run_shell_command('sudo vgchange --refresh')
check_selinux()
osds = int(Config.get('osds'))
hosts = int(Config.get('hosts'))
+ engine = get_container_engine()
# ensure boxes don't exist
self.down()
# podman is ran without sudo
- if engine() == 'podman':
+ if isinstance(engine, PodmanEngine):
I_am = run_shell_command('whoami')
if 'root' in I_am:
print(root_error_msg)
print('Starting containers')
- if engine() == 'docker':
- dcflags = f'-f {Config.get("docker_yaml")}'
- if not os.path.exists('/sys/fs/cgroup/cgroup.controllers'):
- dcflags += f' -f {Config.get("docker_v1_yaml")}'
- run_shell_command(f'{engine_compose()} {dcflags} up --scale hosts={hosts} -d')
- else:
- setup_podman_env(hosts=hosts, osd_devs=osd.load_osd_devices())
+ engine.up(hosts)
+ containers = engine.get_containers()
+ seed = engine.get_seed()
# Umounting somehow brings back the contents of the host /sys/dev/block.
# On startup /sys/dev/block is empty. After umount, we can see symlinks again
# so that lsblk is able to run as expected
- run_dc_shell_command('umount /sys/dev/block', 1, BoxType.SEED)
+ run_dc_shell_command('umount /sys/dev/block', seed)
run_shell_command('sudo sysctl net.ipv4.conf.all.forwarding=1')
run_shell_command('sudo iptables -P FORWARD ACCEPT')
systemctl start chronyd
systemctl status --no-pager chronyd
"""
- for h in range(hosts):
- run_dc_shell_commands(h + 1, BoxType.HOST, chronyd_setup)
- run_dc_shell_commands(1, BoxType.SEED, chronyd_setup)
+ for container in containers:
+ print(colored('Got container:', Colors.OKCYAN), str(container))
+ for container in containers:
+ run_dc_shell_commands(chronyd_setup, container)
print('Seting up host ssh servers')
- for h in range(hosts):
- host._setup_ssh(BoxType.HOST, h + 1)
-
- host._setup_ssh(BoxType.SEED, 1)
+ for container in containers:
+ print(colored('Setting up ssh server for:', Colors.OKCYAN), str(container))
+ host._setup_ssh(container)
verbose = '-v' if Config.get('verbose') else ''
skip_deploy = '--skip-deploy-osds' if Config.get('skip-deploy-osds') else ''
)
skip_dashboard = '--skip-dashboard' if Config.get('skip-dashboard') else ''
box_bootstrap_command = (
- f'/cephadm/box/box.py {verbose} --engine {engine()} cluster bootstrap '
+ f'/cephadm/box/box.py {verbose} --engine {engine.command} cluster bootstrap '
f'--osds {osds} '
f'--hosts {hosts} '
f'{skip_deploy} '
f'{skip_dashboard} '
f'{skip_monitoring_stack} '
)
- run_dc_shell_command(box_bootstrap_command, 1, BoxType.SEED)
-
+ print(box_bootstrap_command)
+ run_dc_shell_command(box_bootstrap_command, seed)
expanded = Config.get('expanded')
if expanded:
dashboard_ip = 'localhost'
info = get_boxes_container_info(with_seed=True)
- if engine() == 'docker':
+ if isinstance(engine, DockerEngine):
for i in range(info['size']):
if get_seed_name() in info['container_names'][i]:
dashboard_ip = info["ips"][i]
print('Bootstrap finished successfully')
- @ensure_outside_container
- def doctor(self):
- pass
-
@ensure_outside_container
def down(self):
- if engine() == 'podman':
- containers = json.loads(run_shell_command('podman container ls --format json'))
+ engine = get_container_engine()
+ if isinstance(engine, PodmanEngine):
+ containers = json.loads(engine.run('container ls --format json'))
for container in containers:
for name in container['Names']:
if name.startswith('box_hosts_'):
- run_shell_command(f'podman container kill {name}')
- run_shell_command(f'podman container rm {name}')
- pods = json.loads(run_shell_command('podman pod ls --format json'))
+ engine.run(f'container kill {name}')
+ engine.run(f'container rm {name}')
+ pods = json.loads(engine.run('pod ls --format json'))
for pod in pods:
if 'Name' in pod and pod['Name'].startswith('box_pod_host'):
name = pod['Name']
- run_shell_command(f'podman pod kill {name}')
- run_shell_command(f'podman pod rm {name}')
+ engine.run(f'pod kill {name}')
+ engine.run(f'pod rm {name}')
else:
run_shell_command(f'{engine_compose()} -f {Config.get("docker_yaml")} down')
print('Successfully killed all boxes')
from util import (
Config,
+ HostContainer,
Target,
get_boxes_container_info,
+ get_container_engine,
inside_container,
run_cephadm_shell_command,
run_dc_shell_command,
)
-def _setup_ssh(container_type: BoxType, container_index):
+def _setup_ssh(container: HostContainer):
if inside_container():
if not os.path.exists('/root/.ssh/known_hosts'):
- run_shell_command('ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N ""')
+ run_shell_command('echo "y" | ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N ""',
+ expect_error=True)
run_shell_command('echo "root:root" | chpasswd')
with open('/etc/ssh/sshd_config', 'a+') as f:
print('Redirecting to _setup_ssh to container')
verbose = '-v' if Config.get('verbose') else ''
run_dc_shell_command(
- f'/cephadm/box/box.py {verbose} --engine {engine()} host setup_ssh {BoxType.to_string(container_type)} {container_index}',
- container_index,
- container_type,
+ f'/cephadm/box/box.py {verbose} --engine {engine()} host setup_ssh {container.name}',
+ container
)
ips = f'{ips}'
hostnames = ' '.join(hostnames)
hostnames = f'{hostnames}'
+ seed = get_container_engine().get_seed()
run_dc_shell_command(
- f'/cephadm/box/box.py {verbose} --engine {engine()} host add_hosts seed {BoxType.to_string(BoxType.SEED)} --ips {ips} --hostnames {hostnames}',
- 1,
- BoxType.SEED,
- )
+ f'/cephadm/box/box.py {verbose} --engine {engine()} host add_hosts {seed.name} --ips {ips} --hostnames {hostnames}',
+ seed
+ )
def _copy_cluster_ssh_key(ips: Union[List[str], str]):
ips = ' '.join(ips)
ips = f'{ips}'
# assume we only have one seed
+ seed = get_container_engine().get_seed()
run_dc_shell_command(
- f'/cephadm/box/box.py {verbose} --engine {engine()} host copy_cluster_ssh_key {BoxType.to_string(BoxType.SEED)} 1 --ips {ips}',
- 1,
- BoxType.SEED,
+ f'/cephadm/box/box.py {verbose} --engine {engine()} host copy_cluster_ssh_key {seed.name} --ips {ips}',
+ seed
)
def set_args(self):
self.parser.add_argument('action', choices=Host.actions)
self.parser.add_argument(
- 'container_type', type=str, help='box_{type}_{index}'
- )
- self.parser.add_argument(
- 'container_index', type=str, help='box_{type}_{index}'
+ 'container_name',
+ type=str,
+ help='box_{type}_{index}. In docker, type can be seed or hosts. In podman only hosts.'
)
self.parser.add_argument('--ips', nargs='*', help='List of host ips')
self.parser.add_argument(
)
def setup_ssh(self):
- type_ = Config.get('container_type')
- index = Config.get('container_index')
- _setup_ssh(type_, index)
+ container_name = Config.get('container_name')
+ engine = get_container_engine()
+ _setup_ssh(engine.get_container(container_name))
def add_hosts(self):
ips = Config.get('ips')
get_orch_hosts,
run_cephadm_shell_command,
run_dc_shell_command,
+ get_container_engine,
run_shell_command,
)
osd_devs = load_osd_devices()
hosts = get_orch_hosts()
host_index = 0
+ seed = get_container_engine().get_seed()
v = '-v' if Config.get('verbose') else ''
for osd in osd_devs.values():
deployed = False
hostname = hosts[host_index]['hostname']
deployed = run_dc_shell_command(
f'/cephadm/box/box.py {v} osd deploy --data {osd["device"]} --hostname {hostname}',
- 1,
- BoxType.SEED
+ seed
)
deployed = 'created osd' in deployed.lower() or 'already created?' in deployed.lower()
print('Waiting 5 seconds to re-run deploy osd...')
import os
import subprocess
import sys
-import enum
-from typing import Any, Callable, Dict
+import copy
+from abc import ABCMeta, abstractmethod
+from enum import Enum
+from typing import Any, Callable, Dict, List
class Colors:
HEADER = '\033[95m'
def colored(msg, color: Colors):
return color + msg + Colors.ENDC
-def run_shell_command(command: str, expect_error=False, verbose=True) -> str:
+class BoxType(str, Enum):
+ SEED = 'seed'
+ HOST = 'host'
+
+class HostContainer:
+ def __init__(self, _name, _type) -> None:
+ self._name: str = _name
+ self._type: BoxType = _type
+
+ @property
+ def name(self) -> str:
+ return self._name
+
+ @property
+ def type(self) -> BoxType:
+ return self._type
+ def __str__(self) -> str:
+ return f'{self.name} {self.type}'
+
+def run_shell_command(command: str, expect_error=False, verbose=True, expect_exit_code=0) -> str:
if Config.get('verbose'):
print(f'{colored("Running command", Colors.HEADER)}: {colored(command, Colors.OKBLUE)}')
err += process.stderr.read().decode('latin1').strip()
out = out.strip()
- if process.returncode != 0 and not expect_error:
+ if process.returncode != 0 and not expect_error and process.returncode != expect_exit_code:
err = colored(err, Colors.FAIL);
- raise RuntimeError(f'Failed command: {command}\n{err}')
+
+ raise RuntimeError(f'Failed command: {command}\n{err}\nexit code: {process.returncode}')
sys.exit(1)
return out
-class BoxType(enum.IntEnum):
- SEED = 0 # where we bootstrap cephadm
- HOST = 1
- @staticmethod
- def to_enum(value: str):
- if value == 'seed':
- return BoxType.SEED
- elif value == 'host':
- return BoxType.HOST
- else:
- print(f'Wrong container type {value}')
- sys.exit(1)
-
- @staticmethod
- def to_string(box_type):
- if box_type == BoxType.SEED:
- return 'seed'
- elif box_type == BoxType.HOST:
- return 'host'
- else:
- print(f'Wrong container type {type_}')
- sys.exit(1)
-
-
-def run_dc_shell_commands(index, box_type: BoxType, commands: str, expect_error=False) -> str:
+def run_dc_shell_commands(commands: str, container: HostContainer, expect_error=False) -> str:
for command in commands.split('\n'):
command = command.strip()
if not command:
continue
- run_dc_shell_command(command.strip(), index, box_type, expect_error=expect_error)
+ run_dc_shell_command(command.strip(), container, expect_error=expect_error)
def run_shell_commands(commands: str, expect_error=False) -> str:
for command in commands.split('\n'):
def run_dc_shell_command(
- command: str, index: int, box_type: BoxType, expect_error=False
+ command: str, container: HostContainer, expect_error=False
) -> str:
- box_type_str = 'box_hosts'
- if box_type == BoxType.SEED:
- index = 0
- if engine() == 'docker':
- box_type_str = 'seed'
- index = 1
-
- container_id = get_container_id(f'{box_type_str}_{index}')
- print(container_id)
- out = run_shell_command(
- f'{engine()} exec -it {container_id} {command}', expect_error
- )
+ out = get_container_engine().run_exec(container, command, expect_error=expect_error)
return out
def inside_container() -> bool:
if inside_container():
orch_host_ls_out = run_cephadm_shell_command('ceph orch host ls --format json')
else:
- orch_host_ls_out = run_dc_shell_command(f'cephadm shell --keyring /etc/ceph/ceph.keyring --config /etc/ceph/ceph.conf -- ceph orch host ls --format json', 1, BoxType.SEED)
+ orch_host_ls_out = run_dc_shell_command(f'cephadm shell --keyring /etc/ceph/ceph.keyring --config /etc/ceph/ceph.conf -- ceph orch host ls --format json',
+ get_container_engine().get_seed())
sp = orch_host_ls_out.split('\n')
orch_host_ls_out = sp[len(sp) - 1]
hosts = json.loads(orch_host_ls_out)
return hosts
+
+
+class ContainerEngine(metaclass=ABCMeta):
+ @property
+ @abstractmethod
+ def command(self) -> str: pass
+
+ @property
+ @abstractmethod
+ def seed_name(self) -> str: pass
+
+ @property
+ @abstractmethod
+ def dockerfile(self) -> str: pass
+
+ @property
+ def host_name_prefix(self) -> str:
+ return 'box_hosts_'
+
+ @abstractmethod
+ def up(self, hosts: int): pass
+
+ def run_exec(self, container: HostContainer, command: str, expect_error: bool = False):
+ return run_shell_command(' '.join([self.command, 'exec', container.name, command]),
+ expect_error=expect_error)
+
+ def run(self, engine_command: str, expect_error: bool = False):
+ return run_shell_command(' '.join([self.command, engine_command]), expect_error=expect_error)
+
+ def get_containers(self) -> List[HostContainer]:
+ ps_out = json.loads(run_shell_command('podman ps --format json'))
+ containers = []
+ for container in ps_out:
+ if not container['Names']:
+ raise RuntimeError(f'Container {container} missing name')
+ name = container['Names'][0]
+ if name == self.seed_name:
+ containers.append(HostContainer(name, BoxType.SEED))
+ elif name.startswith(self.host_name_prefix):
+ containers.append(HostContainer(name, BoxType.HOST))
+ return containers
+
+ def get_seed(self) -> HostContainer:
+ for container in self.get_containers():
+ if container.type == BoxType.SEED:
+ return container
+ raise RuntimeError('Missing seed container')
+
+ def get_container(self, container_name: str):
+ containers = self.get_containers()
+ for container in containers:
+ if container.name == container_name:
+ return container
+ return None
+
+
+ def restart(self):
+ pass
+
+
+class DockerEngine(ContainerEngine):
+ command = 'docker'
+ seed_name = 'seed'
+ dockerfile = 'DockerfileDocker'
+
+ def restart(self):
+ run_shell_command('systemctl restart docker')
+
+ def up(self, hosts: int):
+ dcflags = f'-f {Config.get("docker_yaml")}'
+ if not os.path.exists('/sys/fs/cgroup/cgroup.controllers'):
+ dcflags += f' -f {Config.get("docker_v1_yaml")}'
+ run_shell_command(f'{engine_compose()} {dcflags} up --scale hosts={hosts} -d')
+
+class PodmanEngine(ContainerEngine):
+ command = 'podman'
+ seed_name = 'box_hosts_0'
+ dockerfile = 'DockerfilePodman'
+
+ CAPS = [
+ "SYS_ADMIN",
+ "NET_ADMIN",
+ "SYS_TIME",
+ "SYS_RAWIO",
+ "MKNOD",
+ "NET_RAW",
+ "SETUID",
+ "SETGID",
+ "CHOWN",
+ "SYS_PTRACE",
+ "SYS_TTY_CONFIG",
+ "CAP_AUDIT_WRITE",
+ "CAP_AUDIT_CONTROL",
+ ]
+
+ VOLUMES = [
+ '../../../:/ceph:z',
+ '../:/cephadm:z',
+ '/run/udev:/run/udev',
+ '/sys/dev/block:/sys/dev/block',
+ '/sys/fs/cgroup:/sys/fs/cgroup:ro',
+ '/dev/fuse:/dev/fuse',
+ '/dev/disk:/dev/disk',
+ '/sys/devices/virtual/block:/sys/devices/virtual/block',
+ '/sys/block:/dev/block',
+ '/dev/mapper:/dev/mapper',
+ '/dev/mapper/control:/dev/mapper/control',
+ ]
+
+ TMPFS = ['/run', '/tmp']
+
+ # FIXME: right now we are assuming every service will be exposed through the seed, but this is far
+ # from the truth. Services can be deployed on different hosts so we need a system to manage this.
+ SEED_PORTS = [
+ 8443, # dashboard
+ 3000, # grafana
+ 9093, # alertmanager
+ 9095 # prometheus
+ ]
+
+
+ def setup_podman_env(self, hosts: int = 1, osd_devs={}):
+ network_name = 'box_network'
+ networks = run_shell_command('podman network ls')
+ if network_name not in networks:
+ run_shell_command(f'podman network create -d bridge {network_name}')
+
+ args = [
+ '--group-add', 'keep-groups',
+ '--device', '/dev/fuse' ,
+ '-it' ,
+ '-d',
+ '-e', 'CEPH_BRANCH=main',
+ '--stop-signal', 'RTMIN+3'
+ ]
+
+ for cap in self.CAPS:
+ args.append('--cap-add')
+ args.append(cap)
+
+ for volume in self.VOLUMES:
+ args.append('-v')
+ args.append(volume)
+
+ for tmp in self.TMPFS:
+ args.append('--tmpfs')
+ args.append(tmp)
+
+
+ for osd_dev in osd_devs.values():
+ device = osd_dev["device"]
+ args.append('--device')
+ args.append(f'{device}:{device}')
+
+
+ for host in range(hosts+1): # 0 will be the seed
+ options = copy.copy(args)
+ options.append('--name')
+ options.append(f'box_hosts_{host}')
+ options.append('--network')
+ options.append(f'{network_name}')
+ if host == 0:
+ for port in self.SEED_PORTS:
+ options.append('-p')
+ options.append(f'{port}:{port}')
+
+ options.append('cephadm-box')
+ options = ' '.join(options)
+
+ run_shell_command(f'podman run {options}')
+
+ def up(self, hosts: int):
+ import osd
+ self.setup_podman_env(hosts=hosts, osd_devs=osd.load_osd_devices())
+
+def get_container_engine() -> ContainerEngine:
+ if engine() == 'docker':
+ return DockerEngine()
+ else:
+ return PodmanEngine()
projects / ceph.git / commitdiff