From 957903d561680d366493cb5d6fd36b19aa3fb539 Mon Sep 17 00:00:00 2001 From: Dimitri Savineau Date: Fri, 10 Jul 2020 17:52:38 -0400 Subject: [PATCH] cephadm: add playbook This adds a new playbook for deploying ceph via cephadm. This also adds a new dedicated tox file for CI purpose. Signed-off-by: Dimitri Savineau --- infrastructure-playbooks/cephadm.yml | 352 ++++++++++++++++++ tests/functional/cephadm/Vagrantfile | 1 + tests/functional/cephadm/group_vars/all | 5 + tests/functional/cephadm/hosts | 31 ++ .../functional/cephadm/vagrant_variables.yml | 33 ++ tox-cephadm.ini | 44 +++ 6 files changed, 466 insertions(+) create mode 100644 infrastructure-playbooks/cephadm.yml create mode 120000 tests/functional/cephadm/Vagrantfile create mode 100644 tests/functional/cephadm/group_vars/all create mode 100644 tests/functional/cephadm/hosts create mode 100644 tests/functional/cephadm/vagrant_variables.yml create mode 100644 tox-cephadm.ini diff --git a/infrastructure-playbooks/cephadm.yml b/infrastructure-playbooks/cephadm.yml new file mode 100644 index 000000000..b90eacbdd --- /dev/null +++ b/infrastructure-playbooks/cephadm.yml @@ -0,0 +1,352 @@ +--- +- name: gather facts and prepare system for cephadm + hosts: + - "{{ mon_group_name|default('mons') }}" + - "{{ osd_group_name|default('osds') }}" + - "{{ mds_group_name|default('mdss') }}" + - "{{ rgw_group_name|default('rgws') }}" + - "{{ mgr_group_name|default('mgrs') }}" + - "{{ rbdmirror_group_name|default('rbdmirrors') }}" + - "{{ nfs_group_name|default('nfss') }}" + - "{{ iscsi_gw_group_name|default('iscsigws') }}" + - "{{ grafana_server_group_name|default('grafana-server') }}" + become: true + gather_facts: false + vars: + delegate_facts_host: true + tasks: + - import_role: + name: ceph-defaults + + - name: validate if monitor group doesn't exist or empty + fail: + msg: "you must add a [mons] group and add at least one node." + run_once: true + when: groups[mon_group_name] is undefined or groups[mon_group_name] | length == 0 + + - name: validate if manager group doesn't exist or empty + fail: + msg: "you must add a [mgrs] group and add at least one node." + run_once: true + when: groups[mgr_group_name] is undefined or groups[mgr_group_name] | length == 0 + + - name: validate monitor network configuration + fail: + msg: "Either monitor_address, monitor_address_block or monitor_interface must be provided" + when: + - mon_group_name in group_names + - monitor_address == 'x.x.x.x' + - monitor_address_block == 'subnet' + - monitor_interface == 'interface' + + - name: validate dashboard configuration + when: dashboard_enabled | bool + run_once: true + block: + - name: fail if [grafana-server] group doesn't exist or empty + fail: + msg: "you must add a [grafana-server] group and add at least one node." + when: groups[grafana_server_group_name] is undefined or groups[grafana_server_group_name] | length == 0 + + - name: fail when dashboard_admin_password is not set + fail: + msg: "you must set dashboard_admin_password." + when: dashboard_admin_password is undefined + + - name: validate container registry credentials + fail: + msg: 'ceph_docker_registry_username and/or ceph_docker_registry_password variables need to be set' + when: + - ceph_docker_registry_auth | bool + - (ceph_docker_registry_username is not defined or ceph_docker_registry_password is not defined) or + (ceph_docker_registry_username | length == 0 or ceph_docker_registry_password | length == 0) + + - name: gather facts + setup: + gather_subset: + - 'all' + - '!facter' + - '!ohai' + when: not delegate_facts_host | bool + + - name: gather and delegate facts + setup: + gather_subset: + - 'all' + - '!facter' + - '!ohai' + delegate_to: "{{ item }}" + delegate_facts: true + with_items: "{{ groups['all'] }}" + run_once: true + when: delegate_facts_host | bool + + - import_role: + name: ceph-facts + tasks_from: container_binary.yml + + - name: check if it is atomic host + stat: + path: /run/ostree-booted + register: stat_ostree + + - name: set_fact is_atomic + set_fact: + is_atomic: "{{ stat_ostree.stat.exists }}" + + - import_role: + name: ceph-container-engine + + - import_role: + name: ceph-container-common + tasks_from: registry.yml + when: ceph_docker_registry_auth | bool + + - name: install cephadm requirements + package: + name: ['python3', 'lvm2'] + register: result + until: result is succeeded + + - name: create a cephadm container + command: "{{ container_binary }} create --name cephadm {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}" + changed_when: false + + - name: cp the cephadm cli file + command: "{{ container_binary }} cp cephadm:/usr/sbin/cephadm /usr/sbin/cephadm" + args: + creates: /usr/sbin/cephadm + + - name: remove the cephadm container + command: "{{ container_binary }} rm cephadm" + changed_when: false + +- name: bootstrap the cluster + hosts: "{{ mon_group_name|default('mons') }}[0]" + become: true + gather_facts: false + tasks: + - import_role: + name: ceph-defaults + + - import_role: + name: ceph-facts + tasks_from: set_monitor_address.yml + + - name: create /etc/ceph directory + file: + path: /etc/ceph + state: directory + + - name: bootstrap the new cluster + command: "cephadm bootstrap --mon-ip {{ _current_monitor_address }} --skip-pull --skip-monitoring-stack {{ '--docker' if container_binary == 'docker' else '' }} {{ '--initial-dashboard-user ' + dashboard_admin_user + ' --initial-dashboard-password ' + dashboard_admin_password if dashboard_enabled | bool else '--skip-dashboard' }}" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: set default container image in ceph configuration + command: "cephadm shell -- ceph --cluster {{ cluster }} config set global container_image {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: set container image base in ceph configuration + command: "cephadm shell -- ceph --cluster {{ cluster }} config set mgr mgr/cephadm/container_image_base {{ ceph_docker_registry }}/{{ ceph_docker_image }}" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: set dashboard container image in ceph mgr configuration + when: dashboard_enabled | bool + block: + - name: set alertmanager container image in ceph configuration + command: "cephadm shell -- ceph --cluster {{ cluster }} config set mgr mgr/cephadm/container_image_alertmanager {{ alertmanager_container_image }}" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: set grafana container image in ceph configuration + command: "cephadm shell -- ceph --cluster {{ cluster }} config set mgr mgr/cephadm/container_image_grafana {{ grafana_container_image }}" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: set node-exporter container image in ceph configuration + command: "cephadm shell -- ceph --cluster {{ cluster }} config set mgr mgr/cephadm/container_image_node_exporter {{ node_exporter_container_image }}" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: set prometheus container image in ceph configuration + command: "cephadm shell -- ceph --cluster {{ cluster }} config set mgr mgr/cephadm/container_image_prometheus {{ prometheus_container_image }}" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + +- name: add the other nodes + hosts: + - "{{ mon_group_name|default('mons') }}" + - "{{ osd_group_name|default('osds') }}" + - "{{ mds_group_name|default('mdss') }}" + - "{{ rgw_group_name|default('rgws') }}" + - "{{ mgr_group_name|default('mgrs') }}" + - "{{ rbdmirror_group_name|default('rbdmirrors') }}" + - "{{ nfs_group_name|default('nfss') }}" + - "{{ iscsi_gw_group_name|default('iscsigws') }}" + - "{{ grafana_server_group_name|default('grafana-server') }}" + become: true + gather_facts: false + tasks: + - import_role: + name: ceph-defaults + + - name: get the cephadm ssh pub key + command: "cephadm shell -- ceph --cluster {{ cluster }} cephadm get-pub-key" + changed_when: false + run_once: true + register: cephadm_pubpkey + delegate_to: '{{ groups[mon_group_name][0] }}' + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: allow cephadm key for root account + authorized_key: + user: root + key: '{{ cephadm_pubpkey.stdout }}' + + - name: run cephadm prepare-host + command: cephadm prepare-host + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: manage nodes with cephadm + command: "cephadm shell -- ceph --cluster {{ cluster }} orch host add {{ hostvars[item]['ansible_hostname'] }} {{ hostvars[item]['ansible_default_ipv4']['address'] }} {{ hostvars[item]['group_names'] | join(' ') }}" + changed_when: false + run_once: true + loop: '{{ ansible_play_hosts_all }}' + delegate_to: '{{ groups[mon_group_name][0] }}' + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: add ceph label for core component + command: "cephadm shell -- ceph --cluster {{ cluster }} orch host label add {{ hostvars[item]['ansible_hostname'] }} ceph" + changed_when: false + run_once: true + loop: '{{ ansible_play_hosts_all }}' + delegate_to: '{{ groups[mon_group_name][0] }}' + when: item in groups.get(mon_group_name, []) or + item in groups.get(osd_group_name, []) or + item in groups.get(mds_group_name, []) or + item in groups.get(rgw_group_name, []) or + item in groups.get(mgr_group_name, []) or + item in groups.get(rbdmirror_group_name, []) + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + +- name: adjust service placement + hosts: "{{ mon_group_name|default('mons') }}[0]" + become: true + gather_facts: false + tasks: + - import_role: + name: ceph-defaults + + - name: update the placement of monitor hosts + command: "cephadm shell -- ceph --cluster {{ cluster }} orch apply mon --placement='label:{{ mon_group_name }}'" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: waiting for the monitor to join the quorum... + command: "cephadm shell -- ceph --cluster {{ cluster }} -s --format json" + changed_when: false + register: ceph_health_raw + until: (ceph_health_raw.stdout | from_json)["quorum_names"] | length == groups.get(mon_group_name, []) | length + retries: "{{ health_mon_check_retries }}" + delay: "{{ health_mon_check_delay }}" + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: update the placement of manager hosts + command: "cephadm shell -- ceph --cluster {{ cluster }} orch apply mgr --placement='label:{{ mgr_group_name }}'" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: update the placement of crash hosts + command: "cephadm shell -- ceph --cluster {{ cluster }} orch apply crash --placement='label:ceph'" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + +- name: adjust monitoring service placement + hosts: "{{ grafana_server_group_name|default('grafana-server') }}" + become: true + gather_facts: false + tasks: + - import_role: + name: ceph-defaults + + - name: with dashboard enabled + when: dashboard_enabled | bool + delegate_to: '{{ groups[mon_group_name][0] }}' + run_once: true + block: + - name: enable the prometheus module + command: "cephadm shell -- ceph --cluster {{ cluster }} mgr module enable prometheus" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: update the placement of alertmanager hosts + command: "cephadm shell -- ceph --cluster {{ cluster }} orch apply alertmanager --placement='label:{{ grafana_server_group_name }}'" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: update the placement of grafana hosts + command: "cephadm shell -- ceph --cluster {{ cluster }} orch apply grafana --placement='label:{{ grafana_server_group_name }}'" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: update the placement of prometheus hosts + command: "cephadm shell -- ceph --cluster {{ cluster }} orch apply prometheus --placement='label:{{ grafana_server_group_name }}'" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: update the placement of node-exporter hosts + command: "cephadm shell -- ceph --cluster {{ cluster }} orch apply node-exporter --placement='*'" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + +- name: print information + hosts: "{{ mon_group_name|default('mons') }}[0]" + become: true + gather_facts: false + tasks: + - import_role: + name: ceph-defaults + + - name: show ceph orchestrator services + command: "cephadm shell -- ceph --cluster {{ cluster }} orch ls --refresh" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: show ceph orchestrator daemons + command: "cephadm shell -- ceph --cluster {{ cluster }} orch ps --refresh" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + + - name: inform users about cephadm + debug: + msg: | + This Ceph cluster is now ready to receive more configuration like + adding OSD, MDS daemons, create pools or keyring. + You can do this by using the cephadm CLI and you don't need to use + ceph-ansible playbooks anymore. diff --git a/tests/functional/cephadm/Vagrantfile b/tests/functional/cephadm/Vagrantfile new file mode 120000 index 000000000..706a5bb47 --- /dev/null +++ b/tests/functional/cephadm/Vagrantfile @@ -0,0 +1 @@ +../../../Vagrantfile \ No newline at end of file diff --git a/tests/functional/cephadm/group_vars/all b/tests/functional/cephadm/group_vars/all new file mode 100644 index 000000000..b79aca504 --- /dev/null +++ b/tests/functional/cephadm/group_vars/all @@ -0,0 +1,5 @@ +--- +monitor_interface: eth1 +public_network: "192.168.30.0/24" +cluster_network: "192.168.31.0/24" +dashboard_admin_password: $sX!cD$rYU6qR^B! diff --git a/tests/functional/cephadm/hosts b/tests/functional/cephadm/hosts new file mode 100644 index 000000000..84a20bdb6 --- /dev/null +++ b/tests/functional/cephadm/hosts @@ -0,0 +1,31 @@ +[mons] +mon0 +mon1 +mon2 + +[mgrs] +mon0 +mon1 +mon2 + +[osds] +osd0 +osd1 + +[mdss] +mds0 + +[rgws] +rgw0 + +[nfss] +nfs0 + +[rbdmirrors] +rbd-mirror0 + +[iscsigws] +iscsi-gw0 + +[grafana-server] +mon0 diff --git a/tests/functional/cephadm/vagrant_variables.yml b/tests/functional/cephadm/vagrant_variables.yml new file mode 100644 index 000000000..b2c2e7cc8 --- /dev/null +++ b/tests/functional/cephadm/vagrant_variables.yml @@ -0,0 +1,33 @@ +--- + +# DEPLOY CONTAINERIZED DAEMONS +docker: True + +# DEFINE THE NUMBER OF VMS TO RUN +mon_vms: 3 +osd_vms: 2 +mds_vms: 1 +rgw_vms: 1 +nfs_vms: 1 +grafana_server_vms: 0 +rbd_mirror_vms: 1 +client_vms: 0 +iscsi_gw_vms: 1 +mgr_vms: 0 + +# SUBNETS TO USE FOR THE VMS +public_subnet: 192.168.30 +cluster_subnet: 192.168.31 + +# MEMORY +# set 1024 for CentOS +memory: 2048 + +vagrant_box: centos/8 +# The sync directory changes based on vagrant box +# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant +#vagrant_sync_dir: /home/vagrant/sync +vagrant_sync_dir: /vagrant +# Disables synced folder creation. Not needed for testing, will skip mounting +# the vagrant directory on the remote box regardless of the provider. +vagrant_disable_synced_folder: true diff --git a/tox-cephadm.ini b/tox-cephadm.ini new file mode 100644 index 000000000..b85a97bab --- /dev/null +++ b/tox-cephadm.ini @@ -0,0 +1,44 @@ +[tox] +envlist = centos-container-cephadm + +skipsdist = True + +[testenv] +whitelist_externals = + vagrant + bash + pip + sleep + rm +passenv=* +sitepackages=True +setenv= + ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config -o ControlMaster=auto -o ControlPersist=600s -o PreferredAuthentications=publickey + ANSIBLE_CONFIG = {toxinidir}/ansible.cfg + ANSIBLE_CALLBACK_WHITELIST = profile_tasks + ANSIBLE_KEEP_REMOTE_FILES = 1 + ANSIBLE_CACHE_PLUGIN = memory + ANSIBLE_GATHERING = implicit + # only available for ansible >= 2.5 + ANSIBLE_STDOUT_CALLBACK = yaml + # Set the vagrant box image to use + CEPH_ANSIBLE_VAGRANT_BOX = centos/8 + +deps= -r{toxinidir}/tests/requirements.txt +changedir= {toxinidir}/tests/functional/cephadm + +commands= + bash {toxinidir}/tests/scripts/vagrant_up.sh --no-provision {posargs:--provider=virtualbox} + bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir} + + ansible-playbook -vv -i {changedir}/hosts {toxinidir}/infrastructure-playbooks/cephadm.yml --extra-vars "\ + delegate_facts_host={env:DELEGATE_FACTS_HOST:True} \ + ceph_docker_registry={env:CEPH_DOCKER_REGISTRY:docker.io} \ + ceph_docker_image={env:CEPH_DOCKER_IMAGE:ceph/daemon-base} \ + ceph_docker_image_tag={env:CEPH_DOCKER_IMAGE_TAG:latest-master-devel} \ + ceph_docker_registry_auth=True \ + ceph_docker_registry_username={env:DOCKER_HUB_USERNAME} \ + ceph_docker_registry_password={env:DOCKER_HUB_PASSWORD} \ + " + + vagrant destroy -f -- 2.39.5