From: Sébastien Han Date: Tue, 12 Jul 2016 10:15:22 +0000 (+0200) Subject: rbd-mirror: container deployment support X-Git-Tag: v1.0.6~53^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=e6097469bd9a34daeab55a7132e7c403ca52bf85;p=ceph-ansible.git rbd-mirror: container deployment support add the ability to deploy a containerized version of the rbd mirror daemon. Signed-off-by: Sébastien Han --- diff --git a/Vagrantfile b/Vagrantfile index df44fbce0..e3197c98e 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -7,20 +7,21 @@ VAGRANTFILE_API_VERSION = '2' config_file=File.expand_path(File.join(File.dirname(__FILE__), 'vagrant_variables.yml')) settings=YAML.load_file(config_file) -NMONS = settings['mon_vms'] -NOSDS = settings['osd_vms'] -NMDSS = settings['mds_vms'] -NRGWS = settings['rgw_vms'] -NNFSS = settings['nfs_vms'] -RESTAPI = settings['restapi'] -CLIENTS = settings['client_vms'] -SUBNET = settings['subnet'] -BOX = settings['vagrant_box'] -BOX_URL = settings['vagrant_box_url'] -MEMORY = settings['memory'] -STORAGECTL = settings['vagrant_storagectl'] -ETH = settings['eth'] -DOCKER = settings['docker'] +NMONS = settings['mon_vms'] +NOSDS = settings['osd_vms'] +NMDSS = settings['mds_vms'] +NRGWS = settings['rgw_vms'] +NNFSS = settings['nfs_vms'] +RESTAPI = settings['restapi'] +NRBD_MIRRORS = settings['rbd_mirror_vms'] +CLIENTS = settings['client_vms'] +SUBNET = settings['subnet'] +BOX = settings['vagrant_box'] +BOX_URL = settings['vagrant_box_url'] +MEMORY = settings['memory'] +STORAGECTL = settings['vagrant_storagectl'] +ETH = settings['eth'] +DOCKER = settings['docker'] if BOX == 'openstack' require 'vagrant-openstack-provider' @@ -47,12 +48,13 @@ ansible_provision = proc do |ansible| # these aren't supported by Vagrant, see # https://github.com/mitchellh/vagrant/issues/3539 ansible.groups = { - 'mons' => (0..NMONS - 1).map { |j| "#{OSPREFIX}mon#{j}" }, - 'osds' => (0..NOSDS - 1).map { |j| "#{OSPREFIX}osd#{j}" }, - 'mdss' => (0..NMDSS - 1).map { |j| "#{OSPREFIX}mds#{j}" }, - 'rgws' => (0..NRGWS - 1).map { |j| "#{OSPREFIX}rgw#{j}" }, - 'nfss' => (0..NNFSS - 1).map { |j| "#{OSPREFIX}nfs#{j}" }, - 'clients' => (0..CLIENTS - 1).map { |j| "#{OSPREFIX}client#{j}" } + 'mons' => (0..NMONS - 1).map { |j| "#{OSPREFIX}mon#{j}" }, + 'osds' => (0..NOSDS - 1).map { |j| "#{OSPREFIX}osd#{j}" }, + 'mdss' => (0..NMDSS - 1).map { |j| "#{OSPREFIX}mds#{j}" }, + 'rgws' => (0..NRGWS - 1).map { |j| "#{OSPREFIX}rgw#{j}" }, + 'nfss' => (0..NNFSS - 1).map { |j| "#{OSPREFIX}nfs#{j}" }, + 'rbd_mirrors' => (0..NRBD_MIRRORS - 1).map { |j| "#{OSPREFIX}rbd_mirror#{j}" }, + 'clients' => (0..CLIENTS - 1).map { |j| "#{OSPREFIX}client#{j}" } } if RESTAPI then @@ -69,6 +71,7 @@ ansible_provision = proc do |ansible| rgw_containerized_deployment: 'true', nfs_containerized_deployment: 'true', restapi_containerized_deployment: 'true', + rbd_mirror_containerized_deployment: 'true', ceph_mon_docker_interface: ETH, ceph_mon_docker_subnet: "#{SUBNET}.0/24", ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK,OSD_JOURNAL_SIZE=100", @@ -249,6 +252,34 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end end + (0..NRBD_MIRRORS - 1).each do |i| + config.vm.define "#{OSPREFIX}rbd_mirror#{i}" do |rbd_mirror| + rbd_mirror.vm.hostname = "#{OSPREFIX}ceph-rbd-mirror#{i}" + if !OSVM + rbd_mirror.vm.network :private_network, ip: "#{SUBNET}.8#{i}" + end + # Virtualbox + rbd_mirror.vm.provider :virtualbox do |vb| + vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"] + end + + # VMware + rbd_mirror.vm.provider :vmware_fusion do |v| + v.vmx['memsize'] = "#{MEMORY}" + end + + # Libvirt + rbd_mirror.vm.provider :libvirt do |lv| + lv.memory = MEMORY + end + # Parallels + rbd_mirror.vm.provider "parallels" do |prl| + prl.name = "ceph-rbd-mirror#{i}" + prl.memory = "#{MEMORY}" + end + end + end + (0..NMONS - 1).each do |i| config.vm.define "#{OSPREFIX}mon#{i}" do |mon| mon.vm.hostname = "#{OSPREFIX}ceph-mon#{i}" diff --git a/group_vars/all.docker.sample b/group_vars/all.docker.sample index eb1943af2..48cf7066a 100644 --- a/group_vars/all.docker.sample +++ b/group_vars/all.docker.sample @@ -4,6 +4,7 @@ dummy: ########## # GLOBAL # ########## +#docker: true #ceph_docker_dev_image: false ####### @@ -42,7 +43,7 @@ dummy: ####### # MDS # ####### -#mds_containerized_deployment: false +#mds_containerized_deployment: true #mds_containerized_deployment_with_kv: false #kv_type: etcd #kv_endpoint: 127.0.0.1 @@ -67,10 +68,24 @@ dummy: ########### # RESTAPI # ########### -#restapi_containerized_deployment: false +#restapi_containerized_deployment: true #ceph_restapi_docker_interface: eth0 #ceph_restapi_port: 5000 #ceph_restapi_docker_username: ceph #ceph_restapi_docker_imagename: daemon #ceph_restapi_docker_image_tag: latest #ceph_restapi_docker_extra_env: "RESTAPI_IP=0.0.0.0" # comma separated variables + + +############## +# RBD MIRROR # +############## +#rbd_mirror_containerized_deployment: true +#rbd_mirror_containerized_deployment_with_kv: false +#kv_type: etcd +#kv_endpoint: 127.0.0.1 +#ceph_rbd_mirror_docker_username: ceph +#ceph_rbd_mirror_docker_imagename: daemon +#ceph_rbd_mirror_docker_image_tag: latest +#ceph_docker_on_openstack: false + diff --git a/group_vars/osds.sample b/group_vars/osds.sample index 7fb63f381..8e817d115 100644 --- a/group_vars/osds.sample +++ b/group_vars/osds.sample @@ -131,6 +131,7 @@ dummy: #osd_containerized_deployment_with_kv: false #kv_type: etcd #kv_endpoint: 127.0.0.1 +#kv_port: 4001 #ceph_osd_docker_prepare_env: "OSD_FORCE_ZAP=1" #ceph_osd_docker_username: ceph #ceph_osd_docker_imagename: daemon diff --git a/group_vars/rbd-mirrors.sample b/group_vars/rbd-mirrors.sample index ad0f46f29..a32435e79 100644 --- a/group_vars/rbd-mirrors.sample +++ b/group_vars/rbd-mirrors.sample @@ -28,6 +28,7 @@ dummy: # Thus if cluster names are identical we can not have them under /etc/ceph #ceph_rbd_mirror_remote_cluster: "" + ################# # CONFIGURATION # ################# @@ -35,3 +36,17 @@ dummy: #ceph_rbd_mirror_configure: false #ceph_rbd_mirror_pool: "" + +########## +# DOCKER # +########## + +#rbd_mirror_containerized_deployment: false +#rbd_mirror_containerized_deployment_with_kv: false +#kv_type: etcd +#kv_endpoint: 127.0.0.1 +#ceph_rbd_mirror_docker_username: ceph +#ceph_rbd_mirror_docker_imagename: daemon +#ceph_rbd_mirror_docker_image_tag: latest +#ceph_docker_on_openstack: false + diff --git a/roles/ceph-rbd-mirror/defaults/main.yml b/roles/ceph-rbd-mirror/defaults/main.yml index 12207b179..2292a238a 100644 --- a/roles/ceph-rbd-mirror/defaults/main.yml +++ b/roles/ceph-rbd-mirror/defaults/main.yml @@ -20,9 +20,24 @@ ceph_rbd_mirror_remote_user: admin # Thus if cluster names are identical we can not have them under /etc/ceph ceph_rbd_mirror_remote_cluster: "" + ################# # CONFIGURATION # ################# ceph_rbd_mirror_configure: false ceph_rbd_mirror_pool: "" + + +########## +# DOCKER # +########## + +rbd_mirror_containerized_deployment: false +rbd_mirror_containerized_deployment_with_kv: false +kv_type: etcd +kv_endpoint: 127.0.0.1 +ceph_rbd_mirror_docker_username: ceph +ceph_rbd_mirror_docker_imagename: daemon +ceph_rbd_mirror_docker_image_tag: latest +ceph_docker_on_openstack: false diff --git a/roles/ceph-rbd-mirror/meta/main.yml b/roles/ceph-rbd-mirror/meta/main.yml index 648af5aa6..bb297c9ca 100644 --- a/roles/ceph-rbd-mirror/meta/main.yml +++ b/roles/ceph-rbd-mirror/meta/main.yml @@ -14,4 +14,4 @@ galaxy_info: categories: - system dependencies: - - { role: ceph.ceph-common } + - { role: ceph.ceph-common, when: not rbd_mirror_containerized_deployment } diff --git a/roles/ceph-rbd-mirror/tasks/docker/checks.yml b/roles/ceph-rbd-mirror/tasks/docker/checks.yml new file mode 100644 index 000000000..255a6d01a --- /dev/null +++ b/roles/ceph-rbd-mirror/tasks/docker/checks.yml @@ -0,0 +1,27 @@ +--- +- name: set config and keys paths + set_fact: + ceph_config_keys: + - /etc/ceph/ceph.client.admin.keyring + - /etc/ceph/ceph.conf + - /etc/ceph/monmap + - /etc/ceph/ceph.mon.keyring + - /var/lib/ceph/bootstrap-osd/ceph.keyring + - /var/lib/ceph/bootstrap-rgw/ceph.keyring + - /var/lib/ceph/bootstrap-mds/ceph.keyring + +- name: stat for ceph config and keys + stat: + path: "{{ item }}" + with_items: ceph_config_keys + changed_when: false + failed_when: false + register: statleftover + +- name: fail if we find existing cluster files + fail: + msg: "looks like no cluster is running but ceph files are present, please remove them" + with_together: + - ceph_config_keys + - statleftover.results + when: item.1.stat.exists == true diff --git a/roles/ceph-rbd-mirror/tasks/docker/dirs_permissions.yml b/roles/ceph-rbd-mirror/tasks/docker/dirs_permissions.yml new file mode 100644 index 000000000..3a85017f2 --- /dev/null +++ b/roles/ceph-rbd-mirror/tasks/docker/dirs_permissions.yml @@ -0,0 +1,35 @@ +--- +# NOTE (leseb): we can not use docker inspect with 'format filed' because of +# https://github.com/ansible/ansible/issues/10156 +- name: inspect ceph version + shell: docker inspect "docker.io/{{ ceph_rbd_mirror_docker_username }}/{{ ceph_rbd_mirror_docker_imagename }}:{{ ceph_rbd_mirror_docker_image_tag }}" | awk -F '=' '/CEPH_VERSION/ { gsub ("\",", "", $2); print $2 }' | uniq + changed_when: false + failed_when: false + run_once: true + register: ceph_version + +- set_fact: + after_hammer=True + when: ceph_version.stdout not in ['firefly','giant', 'hammer'] + +- name: create bootstrap directories (for or before hammer) + file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + with_items: + - /etc/ceph/ + when: not after_hammer + +- name: create bootstrap directories (after hammer) + file: + path: "{{ item }}" + state: directory + owner: "64045" + group: "64045" + mode: "0755" + with_items: + - /etc/ceph/ + when: after_hammer diff --git a/roles/ceph-rbd-mirror/tasks/docker/fetch_configs.yml b/roles/ceph-rbd-mirror/tasks/docker/fetch_configs.yml new file mode 100644 index 000000000..b8b1ea6ce --- /dev/null +++ b/roles/ceph-rbd-mirror/tasks/docker/fetch_configs.yml @@ -0,0 +1,29 @@ +--- +# NOTE (leseb): the mds container needs the admin key +# so it can create the mds pools for cephfs +- name: set config and keys paths + set_fact: + ceph_config_keys: + - /etc/ceph/ceph.conf + - /etc/ceph/ceph.client.admin.keyring + +- name: stat for ceph config and keys + local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }} + with_items: ceph_config_keys + changed_when: false + become: false + failed_when: false + register: statconfig + +- name: try to fetch ceph config and keys + copy: + src: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" + dest: "{{ item.0 }}" + owner: root + group: root + mode: 0644 + changed_when: false + with_together: + - ceph_config_keys + - statconfig.results + when: item.1.stat.exists == true diff --git a/roles/ceph-rbd-mirror/tasks/docker/main.yml b/roles/ceph-rbd-mirror/tasks/docker/main.yml new file mode 100644 index 000000000..a9b6df218 --- /dev/null +++ b/roles/ceph-rbd-mirror/tasks/docker/main.yml @@ -0,0 +1,31 @@ +--- +- name: check if a cluster is already running + shell: "docker ps | grep -sq 'ceph/daemon'" + register: ceph_health + changed_when: false + failed_when: false + +- name: check if it is Atomic host + stat: path=/run/ostree-booted + register: stat_ostree + +- name: set fact for using Atomic host + set_fact: + is_atomic='{{ stat_ostree.stat.exists }}' + +- include: checks.yml + when: ceph_health.rc != 0 + +- include: pre_requisite.yml +- include: "{{ playbook_dir }}/roles/ceph-common/tasks/docker/fetch_image.yml" + vars: + ceph_docker_username: "{{ ceph_rbd_mirror_docker_username }}" + ceph_docker_imagename: "{{ ceph_rbd_mirror_docker_imagename }}" + ceph_docker_image_tag: "{{ ceph_rbd_mirror_docker_image_tag }}" +- include: dirs_permissions.yml +- include: fetch_configs.yml + +- include: selinux.yml + when: ansible_os_family == 'RedHat' + +- include: start_docker_rbd_mirror.yml diff --git a/roles/ceph-rbd-mirror/tasks/docker/pre_requisite.yml b/roles/ceph-rbd-mirror/tasks/docker/pre_requisite.yml new file mode 100644 index 000000000..4b060aed6 --- /dev/null +++ b/roles/ceph-rbd-mirror/tasks/docker/pre_requisite.yml @@ -0,0 +1,126 @@ +--- +- name: install pip and docker on ubuntu + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - python-pip + - docker + - docker.io + when: ansible_distribution == 'Ubuntu' + tags: + with_pkg + +- name: install pip and docker on debian + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - python-pip + - docker-engine + when: ansible_distribution == 'Debian' + tags: + with_pkg + +# install epel for pip +- name: install epel on redhat + yum: + name: "{{ item }}" + state: present + with_items: + - epel-release + when: + - ansible_os_family == 'RedHat' + - ansible_pkg_mgr == "yum" + tags: + with_pkg + failed_when: false + +- name: install pip on redhat + yum: + name: "{{ item }}" + state: present + with_items: + - python-pip + when: + - ansible_os_family == 'RedHat' + - ansible_pkg_mgr == "yum" + tags: + with_pkg + +- name: install docker-engine on redhat + yum: + name: "{{ item }}" + state: present + with_items: + - docker-engine + when: + - ansible_os_family == 'RedHat' + - ansible_pkg_mgr == "yum" + tags: + with_pkg + failed_when: false + +# for CentOS +- name: install docker on redhat + yum: + name: "{{ item }}" + state: present + with_items: + - docker + when: + - ansible_os_family == 'RedHat' + - ansible_pkg_mgr == "yum" + tags: + with_pkg + failed_when: false + +# docker package could be docker-enginer or docker +- name: install pip and docker on redhat + dnf: + name: "{{ item }}" + state: present + with_items: + - python-pip + - docker-engine + when: + - ansible_os_family == 'RedHat' + - ansible_pkg_mgr == "dnf" + tags: + with_pkg + +- name: start docker service + service: + name: docker + state: started + enabled: yes + tags: + with_pkg + +# NOTE (jimcurtis): need at least version 1.9.0 of six or we get: +# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined +- name: install six + pip: + name: six + version: 1.9.0 + tags: + with_pkg + +# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227 +- name: install docker-py + pip: + name: docker-py + version: 1.1.0 + tags: + with_pkg + when: ansible_version['full'] | version_compare('2.1.0.0', '<') + +- name: install docker-py + pip: + name: docker-py + state: latest + tags: + with_pkg + when: ansible_version['full'] | version_compare('2.1.0.0', '>=') diff --git a/roles/ceph-rbd-mirror/tasks/docker/selinux.yml b/roles/ceph-rbd-mirror/tasks/docker/selinux.yml new file mode 100644 index 000000000..3630824d7 --- /dev/null +++ b/roles/ceph-rbd-mirror/tasks/docker/selinux.yml @@ -0,0 +1,13 @@ +--- +- name: check if selinux is enabled + command: getenforce + register: sestatus + changed_when: false + +- name: set selinux permissions + shell: chcon -Rt svirt_sandbox_file_t {{ item }} + with_items: + - /etc/ceph + - /var/lib/ceph + changed_when: false + when: sestatus.stdout != 'Disabled' diff --git a/roles/ceph-rbd-mirror/tasks/docker/start_docker_rbd_mirror.yml b/roles/ceph-rbd-mirror/tasks/docker/start_docker_rbd_mirror.yml new file mode 100644 index 000000000..cb4ff3b4a --- /dev/null +++ b/roles/ceph-rbd-mirror/tasks/docker/start_docker_rbd_mirror.yml @@ -0,0 +1,47 @@ +--- +# Use systemd to manage container on Atomic host +- name: generate systemd unit file + become: true + template: + src: "{{ role_path }}/templates/ceph-rbd-mirror.service.j2" + dest: /var/lib/ceph/ceph-rbd-mirror@.service + owner: "root" + group: "root" + mode: "0644" + when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS' + +- name: link systemd unit file for rbd mirror instance + file: + src: /var/lib/ceph/ceph-rbd-mirror@.service + dest: /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror@{{ ansible_hostname }}.service + state: link + when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS' + +- name: enable systemd unit file for rbd mirror instance + command: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror@{{ ansible_hostname }}.service + failed_when: false + changed_when: false + when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS' + +- name: reload systemd unit files + command: systemctl daemon-reload + changed_when: false + failed_when: false + when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS' + +- name: systemd start rbd mirror container + service: + name: ceph-rbd-mirror@{{ ansible_hostname }} + state: started + enabled: yes + changed_when: false + when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS' + +- name: run the ceph rbd mirror docker image + docker: + image: "{{ ceph_rbd_mirror_docker_username }}/{{ ceph_rbd_mirror_docker_imagename }}" + name: ceph-{{ ansible_hostname }}-rbd-mirror + net: host + state: running + volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph" + when: ansible_os_family != 'RedHat' and ansible_os_family != 'CoreOS' diff --git a/roles/ceph-rbd-mirror/tasks/main.yml b/roles/ceph-rbd-mirror/tasks/main.yml index 05fbfff74..abb3c5304 100644 --- a/roles/ceph-rbd-mirror/tasks/main.yml +++ b/roles/ceph-rbd-mirror/tasks/main.yml @@ -1,5 +1,14 @@ --- - include: pre_requisite.yml + when: not rbd_mirror_containerized_deployment + - include: start_rbd_mirror.yml + when: not rbd_mirror_containerized_deployment + - include: configure_mirroring.yml - when: ceph_rbd_mirror_configure + when: + - ceph_rbd_mirror_configure + - not rbd_mirror_containerized_deployment + +- include: ./docker/main.yml + when: rbd_mirror_containerized_deployment diff --git a/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 b/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 new file mode 100644 index 000000000..69aa5b605 --- /dev/null +++ b/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 @@ -0,0 +1,28 @@ +[Unit] +Description=Ceph RBD mirror +After=docker.service + +[Service] +EnvironmentFile=-/etc/environment +ExecStartPre=-/usr/bin/docker stop {{ ansible_hostname }} +ExecStartPre=-/usr/bin/docker rm {{ ansible_hostname }} +ExecStart=/usr/bin/docker run --rm --net=host \ + {% if not rbd_mirror_containerized_deployment_with_kv -%} + -v /var/lib/ceph:/var/lib/ceph \ + -v /etc/ceph:/etc/ceph \ + {% else -%} + -e KV_TYPE={{kv_type}} \ + -e KV_IP={{kv_endpoint}} \ + {% endif -%} + --privileged \ + -e CEPH_DAEMON=RBD_MIRROR \ + --name={{ ansible_hostname }} \ + {{ ceph_rbd_mirror_docker_username }}/{{ ceph_rbd_mirror_docker_imagename }} +ExecStopPost=-/usr/bin/docker stop {{ ansible_hostname }} +Restart=always +RestartSec=10s +TimeoutStartSec=120 +TimeoutStopSec=15 + +[Install] +WantedBy=multi-user.target diff --git a/vagrant_variables.yml.sample b/vagrant_variables.yml.sample index ed4b44013..23c4ef26a 100644 --- a/vagrant_variables.yml.sample +++ b/vagrant_variables.yml.sample @@ -9,6 +9,7 @@ osd_vms: 3 mds_vms: 0 rgw_vms: 0 nfs_vms: 0 +rbd_mirror_vms: 0 client_vms: 0 # Deploy RESTAPI on each of the Monitors