From 42e748a51479e6324d5cd0b2b27a4bd9bc2a5346 Mon Sep 17 00:00:00 2001 From: pprokop Date: Tue, 15 Mar 2016 17:32:17 +0100 Subject: [PATCH] Adding dockerized ceph-mon deployment with kv backend --- group_vars/all.docker.sample | 4 ++ roles/ceph-common/templates/ceph.conf.j2 | 3 +- roles/ceph-mon/defaults/main.yml | 4 ++ roles/ceph-mon/tasks/docker/main.yml | 10 ++++- .../tasks/docker/start_docker_monitor.yml | 41 +++++++++++++++---- .../{tasks => }/templates/ceph-mon.service.j2 | 9 +++- 6 files changed, 57 insertions(+), 14 deletions(-) rename roles/ceph-mon/{tasks => }/templates/ceph-mon.service.j2 (72%) diff --git a/group_vars/all.docker.sample b/group_vars/all.docker.sample index c0d84362e..fce0c22a9 100644 --- a/group_vars/all.docker.sample +++ b/group_vars/all.docker.sample @@ -5,6 +5,10 @@ dummy: # MON # ####### #mon_containerized_deployment: true +#mon_containerized_deployment_with_kv: false +#kv_type: etcd +#kv_endpoint: 127.0.0.1 +#kv_port: 4001 #mon_docker_privileged: true #ceph_mon_docker_username: ceph #ceph_mon_docker_imagename: daemon diff --git a/roles/ceph-common/templates/ceph.conf.j2 b/roles/ceph-common/templates/ceph.conf.j2 index 63f8140a4..c77e127a9 100644 --- a/roles/ceph-common/templates/ceph.conf.j2 +++ b/roles/ceph-common/templates/ceph.conf.j2 @@ -15,7 +15,6 @@ auth service required = none auth client required = none auth supported = none {% endif %} -fsid = {{ fsid }} max open files = {{ max_open_files }} osd pool default pg num = {{ pool_default_pg_num }} osd pool default pgp num = {{ pool_default_pgp_num }} @@ -88,6 +87,7 @@ debug mon = {{ debug_mon_level }} debug paxos = {{ debug_mon_level }} debug auth = {{ debug_mon_level }} {% endif %} +{% if not mon_containerized_deployment_with_kv %} {% for host in groups[mon_group_name] %} {% if hostvars[host]['ansible_fqdn'] is defined and mon_use_fqdn %} [mon.{{ hostvars[host]['ansible_fqdn'] }}] @@ -102,6 +102,7 @@ host = {{ hostvars[host]['ansible_hostname'] }} {% include 'mon_addr_address.j2' %} {% endif %} {% endfor %} +{% endif %} [osd] osd mkfs type = {{ osd_mkfs_type }} diff --git a/roles/ceph-mon/defaults/main.yml b/roles/ceph-mon/defaults/main.yml index ff638c18f..abd30e505 100644 --- a/roles/ceph-mon/defaults/main.yml +++ b/roles/ceph-mon/defaults/main.yml @@ -63,8 +63,12 @@ openstack_keys: ########## mon_containerized_deployment: false +mon_containerized_deployment_with_kv: false +mon_containerized_default_ceph_conf_with_kv: false ceph_mon_docker_interface: eth0 #ceph_mon_docker_subnet: # subnet of the ceph_mon_docker_interface ceph_mon_docker_username: ceph ceph_mon_docker_imagename: daemon ceph_mon_extra_envs: "MON_NAME={{ ansible_hostname }}" # comma separated variables +ceph_docker_on_openstack: false +mon_docker_privileged: true diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml index c5d82070f..e59dbba11 100644 --- a/roles/ceph-mon/tasks/docker/main.yml +++ b/roles/ceph-mon/tasks/docker/main.yml @@ -14,15 +14,21 @@ is_atomic='{{ stat_ostree.stat.exists }}' - include: checks.yml - when: ceph_health.rc != 0 + when: ceph_health.rc != 0 and not mon_containerized_deployment_with_kv - include: pre_requisite.yml + - include: selinux.yml when: ansible_os_family == 'RedHat' # let the first mon create configs and keyrings - include: create_configs.yml when: inventory_hostname == groups.mons[0] + - include: fetch_configs.yml + when: not mon_containerized_deployment_with_kv + - include: start_docker_monitor.yml -- include: ../ceph_keys.yml \ No newline at end of file + +- include: ../ceph_keys.yml + when: not mon_containerized_deployment_with_kv \ No newline at end of file diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml index 3dad4f710..3fa696f59 100644 --- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml +++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml @@ -6,11 +6,33 @@ - /var/lib/ceph/bootstrap-osd/ceph.keyring - /var/lib/ceph/bootstrap-rgw/ceph.keyring - /var/lib/ceph/bootstrap-mds/ceph.keyring - + when: not mon_containerized_deployment_with_kv + - name: pull ceph daemon image shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}" -# Use systemd to manage container on Atomic host +- name: populate kv_store + docker: + name: populate-kv-store + image: ceph/daemon + command: populate_kvstore + net: host + env: + KV_TYPE: "{{kv_type}}" + KV_IP: "{{kv_endpoint}}" + KV_PORT: "{{kv_port}}" + volumes: + - /etc/ceph/ceph.conf:/etc/ceph/ceph.defaults + run_once: true + when: inventory_hostname == groups.mons[0] and mon_containerized_deployment_with_kv + +- name: delete populate-kv-store docker + docker: + name: populate-kv-store + state: absent + image: ceph/daemon + +# Use systemd to manage container on Atomic host and CoreOS - name: generate systemd unit file sudo: true template: @@ -19,23 +41,23 @@ owner: "root" group: "root" mode: "0644" - + - name: link systemd unit file for mon instance file: src: /var/lib/ceph/ceph-mon@.service dest: /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service state: link - when: is_atomic + when: is_atomic or ansible_os_family == 'CoreOS' - name: enable systemd unit file for mon instance shell: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service failed_when: false changed_when: false - when: is_atomic + when: is_atomic or ansible_os_family == 'CoreOS' - name: reload systemd unit files shell: systemctl daemon-reload - when: is_atomic + when: is_atomic or ansible_os_family == 'CoreOS' - name: systemd start mon container service: @@ -43,7 +65,7 @@ state: started enabled: yes changed_when: false - when: is_atomic + when: is_atomic or ansible_os_family == 'CoreOS' - name: wait for ceph.conf exists wait_for: @@ -59,7 +81,7 @@ privileged: "{{ mon_docker_privileged }}" env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}" volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph" - when: not is_atomic + when: not is_atomic and ansible_os_family != 'CoreOS' - name: stat for ceph config and keys stat: path="{{ item }}" @@ -67,6 +89,7 @@ changed_when: false failed_when: false register: statmonconfig + when: not mon_containerized_deployment_with_kv - name: fetch boostrap keys and conf from mon fetch: @@ -76,5 +99,5 @@ with_together: - ceph_bootstrap_config_keys - statmonconfig.results - when: item.1.stat.exists == true + when: not mon_containerized_deployment_with_kv and item.1.stat.exists == true and inventory_hostname == groups.mons[0] diff --git a/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 b/roles/ceph-mon/templates/ceph-mon.service.j2 similarity index 72% rename from roles/ceph-mon/tasks/templates/ceph-mon.service.j2 rename to roles/ceph-mon/templates/ceph-mon.service.j2 index 484b2e3a1..76f04226e 100644 --- a/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 +++ b/roles/ceph-mon/templates/ceph-mon.service.j2 @@ -3,12 +3,17 @@ Description=Ceph Monitor After=docker.service [Service] -EnvironmentFile=/etc/environment +EnvironmentFile=-/etc/environment ExecStartPre=-/usr/bin/docker rm %i ExecStartPre=/usr/bin/mkdir -p /etc/ceph /var/lib/ceph/mon -ExecStart=/usr/bin/docker run --rm --name %p --net=host \ +ExecStart=/usr/bin/docker run --rm --name %i --net=host \ + {% if not mon_containerized_deployment_with_kv -%} -v /var/lib/ceph:/var/lib/ceph \ -v /etc/ceph:/etc/ceph \ + {% else -%} + -e KV_TYPE={{kv_type}} \ + -e KV_IP={{kv_endpoint}}\ + {% endif -%} --privileged \ -e CEPH_DAEMON=MON \ -e MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }} \ -- 2.39.5