Adding dockerized ceph-mon deployment with kv backend

diff --git a/group_vars/all.docker.sample b/group_vars/all.docker.sample
index c0d84362ea2f6cb24046118857ba52e7bb50048c..fce0c22a9ca14ee7c1a077a66db51b44cc18379c 100644 (file)
--- a/group_vars/all.docker.sample
+++ b/group_vars/all.docker.sample
@@ -5,6 +5,10 @@ dummy:
 # MON #
 #######
 #mon_containerized_deployment: true
+#mon_containerized_deployment_with_kv: false
+#kv_type: etcd
+#kv_endpoint: 127.0.0.1
+#kv_port: 4001
 #mon_docker_privileged: true
 #ceph_mon_docker_username: ceph
 #ceph_mon_docker_imagename: daemon

diff --git a/roles/ceph-common/templates/ceph.conf.j2 b/roles/ceph-common/templates/ceph.conf.j2
index 63f8140a468a593d9eb554dee4676a0be39901e0..c77e127a95da015a445a3bab208d81de9c4e6a2b 100644 (file)
--- a/roles/ceph-common/templates/ceph.conf.j2
+++ b/roles/ceph-common/templates/ceph.conf.j2
@@ -15,7 +15,6 @@ auth service required = none
 auth client required = none
 auth supported = none
 {% endif %}
-fsid = {{ fsid }}
 max open files = {{ max_open_files }}
 osd pool default pg num = {{ pool_default_pg_num }}
 osd pool default pgp num = {{ pool_default_pgp_num }}
@@ -88,6 +87,7 @@ debug mon = {{ debug_mon_level }}
 debug paxos = {{ debug_mon_level }}
 debug auth = {{ debug_mon_level }}
 {% endif %}
+{% if not mon_containerized_deployment_with_kv %}
 {% for host in groups[mon_group_name] %}
 {% if hostvars[host]['ansible_fqdn'] is defined and mon_use_fqdn %}
 [mon.{{ hostvars[host]['ansible_fqdn'] }}]
@@ -102,6 +102,7 @@ host = {{ hostvars[host]['ansible_hostname'] }}
 {% include 'mon_addr_address.j2' %}
 {% endif %}
 {% endfor %}
+{% endif %}
 
 [osd]
 osd mkfs type = {{ osd_mkfs_type }}

diff --git a/roles/ceph-mon/defaults/main.yml b/roles/ceph-mon/defaults/main.yml
index ff638c18f44fc143b09b8fbfa7fdf9b2f6596c9d..abd30e5059c4458c0970e638637373a37267dc27 100644 (file)
--- a/roles/ceph-mon/defaults/main.yml
+++ b/roles/ceph-mon/defaults/main.yml
@@ -63,8 +63,12 @@ openstack_keys:
 ##########
 
 mon_containerized_deployment: false
+mon_containerized_deployment_with_kv: false
+mon_containerized_default_ceph_conf_with_kv: false
 ceph_mon_docker_interface: eth0
 #ceph_mon_docker_subnet: # subnet of the ceph_mon_docker_interface
 ceph_mon_docker_username: ceph
 ceph_mon_docker_imagename: daemon
 ceph_mon_extra_envs: "MON_NAME={{ ansible_hostname }}" # comma separated variables
+ceph_docker_on_openstack: false
+mon_docker_privileged: true

diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml
index c5d82070f023709410945bbca9fc24f9e2f3109a..e59dbba11ac983aab9980cffa7615f32f523501a 100644 (file)
--- a/roles/ceph-mon/tasks/docker/main.yml
+++ b/roles/ceph-mon/tasks/docker/main.yml
@@ -14,15 +14,21 @@
       is_atomic='{{ stat_ostree.stat.exists }}'
                                   
 - include: checks.yml
-  when: ceph_health.rc != 0
+  when: ceph_health.rc != 0 and not mon_containerized_deployment_with_kv
 
 - include: pre_requisite.yml
+
 - include: selinux.yml
   when: ansible_os_family == 'RedHat'
 
 # let the first mon create configs and keyrings
 - include: create_configs.yml
   when: inventory_hostname == groups.mons[0]
+
 - include: fetch_configs.yml
+  when: not mon_containerized_deployment_with_kv
+
 - include: start_docker_monitor.yml
-- include: ../ceph_keys.yml
\ No newline at end of file
+
+- include: ../ceph_keys.yml
+  when: not mon_containerized_deployment_with_kv
\ No newline at end of file

diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml
index 3dad4f710124577cf96502a12f2e8707f7663d7d..3fa696f591674a5a754293eefd698a3e5e76fcb2 100644 (file)
--- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml
+++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml
@@ -6,11 +6,33 @@
           - /var/lib/ceph/bootstrap-osd/ceph.keyring
           - /var/lib/ceph/bootstrap-rgw/ceph.keyring
           - /var/lib/ceph/bootstrap-mds/ceph.keyring
-                                                
+  when: not mon_containerized_deployment_with_kv
+
 - name: pull ceph daemon image
   shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
 
-# Use systemd to manage container on Atomic host
+- name: populate kv_store
+  docker:
+    name: populate-kv-store
+    image: ceph/daemon
+    command: populate_kvstore
+    net: host
+    env:
+      KV_TYPE: "{{kv_type}}"
+      KV_IP: "{{kv_endpoint}}"
+      KV_PORT: "{{kv_port}}"
+    volumes:
+      - /etc/ceph/ceph.conf:/etc/ceph/ceph.defaults
+  run_once: true
+  when: inventory_hostname == groups.mons[0] and mon_containerized_deployment_with_kv
+
+- name: delete populate-kv-store docker
+  docker:
+    name: populate-kv-store
+    state: absent
+    image: ceph/daemon
+
+# Use systemd to manage container on Atomic host and CoreOS
 - name: generate systemd unit file
   sudo: true
   template:
@@ -19,23 +41,23 @@
     owner: "root"
     group: "root"
     mode: "0644"
-  
+
 - name: link systemd unit file for mon instance
   file:
     src: /var/lib/ceph/ceph-mon@.service
     dest: /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service
     state: link
-  when: is_atomic
+  when: is_atomic or ansible_os_family == 'CoreOS'
 
 - name: enable systemd unit file for mon instance
   shell: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service
   failed_when: false
   changed_when: false
-  when: is_atomic
+  when: is_atomic or ansible_os_family == 'CoreOS'
 
 - name: reload systemd unit files
   shell: systemctl daemon-reload
-  when: is_atomic
+  when: is_atomic or ansible_os_family == 'CoreOS'
 
 - name: systemd start mon container
   service:
@@ -43,7 +65,7 @@
     state: started
     enabled: yes
   changed_when: false
-  when: is_atomic
+  when: is_atomic or ansible_os_family == 'CoreOS'
 
 - name: wait for ceph.conf exists
   wait_for:
@@ -59,7 +81,7 @@
     privileged: "{{ mon_docker_privileged }}"
     env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}"
     volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
-  when: not is_atomic 
+  when: not is_atomic and ansible_os_family != 'CoreOS'
 
 - name: stat for ceph config and keys
   stat: path="{{ item }}"
@@ -67,6 +89,7 @@
   changed_when: false
   failed_when: false
   register: statmonconfig
+  when: not mon_containerized_deployment_with_kv
 
 - name: fetch boostrap keys and conf from mon
   fetch:
@@ -76,5 +99,5 @@
   with_together:
     - ceph_bootstrap_config_keys
     - statmonconfig.results
-  when: item.1.stat.exists == true
+  when: not mon_containerized_deployment_with_kv and item.1.stat.exists == true
         and inventory_hostname == groups.mons[0]

diff --git a/roles/ceph-mon/tasks/templates/ceph-mon.service.j2 b/roles/ceph-mon/tasks/templates/ceph-mon.service.j2
deleted file mode 100644 (file)
index 484b2e3..0000000
--- a/roles/ceph-mon/tasks/templates/ceph-mon.service.j2
+++ /dev/null
@@ -1,26 +0,0 @@
-[Unit]
-Description=Ceph Monitor
-After=docker.service
-
-[Service]
-EnvironmentFile=/etc/environment
-ExecStartPre=-/usr/bin/docker rm %i
-ExecStartPre=/usr/bin/mkdir -p /etc/ceph /var/lib/ceph/mon
-ExecStart=/usr/bin/docker run --rm --name %p --net=host \
-   -v /var/lib/ceph:/var/lib/ceph \
-   -v /etc/ceph:/etc/ceph \
-   --privileged \
-   -e CEPH_DAEMON=MON \
-   -e MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }} \
-   -e CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }} \
-   -e MON_NAME={{ ansible_hostname }} \
-   --name={{ ansible_hostname }} \
-    {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}
-ExecStopPost=-/usr/bin/docker stop %i
-Restart=always
-RestartSec=10s
-TimeoutStartSec=120
-TimeoutStopSec=15
-
-[Install]
-WantedBy=multi-user.target
\ No newline at end of file

diff --git a/roles/ceph-mon/templates/ceph-mon.service.j2 b/roles/ceph-mon/templates/ceph-mon.service.j2
new file mode 100644 (file)
index 0000000..76f0422
--- /dev/null
+++ b/roles/ceph-mon/templates/ceph-mon.service.j2
@@ -0,0 +1,31 @@
+[Unit]
+Description=Ceph Monitor
+After=docker.service
+
+[Service]
+EnvironmentFile=-/etc/environment
+ExecStartPre=-/usr/bin/docker rm %i
+ExecStartPre=/usr/bin/mkdir -p /etc/ceph /var/lib/ceph/mon
+ExecStart=/usr/bin/docker run --rm --name %i --net=host \
+   {% if not mon_containerized_deployment_with_kv -%}
+   -v /var/lib/ceph:/var/lib/ceph \
+   -v /etc/ceph:/etc/ceph \
+   {% else -%}
+   -e KV_TYPE={{kv_type}} \
+   -e KV_IP={{kv_endpoint}}\
+   {% endif -%}
+   --privileged \
+   -e CEPH_DAEMON=MON \
+   -e MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }} \
+   -e CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }} \
+   -e MON_NAME={{ ansible_hostname }} \
+   --name={{ ansible_hostname }} \
+    {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}
+ExecStopPost=-/usr/bin/docker stop %i
+Restart=always
+RestartSec=10s
+TimeoutStartSec=120
+TimeoutStopSec=15
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file