From: Sébastien Han <seb@redhat.com>
Date: Tue, 22 Mar 2016 15:58:20 +0000 (+0100)
Subject: docker: fix tons of issues
X-Git-Tag: v1.0.3~11^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F648%2Fhead;p=ceph-ansible.git

docker: fix tons of issues

Signed-off-by: Sébastien Han <seb@redhat.com>
---

diff --git a/group_vars/all.sample b/group_vars/all.sample
index 4284df595..48375f711 100644
--- a/group_vars/all.sample
+++ b/group_vars/all.sample
@@ -337,3 +337,4 @@ dummy:
 ##########
 
 #docker: false
+
diff --git a/group_vars/common-coreoss.sample b/group_vars/common-coreoss.sample
index 4cfc5bfa3..32ae29553 100644
--- a/group_vars/common-coreoss.sample
+++ b/group_vars/common-coreoss.sample
@@ -1,7 +1,7 @@
 ---
 # Variables here are applicable to all host groups NOT roles
 
-# This sample file generated by generate-group-vars.sh
+# This sample file generated by generate_group_vars_sample.sh
 
 # Dummy variable to avoid error because ansible does not recognize the
 # file as a good configuration file when no variable in it.
@@ -14,3 +14,4 @@ dummy:
 #pypy_binary_directory: /opt/bin
 #pip_url: https://bootstrap.pypa.io/get-pip.py
 #local_temp_directory: /tmp
+
diff --git a/group_vars/mdss.sample b/group_vars/mdss.sample
index 3cb0d0e2e..aa7729bdf 100644
--- a/group_vars/mdss.sample
+++ b/group_vars/mdss.sample
@@ -32,4 +32,5 @@ dummy:
 #ceph_mds_docker_username: ceph
 #ceph_mds_docker_imagename: daemon
 #ceph_mds_docker_extra_env: "MDS_NAME={{ ansible_hostname }}" # comma separated variables
+#ceph_docker_on_openstack: false
 
diff --git a/group_vars/osds.sample b/group_vars/osds.sample
index fb60f230c..11f0caee1 100644
--- a/group_vars/osds.sample
+++ b/group_vars/osds.sample
@@ -128,4 +128,5 @@ dummy:
 #ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK" # comma separated variables
 #ceph_osd_docker_devices:
 # - /dev/sdb
+#ceph_docker_on_openstack: false
 
diff --git a/group_vars/restapis.sample b/group_vars/restapis.sample
index f87d0a85f..e58bfbb31 100644
--- a/group_vars/restapis.sample
+++ b/group_vars/restapis.sample
@@ -24,4 +24,5 @@ dummy:
 #ceph_restapi_docker_username: ceph
 #ceph_restapi_docker_imagename: daemon
 #ceph_restapi_docker_extra_env: "RESTAPI_IP=0.0.0.0" # comma separated variables
+#ceph_docker_on_openstack: false
 
diff --git a/group_vars/rgws.sample b/group_vars/rgws.sample
index 6c4adc9b1..f7d6425c6 100644
--- a/group_vars/rgws.sample
+++ b/group_vars/rgws.sample
@@ -39,4 +39,5 @@ dummy:
 #ceph_rgw_docker_username: ceph
 #ceph_rgw_docker_imagename: daemon
 #ceph_rgw_docker_extra_env: "RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}" # comma separated variables
+#ceph_docker_on_openstack: false
 
diff --git a/roles/ceph-common-coreos/README.md b/roles/ceph-common-coreos/README.md
index 297650853..8574a6f2d 100644
--- a/roles/ceph-common-coreos/README.md
+++ b/roles/ceph-common-coreos/README.md
@@ -24,7 +24,7 @@ has to be used with pypy in version 2.4 and below due to some dependency issues.
 ```
 - hosts: servers
   remote_user: core
-  sudo: True
+  become: True
   gather_facts: false
   roles:
      - { role: ceph-common-coreos }
diff --git a/roles/ceph-common-coreos/tasks/install_pip.yml b/roles/ceph-common-coreos/tasks/install_pip.yml
index bac7d6975..366d93f85 100644
--- a/roles/ceph-common-coreos/tasks/install_pip.yml
+++ b/roles/ceph-common-coreos/tasks/install_pip.yml
@@ -7,11 +7,11 @@
 
 - name: create local temp directory
   local_action: raw mkdir -p {{local_temp_directory}}
-  sudo: no
+  become: no
 
 - name: prepare install_pip.sh
   local_action: template src=install_pip.sh.j2 dest={{local_temp_directory}}/install_pip.sh
-  sudo: no
+  become: no
 
 - name: run pip.sh
   script: "{{local_temp_directory}}/install_pip.sh"
@@ -27,4 +27,4 @@
 
 - name: remove pip.sh
   local_action: file path="{{local_temp_directory}}/pip.sh" state=absent
-  sudo: no
+  become: no
diff --git a/roles/ceph-common-coreos/tasks/install_pypy.yml b/roles/ceph-common-coreos/tasks/install_pypy.yml
index fb0ad0c3c..a441101a9 100644
--- a/roles/ceph-common-coreos/tasks/install_pypy.yml
+++ b/roles/ceph-common-coreos/tasks/install_pypy.yml
@@ -7,11 +7,11 @@
 
 - name: create local temp directory
   local_action: raw mkdir -p {{local_temp_directory}}
-  sudo: no
+  become: no
 
 - name: prepare python executable
   local_action: template src=install_python.sh.j2 dest={{local_temp_directory}}/install_python.sh
-  sudo: no
+  become: no
 
 - name: fix library
   raw: ln -s /lib64/libncurses.so.5.9 {{pypy_directory}}/lib_pypy/libtinfo.so.5
@@ -30,4 +30,4 @@
 
 - name: remove install_python.sh
   local_action: file path="{{local_temp_directory}}/install_python.sh" state=absent
-  sudo: no
+  become: no
diff --git a/roles/ceph-common/defaults/main.yml b/roles/ceph-common/defaults/main.yml
index b810aae1b..1a66d3c46 100644
--- a/roles/ceph-common/defaults/main.yml
+++ b/roles/ceph-common/defaults/main.yml
@@ -42,6 +42,7 @@ centos_package_dependencies:
   - epel-release
   - ntp
   - python-setuptools
+  - libselinux-python
 
 redhat_package_dependencies:
   - python-pycurl
@@ -182,7 +183,10 @@ rbd_default_format: 2
 
 ## Monitor options
 #
+# You must define either monitor_interface or monitor_address. Preference
+# will go to monitor_interface if both are defined.
 monitor_interface: interface
+monitor_address: 0.0.0.0
 mon_use_fqdn: false # if set to true, the MON name used will be the fqdn in the ceph.conf
 mon_osd_down_out_interval: 600
 mon_osd_min_down_reporters: 7 # number of OSDs per host + 1
@@ -277,6 +281,7 @@ email_address: foo@bar.com
 ## REST API options
 #
 restapi_interface: "{{ monitor_interface }}"
+restapi_address: "{{ monitor_address }}"
 restapi_port: 5000
 restapi_base_url: /api/v0.1
 restapi_log_level: warning # available level are: critical, error, warning, info, debug
diff --git a/roles/ceph-common/tasks/installs/install_on_redhat.yml b/roles/ceph-common/tasks/installs/install_on_redhat.yml
index 670e529ef..da0ce967e 100644
--- a/roles/ceph-common/tasks/installs/install_on_redhat.yml
+++ b/roles/ceph-common/tasks/installs/install_on_redhat.yml
@@ -46,7 +46,9 @@
   when:
     (ceph_origin == "distro" or ceph_stable_rh_storage) and
     mon_group_name in group_names and
-    ansible_pkg_mgr == "yum"
+    ansible_pkg_mgr == "yum" and
+    ceph_stable and
+    ceph_stable_release not in ceph_stable_releases
 
 - name: install distro or red hat storage ceph mon
   dnf:
@@ -58,7 +60,9 @@
   when:
     (ceph_origin == "distro" or ceph_stable_rh_storage) and
     mon_group_name in group_names and
-    ansible_pkg_mgr == "dnf"
+    ansible_pkg_mgr == "dnf" and
+    ceph_stable and
+    ceph_stable_release not in ceph_stable_releases
 
 - name: install distro or red hat storage ceph osd
   yum:
@@ -70,7 +74,9 @@
   when:
     (ceph_origin == "distro" or ceph_stable_rh_storage) and
     osd_group_name in group_names and
-    ansible_pkg_mgr == "yum"
+    ansible_pkg_mgr == "yum" and
+    ceph_stable and
+    ceph_stable_release not in ceph_stable_releases
 
 - name: install distro or red hat storage ceph osd
   dnf:
@@ -82,7 +88,9 @@
   when:
     (ceph_origin == "distro" or ceph_stable_rh_storage) and
     osd_group_name in group_names and
-    ansible_pkg_mgr == "dnf"
+    ansible_pkg_mgr == "dnf" and
+    ceph_stable and
+    ceph_stable_release not in ceph_stable_releases
 
 - name: install ceph-test
   yum:
diff --git a/roles/ceph-mds/defaults/main.yml b/roles/ceph-mds/defaults/main.yml
index 2df43d9dd..9229d2a2d 100644
--- a/roles/ceph-mds/defaults/main.yml
+++ b/roles/ceph-mds/defaults/main.yml
@@ -24,3 +24,4 @@ mds_containerized_deployment: false
 ceph_mds_docker_username: ceph
 ceph_mds_docker_imagename: daemon
 ceph_mds_docker_extra_env: "MDS_NAME={{ ansible_hostname }}" # comma separated variables
+ceph_docker_on_openstack: false
diff --git a/roles/ceph-mon/tasks/ceph_keys.yml b/roles/ceph-mon/tasks/ceph_keys.yml
index c9ddc6854..d28d8776b 100644
--- a/roles/ceph-mon/tasks/ceph_keys.yml
+++ b/roles/ceph-mon/tasks/ceph_keys.yml
@@ -12,21 +12,8 @@
   changed_when: false
   when:
     cephx and
-    not mon_containerized_deployment and
     groups[restapi_group_name] is defined
 
-- name: create ceph rest api keyring when mon is containerized
-  command: docker exec {{ ansible_hostname }} ceph auth get-or-create client.restapi osd 'allow *' mon 'allow *' -o /etc/ceph/ceph.client.restapi.keyring
-  args:
-     creates: /etc/ceph/ceph.client.restapi.keyring
-  changed_when: false
-  when:
-    cephx and
-    mon_containerized_deployment and
-    groups[restapi_group_name] is defined and
-    inventory_hostname == groups.mons|last
-
-
 - include: openstack_config.yml
   when: openstack_config
 
diff --git a/roles/ceph-mon/tasks/docker/copy_configs.yml b/roles/ceph-mon/tasks/docker/copy_configs.yml
new file mode 100644
index 000000000..f2ba50e81
--- /dev/null
+++ b/roles/ceph-mon/tasks/docker/copy_configs.yml
@@ -0,0 +1,10 @@
+---
+- name: push ceph files to the ansible server
+  fetch:
+    src: "{{ item.0 }}"
+    dest: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
+    flat: yes
+  with_together:
+    - ceph_config_keys
+    - statconfig.results
+  when: item.1.stat.exists == false
diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml
index 469cf9357..49a381886 100644
--- a/roles/ceph-mon/tasks/docker/main.yml
+++ b/roles/ceph-mon/tasks/docker/main.yml
@@ -12,7 +12,7 @@
 - name: set fact for using Atomic host
   set_fact:
       is_atomic='{{ stat_ostree.stat.exists }}'
-                                  
+
 - include: checks.yml
   when: ceph_health.rc != 0 and not mon_containerized_deployment_with_kv
 
@@ -23,12 +23,24 @@
 
 # let the first mon create configs and keyrings
 - include: create_configs.yml
-  when: inventory_hostname == groups.mons[0] and not mon_containerized_default_ceph_conf_with_kv
+  when: inventory_hostname == groups.mons[0] and mon_containerized_default_ceph_conf_with_kv
 
 - include: fetch_configs.yml
   when: not mon_containerized_deployment_with_kv
 
 - include: start_docker_monitor.yml
 
-- include: ../ceph_keys.yml
-  when: not mon_containerized_deployment_with_kv
\ No newline at end of file
+- include: copy_configs.yml
+  when: not mon_containerized_deployment_with_kv
+
+- name: create ceph rest api keyring when mon is containerized
+  command: docker exec {{ ansible_hostname }} ceph auth get-or-create client.restapi osd 'allow *' mon 'allow *' -o /etc/ceph/ceph.client.restapi.keyring
+  args:
+     creates: /etc/ceph/ceph.client.restapi.keyring
+  changed_when: false
+  when:
+    cephx and
+    mon_containerized_deployment and
+    groups[restapi_group_name] is defined and
+    inventory_hostname == groups.mons|last and
+    not mon_containerized_deployment_with_kv
diff --git a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml
index 1b5602cf5..48fb8ad6d 100644
--- a/roles/ceph-mon/tasks/docker/start_docker_monitor.yml
+++ b/roles/ceph-mon/tasks/docker/start_docker_monitor.yml
@@ -1,13 +1,4 @@
 ---
-- name: set config and keys paths
-  set_fact:
-      ceph_bootstrap_config_keys:
-          - /etc/ceph/ceph.conf
-          - /var/lib/ceph/bootstrap-osd/ceph.keyring
-          - /var/lib/ceph/bootstrap-rgw/ceph.keyring
-          - /var/lib/ceph/bootstrap-mds/ceph.keyring
-  when: not mon_containerized_deployment_with_kv
-
 - name: pull ceph daemon image
   shell: "docker pull {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
 
@@ -22,8 +13,10 @@
       KV_IP: "{{kv_endpoint}}"
       KV_PORT: "{{kv_port}}"
   run_once: true
-  when: inventory_hostname == groups.mons[0] and mon_containerized_deployment_with_kv
-        and mon_containerized_default_ceph_conf_with_kv
+  when:
+    inventory_hostname == groups.mons[0] and
+    mon_containerized_deployment_with_kv and
+    mon_containerized_default_ceph_conf_with_kv
 
 - name: populate kv_store with custom ceph.conf
   docker:
@@ -38,8 +31,10 @@
     volumes:
       - /etc/ceph/ceph.conf:/etc/ceph/ceph.defaults
   run_once: true
-  when: inventory_hostname == groups.mons[0] and mon_containerized_deployment_with_kv
-        and not mon_containerized_default_ceph_conf_with_kv
+  when:
+    inventory_hostname == groups.mons[0] and
+    mon_containerized_deployment_with_kv and
+    not mon_containerized_default_ceph_conf_with_kv
 
 - name: delete populate-kv-store docker
   docker:
@@ -49,7 +44,7 @@
 
 # Use systemd to manage container on Atomic host and CoreOS
 - name: generate systemd unit file
-  sudo: true
+  become: true
   template:
     src: ceph-mon.service.j2
     dest: /var/lib/ceph/ceph-mon@.service
@@ -62,17 +57,23 @@
     src: /var/lib/ceph/ceph-mon@.service
     dest: /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service
     state: link
-  when: is_atomic or ansible_os_family == 'CoreOS'
+  when:
+    is_atomic or
+    ansible_os_family == 'CoreOS'
 
 - name: enable systemd unit file for mon instance
   shell: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service
   failed_when: false
   changed_when: false
-  when: is_atomic or ansible_os_family == 'CoreOS'
+  when:
+    is_atomic or
+    ansible_os_family == 'CoreOS'
 
 - name: reload systemd unit files
   shell: systemctl daemon-reload
-  when: is_atomic or ansible_os_family == 'CoreOS'
+  when:
+    is_atomic or
+    ansible_os_family == 'CoreOS'
 
 - name: systemd start mon container
   service:
@@ -80,14 +81,16 @@
     state: started
     enabled: yes
   changed_when: false
-  when: is_atomic or ansible_os_family == 'CoreOS'
+  when:
+    is_atomic or
+    ansible_os_family == 'CoreOS'
 
 - name: wait for ceph.conf exists
   wait_for:
       path: /etc/ceph/ceph.conf
   when: is_atomic
 
-- name: run the ceph Monitor docker image
+- name: run the ceph monitor docker image
   docker:
     image: "{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}"
     name: "{{ ansible_hostname }}"
@@ -96,23 +99,6 @@
     privileged: "{{ mon_docker_privileged }}"
     env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}"
     volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
-  when: not is_atomic and ansible_os_family != 'CoreOS'
-
-- name: stat for ceph config and keys
-  stat: path="{{ item }}"
-  with_items: ceph_bootstrap_config_keys
-  changed_when: false
-  failed_when: false
-  register: statmonconfig
-  when: not mon_containerized_deployment_with_kv
-
-- name: fetch boostrap keys and conf from mon
-  fetch:
-     dest: "{{ playbook_dir }}/{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
-     src: "{{ item.0 }}"
-     flat: yes
-  with_together:
-    - ceph_bootstrap_config_keys
-    - statmonconfig.results
-  when: not mon_containerized_deployment_with_kv and item.1.stat.exists == true
-        and inventory_hostname == groups.mons[0]
+  when:
+    not is_atomic and
+    ansible_os_family != 'CoreOS'
diff --git a/roles/ceph-osd/defaults/main.yml b/roles/ceph-osd/defaults/main.yml
index d7c13e0b8..0b86c1fe3 100644
--- a/roles/ceph-osd/defaults/main.yml
+++ b/roles/ceph-osd/defaults/main.yml
@@ -120,3 +120,4 @@ ceph_osd_docker_imagename: daemon
 ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK" # comma separated variables
 #ceph_osd_docker_devices:
 # - /dev/sdb
+ceph_docker_on_openstack: false
diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml
index 919b98de9..2892f9262 100644
--- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml
+++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml
@@ -2,13 +2,13 @@
 # For openstack VMs modify the mount point below depending on if the Openstack
 # VM deploy tool defaults to mounting ephemeral disks
 - name: umount ceph disk (if on openstack)
-  mount: 
+  mount:
     name: /mnt
     src: /dev/vdb
     fstype: ext3
     state: unmounted
   when: ceph_docker_on_openstack
-  
+
 # (rootfs) for reasons I haven't figured out, docker pull and run will fail.
 - name: pull ceph daemon image
   shell: "docker pull {{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
@@ -28,7 +28,7 @@
 
 # Use systemd to manage container on Atomic host
 - name: generate systemd unit file
-  sudo: true
+  become: true
   template:
     src: ceph-osd.service.j2
     dest: /var/lib/ceph/ceph-osd@.service
diff --git a/roles/ceph-restapi/defaults/main.yml b/roles/ceph-restapi/defaults/main.yml
index c8320c114..4ac192eed 100644
--- a/roles/ceph-restapi/defaults/main.yml
+++ b/roles/ceph-restapi/defaults/main.yml
@@ -16,3 +16,4 @@ ceph_restapi_port: 5000
 ceph_restapi_docker_username: ceph
 ceph_restapi_docker_imagename: daemon
 ceph_restapi_docker_extra_env: "RESTAPI_IP=0.0.0.0" # comma separated variables
+ceph_docker_on_openstack: false
diff --git a/roles/ceph-restapi/tasks/docker/fetch_configs.yml b/roles/ceph-restapi/tasks/docker/fetch_configs.yml
index 2faf8ade5..8af56884e 100644
--- a/roles/ceph-restapi/tasks/docker/fetch_configs.yml
+++ b/roles/ceph-restapi/tasks/docker/fetch_configs.yml
@@ -15,7 +15,7 @@
 
 - name: try to fetch ceph config and keys
   copy:
-    src: "{{ playbook_dir }}/{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
+    src: "{{ playbook_dir }}/{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
     dest: "{{ item.0 }}"
     owner: root
     group: root
diff --git a/roles/ceph-rgw/defaults/main.yml b/roles/ceph-rgw/defaults/main.yml
index 26b2b0913..076c93fe7 100644
--- a/roles/ceph-rgw/defaults/main.yml
+++ b/roles/ceph-rgw/defaults/main.yml
@@ -31,3 +31,4 @@ ceph_rgw_civetweb_port: 80
 ceph_rgw_docker_username: ceph
 ceph_rgw_docker_imagename: daemon
 ceph_rgw_docker_extra_env: "RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}" # comma separated variables
+ceph_docker_on_openstack: false
diff --git a/roles/ceph-rgw/tasks/docker/fetch_configs.yml b/roles/ceph-rgw/tasks/docker/fetch_configs.yml
index 45dd3bebc..6902b5919 100644
--- a/roles/ceph-rgw/tasks/docker/fetch_configs.yml
+++ b/roles/ceph-rgw/tasks/docker/fetch_configs.yml
@@ -15,7 +15,7 @@
 
 - name: try to fetch ceph config and keys
   copy:
-    src: "{{ playbook_dir }}/{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
+    src: "{{ playbook_dir }}/{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
     dest: "{{ item.0 }}"
     owner: root
     group: root