From: Sébastien Han Date: Thu, 16 Mar 2017 09:17:08 +0000 (+0100) Subject: mgr: add new role for ceph-mgr X-Git-Tag: v2.2.0~1^2~2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=2aa5286544d451b45e77b2605443e5eb16ceefc4;p=ceph-ansible.git mgr: add new role for ceph-mgr The Ceph Manager daemon (ceph-mgr) runs alongside monitor daemons, to provide additional monitoring and interfaces to external monitoring and management systems. Only works as of the Kraken release. Co-Authored-By: Guillaume Abrioux Signed-off-by: Sébastien Han --- diff --git a/Vagrantfile b/Vagrantfile index fd1bfbedc..beb17fe5e 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -18,6 +18,7 @@ RESTAPI = settings['restapi'] NRBD_MIRRORS = settings['rbd_mirror_vms'] CLIENTS = settings['client_vms'] NISCSI_GWS = settings['iscsi_gw_vms'] +MGRS = settings['mgr_vms'] PUBLIC_SUBNET = settings['public_subnet'] CLUSTER_SUBNET = settings['cluster_subnet'] BOX = settings['vagrant_box'] @@ -55,7 +56,8 @@ ansible_provision = proc do |ansible| 'nfss' => (0..NNFSS - 1).map { |j| "#{LABEL_PREFIX}nfs#{j}" }, 'rbd_mirrors' => (0..NRBD_MIRRORS - 1).map { |j| "#{LABEL_PREFIX}rbd_mirror#{j}" }, 'clients' => (0..CLIENTS - 1).map { |j| "#{LABEL_PREFIX}client#{j}" }, - 'iscsi_gw' => (0..NISCSI_GWS - 1).map { |j| "#{LABEL_PREFIX}iscsi_gw#{j}" } + 'iscsi_gw' => (0..NISCSI_GWS - 1).map { |j| "#{LABEL_PREFIX}iscsi_gw#{j}" }, + 'mgrs' => (0..MGRS - 1).map { |j| "#{LABEL_PREFIX}mgr#{j}" } } if RESTAPI then @@ -78,6 +80,7 @@ ansible_provision = proc do |ansible| nfs_containerized_deployment: 'true', restapi_containerized_deployment: 'true', rbd_mirror_containerized_deployment: 'true', + mgr_containerized_deployment: 'true', ceph_mon_docker_interface: ETH, ceph_mon_docker_subnet: "#{PUBLIC_SUBNET}.0/24", ceph_osd_docker_devices: settings['disks'], @@ -178,6 +181,41 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end end + (0..MGRS - 1).each do |i| + config.vm.define "#{LABEL_PREFIX}mgr#{i}" do |mgr| + mgr.vm.hostname = "#{LABEL_PREFIX}ceph-mgr#{i}" + if ASSIGN_STATIC_IP + mgr.vm.network :private_network, + ip: "#{PUBLIC_SUBNET}.3#{i}" + end + # Virtualbox + mgr.vm.provider :virtualbox do |vb| + vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"] + end + + # VMware + mgr.vm.provider :vmware_fusion do |v| + v.vmx['memsize'] = "#{MEMORY}" + end + + # Libvirt + mgr.vm.provider :libvirt do |lv| + lv.memory = MEMORY + lv.random_hostname = true + end + + # Parallels + mgr.vm.provider "parallels" do |prl| + prl.name = "ceph-mgr#{i}" + prl.memory = "#{MEMORY}" + end + + mgr.vm.provider :linode do |provider| + provider.label = mgr.vm.hostname + end + end + end + (0..CLIENTS - 1).each do |i| config.vm.define "#{LABEL_PREFIX}client#{i}" do |client| client.vm.hostname = "#{LABEL_PREFIX}ceph-client#{i}" diff --git a/contrib/push-roles-to-ansible-galaxy.sh b/contrib/push-roles-to-ansible-galaxy.sh index b1dbf9f74..3d15ab07d 100755 --- a/contrib/push-roles-to-ansible-galaxy.sh +++ b/contrib/push-roles-to-ansible-galaxy.sh @@ -5,7 +5,7 @@ set -xe BASEDIR=$(dirname "$0") LOCAL_BRANCH=$(cd $BASEDIR && git rev-parse --abbrev-ref HEAD) BRANCHES="master ansible-1.9" -ROLES="ceph-common ceph-mon ceph-osd ceph-mds ceph-rgw ceph-restapi ceph-agent ceph-fetch-keys ceph-rbd-mirror ceph-client ceph-docker-common" +ROLES="ceph-common ceph-mon ceph-osd ceph-mds ceph-rgw ceph-restapi ceph-agent ceph-fetch-keys ceph-rbd-mirror ceph-client ceph-docker-common ceph-mgr" # FUNCTIONS diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample index 870c421ab..363e6aaf8 100644 --- a/group_vars/all.yml.sample +++ b/group_vars/all.yml.sample @@ -39,6 +39,7 @@ dummy: #rbdmirror_group_name: rbdmirrors #client_group_name: clients #iscsi_group_name: iscsigws +#mgr_group_name: mgrs # If check_firewall is true, then ansible will try to determine if the # Ceph ports are blocked by a firewall. If the machine running ansible diff --git a/group_vars/mgrs.yml.sample b/group_vars/mgrs.yml.sample new file mode 100644 index 000000000..c4c60fc66 --- /dev/null +++ b/group_vars/mgrs.yml.sample @@ -0,0 +1,23 @@ +--- +# Variables here are applicable to all host groups NOT roles + +# This sample file generated by generate_group_vars_sample.sh + +# Dummy variable to avoid error because ansible does not recognize the +# file as a good configuration file when no variable in it. +dummy: + +########## +# DOCKER # +########## + +#mgr_containerized_deployment: false +#mgr_containerized_deployment_with_kv: false +#kv_type: etcd +#kv_endpoint: 127.0.0.1 +#ceph_docker_image: "ceph/daemon" +#ceph_docker_image_tag: latest +#ceph_mgr_docker_extra_env: -e CLUSTER={{ cluster }} -e MGR_NAME={{ ansible_hostname }} +#ceph_docker_on_openstack: false +#ceph_config_keys: [] # DON'T TOUCH ME + diff --git a/roles/ceph-common/defaults/main.yml b/roles/ceph-common/defaults/main.yml index 7cdb24fca..b6d3b848c 100644 --- a/roles/ceph-common/defaults/main.yml +++ b/roles/ceph-common/defaults/main.yml @@ -31,6 +31,7 @@ restapi_group_name: restapis rbdmirror_group_name: rbdmirrors client_group_name: clients iscsi_group_name: iscsigws +mgr_group_name: mgrs # If check_firewall is true, then ansible will try to determine if the # Ceph ports are blocked by a firewall. If the machine running ansible diff --git a/roles/ceph-common/tasks/installs/install_on_debian.yml b/roles/ceph-common/tasks/installs/install_on_debian.yml index c6ab701ac..f25776399 100644 --- a/roles/ceph-common/tasks/installs/install_on_debian.yml +++ b/roles/ceph-common/tasks/installs/install_on_debian.yml @@ -57,3 +57,10 @@ state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" update_cache: yes when: nfs_group_name in group_names + +- name: install ceph mgr + apt: + pkg: ceph-mgr + state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" + default_release: "{{ ceph_stable_release_uca | default(ansible_distribution_release) }}{{ '-backports' if ceph_origin == 'distro' and ceph_use_distro_backports else ''}}" + when: mgr_group_name in group_names diff --git a/roles/ceph-common/tasks/installs/install_on_redhat.yml b/roles/ceph-common/tasks/installs/install_on_redhat.yml index 8df9ebca9..8a8d82a77 100644 --- a/roles/ceph-common/tasks/installs/install_on_redhat.yml +++ b/roles/ceph-common/tasks/installs/install_on_redhat.yml @@ -142,3 +142,9 @@ when: - nfs_group_name in group_names - nfs_obj_gw + +- name: install ceph mgr + package: + name: ceph-mgr + state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}" + when: mgr_group_name in group_names diff --git a/roles/ceph-mgr/LICENSE b/roles/ceph-mgr/LICENSE new file mode 100644 index 000000000..acee72b2b --- /dev/null +++ b/roles/ceph-mgr/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [2014] [Sébastien Han] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/roles/ceph-mgr/README.md b/roles/ceph-mgr/README.md new file mode 100644 index 000000000..fbb917e4a --- /dev/null +++ b/roles/ceph-mgr/README.md @@ -0,0 +1,41 @@ +# Ansible role: Ceph Manager + +This role bootstraps the Ceph Manager. + +# Requirements + +Nothing, it runs out of the box. + +# Role variables + +Have a look at: `defaults/main.yml`. + +## Mandatory variables + +None. + +# Dependencies + +The role `ceph.ceph-common` must be installed. + +# Example Playbook + +``` +- hosts: servers + remote_user: ubuntu + roles: + - { role: ceph.ceph-mgr } +``` + +# Contribution + +**THIS REPOSITORY DOES NOT ACCEPT PULL REQUESTS** +**PULL REQUESTS MUST GO THROUGH [CEPH-ANSIBLE](https://github.com/ceph/ceph-ansible)** + +# License + +Apache + +# Author Information + +This role was created by Sebastien Han. diff --git a/roles/ceph-mgr/defaults/main.yml b/roles/ceph-mgr/defaults/main.yml new file mode 100644 index 000000000..11227b781 --- /dev/null +++ b/roles/ceph-mgr/defaults/main.yml @@ -0,0 +1,14 @@ +--- +########## +# DOCKER # +########## + +mgr_containerized_deployment: false +mgr_containerized_deployment_with_kv: false +kv_type: etcd +kv_endpoint: 127.0.0.1 +ceph_docker_image: "ceph/daemon" +ceph_docker_image_tag: latest +ceph_mgr_docker_extra_env: -e CLUSTER={{ cluster }} -e MGR_NAME={{ ansible_hostname }} +ceph_docker_on_openstack: false +ceph_config_keys: [] # DON'T TOUCH ME diff --git a/roles/ceph-mgr/meta/main.yml b/roles/ceph-mgr/meta/main.yml new file mode 100644 index 000000000..878df1d1d --- /dev/null +++ b/roles/ceph-mgr/meta/main.yml @@ -0,0 +1,18 @@ +--- +galaxy_info: + author: Sébastien Han + description: Installs Ceph Manager + license: Apache + min_ansible_version: 1.7 + platforms: + - name: Ubuntu + versions: + - trusty + - name: EL + versions: + - 7 + categories: + - system +dependencies: + - { role: ceph.ceph-common, when: not mgr_containerized_deployment } + - { role: ceph.ceph-docker-common, when: mgr_containerized_deployment } diff --git a/roles/ceph-mgr/tasks/docker/checks.yml b/roles/ceph-mgr/tasks/docker/checks.yml new file mode 100644 index 000000000..968fc038c --- /dev/null +++ b/roles/ceph-mgr/tasks/docker/checks.yml @@ -0,0 +1,28 @@ +--- +- name: set config and keys paths + set_fact: + ceph_config_keys: + - /etc/ceph/{{ cluster }}.client.admin.keyring + - /etc/ceph/{{ cluster }}.conf + - /etc/ceph/monmap-{{ cluster }} + - /etc/ceph/{{ cluster }}.mon.keyring + - /var/lib/ceph/bootstrap-osd/{{ cluster }}.keyring + - /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring + - /var/lib/ceph/bootstrap-mds/{{ cluster }}.keyring + +- name: stat for ceph config and keys + stat: + path: "{{ item }}" + with_items: "{{ ceph_config_keys }}" + changed_when: false + failed_when: false + always_run: true + register: statleftover + +- name: fail if we find existing cluster files + fail: + msg: "looks like no cluster is running but ceph files are present, please remove them" + with_together: + - "{{ ceph_config_keys }}" + - "{{ statleftover.results }}" + when: item.1.stat.exists == true diff --git a/roles/ceph-mgr/tasks/docker/dirs_permissions.yml b/roles/ceph-mgr/tasks/docker/dirs_permissions.yml new file mode 100644 index 000000000..faa4631c5 --- /dev/null +++ b/roles/ceph-mgr/tasks/docker/dirs_permissions.yml @@ -0,0 +1,12 @@ +--- +- name: create bootstrap directories + file: + path: "{{ item }}" + state: directory + owner: "64045" + group: "64045" + mode: "0755" + with_items: + - /etc/ceph + - /var/lib/ceph/mgr + - /var/lib/ceph/mgr/{{ cluster }}-{{ ansible_hostname }} diff --git a/roles/ceph-mgr/tasks/docker/fetch_configs.yml b/roles/ceph-mgr/tasks/docker/fetch_configs.yml new file mode 100644 index 000000000..6e43b5e12 --- /dev/null +++ b/roles/ceph-mgr/tasks/docker/fetch_configs.yml @@ -0,0 +1,43 @@ +--- +- name: set config and keys paths + set_fact: + ceph_config_keys: + - /etc/ceph/{{ cluster }}.conf + - /etc/ceph/{{ cluster }}.mgr.{{ ansible_hostname }}.keyring + +- name: stat for ceph config and keys + local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }} + with_items: "{{ ceph_config_keys }}" + changed_when: false + become: false + failed_when: false + always_run: true + register: statconfig + +- name: try to fetch ceph config and keys + copy: + src: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}" + dest: "{{ item.0 }}" + owner: root + group: root + mode: 0644 + changed_when: false + with_together: + - "{{ ceph_config_keys }}" + - "{{ statconfig.results }}" + when: item.1.stat.exists == true + +- name: "copy mgr key to /var/lib/ceph/mgr/{{ cluster }}-{{ ansible_hostname }}/keyring" + command: cp /etc/ceph/{{ cluster }}.mgr.{{ ansible_hostname }}.keyring /var/lib/ceph/mgr/{{ cluster }}-{{ ansible_hostname }}/keyring + changed_when: false + always_run: true + with_items: "{{ statconfig.results }}" + when: item.stat.exists == true + +- name: set ceph mgr key permission + file: + path: "/var/lib/ceph/mgr/{{ cluster }}-{{ ansible_hostname }}/keyring" + owner: "{{ bootstrap_dirs_owner }}" + group: "{{ bootstrap_dirs_group }}" + mode: "0600" + when: cephx diff --git a/roles/ceph-mgr/tasks/docker/main.yml b/roles/ceph-mgr/tasks/docker/main.yml new file mode 100644 index 000000000..1a876d5d8 --- /dev/null +++ b/roles/ceph-mgr/tasks/docker/main.yml @@ -0,0 +1,36 @@ +--- +- name: check if a cluster is already running + command: "docker ps -q -a --filter='ancestor={{ ceph_docker_image }}:{{ ceph_docker_image_tag }}'" + register: ceph_health + changed_when: false + failed_when: false + always_run: true + +- include: checks.yml + when: ceph_health.rc != 0 + +- include: "{{ playbook_dir }}/roles/ceph-common/tasks/misc/ntp_atomic.yml" + when: + - is_atomic + - ansible_os_family == 'RedHat' + - ntp_service_enabled + +- include: "{{ playbook_dir }}/roles/ceph-common/tasks/misc/ntp_redhat.yml" + when: + - not is_atomic + - ansible_os_family == 'RedHat' + - ntp_service_enabled + +- include: "{{ playbook_dir }}/roles/ceph-common/tasks/misc/ntp_debian.yml" + when: + - ansible_os_family == 'Debian' + - ntp_service_enabled + +- include: "{{ playbook_dir }}/roles/ceph-common/tasks/docker/fetch_image.yml" +- include: dirs_permissions.yml +- include: fetch_configs.yml + +- include: selinux.yml + when: ansible_os_family == 'RedHat' + +- include: start_docker_mgr.yml diff --git a/roles/ceph-mgr/tasks/docker/selinux.yml b/roles/ceph-mgr/tasks/docker/selinux.yml new file mode 100644 index 000000000..6d4ee022f --- /dev/null +++ b/roles/ceph-mgr/tasks/docker/selinux.yml @@ -0,0 +1,14 @@ +--- +- name: check if selinux is enabled + command: getenforce + register: sestatus + changed_when: false + always_run: true + +- name: set selinux permissions + shell: chcon -Rt svirt_sandbox_file_t {{ item }} + with_items: + - /etc/ceph + - /var/lib/ceph + changed_when: false + when: sestatus.stdout != 'Disabled' diff --git a/roles/ceph-mgr/tasks/docker/start_docker_mgr.yml b/roles/ceph-mgr/tasks/docker/start_docker_mgr.yml new file mode 100644 index 000000000..254de9f46 --- /dev/null +++ b/roles/ceph-mgr/tasks/docker/start_docker_mgr.yml @@ -0,0 +1,26 @@ +--- +- name: generate systemd unit file + become: true + template: + src: "{{ role_path }}/templates/ceph-mgr.service.j2" + dest: /etc/systemd/system/ceph-mgr@.service + owner: "root" + group: "root" + mode: "0644" + +- name: enable systemd unit file for mgr instance + shell: systemctl enable ceph-mgr@{{ ansible_hostname }}.service + failed_when: false + changed_when: false + +- name: reload systemd unit files + shell: systemctl daemon-reload + changed_when: false + failed_when: false + +- name: systemd start mgr container + service: + name: ceph-mgr@{{ ansible_hostname }} + state: started + enabled: yes + changed_when: false diff --git a/roles/ceph-mgr/tasks/main.yml b/roles/ceph-mgr/tasks/main.yml new file mode 100644 index 000000000..6c31d30cd --- /dev/null +++ b/roles/ceph-mgr/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- include: pre_requisite.yml + when: not mgr_containerized_deployment + +- include: ./docker/main.yml + when: mgr_containerized_deployment diff --git a/roles/ceph-mgr/tasks/pre_requisite.yml b/roles/ceph-mgr/tasks/pre_requisite.yml new file mode 100644 index 000000000..35edcc390 --- /dev/null +++ b/roles/ceph-mgr/tasks/pre_requisite.yml @@ -0,0 +1,42 @@ +--- +- name: create mgr directory + file: + path: /var/lib/ceph/mgr/ + state: directory + owner: "ceph" + group: "ceph" + mode: "0755" + +- name: create mgr directory + file: + path: /var/lib/ceph/mgr/{{ cluster }}-{{ ansible_hostname }} + state: directory + owner: "ceph" + group: "ceph" + mode: "0755" + +- name: copy mgr key + copy: + src: "{{ fetch_directory }}/{{ fsid }}/etc/ceph/{{ cluster }}.mgr.{{ ansible_hostname }}.keyring" + dest: "/var/lib/ceph/mgr/{{ cluster }}-{{ ansible_hostname }}/keyring" + owner: "ceph" + group: "ceph" + mode: "0600" + when: + - cephx + +- name: set mgr key permissions + file: + path: /var/lib/ceph/mgr/{{ cluster }}-{{ ansible_hostname }}/keyring + owner: "ceph" + group: "ceph" + mode: "0600" + when: + - cephx + +- name: start and add that the mgr service to the init sequence + service: + name: "ceph-mgr@{{ ansible_hostname }}" + state: started + enabled: yes + changed_when: false diff --git a/roles/ceph-mgr/templates/ceph-mgr.service.j2 b/roles/ceph-mgr/templates/ceph-mgr.service.j2 new file mode 100644 index 000000000..cdb4e0de0 --- /dev/null +++ b/roles/ceph-mgr/templates/ceph-mgr.service.j2 @@ -0,0 +1,30 @@ +[Unit] +Description=Ceph Manager +After=docker.service + +[Service] +EnvironmentFile=-/etc/environment +ExecStartPre=-/usr/bin/docker stop ceph-mgr-{{ ansible_hostname }} +ExecStartPre=-/usr/bin/docker rm ceph-mgr-{{ ansible_hostname }} +ExecStart=/usr/bin/docker run --rm --net=host \ + {% if not mgr_containerized_deployment_with_kv -%} + -v /var/lib/ceph:/var/lib/ceph \ + -v /etc/ceph:/etc/ceph \ + {% else -%} + -e KV_TYPE={{kv_type}} \ + -e KV_IP={{kv_endpoint}} \ + -e KV_PORT={{kv_port}} \ + {% endif -%} + -v /etc/localtime:/etc/localtime:ro \ + -e CEPH_DAEMON=MGR \ + {{ ceph_mgr_docker_extra_env }} \ + --name=ceph-mgr-{{ ansible_hostname }} \ + {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} +ExecStopPost=-/usr/bin/docker stop ceph-mgr-{{ ansible_hostname }} +Restart=always +RestartSec=10s +TimeoutStartSec=120 +TimeoutStopSec=15 + +[Install] +WantedBy=multi-user.target diff --git a/roles/ceph-mon/tasks/ceph_keys.yml b/roles/ceph-mon/tasks/ceph_keys.yml index 732b5af14..3e7ffe2f1 100644 --- a/roles/ceph-mon/tasks/ceph_keys.yml +++ b/roles/ceph-mon/tasks/ceph_keys.yml @@ -40,7 +40,19 @@ changed_when: false when: - cephx - - groups[restapi_group_name] is defined + - groups.get(restapi_group_name, []) | length > 0 + - inventory_hostname == groups[mon_group_name]|last + +- name: create ceph mgr keyring(s) when mon is not containerized + command: ceph --cluster {{ cluster }} auth get-or-create mgr.{{ hostvars[item]['ansible_hostname'] }} mon 'allow *' -o /etc/ceph/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring + args: + creates: /etc/ceph/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring + changed_when: false + when: + - cephx + - groups.get(mgr_group_name, []) | length > 0 + - inventory_hostname == groups[mon_group_name]|last + with_items: "{{ groups[mgr_group_name] }}" - include: set_osd_pool_default_pg_num.yml @@ -92,13 +104,14 @@ src: "{{ item }}" dest: "{{ fetch_directory }}/{{ fsid }}/{{ item }}" flat: yes - run_once: true with_items: - "{{ ceph_keys.get('stdout_lines') | default([]) }}" - /var/lib/ceph/bootstrap-osd/{{ cluster }}.keyring - /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring - /var/lib/ceph/bootstrap-mds/{{ cluster }}.keyring - when: cephx + when: + - cephx + - inventory_hostname == groups[mon_group_name]|last - name: drop in a motd script to report status when logging in copy: diff --git a/roles/ceph-mon/tasks/docker/fetch_configs.yml b/roles/ceph-mon/tasks/docker/fetch_configs.yml index 3ceac9f97..5b8df68ff 100644 --- a/roles/ceph-mon/tasks/docker/fetch_configs.yml +++ b/roles/ceph-mon/tasks/docker/fetch_configs.yml @@ -10,6 +10,23 @@ - /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring - /var/lib/ceph/bootstrap-mds/{{ cluster }}.keyring +- name: add mgr keys to config and keys paths + set_fact: + tmp_ceph_mgr_keys: /etc/ceph/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring + with_items: "{{ groups[mgr_group_name] }}" + register: tmp_ceph_mgr_keys_result + when: "{{ groups.get(mgr_group_name, []) | length > 0 }}" + +- name: convert mgr keys to an array + set_fact: + ceph_mgr_keys: "{{ tmp_ceph_mgr_keys_result.results | map(attribute='ansible_facts.tmp_ceph_mgr_keys') | list }}" + when: "{{ groups.get(mgr_group_name, []) | length > 0 }}" + +- name: merge mgr keys to config and keys paths + set_fact: + ceph_config_keys: "{{ ceph_config_keys + ceph_mgr_keys }}" + when: "{{ groups.get(mgr_group_name, []) | length > 0 }}" + - name: stat for ceph config and keys local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }} with_items: "{{ ceph_config_keys }}" diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml index d2265eda7..8b321d61c 100644 --- a/roles/ceph-mon/tasks/docker/main.yml +++ b/roles/ceph-mon/tasks/docker/main.yml @@ -88,3 +88,16 @@ when: - openstack_config - "{{ inventory_hostname == groups[mon_group_name] | last }}" + +- name: create ceph mgr keyring(s) when mon is containerized + command: docker exec ceph-mon-{{ ansible_hostname }} ceph --cluster {{ cluster }} auth get-or-create mgr.{{ hostvars[item]['ansible_hostname'] }} mon 'allow *' -o /etc/ceph/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring + args: + creates: /etc/ceph/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring + changed_when: false + when: + - cephx + - mon_containerized_deployment + - "{{ groups.get(mgr_group_name, []) | length > 0 }}" + - inventory_hostname == groups[mon_group_name]|last + - not mon_containerized_deployment_with_kv + with_items: "{{ groups[mgr_group_name] }}" diff --git a/site-docker.yml.sample b/site-docker.yml.sample index 4d0824805..167d8f2c4 100644 --- a/site-docker.yml.sample +++ b/site-docker.yml.sample @@ -40,3 +40,8 @@ become: True roles: - ceph-restapi + +- hosts: mgrs + become: True + roles: + - ceph-mgr diff --git a/site.yml.sample b/site.yml.sample index 437f5ed2c..b1806418a 100644 --- a/site.yml.sample +++ b/site.yml.sample @@ -12,6 +12,7 @@ - rbdmirrors - clients - iscsigws + - mgrs gather_facts: false tags: - always @@ -97,3 +98,9 @@ become: True roles: - ceph-iscsi-gw + +- hosts: mgrs + gather_facts: false + become: True + roles: + - ceph-mgr diff --git a/tests/functional/centos/7/cluster/vagrant_variables.yml b/tests/functional/centos/7/cluster/vagrant_variables.yml index dea5db175..ded9c1540 100644 --- a/tests/functional/centos/7/cluster/vagrant_variables.yml +++ b/tests/functional/centos/7/cluster/vagrant_variables.yml @@ -12,6 +12,7 @@ nfs_vms: 0 rbd_mirror_vms: 0 client_vms: 1 iscsi_gw_vms: 0 +mgr_vms: 0 # Deploy RESTAPI on each of the Monitors restapi: true diff --git a/tests/functional/centos/7/dmcrypt-dedicated-journal/vagrant_variables.yml b/tests/functional/centos/7/dmcrypt-dedicated-journal/vagrant_variables.yml index e399f89e9..8a8fb891d 100644 --- a/tests/functional/centos/7/dmcrypt-dedicated-journal/vagrant_variables.yml +++ b/tests/functional/centos/7/dmcrypt-dedicated-journal/vagrant_variables.yml @@ -12,6 +12,7 @@ nfs_vms: 0 rbd_mirror_vms: 0 client_vms: 0 iscsi_gw_vms: 0 +mgr_vms: 0 # Deploy RESTAPI on each of the Monitors restapi: true diff --git a/tests/functional/centos/7/dmcrypt-journal-collocation/vagrant_variables.yml b/tests/functional/centos/7/dmcrypt-journal-collocation/vagrant_variables.yml index a67131abd..33ff53f7a 100644 --- a/tests/functional/centos/7/dmcrypt-journal-collocation/vagrant_variables.yml +++ b/tests/functional/centos/7/dmcrypt-journal-collocation/vagrant_variables.yml @@ -12,6 +12,7 @@ nfs_vms: 0 rbd_mirror_vms: 0 client_vms: 0 iscsi_gw_vms: 0 +mgr_vms: 0 # Deploy RESTAPI on each of the Monitors restapi: true diff --git a/tests/functional/centos/7/docker-cluster-dedicated-journal/vagrant_variables.yml b/tests/functional/centos/7/docker-cluster-dedicated-journal/vagrant_variables.yml index f77870126..aa03b197c 100644 --- a/tests/functional/centos/7/docker-cluster-dedicated-journal/vagrant_variables.yml +++ b/tests/functional/centos/7/docker-cluster-dedicated-journal/vagrant_variables.yml @@ -12,6 +12,7 @@ nfs_vms: 0 rbd_mirror_vms: 0 client_vms: 0 iscsi_gw_vms: 0 +mgr_vms: 0 # Deploy RESTAPI on each of the Monitors restapi: true diff --git a/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/vagrant_variables.yml b/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/vagrant_variables.yml index f77870126..aa03b197c 100644 --- a/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/vagrant_variables.yml +++ b/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/vagrant_variables.yml @@ -12,6 +12,7 @@ nfs_vms: 0 rbd_mirror_vms: 0 client_vms: 0 iscsi_gw_vms: 0 +mgr_vms: 0 # Deploy RESTAPI on each of the Monitors restapi: true diff --git a/tests/functional/centos/7/docker-cluster/vagrant_variables.yml b/tests/functional/centos/7/docker-cluster/vagrant_variables.yml index 7b3f99cd6..5c4d1081f 100644 --- a/tests/functional/centos/7/docker-cluster/vagrant_variables.yml +++ b/tests/functional/centos/7/docker-cluster/vagrant_variables.yml @@ -9,9 +9,10 @@ osd_vms: 1 mds_vms: 1 rgw_vms: 1 nfs_vms: 0 -rbd_mirror_vms: 0 +rbd_mirror_vms: 1 client_vms: 0 iscsi_gw_vms: 0 +mgr_vms: 1 # Deploy RESTAPI on each of the Monitors restapi: true diff --git a/tests/functional/centos/7/journal-collocation/vagrant_variables.yml b/tests/functional/centos/7/journal-collocation/vagrant_variables.yml index ed27f6b2e..cbb343842 100644 --- a/tests/functional/centos/7/journal-collocation/vagrant_variables.yml +++ b/tests/functional/centos/7/journal-collocation/vagrant_variables.yml @@ -12,6 +12,7 @@ nfs_vms: 0 rbd_mirror_vms: 0 client_vms: 0 iscsi_gw_vms: 0 +mgr_vms: 0 # Deploy RESTAPI on each of the Monitors restapi: true diff --git a/tests/functional/ubuntu/16.04/cluster/vagrant_variables.yml b/tests/functional/ubuntu/16.04/cluster/vagrant_variables.yml index 35a60b9ba..f4e71cd9a 100644 --- a/tests/functional/ubuntu/16.04/cluster/vagrant_variables.yml +++ b/tests/functional/ubuntu/16.04/cluster/vagrant_variables.yml @@ -12,6 +12,7 @@ nfs_vms: 0 rbd_mirror_vms: 0 client_vms: 1 iscsi_gw_vms: 0 +mgr_vms: 0 # Deploy RESTAPI on each of the Monitors restapi: true diff --git a/vagrant_variables.yml.sample b/vagrant_variables.yml.sample index 2f658624b..19fa1a4d5 100644 --- a/vagrant_variables.yml.sample +++ b/vagrant_variables.yml.sample @@ -12,6 +12,7 @@ nfs_vms: 0 rbd_mirror_vms: 0 client_vms: 0 iscsi_gw_vms: 0 +mgr_vms: 0 # Deploy RESTAPI on each of the Monitors restapi: true