dest: "{{ ceph_conf_key_directory }}"
import_key: "{{ condition_copy_admin_key }}"
mode: "{{ item.mode|default(omit) }}"
+ owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+ group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
environment:
CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
- CEPH_UID: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
CEPH_CONTAINER_BINARY: "{{ container_binary }}"
with_items: "{{ keys }}"
delegate_to: "{{ delegated_node }}"
---
- name: waiting for the monitor(s) to form the quorum...
- command: "ceph --cluster {{ cluster }} -n mon. -k /var/lib/ceph/mon/{{ cluster }}-{{ ansible_hostname }}/keyring mon_status --format json"
+ command: >
+ {{ docker_exec_cmd }}
+ ceph
+ --cluster {{ cluster }}
+ -n mon.
+ -k /var/lib/ceph/mon/{{ cluster }}-{{ ansible_hostname }}/keyring
+ mon_status
+ --format json
register: ceph_health_raw
+ run_once: true
until: >
(ceph_health_raw.stdout | default({}) | from_json)['state'] in ['leader', 'peon']
retries: "{{ handler_health_mon_check_retries }}"
delay: "{{ handler_health_mon_check_delay }}"
- when:
- - ceph_release_num[ceph_release] >= ceph_release_num.nautilus
-- name: fetch ceph keys in nautilus and above
+- name: fetch ceph initial keys
ceph_key:
state: fetch_initial_keys
cluster: "{{ cluster }}"
+ owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+ group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+ mode: "0400"
+ environment:
+ CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
+ CEPH_CONTAINER_BINARY: "{{ container_binary }}"
when:
- - ceph_release_num[ceph_release] >= ceph_release_num.nautilus
- - cephx
-
-- name: collect admin and bootstrap keys
- command: ceph-create-keys --cluster {{ cluster }} -i {{ monitor_name }} -t 30
- changed_when: false
- check_mode: no
- when:
- - ceph_release_num[ceph_release] < ceph_release_num.nautilus
- - cephx
-
-# NOTE (leseb): wait for mon discovery and quorum resolution
-# the admin key is not instantaneously created so we have to wait a bit
-# msg: is only supported as of Ansible 2.4.
-- name: "wait for {{ cluster }}.client.admin.keyring exists"
- wait_for:
- path: /etc/ceph/{{ cluster }}.client.admin.keyring
- timeout: 30
- msg: "Timed out while waiting for keyring creation. Check network settings on mon nodes."
- when:
- - ceph_release_num[ceph_release] < ceph_release_num.nautilus
- cephx
- - (ansible_version.major == 2 and ansible_version.minor >= 4) or
- ansible_version.major > 2
-- name: "wait for {{ cluster }}.client.admin.keyring exists"
- wait_for:
- path: /etc/ceph/{{ cluster }}.client.admin.keyring
- timeout: 30
- when:
- - ceph_release_num[ceph_release] < ceph_release_num.nautilus
- - cephx
- - ansible_version.major == 2 and ansible_version.minor < 4
+- block:
+ - name: create ceph mgr keyring(s)
+ ceph_key:
+ name: "mgr.{{ hostvars[item]['ansible_hostname'] }}"
+ state: present
+ caps:
+ mon: allow profile mgr
+ osd: allow *
+ mds: allow *
+ cluster: "{{ cluster }}"
+ secret: "{{ (mgr_secret != 'mgr_secret') | ternary(mgr_secret, omit) }}"
+ owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+ group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+ mode: "0400"
+ environment:
+ CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
+ CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+ with_items:
+ - "{{ groups.get(mgr_group_name, []) }}" # this honors the condition where mgrs run on separate machines
+ - "{{ groups.get(mon_group_name, []) }}" # this honors the new rule where mgrs are always collocated with mons
-- name: create ceph mgr keyring(s) when mon is not containerized
- ceph_key:
- name: "mgr.{{ hostvars[item]['ansible_hostname'] }}"
- state: present
- caps:
- mon: allow profile mgr
- osd: allow *
- mds: allow *
- cluster: "{{ cluster }}"
- secret: "{{ (mgr_secret != 'mgr_secret') | ternary(mgr_secret, omit) }}"
- when:
- - cephx
- - inventory_hostname == groups[mon_group_name]|last
- with_items:
- - "{{ groups.get(mgr_group_name, []) }}" # this honors the condition where mgrs run on separate machines
- - "{{ groups.get(mon_group_name, []) }}" # this honors the new rule where mgrs are always collocated with mons
-
-# once this gets backported github.com/ceph/ceph/pull/20983
-# we will be able to remove these 2 tasks below
-- name: find ceph keys
- shell: ls -1 /etc/ceph/*.keyring
- changed_when: false
- register: ceph_keys
- check_mode: no
- when:
- - cephx
-
-- name: set keys permissions
- file:
- path: "{{ item }}"
- owner: "ceph"
- group: "ceph"
- mode: "{{ ceph_keyring_permissions }}"
- with_items:
- - "{{ ceph_keys.get('stdout_lines') | default([]) }}"
+ - name: copy ceph mgr key(s) to the ansible server
+ fetch:
+ src: "{{ ceph_conf_key_directory }}/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring"
+ dest: "{{ fetch_directory }}/{{ fsid }}/{{ ceph_conf_key_directory }}/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring"
+ flat: yes
+ with_items:
+ - "{{ groups.get(mgr_group_name, []) }}"
+ when:
+ - groups.get(mgr_group_name, []) | length > 0
when:
- cephx
+ - not rolling_update
+ - inventory_hostname == groups[mon_group_name]|last # only on the last to avoid key creation collision
- name: copy keys to the ansible server
fetch:
dest: "{{ fetch_directory }}/{{ fsid }}/{{ item }}"
flat: yes
with_items:
- - "{{ ceph_keys.get('stdout_lines') | default([]) }}"
- /var/lib/ceph/bootstrap-osd/{{ cluster }}.keyring
- /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring
- /var/lib/ceph/bootstrap-mds/{{ cluster }}.keyring
- /var/lib/ceph/bootstrap-rbd/{{ cluster }}.keyring
- /var/lib/ceph/bootstrap-rbd-mirror/{{ cluster }}.keyring
+ - /etc/ceph/{{ cluster }}.client.admin.keyring
when:
- cephx
- inventory_hostname == groups[mon_group_name] | last
--- /dev/null
+---
+- name: configure ceph profile.d aliases
+ template:
+ src: ceph-aliases.sh.j2
+ dest: "/etc/profile.d/ceph-aliases.sh"
+ mode: 0755
+ owner: root
+ group: root
python -c "import os ; import struct ;
import time; import base64 ; key = os.urandom(16) ;
header = struct.pack('<hiih',1,int(time.time()),0,len(key)) ;
- print(base64.b64encode(header + key).decode())" | tee {{ fetch_directory }}/monitor_keyring.conf
- args:
- creates: "{{ fetch_directory }}/monitor_keyring.conf"
- delegate_to: localhost
+ print(base64.b64encode(header + key).decode())"
register: monitor_keyring
- become: false
when: cephx
+ run_once: true # must run on a single mon only
-- name: read monitor initial keyring if it already exists
- command: "cat {{ fetch_directory }}/monitor_keyring.conf"
- args:
- removes: "{{ fetch_directory }}/monitor_keyring.conf"
- delegate_to: localhost
- changed_when: false
- register: monitor_keyring
- become: false
- check_mode: no
- when: cephx
+ # add code to read the key or/and try to find it on other nodes
- name: create monitor initial keyring
ceph_key:
name: mon.
state: present
dest: "/var/lib/ceph/tmp/"
- secret: "{{ monitor_secret }}"
+ secret: "{{ monitor_keyring.stdout }}"
cluster: "{{ cluster }}"
caps:
mon: allow *
import_key: False
+ owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+ group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+ mode: "0400"
+ environment:
+ CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
+ CEPH_CONTAINER_BINARY: "{{ container_binary }}"
when: cephx
-- name: set initial monitor key permissions
- file:
- path: "/var/lib/ceph/tmp/{{ cluster }}.mon..keyring"
- owner: "ceph"
- group: "ceph"
- mode: "0600"
- when: cephx
+- name: copy the initial key in /etc/ceph (for containers)
+ command: >
+ cp /var/lib/ceph/tmp/{{ cluster }}.mon..keyring /etc/ceph/{{ cluster }}.mon.keyring
+ changed_when: false
+ when:
+ - cephx
+ - containerized_deployment
- name: create (and fix ownership of) monitor directory
file:
path: /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }}
state: directory
- owner: "ceph"
- group: "ceph"
+ owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+ group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
mode: "0755"
recurse: true
name: client.admin
state: present
secret: "{{ admin_secret }}"
- auid: 0
caps: "{{ client_admin_ceph_authtool_cap }}"
import_key: False
cluster: "{{ cluster }}"
+ owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+ group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+ mode: "0400"
+ environment:
+ CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
+ CEPH_CONTAINER_BINARY: "{{ container_binary }}"
register: create_custom_admin_secret
when:
- cephx
- admin_secret != 'admin_secret'
-- name: set ownership of admin keyring
- file:
- path: /etc/ceph/{{ cluster }}.client.admin.keyring
- state: file
- owner: 'ceph'
- group: 'ceph'
- mode: "{{ ceph_keyring_permissions }}"
- when:
- - cephx
- - admin_secret != 'admin_secret'
+- name: set_fact ceph-authtool container command
+ set_fact:
+ ceph_authtool_cmd: "{{ container_binary + ' run --rm -v /var/lib/ceph:/var/lib/ceph:z -v /etc/ceph/:/etc/ceph/:z --entrypoint=ceph-authtool ' + ceph_client_docker_registry + '/' + ceph_client_docker_image + ':' + ceph_client_docker_image_tag if containerized_deployment else 'ceph-authtool' }}"
- name: import admin keyring into mon keyring
- command: ceph-authtool /var/lib/ceph/tmp/{{ cluster }}.mon..keyring --import-keyring /etc/ceph/{{ cluster }}.client.admin.keyring
+ command: >
+ {{ ceph_authtool_cmd }}
+ /var/lib/ceph/tmp/{{ cluster }}.mon.keyring --import-keyring /etc/ceph/{{ cluster }}.client.admin.keyring
when:
- not create_custom_admin_secret.get('skipped')
- cephx
- admin_secret != 'admin_secret'
+- name: set_fact ceph-mon container command
+ set_fact:
+ ceph_mon_cmd: "{{ container_binary + ' run --rm --net=host -v /var/lib/ceph/:/var/lib/ceph:z -v /etc/ceph/:/etc/ceph/:z --entrypoint=ceph-mon ' + ceph_client_docker_registry + '/' + ceph_client_docker_image + ':' +ceph_client_docker_image_tag if containerized_deployment else 'ceph-mon' }}"
+
- name: ceph monitor mkfs with keyring
- command: ceph-mon --cluster {{ cluster }} --setuser ceph --setgroup ceph --mkfs -i {{ monitor_name }} --fsid {{ fsid }} --keyring /var/lib/ceph/tmp/{{ cluster }}.mon..keyring
+ command: >
+ {{ ceph_mon_cmd }}
+ --cluster {{ cluster }}
+ --setuser ceph
+ --setgroup ceph
+ --mkfs
+ -i {{ monitor_name }}
+ --fsid {{ fsid }}
+ --keyring /var/lib/ceph/tmp/{{ cluster }}.mon..keyring
args:
creates: /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }}/keyring
when:
- cephx
- name: ceph monitor mkfs without keyring
- command: ceph-mon --cluster {{ cluster }} --setuser ceph --setgroup ceph --mkfs -i {{ monitor_name }} --fsid {{ fsid }}
+ command: >
+ {{ ceph_mon_cmd }}
+ --cluster {{ cluster }}
+ --setuser ceph
+ --setgroup ceph
+ --mkfs
+ -i {{ monitor_name }}
+ --fsid {{ fsid }}
args:
creates: /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }}/store.db
when:
- - not cephx
+ - not cephx
\ No newline at end of file
+++ /dev/null
----
-- name: configure ceph profile.d aliases
- template:
- src: ceph-aliases.sh.j2
- dest: "/etc/profile.d/ceph-aliases.sh"
- mode: 0755
- owner: root
- group: root
+++ /dev/null
----
-- name: set_fact ceph_config_keys
- set_fact:
- ceph_config_keys:
- - /etc/ceph/{{ cluster }}.client.admin.keyring
- - /etc/ceph/{{ cluster }}.mon.keyring
- - /var/lib/ceph/bootstrap-osd/{{ cluster }}.keyring
- - /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring
- - /var/lib/ceph/bootstrap-mds/{{ cluster }}.keyring
- - /var/lib/ceph/bootstrap-rbd/{{ cluster }}.keyring
- - /var/lib/ceph/bootstrap-rbd-mirror/{{ cluster }}.keyring
-
-- name: stat for ceph config and keys
- stat:
- path: "{{ fetch_directory }}/{{ fsid }}/{{ item }}"
- delegate_to: localhost
- with_items: "{{ ceph_config_keys }}"
- changed_when: false
- become: false
- failed_when: false
- register: statconfig
- check_mode: no
-
-- name: try to copy ceph keys
- copy:
- src: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
- dest: "{{ item.0 }}"
- owner: "{{ ceph_uid }}"
- group: "{{ ceph_uid }}"
- mode: 0600
- changed_when: false
- with_together:
- - "{{ ceph_config_keys }}"
- - "{{ statconfig.results }}"
- when:
- - item.1.stat.exists == true
- - item.0 is search("keyring")
+++ /dev/null
----
-- name: push ceph files to the ansible server
- fetch:
- src: "{{ item.0 }}"
- dest: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
- flat: yes
- with_together:
- - "{{ ceph_config_keys }}"
- - "{{ statconfig.results }}"
- when: item.1.stat.exists == false
+++ /dev/null
----
-- name: include copy_configs.yml
- include_tasks: copy_configs.yml
-
-- name: include start_docker_monitor.yml
- include_tasks: start_docker_monitor.yml
-
-- name: include configure_ceph_command_aliases.yml
- include_tasks: configure_ceph_command_aliases.yml
-
-- name: wait for monitor socket to exist
- command: "{{ docker_exec_cmd }} sh -c 'stat /var/run/ceph/{{ cluster }}-mon.{{ ansible_hostname }}.asok || stat /var/run/ceph/{{ cluster }}-mon.{{ ansible_fqdn }}.asok'"
- changed_when: false
- register: monitor_socket
- retries: 5
- delay: 15
- until: monitor_socket.rc == 0
-
-- name: force peer addition as potential bootstrap peer for cluster bringup
- command: "{{ docker_exec_cmd }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ monitor_name }}.asok add_bootstrap_peer_hint {{ _current_monitor_address }}"
- changed_when: false
- failed_when: false
-
-- name: include fetch_configs.yml
- include_tasks: fetch_configs.yml
- run_once: true
-
-- block:
- - name: create ceph mgr keyring(s) when mon is containerized
- command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} auth get-or-create mgr.{{ hostvars[item]['ansible_hostname'] }} mon 'allow profile mgr' osd 'allow *' mds 'allow *' -o /etc/ceph/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring"
- args:
- creates: "{{ ceph_conf_key_directory }}/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring"
- changed_when: false
- when:
- - cephx
- - containerized_deployment
- with_items:
- - "{{ groups.get(mgr_group_name, []) }}" # this honors the condition where mgrs run on separate machines
- - "{{ groups.get(mon_group_name, []) }}" # this honors the new rule where mgrs are always collocated with mons
-
- - name: fetch ceph mgr key(s)
- fetch:
- src: "{{ ceph_conf_key_directory }}/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring"
- dest: "{{ fetch_directory }}/{{ fsid }}/{{ ceph_conf_key_directory }}/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring"
- flat: yes
- with_items:
- - "{{ groups.get(mgr_group_name, []) }}"
- when:
- - groups.get(mgr_group_name, []) | length > 0
-
- when:
- - not rolling_update
- - inventory_hostname == groups[mon_group_name]|last
+++ /dev/null
----
-- name: generate systemd unit file
- become: true
- template:
- src: "{{ role_path }}/templates/ceph-mon.service.j2"
- dest: /etc/systemd/system/ceph-mon@.service
- owner: "root"
- group: "root"
- mode: "0644"
- notify:
- - restart ceph mons
-
-- name: systemd start mon container
- systemd:
- name: ceph-mon@{{ ansible_hostname }}
- state: started
- enabled: yes
- daemon_reload: yes
\ No newline at end of file
- name: include deploy_monitors.yml
include_tasks: deploy_monitors.yml
- when: not containerized_deployment
+ when:
+ # we test for both container and non-container
+ - (mon_socket_stat is defined and mon_socket_stat.get('rc') != 0) or (ceph_mon_container_stat is defined and ceph_mon_container_stat.get('stdout_lines', [])|length == 0)
- name: include start_monitor.yml
include_tasks: start_monitor.yml
- name: include_tasks ceph_keys.yml
include_tasks: ceph_keys.yml
- when: not containerized_deployment
- name: include secure_cluster.yml
include_tasks: secure_cluster.yml
when:
- secure_cluster
-- name: include docker/main.yml
- include_tasks: docker/main.yml
- when: containerized_deployment
-
- name: crush_rules.yml
include_tasks: crush_rules.yml
when:
- crush_rule_config
+
+- name: include configure_ceph_command_aliases.yml
+ include_tasks: configure_ceph_command_aliases.yml
+ when:
+ - containerized_deployment
\ No newline at end of file
state: directory
path: "/etc/systemd/system/ceph-mon@.service.d/"
when:
+ - not containerized_deployment
- ceph_mon_systemd_overrides is defined
- ansible_service_mgr == 'systemd'
config_overrides: "{{ ceph_mon_systemd_overrides | default({}) }}"
config_type: "ini"
when:
+ - not containerized_deployment
- ceph_mon_systemd_overrides is defined
- ansible_service_mgr == 'systemd'
-- name: start the monitor service
- service:
- name: ceph-mon@{{ monitor_name }}
- state: started
- enabled: yes
- changed_when: false
+- name: generate systemd unit file for mon container
+ become: true
+ template:
+ src: "{{ role_path }}/templates/ceph-mon.service.j2"
+ dest: /etc/systemd/system/ceph-mon@.service
+ owner: "root"
+ group: "root"
+ mode: "0644"
+ notify:
+ - restart ceph mons
+ when: containerized_deployment
-# NOTE: This patch only affects Kraken. This fixes a bug when the monitor service
-# does not start automatically after a reboot. Fix proposed upstream: ceph/ceph#14226
-- name: enable the ceph-mon.target service
+- name: start the monitor service
systemd:
- name: ceph-mon.target
+ name: ceph-mon@{{ monitor_name if not containerized_deployment else ansible_hostname }}
+ state: started
enabled: yes
- masked: no
- changed_when: false
+ daemon_reload: yes
\ No newline at end of file
mode: "{{ item.mode|default(omit) }}"
environment:
CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
- CEPH_UID: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
CEPH_CONTAINER_BINARY: "{{ container_binary }}"
with_items: "{{ openstack_keys }}"
delegate_to: "{{ groups[mon_group_name][0] }}"
name: ceph-mgr
private: false
- serial: 1 # MUST be '1' WHEN DEPLOYING MONITORS ON DOCKER CONTAINERS
-
- hosts: mons
tasks:
- name: set ceph monitor install 'Complete'
projects / ceph-ansible.git / commitdiff