From 82764afe8d23da6cf3ca56ecb114c334fee0272a Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Wed, 20 Mar 2019 11:35:36 +0100 Subject: [PATCH] update: mask systemd service units during upgrade This prevents the packaging from restarting services before we do need to restart them in the rolling update sequence. We want to handle services restart at rolling_update playbook. Signed-off-by: Guillaume Abrioux --- infrastructure-playbooks/rolling_update.yml | 80 +++++++++---------- .../tasks/container/containerized.yml | 1 + .../tasks/non-container/prerequisites.yml | 2 + roles/ceph-mds/tasks/containerized.yml | 1 + roles/ceph-mds/tasks/non_containerized.yml | 1 + roles/ceph-mgr/tasks/start_mgr.yml | 3 +- roles/ceph-mon/tasks/start_monitor.yml | 3 +- roles/ceph-nfs/tasks/start_nfs.yml | 2 + roles/ceph-osd/tasks/start_osds.yml | 1 + .../tasks/docker/start_docker_rbd_mirror.yml | 3 +- .../tasks/start_rbd_mirror.yml | 4 +- .../tasks/docker/start_docker_rgw.yml | 2 + roles/ceph-rgw/tasks/start_radosgw.yml | 1 + 13 files changed, 59 insertions(+), 45 deletions(-) diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml index 15eee8f6e..7ee831ce0 100644 --- a/infrastructure-playbooks/rolling_update.yml +++ b/infrastructure-playbooks/rolling_update.yml @@ -96,32 +96,39 @@ set_fact: mon_host: "{{ groups[mon_group_name] | difference([inventory_hostname]) | last }}" + # NOTE: we mask the service so the RPM can't restart it + # after the package gets upgraded - name: stop ceph mon - shortname systemd: name: ceph-mon@{{ ansible_hostname }} state: stopped - enabled: yes + enabled: no + masked: yes ignore_errors: True when: - not containerized_deployment + # NOTE: we mask the service so the RPM can't restart it + # after the package gets upgraded - name: stop ceph mon - fqdn systemd: name: ceph-mon@{{ ansible_fqdn }} state: stopped - enabled: yes + enabled: no + masked: yes ignore_errors: True when: - not containerized_deployment - - name: stop ceph mgr + # only mask the service for mgr because it must be upgraded + # after ALL monitors, even when collocated + - name: mask the mgr service systemd: name: ceph-mgr@{{ ansible_hostname }} - state: stopped - enabled: yes - ignore_errors: True # if no mgr collocated with mons + masked: yes when: - - not containerized_deployment + - inventory_hostname in groups[mgr_group_name] + or groups[mgr_group_name] | length == 0 - import_role: name: ceph-defaults @@ -190,16 +197,6 @@ when: - containerized_deployment - - name: restart containerized ceph mgr - systemd: - name: ceph-mgr@{{ monitor_name }} - state: restarted - enabled: yes - daemon_reload: yes - ignore_errors: True # if no mgr collocated with mons - when: - - containerized_deployment - - name: non container | waiting for the monitor to join the quorum... command: ceph --cluster "{{ cluster }}" -s --format json register: ceph_health_raw @@ -329,7 +326,8 @@ systemd: name: ceph-mgr@{{ ansible_hostname }} state: stopped - enabled: yes + enabled: no + masked: yes failed_when: false when: - not containerized_deployment @@ -351,23 +349,6 @@ - import_role: name: ceph-mgr - - name: start ceph mgr - systemd: - name: ceph-mgr@{{ ansible_hostname }} - state: started - enabled: yes - when: - - not containerized_deployment - - - name: restart containerized ceph mgr - systemd: - name: ceph-mgr@{{ ansible_hostname }} - state: restarted - enabled: yes - daemon_reload: yes - when: - - containerized_deployment - - name: upgrade ceph osds cluster vars: @@ -396,7 +377,8 @@ systemd: name: ceph-osd@{{ item }} state: stopped - enabled: yes + enabled: no + masked: yes with_items: "{{ osd_ids.stdout_lines }}" when: - not containerized_deployment @@ -429,6 +411,7 @@ name: ceph-osd@{{ item }} state: started enabled: yes + masked: no with_items: "{{ osd_ids.stdout_lines }}" when: - not containerized_deployment @@ -438,6 +421,7 @@ name: "{{ item }}" state: restarted enabled: yes + masked: no daemon_reload: yes with_items: "{{ osd_names.stdout_lines }}" when: @@ -544,7 +528,8 @@ systemd: name: ceph-mds@{{ ansible_hostname }} state: stopped - enabled: yes + enabled: no + masked: yes when: - not containerized_deployment @@ -570,6 +555,7 @@ name: ceph-mds@{{ ansible_hostname }} state: started enabled: yes + masked: no when: - not containerized_deployment @@ -578,6 +564,7 @@ name: ceph-mds@{{ ansible_hostname }} state: restarted enabled: yes + masked: no daemon_reload: yes when: - containerized_deployment @@ -595,7 +582,8 @@ systemd: name: ceph-radosgw@rgw.{{ ansible_hostname }} state: stopped - enabled: yes + enabled: no + masked: yes when: - not containerized_deployment @@ -621,6 +609,7 @@ name: ceph-radosgw@rgw.{{ ansible_hostname }} state: started enabled: yes + masked: no when: - not containerized_deployment @@ -629,6 +618,7 @@ name: ceph-radosgw@rgw.{{ ansible_hostname }}.{{ item.instance_name }} state: restarted enabled: yes + masked: no daemon_reload: yes with_items: "{{ rgw_instances }}" when: @@ -656,7 +646,8 @@ systemd: name: "ceph-rbd-mirror@rbd-mirror.{{ ansible_hostname }}" state: stopped - enabled: yes + enabled: no + masked: yes failed_when: false - import_role: @@ -681,6 +672,7 @@ name: "ceph-rbd-mirror@rbd-mirror.{{ ansible_hostname }}" state: started enabled: yes + masked: no when: - not containerized_deployment @@ -689,6 +681,7 @@ name: ceph-rbd-mirror@rbd-mirror.{{ ansible_hostname }} state: restarted enabled: yes + masked: no daemon_reload: yes when: - containerized_deployment @@ -709,7 +702,8 @@ systemd: name: nfs-ganesha state: stopped - enabled: yes + enabled: no + masked: yes failed_when: false when: - not containerized_deployment @@ -736,6 +730,7 @@ name: nfs-ganesha state: started enabled: yes + masked: no when: - not containerized_deployment - ceph_nfs_enable_service @@ -745,6 +740,7 @@ name: ceph-nfs@{{ ceph_nfs_service_suffix | default(ansible_hostname) }} state: restarted enabled: yes + masked: no daemon_reload: yes when: - ceph_nfs_enable_service @@ -767,7 +763,8 @@ systemd: name: rbd-target-gw state: stopped - enabled: yes + enabled: no + masked: yes failed_when: false when: - not containerized_deployment @@ -794,6 +791,7 @@ name: rbd-target-gw state: started enabled: yes + masked: no when: - not containerized_deployment diff --git a/roles/ceph-iscsi-gw/tasks/container/containerized.yml b/roles/ceph-iscsi-gw/tasks/container/containerized.yml index a7316c475..7610d2bde 100644 --- a/roles/ceph-iscsi-gw/tasks/container/containerized.yml +++ b/roles/ceph-iscsi-gw/tasks/container/containerized.yml @@ -19,6 +19,7 @@ name: "{{ item }}" state: started enabled: yes + masked: no daemon_reload: yes with_items: - tcmu-runner diff --git a/roles/ceph-iscsi-gw/tasks/non-container/prerequisites.yml b/roles/ceph-iscsi-gw/tasks/non-container/prerequisites.yml index 3fa2dbbdd..cbd7d002f 100644 --- a/roles/ceph-iscsi-gw/tasks/non-container/prerequisites.yml +++ b/roles/ceph-iscsi-gw/tasks/non-container/prerequisites.yml @@ -66,10 +66,12 @@ service: name: rbd-target-gw enabled: yes + masked: no state: started - name: enable the rbd-target-api service and make sure it is running service: name: rbd-target-api enabled: yes + masked: no state: started diff --git a/roles/ceph-mds/tasks/containerized.yml b/roles/ceph-mds/tasks/containerized.yml index 944010bca..c59ceb14c 100644 --- a/roles/ceph-mds/tasks/containerized.yml +++ b/roles/ceph-mds/tasks/containerized.yml @@ -61,6 +61,7 @@ name: ceph-mds@{{ ansible_hostname }} state: started enabled: yes + masked: no daemon_reload: yes - name: wait for mds socket to exist diff --git a/roles/ceph-mds/tasks/non_containerized.yml b/roles/ceph-mds/tasks/non_containerized.yml index d0c77a338..94123980f 100644 --- a/roles/ceph-mds/tasks/non_containerized.yml +++ b/roles/ceph-mds/tasks/non_containerized.yml @@ -58,4 +58,5 @@ name: ceph-mds@{{ mds_name }} state: started enabled: yes + masked: no changed_when: false diff --git a/roles/ceph-mgr/tasks/start_mgr.yml b/roles/ceph-mgr/tasks/start_mgr.yml index 19c1c8b0c..3d7fa3625 100644 --- a/roles/ceph-mgr/tasks/start_mgr.yml +++ b/roles/ceph-mgr/tasks/start_mgr.yml @@ -35,4 +35,5 @@ name: ceph-mgr@{{ ansible_hostname }} state: started enabled: yes - daemon_reload: yes \ No newline at end of file + masked: no + daemon_reload: yes diff --git a/roles/ceph-mon/tasks/start_monitor.yml b/roles/ceph-mon/tasks/start_monitor.yml index e5d2d2387..b1be39945 100644 --- a/roles/ceph-mon/tasks/start_monitor.yml +++ b/roles/ceph-mon/tasks/start_monitor.yml @@ -36,4 +36,5 @@ name: ceph-mon@{{ monitor_name if not containerized_deployment else ansible_hostname }} state: started enabled: yes - daemon_reload: yes \ No newline at end of file + masked: no + daemon_reload: yes diff --git a/roles/ceph-nfs/tasks/start_nfs.yml b/roles/ceph-nfs/tasks/start_nfs.yml index 8a0a7d507..6536881ea 100644 --- a/roles/ceph-nfs/tasks/start_nfs.yml +++ b/roles/ceph-nfs/tasks/start_nfs.yml @@ -83,6 +83,7 @@ name: ceph-nfs@{{ ceph_nfs_service_suffix | default(ansible_hostname) }} state: started enabled: yes + masked: no daemon_reload: yes when: - containerized_deployment @@ -93,6 +94,7 @@ name: nfs-ganesha state: started enabled: yes + masked: no when: - not containerized_deployment - ceph_nfs_enable_service diff --git a/roles/ceph-osd/tasks/start_osds.yml b/roles/ceph-osd/tasks/start_osds.yml index 5460a5e57..06215f20f 100644 --- a/roles/ceph-osd/tasks/start_osds.yml +++ b/roles/ceph-osd/tasks/start_osds.yml @@ -76,6 +76,7 @@ name: ceph-osd@{{ item | regex_replace('/dev/', '') if osd_scenario != 'lvm' and containerized_deployment else item }} state: started enabled: yes + masked: no daemon_reload: yes with_items: "{{ devices if osd_scenario != 'lvm' and containerized_deployment else ((ceph_osd_ids.stdout | from_json).keys() | list) if osd_scenario == 'lvm' and not containerized_deployment else osd_ids_non_container.stdout_lines }}" diff --git a/roles/ceph-rbd-mirror/tasks/docker/start_docker_rbd_mirror.yml b/roles/ceph-rbd-mirror/tasks/docker/start_docker_rbd_mirror.yml index 6bb2844f6..68432bd1d 100644 --- a/roles/ceph-rbd-mirror/tasks/docker/start_docker_rbd_mirror.yml +++ b/roles/ceph-rbd-mirror/tasks/docker/start_docker_rbd_mirror.yml @@ -16,4 +16,5 @@ name: ceph-rbd-mirror@rbd-mirror.{{ ansible_hostname }} state: started enabled: yes - daemon_reload: yes \ No newline at end of file + masked: no + daemon_reload: yes diff --git a/roles/ceph-rbd-mirror/tasks/start_rbd_mirror.yml b/roles/ceph-rbd-mirror/tasks/start_rbd_mirror.yml index 6c3d11c62..70140c5d1 100644 --- a/roles/ceph-rbd-mirror/tasks/start_rbd_mirror.yml +++ b/roles/ceph-rbd-mirror/tasks/start_rbd_mirror.yml @@ -29,6 +29,7 @@ name: "ceph-rbd-mirror.target" state: started enabled: yes + masked: no changed_when: false - name: start and add the rbd-mirror service instance @@ -36,4 +37,5 @@ name: "ceph-rbd-mirror@rbd-mirror.{{ ansible_hostname }}" state: started enabled: yes - changed_when: false \ No newline at end of file + masked: no + changed_when: false diff --git a/roles/ceph-rgw/tasks/docker/start_docker_rgw.yml b/roles/ceph-rgw/tasks/docker/start_docker_rgw.yml index 90db90925..994a6dbb7 100644 --- a/roles/ceph-rgw/tasks/docker/start_docker_rgw.yml +++ b/roles/ceph-rgw/tasks/docker/start_docker_rgw.yml @@ -28,6 +28,7 @@ name: "{{ item }}" state: stopped enabled: no + masked: yes daemon_reload: yes with_items: - "ceph-rgw@{{ ansible_hostname }}" @@ -41,5 +42,6 @@ name: ceph-radosgw@rgw.{{ ansible_hostname }}.{{ item.instance_name }} state: started enabled: yes + masked: no daemon_reload: yes with_items: "{{ rgw_instances }}" diff --git a/roles/ceph-rgw/tasks/start_radosgw.yml b/roles/ceph-rgw/tasks/start_radosgw.yml index 4bac51cc1..0419c1247 100644 --- a/roles/ceph-rgw/tasks/start_radosgw.yml +++ b/roles/ceph-rgw/tasks/start_radosgw.yml @@ -20,6 +20,7 @@ name: ceph-radosgw@rgw.{{ ansible_hostname }}.{{ item.instance_name }} state: started enabled: yes + masked: no with_items: "{{ rgw_instances }}" - name: enable the ceph-radosgw.target service -- 2.39.5