systemd: Support Graceful Reboot for AIO Node

Ceph AIO installation with single/multiple node is not friendly for
loopback mount, especially always get deadlock issue during graceful
system reboot.

We already have `rbdmap.service` with graceful system reboot friendly as
below:

    [Unit]
    After=network-online.target
    Before=remote-fs-pre.target
    Wants=network-online.target remote-fs-pre.target

    [Service]
    ExecStart=/usr/bin/rbdmap map
    ExecReload=/usr/bin/rbdmap map
    ExecStop=/usr/bin/rbdmap unmap-all

This PR introduce:

  - `ceph-mon.target`: Ensure startup after `network-online.target` and
    before `remote-fs-pre.target`
  - `ceph-*.target`: Ensure startup after `ceph-mon.target` and before
    `remote-fs-pre.target`
  - `rbdmap.service`: Once all `_netdev` get unmount by
    `remote-fs.target`, ensure unmap all RBD BEFORE any Ceph components
    under `ceph.target` get stopped during shutdown

The logic is concept proof by
<https://github.com/alvistack/ansible-role-ceph_common/tree/develop>;
also works as expected with Ceph + Kubernetes deployment by
<https://github.com/alvistack/ansible-collection-kubernetes/tree/develop>.
No more deadlock happened during graceful system reboot, both AIO
single/multiple no de with loopback mount.

Also see:

  - <https://github.com/ceph/ceph/pull/36776>
  - <https://github.com/etcd-io/etcd/pull/12259>
  - <https://github.com/cri-o/cri-o/pull/4128>
  - <https://github.com/kubernetes/release/pull/1504>

Fixes: https://tracker.ceph.com/issues/47528
Signed-off-by: Wong Hoi Sing Edison <hswong3i@gmail.com>

diff --git a/systemd/ceph-fuse.target b/systemd/ceph-fuse.target
index 70f5cb6e16b63759542916766dc14ade274b5ab9..c31fdfa8dce7a2ae5ccd110131e2498e479ff37f 100644 (file)
--- a/systemd/ceph-fuse.target
+++ b/systemd/ceph-fuse.target
@@ -2,5 +2,6 @@
 Description=ceph target allowing to start/stop all ceph-fuse@.service instances at once
 PartOf=ceph.target
 Before=ceph.target
+
 [Install]
 WantedBy=remote-fs.target ceph.target

diff --git a/systemd/ceph-immutable-object-cache.target b/systemd/ceph-immutable-object-cache.target
index ed512953b0b0a0ee9da9fcb4fbc18674157107d3..f041bad6013b82b1e3ed23e3bc474d4e4ce453d1 100644 (file)
--- a/systemd/ceph-immutable-object-cache.target
+++ b/systemd/ceph-immutable-object-cache.target
@@ -2,5 +2,6 @@
 Description=ceph target allowing to start/stop all ceph-immutable-object-cache@.service instances at once
 PartOf=ceph.target
 Before=ceph.target
+
 [Install]
 WantedBy=multi-user.target ceph.target

diff --git a/systemd/ceph-mds.target b/systemd/ceph-mds.target
index 238f3ab90c73e5421a409e62773141f6d148e674..1101f21f699cdbea2edcc6f7480a5dbe59a35a92 100644 (file)
--- a/systemd/ceph-mds.target
+++ b/systemd/ceph-mds.target
@@ -1,6 +1,9 @@
 [Unit]
 Description=ceph target allowing to start/stop all ceph-mds@.service instances at once
 PartOf=ceph.target
+After=ceph-mon.target
 Before=ceph.target
+Wants=ceph.target ceph-mon.target
+
 [Install]
 WantedBy=multi-user.target ceph.target

diff --git a/systemd/ceph-mds@.service.in b/systemd/ceph-mds@.service.in
index 6a235ec1dfa073e430828afd4cc795c005ac6848..2884f587f9768f876facd895fb13f607443f7248 100644 (file)
--- a/systemd/ceph-mds@.service.in
+++ b/systemd/ceph-mds@.service.in
@@ -1,8 +1,9 @@
 [Unit]
 Description=Ceph metadata server daemon
-After=network-online.target local-fs.target time-sync.target
-Wants=network-online.target local-fs.target time-sync.target
 PartOf=ceph-mds.target
+After=network-online.target local-fs.target time-sync.target
+Before=remote-fs-pre.target ceph-mds.target
+Wants=network-online.target local-fs.target time-sync.target remote-fs-pre.target ceph-mds.target
 
 [Service]
 Environment=CLUSTER=ceph

diff --git a/systemd/ceph-mgr.target b/systemd/ceph-mgr.target
index f25e494b1d382cc2af0f638a70c48e7cc10b9b3a..288888b0d8d5196f1b1cd6254628fe811221a97d 100644 (file)
--- a/systemd/ceph-mgr.target
+++ b/systemd/ceph-mgr.target
@@ -1,6 +1,9 @@
 [Unit]
 Description=ceph target allowing to start/stop all ceph-mgr@.service instances at once
 PartOf=ceph.target
+After=ceph-mon.target
 Before=ceph.target
+Wants=ceph.target ceph-mon.target
+
 [Install]
 WantedBy=multi-user.target ceph.target

diff --git a/systemd/ceph-mgr@.service.in b/systemd/ceph-mgr@.service.in
index f0f3fefc87fb8c28c9a3f4a614dfdc075e13f961..1ee28285209ba6c801d0c410b9a09859eccb0a33 100644 (file)
--- a/systemd/ceph-mgr@.service.in
+++ b/systemd/ceph-mgr@.service.in
@@ -1,8 +1,9 @@
 [Unit]
 Description=Ceph cluster manager daemon
-After=network-online.target local-fs.target time-sync.target
-Wants=network-online.target local-fs.target time-sync.target
 PartOf=ceph-mgr.target
+After=network-online.target local-fs.target time-sync.target
+Before=remote-fs-pre.target ceph-mgr.target
+Wants=network-online.target local-fs.target time-sync.target remote-fs-pre.target ceph-mgr.target
 
 [Service]
 Environment=CLUSTER=ceph
@@ -28,11 +29,9 @@ RestartSec=10
 RestrictSUIDSGID=true
 StartLimitBurst=3
 StartLimitInterval=30min
-
 # We need to disable this protection as some python libraries generate
 # dynamic code, like python-cffi, and require mmap calls to succeed
 MemoryDenyWriteExecute=false
 
-
 [Install]
 WantedBy=ceph-mgr.target

diff --git a/systemd/ceph-mon.target b/systemd/ceph-mon.target
index 097c83ba3be8322ba92e200d1efdffa228ac9369..4325bac7afeb6102fcee7cb1dd49aae73d173972 100644 (file)
--- a/systemd/ceph-mon.target
+++ b/systemd/ceph-mon.target
@@ -2,5 +2,7 @@
 Description=ceph target allowing to start/stop all ceph-mon@.service instances at once
 PartOf=ceph.target
 Before=ceph.target
+Wants=ceph.target
+
 [Install]
 WantedBy=multi-user.target ceph.target

diff --git a/systemd/ceph-mon@.service.in b/systemd/ceph-mon@.service.in
index 6fbdabec2ca603639d499e85ce738f7f4200ed04..994cdfd2869593f65148f39f7f096fccb206fab0 100644 (file)
--- a/systemd/ceph-mon@.service.in
+++ b/systemd/ceph-mon@.service.in
@@ -1,13 +1,13 @@
 [Unit]
 Description=Ceph cluster monitor daemon
-
+PartOf=ceph-mon.target
 # According to:
 #   http://www.freedesktop.org/wiki/Software/systemd/NetworkTarget
 # these can be removed once ceph-mon will dynamically change network
 # configuration.
 After=network-online.target local-fs.target time-sync.target
-Wants=network-online.target local-fs.target time-sync.target
-PartOf=ceph-mon.target
+Before=remote-fs-pre.target ceph-mon.target
+Wants=network-online.target local-fs.target time-sync.target remote-fs-pre.target ceph-mon.target
 
 [Service]
 Environment=CLUSTER=ceph

diff --git a/systemd/ceph-osd.target b/systemd/ceph-osd.target
index 7f677f54d5ece2bb256369555bbc2df05b76e391..e4d1b9f07fc6ae5ac2ba741201c73de3fc575b31 100644 (file)
--- a/systemd/ceph-osd.target
+++ b/systemd/ceph-osd.target
@@ -1,6 +1,9 @@
 [Unit]
 Description=ceph target allowing to start/stop all ceph-osd@.service instances at once
 PartOf=ceph.target
+After=ceph-mon.target
 Before=ceph.target
+Wants=ceph.target ceph-mon.target
+
 [Install]
 WantedBy=multi-user.target ceph.target

diff --git a/systemd/ceph-osd@.service.in b/systemd/ceph-osd@.service.in
index 4a2254e99478edebdbf2b0852d24c4ba1824bcd4..4981417d6202cb0484c23a39a4736f7ee3d2eedf 100644 (file)
--- a/systemd/ceph-osd@.service.in
+++ b/systemd/ceph-osd@.service.in
@@ -1,8 +1,9 @@
 [Unit]
 Description=Ceph object storage daemon osd.%i
-After=network-online.target local-fs.target time-sync.target ceph-mon.target
-Wants=network-online.target local-fs.target time-sync.target
 PartOf=ceph-osd.target
+After=network-online.target local-fs.target time-sync.target
+Before=remote-fs-pre.target ceph-osd.target
+Wants=network-online.target local-fs.target time-sync.target remote-fs-pre.target ceph-osd.target
 
 [Service]
 Environment=CLUSTER=ceph

diff --git a/systemd/ceph-radosgw.target b/systemd/ceph-radosgw.target
index 1799e29e84d7450afc3547c04e9d1dd9caf7cb2c..8ea707a0bbdf6f65cebe31707cb979a5c6c36f56 100644 (file)
--- a/systemd/ceph-radosgw.target
+++ b/systemd/ceph-radosgw.target
@@ -1,6 +1,9 @@
 [Unit]
 Description=ceph target allowing to start/stop all ceph-radosgw@.service instances at once
 PartOf=ceph.target
+After=ceph-mon.target
 Before=ceph.target
+Wants=ceph.target ceph-mon.target
+
 [Install]
 WantedBy=multi-user.target ceph.target

diff --git a/systemd/ceph-radosgw@.service.in b/systemd/ceph-radosgw@.service.in
index 040a9ccdafa0e5bae2bc380eeb005cd4daff16c0..cfff60c18b8ccc754b630540e2813d8f72e42f8c 100644 (file)
--- a/systemd/ceph-radosgw@.service.in
+++ b/systemd/ceph-radosgw@.service.in
@@ -1,8 +1,9 @@
 [Unit]
 Description=Ceph rados gateway
-After=network-online.target local-fs.target time-sync.target
-Wants=network-online.target local-fs.target time-sync.target
 PartOf=ceph-radosgw.target
+After=network-online.target local-fs.target time-sync.target
+Before=remote-fs-pre.target ceph-radosgw.target
+Wants=network-online.target local-fs.target time-sync.target remote-fs-pre.target ceph-radosgw.target
 
 [Service]
 Environment=CLUSTER=ceph

diff --git a/systemd/ceph-rbd-mirror.target b/systemd/ceph-rbd-mirror.target
index 43e9a4c816478244154b55729d1e9bbd65b9582f..57ea09f1dc510901eb81dede8be13cfed6563bea 100644 (file)
--- a/systemd/ceph-rbd-mirror.target
+++ b/systemd/ceph-rbd-mirror.target
@@ -2,5 +2,6 @@
 Description=ceph target allowing to start/stop all ceph-rbd-mirror@.service instances at once
 PartOf=ceph.target
 Before=ceph.target
+
 [Install]
 WantedBy=multi-user.target ceph.target

diff --git a/systemd/ceph.target b/systemd/ceph.target
index 60734baff689d6603c520e14d11ed224b0ef31e7..67a982c5b446abe17832e5aa24466252dec0880c 100644 (file)
--- a/systemd/ceph.target
+++ b/systemd/ceph.target
@@ -1,4 +1,5 @@
 [Unit]
 Description=ceph target allowing to start/stop all ceph*@.service instances at once
+
 [Install]
 WantedBy=multi-user.target

diff --git a/systemd/rbdmap.service.in b/systemd/rbdmap.service.in
index 4757ee6ccb29de1d892b24c26762265866ac5c5b..6644508cf0dea9ffb617bcf55e37d9315109926a 100644 (file)
--- a/systemd/rbdmap.service.in
+++ b/systemd/rbdmap.service.in
@@ -1,9 +1,8 @@
 [Unit]
 Description=Map RBD devices
-
-After=network-online.target
+After=network-online.target ceph.target
 Before=remote-fs-pre.target
-Wants=network-online.target remote-fs-pre.target
+Wants=network-online.target remote-fs-pre.target ceph.target
 
 [Service]
 EnvironmentFile=-@SYSTEMD_ENV_FILE@