ceph-disk: OSD hotplug fixes for Centos

Two fixes for Centos 6.3 and other systems with udev versions
prior to 172.  The disk peristant name using the GPT UUID does
not exist, so use the by_path persistent name instead for the
journal symlink.

The gpt label fields are not available for use in udev rules. Add
ceph-disk-udev wrapper script that extracts the partition
type guid from the label and calls ceph-disk-activate if it is
a ceph guid type. (Bug #4632)

Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>

diff --git a/Makefile.am b/Makefile.am
index adeb4e577289bf39dcced2a547e23740a0c2898a..03cb914079fe9b95d56cc5a1d5777a51434d793e 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -10,7 +10,8 @@ EXTRA_DIST += \
        src/test/cli \
        src/test/downloads \
        udev/50-rbd.rules \
-       udev/95-ceph-osd.rules
+       udev/95-ceph-osd.rules \
+       udev/95-ceph-osd-alt.rules
 
 
 all-local:

diff --git a/ceph.spec.in b/ceph.spec.in
index e52b307ecd44ab942b1460850335901741b33501..dbd662b543f60c7e14be28df751182fcc4b09d70 100644 (file)
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -25,10 +25,11 @@ Requires:   librbd1 = %{version}-%{release}
 Requires:      librados2 = %{version}-%{release}
 Requires:      libcephfs1 = %{version}-%{release}
 Requires:      python
+Requires:      python-argparse
+Requires:      python-lockfile
 Requires:      cryptsetup
 Requires:      parted
 Requires:      util-linux
-Requires:      python-argparse
 Requires:      hdparm
 Requires(post):        binutils
 BuildRoot:      %{_tmppath}/%{name}-%{version}-build
@@ -301,7 +302,11 @@ chmod 0644 $RPM_BUILD_ROOT%{_docdir}/ceph/sample.fetch_config
 
 # udev rules
 install -m 0644 -D udev/50-rbd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/50-rbd.rules
+%if 0%{?centos}
+install -m 0644 -D udev/95-ceph-osd-alt.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules
+%else
 install -m 0644 -D udev/95-ceph-osd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules
+%endif
 
 #set up placeholder directories
 mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/ceph
@@ -386,6 +391,7 @@ fi
 %{_sbindir}/ceph-disk
 %{_sbindir}/ceph-disk-activate
 %{_sbindir}/ceph-disk-prepare
+%{_sbindir}/ceph-disk-udev
 %{_sbindir}/ceph-create-keys
 %{_sbindir}/rcceph
 /sbin/mkcephfs

diff --git a/debian/control b/debian/control
index 98d0b37ffc1e9befe036582655151e90c452ee30..efa86377c52ede0e232e8bfab471da4442e35149 100644 (file)
--- a/debian/control
+++ b/debian/control
@@ -45,6 +45,8 @@ Depends: binutils,
          gdisk,
          parted,
          python,
+         python-argparse,
+         python-lockfile,
          sdparm | hdparm,
          uuid-runtime,
          xfsprogs,

diff --git a/src/Makefile.am b/src/Makefile.am
index cd360fcd147a048169b196f95c1bb5d61cffd70b..f31cd774ae02352c6b854998c4d574f47982e9ce 100644 (file)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -33,6 +33,7 @@ ceph_sbin_SCRIPTS = \
        ceph-disk \
        ceph-disk-prepare \
        ceph-disk-activate \
+       ceph-disk-udev \
        ceph-create-keys
 
 sbin_SCRIPTS = \
@@ -1192,6 +1193,7 @@ EXTRA_DIST += \
        ceph-disk \
        ceph-disk-prepare \
        ceph-disk-activate \
+       ceph-disk-udev \
        ceph-create-keys \
        mount.fuse.ceph
 

diff --git a/src/ceph-disk b/src/ceph-disk
index 73c158a6ece1a14ee425682862ef21a9e632523e..8694ee1eefaf524a61c0909033acc120652a7df7 100755 (executable)
--- a/src/ceph-disk
+++ b/src/ceph-disk
@@ -12,6 +12,7 @@ import stat
 import sys
 import tempfile
 import uuid
+import lockfile
 
 CEPH_OSD_ONDISK_MAGIC = 'ceph osd volume v026'
 
@@ -62,6 +63,7 @@ if LOG_NAME == '__main__':
     LOG_NAME = os.path.basename(sys.argv[0])
 LOG = logging.getLogger(LOG_NAME)
 
+lock = lockfile.FileLock('/var/lib/ceph/tmp/ceph-disk.lock')
 
 ###### exceptions ########
 
@@ -683,6 +685,16 @@ def zap(dev):
         raise Error(e)
 
 
+def get_udev_version():
+    version = _check_output(
+        args=[
+            'udevadm',
+            '--version',
+        ],
+    )   
+    return int(version)
+
+
 def prepare_journal_dev(
     data,
     journal,
@@ -759,9 +771,24 @@ def prepare_journal_dev(
                 ],
             )
 
-        journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format(
-            journal_uuid=journal_uuid,
-            )
+        if get_udev_version() >= 172:
+            journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format(
+                journal_uuid=journal_uuid,
+                )
+        else:
+            # udev prior to version 172 doesn't create by-partuuid directory
+            # use by-path symlink instead.  This is the third symlink returned
+            # by udevadm when queried.
+            LOG.debug('Using alternate persistant name for journal symlink')
+            symlinks = _check_output(
+                args=[
+                    'udevadm',
+                    'info',
+                    '--query=symlink',
+                    '--name={name}'.format(name=os.path.basename(journal)),
+                    ],
+                )   
+            journal_symlink='/dev/{symlink}-part{num}'.format(symlink=symlinks.split()[2], num=num)
 
         journal_dmcrypt = None
         if journal_dm_keypath:
@@ -1033,6 +1060,7 @@ def main_prepare(args):
     osd_dm_keypath = None
 
     try:
+        lock.acquire()
         if not os.path.exists(args.data):
             raise Error('data path does not exist', args.data)
 
@@ -1161,12 +1189,14 @@ def main_prepare(args):
                 )
         else:
             raise Error('not a dir or block device', args.data)
+        lock.release()
 
     except Error as e:
         if journal_dm_keypath:
             os.unlink(journal_dm_keypath)
         if osd_dm_keypath:
             os.unlink(osd_dm_keypath)
+        lock.release()
         raise e
 
 
@@ -1557,26 +1587,32 @@ def main_activate(args):
     if not os.path.exists(args.path):
         raise Error('%s does not exist', args.path)
 
-    mode = os.stat(args.path).st_mode
-    if stat.S_ISBLK(mode):
-        (cluster, osd_id) = mount_activate(
-            dev=args.path,
-            activate_key_template=args.activate_key_template,
-            init=args.mark_init,
-            )
-    elif stat.S_ISDIR(mode):
-        (cluster, osd_id) = activate_dir(
-            path=args.path,
-            activate_key_template=args.activate_key_template,
-            init=args.mark_init,
+    lock.acquire()
+    try:
+        mode = os.stat(args.path).st_mode
+        if stat.S_ISBLK(mode):
+            (cluster, osd_id) = mount_activate(
+                dev=args.path,
+                activate_key_template=args.activate_key_template,
+                init=args.mark_init,
+                )
+        elif stat.S_ISDIR(mode):
+            (cluster, osd_id) = activate_dir(
+                path=args.path,
+                activate_key_template=args.activate_key_template,
+                init=args.mark_init,
+                )
+        else:
+            raise Error('%s is not a directory or block device', args.path)
+    
+        start_daemon(
+            cluster=cluster,
+            osd_id=osd_id,
             )
-    else:
-        raise Error('%s is not a directory or block device', args.path)
+        lock.release()
 
-    start_daemon(
-        cluster=cluster,
-        osd_id=osd_id,
-        )
+    except:
+        lock.release()
 
 
 

diff --git a/src/ceph-disk-udev b/src/ceph-disk-udev
new file mode 100755 (executable)
index 0000000..a96c216
--- /dev/null
+++ b/src/ceph-disk-udev
@@ -0,0 +1,56 @@
+#! /bin/sh
+
+# Wrapper for the ceph udev rules.  Since older versions of udev do not support gpt label fields, this shell
+# script is invoked from the udev rule to read the needed gpt label fields and call the appropriate ceph
+# OSD functions.
+
+PARTNO=$1
+NAME=$2
+PARENT_NAME=$3
+
+# Get GPT partition type guid
+ID_PART_ENTRY_TYPE=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition GUID code" | awk '{print $4}' | tr '[:upper:]' '[:lower:]')
+case $ID_PART_ENTRY_TYPE in
+
+45b0969e-9b03-4f30-b4c6-b4b80ceff106)
+    # JOURNAL_UUID
+    ;;
+
+45b0969e-9b03-4f30-b4c6-5ec00ceff106)
+    # DMCRYPT_JOURNAL_UUID
+    # Map journal if using dm-crypt
+    ID_PART_ENTRY_UUID=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition unique GUID" | awk '{print $4}' | tr '[:upper:]' '[:lower:]')
+    /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME}
+    ;;
+
+4fbd7e29-9d25-41b8-afd0-062c0ceff05d)
+    # OSD_UUID
+    # activate ceph-tagged partitions.
+    /usr/sbin/ceph-disk -v activate --mount /dev/${NAME}
+    ;;
+
+4fbd7e29-9d25-41b8-afd0-5ec00ceff05d)
+    # DMCRYPT_OSD_UUID
+    # Map data device and activate ceph-tagged partitions
+    # for dm-crypted data devices
+    ID_PART_ENTRY_UUID=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition unique GUID" | awk '{print $4}' | tr '[:upper:]' '[:lower:]')
+    /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME}
+    bash -c 'while [ ! -e /dev/mapper/${ID_PART_ENTRY_UUID} ];do sleep 1; done'
+    /usr/sbin/ceph-disk-activate --mount /dev/mapper/${ID_PART_ENTRY_UUID}
+    ;;
+
+89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be)
+    # TOBE_UUID
+    ;;
+
+89c57f98-2fe5-4dc0-89c1-5ec00ceff2be)
+    # DMCRYPT_TOBE_UUID
+    ;;
+
+*)
+    # Not a Ceph device
+    ;;
+
+esac
+
+exit

diff --git a/udev/95-ceph-osd-alt.rules b/udev/95-ceph-osd-alt.rules
new file mode 100644 (file)
index 0000000..702ceea
--- /dev/null
+++ b/udev/95-ceph-osd-alt.rules
@@ -0,0 +1,5 @@
+#  Check gpt partion for ceph tags and activate
+ACTION=="add", SUBSYSTEM=="block", \
+  ENV{DEVTYPE}=="partition", \
+  ENV{ID_PART_TABLE_TYPE}=="gpt", \
+  RUN+="/usr/sbin/ceph-disk-udev $number $name"