ceph-disk: OSD hotplug fixes for Centos

Two fixes for Centos 6.3 and other systems with udev versions
prior to 172.  The disk peristant name using the GPT UUID does
not exist, so use the by_path persistent name instead for the
journal symlink.

The gpt label fields are not available for use in udev rules. Add
ceph-disk-udev wrapper script that extracts the partition
type guid from the label and calls ceph-disk-activate if it is
a ceph guid type. (Bug #4632)

Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>

7 files changed, 131 insertions, 23 deletions

diff --git a/Makefile.am b/Makefile.am
index adeb4e57728..03cb914079f 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -10,7 +10,8 @@ EXTRA_DIST += \
 	src/test/cli \
 	src/test/downloads \
 	udev/50-rbd.rules \
-	udev/95-ceph-osd.rules
+	udev/95-ceph-osd.rules \
+	udev/95-ceph-osd-alt.rules
 
 
 all-local:
diff --git a/ceph.spec.in b/ceph.spec.in
index e52b307ecd4..dbd662b543f 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -25,10 +25,11 @@ Requires:	librbd1 = %{version}-%{release}
 Requires:	librados2 = %{version}-%{release}
 Requires:	libcephfs1 = %{version}-%{release}
 Requires:	python
+Requires:	python-argparse
+Requires:	python-lockfile
 Requires:	cryptsetup
 Requires:	parted
 Requires:	util-linux
-Requires:	python-argparse
 Requires:	hdparm
 Requires(post):	binutils
 BuildRoot:      %{_tmppath}/%{name}-%{version}-build
@@ -301,7 +302,11 @@ chmod 0644 $RPM_BUILD_ROOT%{_docdir}/ceph/sample.fetch_config
 
 # udev rules
 install -m 0644 -D udev/50-rbd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/50-rbd.rules
+%if 0%{?centos}
+install -m 0644 -D udev/95-ceph-osd-alt.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules
+%else
 install -m 0644 -D udev/95-ceph-osd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules
+%endif
 
 #set up placeholder directories
 mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/ceph
@@ -386,6 +391,7 @@ fi
 %{_sbindir}/ceph-disk
 %{_sbindir}/ceph-disk-activate
 %{_sbindir}/ceph-disk-prepare
+%{_sbindir}/ceph-disk-udev
 %{_sbindir}/ceph-create-keys
 %{_sbindir}/rcceph
 /sbin/mkcephfs
diff --git a/debian/control b/debian/control
index 98d0b37ffc1..efa86377c52 100644
--- a/debian/control
+++ b/debian/control
@@ -45,6 +45,8 @@ Depends: binutils,
          gdisk,
          parted,
          python,
+         python-argparse,
+         python-lockfile,
          sdparm | hdparm,
          uuid-runtime,
          xfsprogs,
diff --git a/src/Makefile.am b/src/Makefile.am
index cd360fcd147..f31cd774ae0 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -33,6 +33,7 @@ ceph_sbin_SCRIPTS = \
 	ceph-disk \
 	ceph-disk-prepare \
 	ceph-disk-activate \
+	ceph-disk-udev \
 	ceph-create-keys
 
 sbin_SCRIPTS = \
@@ -1192,6 +1193,7 @@ EXTRA_DIST += \
 	ceph-disk \
 	ceph-disk-prepare \
 	ceph-disk-activate \
+	ceph-disk-udev \
 	ceph-create-keys \
 	mount.fuse.ceph
 
diff --git a/src/ceph-disk b/src/ceph-disk
index 73c158a6ece..8694ee1eefa 100755
--- a/src/ceph-disk
+++ b/src/ceph-disk
@@ -12,6 +12,7 @@ import stat
 import sys
 import tempfile
 import uuid
+import lockfile
 
 CEPH_OSD_ONDISK_MAGIC = 'ceph osd volume v026'
 
@@ -62,6 +63,7 @@ if LOG_NAME == '__main__':
     LOG_NAME = os.path.basename(sys.argv[0])
 LOG = logging.getLogger(LOG_NAME)
 
+lock = lockfile.FileLock('/var/lib/ceph/tmp/ceph-disk.lock')
 
 ###### exceptions ########
 
@@ -683,6 +685,16 @@ def zap(dev):
         raise Error(e)
 
 
+def get_udev_version():
+    version = _check_output(
+        args=[
+            'udevadm',
+            '--version',
+        ],
+    )   
+    return int(version)
+
+
 def prepare_journal_dev(
     data,
     journal,
@@ -759,9 +771,24 @@ def prepare_journal_dev(
                 ],
             )
 
-        journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format(
-            journal_uuid=journal_uuid,
-            )
+        if get_udev_version() >= 172:
+            journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format(
+                journal_uuid=journal_uuid,
+                )
+        else:
+            # udev prior to version 172 doesn't create by-partuuid directory
+            # use by-path symlink instead.  This is the third symlink returned
+            # by udevadm when queried.
+            LOG.debug('Using alternate persistant name for journal symlink')
+            symlinks = _check_output(
+                args=[
+                    'udevadm',
+                    'info',
+                    '--query=symlink',
+                    '--name={name}'.format(name=os.path.basename(journal)),
+                    ],
+                )   
+            journal_symlink='/dev/{symlink}-part{num}'.format(symlink=symlinks.split()[2], num=num)
 
         journal_dmcrypt = None
         if journal_dm_keypath:
@@ -1033,6 +1060,7 @@ def main_prepare(args):
     osd_dm_keypath = None
 
     try:
+        lock.acquire()
         if not os.path.exists(args.data):
             raise Error('data path does not exist', args.data)
 
@@ -1161,12 +1189,14 @@ def main_prepare(args):
                 )
         else:
             raise Error('not a dir or block device', args.data)
+        lock.release()
 
     except Error as e:
         if journal_dm_keypath:
             os.unlink(journal_dm_keypath)
         if osd_dm_keypath:
             os.unlink(osd_dm_keypath)
+        lock.release()
         raise e
 
 
@@ -1557,26 +1587,32 @@ def main_activate(args):
     if not os.path.exists(args.path):
         raise Error('%s does not exist', args.path)
 
-    mode = os.stat(args.path).st_mode
-    if stat.S_ISBLK(mode):
-        (cluster, osd_id) = mount_activate(
-            dev=args.path,
-            activate_key_template=args.activate_key_template,
-            init=args.mark_init,
-            )
-    elif stat.S_ISDIR(mode):
-        (cluster, osd_id) = activate_dir(
-            path=args.path,
-            activate_key_template=args.activate_key_template,
-            init=args.mark_init,
+    lock.acquire()
+    try:
+        mode = os.stat(args.path).st_mode
+        if stat.S_ISBLK(mode):
+            (cluster, osd_id) = mount_activate(
+                dev=args.path,
+                activate_key_template=args.activate_key_template,
+                init=args.mark_init,
+                )
+        elif stat.S_ISDIR(mode):
+            (cluster, osd_id) = activate_dir(
+                path=args.path,
+                activate_key_template=args.activate_key_template,
+                init=args.mark_init,
+                )
+        else:
+            raise Error('%s is not a directory or block device', args.path)
+    
+        start_daemon(
+            cluster=cluster,
+            osd_id=osd_id,
             )
-    else:
-        raise Error('%s is not a directory or block device', args.path)
+        lock.release()
 
-    start_daemon(
-        cluster=cluster,
-        osd_id=osd_id,
-        )
+    except:
+        lock.release()
 
 
 
diff --git a/src/ceph-disk-udev b/src/ceph-disk-udev
new file mode 100755
index 00000000000..a96c21652aa
--- /dev/null
+++ b/src/ceph-disk-udev
@@ -0,0 +1,56 @@
+#! /bin/sh
+
+# Wrapper for the ceph udev rules.  Since older versions of udev do not support gpt label fields, this shell
+# script is invoked from the udev rule to read the needed gpt label fields and call the appropriate ceph
+# OSD functions.
+
+PARTNO=$1
+NAME=$2
+PARENT_NAME=$3
+
+# Get GPT partition type guid
+ID_PART_ENTRY_TYPE=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition GUID code" | awk '{print $4}' | tr '[:upper:]' '[:lower:]')
+case $ID_PART_ENTRY_TYPE in
+
+45b0969e-9b03-4f30-b4c6-b4b80ceff106)
+    # JOURNAL_UUID
+    ;;
+
+45b0969e-9b03-4f30-b4c6-5ec00ceff106)
+    # DMCRYPT_JOURNAL_UUID
+    # Map journal if using dm-crypt
+    ID_PART_ENTRY_UUID=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition unique GUID" | awk '{print $4}' | tr '[:upper:]' '[:lower:]')
+    /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME}
+    ;;
+
+4fbd7e29-9d25-41b8-afd0-062c0ceff05d)
+    # OSD_UUID
+    # activate ceph-tagged partitions.
+    /usr/sbin/ceph-disk -v activate --mount /dev/${NAME}
+    ;;
+
+4fbd7e29-9d25-41b8-afd0-5ec00ceff05d)
+    # DMCRYPT_OSD_UUID
+    # Map data device and activate ceph-tagged partitions
+    # for dm-crypted data devices
+    ID_PART_ENTRY_UUID=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition unique GUID" | awk '{print $4}' | tr '[:upper:]' '[:lower:]')
+    /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME}
+    bash -c 'while [ ! -e /dev/mapper/${ID_PART_ENTRY_UUID} ];do sleep 1; done'
+    /usr/sbin/ceph-disk-activate --mount /dev/mapper/${ID_PART_ENTRY_UUID}
+    ;;
+
+89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be)
+    # TOBE_UUID
+    ;;
+
+89c57f98-2fe5-4dc0-89c1-5ec00ceff2be)
+    # DMCRYPT_TOBE_UUID
+    ;;
+
+*)
+    # Not a Ceph device
+    ;;
+
+esac
+
+exit
diff --git a/udev/95-ceph-osd-alt.rules b/udev/95-ceph-osd-alt.rules
new file mode 100644
index 00000000000..702ceea5d39
--- /dev/null
+++ b/udev/95-ceph-osd-alt.rules
@@ -0,0 +1,5 @@
+#  Check gpt partion for ceph tags and activate
+ACTION=="add", SUBSYSTEM=="block", \
+  ENV{DEVTYPE}=="partition", \
+  ENV{ID_PART_TABLE_TYPE}=="gpt", \
+  RUN+="/usr/sbin/ceph-disk-udev $number $name"