ceph-disk: OSD hotplug fixes for Centoswip-4632

Two fixes for Centos 6.3 and other systems with udev versions
prior to 172.  The disk peristant name using the GPT UUID does
not exist, so use the by_path persistent name instead for the
journal symlink.

The gpt label fields are not available for use in udev rules. Add
ceph-disk-udev wrapper script that extracts the partition
type guid from the label and calls ceph-disk-activate if it is
a ceph guid type. (Bug #4632)

Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>

7 files changed, 131 insertions, 23 deletions

diff --git a/Makefile.am b/Makefile.am
index adeb4e57728..03cb914079f 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -10,7 +10,8 @@ EXTRA_DIST += \
 	src/test/cli \
 	src/test/downloads \
 	udev/50-rbd.rules \
-	udev/95-ceph-osd.rules
+	udev/95-ceph-osd.rules \
+	udev/95-ceph-osd-alt.rules
 
 
 all-local:
diff --git a/ceph.spec.in b/ceph.spec.in
index a2b504d2d2f..023258cbe61 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -25,10 +25,11 @@ Requires:	librbd1 = %{version}-%{release}
 Requires:	librados2 = %{version}-%{release}
 Requires:	libcephfs1 = %{version}-%{release}
 Requires:	python
+Requires:	python-argparse
+Requires:	python-lockfile
 Requires:	cryptsetup
 Requires:	parted
 Requires:	util-linux
-Requires:	python-argparse
 Requires:	hdparm
 Requires(post):	binutils
 BuildRoot:      %{_tmppath}/%{name}-%{version}-build
@@ -301,7 +302,11 @@ chmod 0644 $RPM_BUILD_ROOT%{_docdir}/ceph/sample.fetch_config
 
 # udev rules
 install -m 0644 -D udev/50-rbd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/50-rbd.rules
+%if 0%{?centos}
+install -m 0644 -D udev/95-ceph-osd-alt.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules
+%else
 install -m 0644 -D udev/95-ceph-osd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules
+%endif
 
 #set up placeholder directories
 mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/ceph
@@ -386,6 +391,7 @@ fi
 %{_sbindir}/ceph-disk
 %{_sbindir}/ceph-disk-activate
 %{_sbindir}/ceph-disk-prepare
+%{_sbindir}/ceph-disk-udev
 %{_sbindir}/ceph-create-keys
 %{_sbindir}/rcceph
 /sbin/mkcephfs
diff --git a/debian/control b/debian/control
index 98d0b37ffc1..efa86377c52 100644
--- a/debian/control
+++ b/debian/control
@@ -45,6 +45,8 @@ Depends: binutils,
          gdisk,
          parted,
          python,
+         python-argparse,
+         python-lockfile,
          sdparm | hdparm,
          uuid-runtime,
          xfsprogs,
diff --git a/src/Makefile.am b/src/Makefile.am
index d528b78a1be..0b223f20299 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -33,6 +33,7 @@ ceph_sbin_SCRIPTS = \
 	ceph-disk \
 	ceph-disk-prepare \
 	ceph-disk-activate \
+	ceph-disk-udev \
 	ceph-create-keys
 
 sbin_SCRIPTS = \
@@ -1192,6 +1193,7 @@ EXTRA_DIST += \
 	ceph-disk \
 	ceph-disk-prepare \
 	ceph-disk-activate \
+	ceph-disk-udev \
 	ceph-create-keys \
 	mount.fuse.ceph
 
diff --git a/src/ceph-disk b/src/ceph-disk
index 68a171938c5..c9515997ded 100755
--- a/src/ceph-disk
+++ b/src/ceph-disk
@@ -12,6 +12,7 @@ import stat
 import sys
 import tempfile
 import uuid
+import lockfile
 
 CEPH_OSD_ONDISK_MAGIC = 'ceph osd volume v026'
 
@@ -62,6 +63,7 @@ if LOG_NAME == '__main__':
     LOG_NAME = os.path.basename(sys.argv[0])
 LOG = logging.getLogger(LOG_NAME)
 
+lock = lockfile.FileLock('/var/lib/ceph/tmp/ceph-disk.lock')
 
 ###### exceptions ########
 
@@ -683,6 +685,16 @@ def zap(dev):
         raise Error(e)
 
 
+def get_udev_version():
+    version = _check_output(
+        args=[
+            'udevadm',
+            '--version',
+        ],
+    )   
+    return int(version)
+
+
 def prepare_journal_dev(
     data,
     journal,
@@ -761,9 +773,24 @@ def prepare_journal_dev(
                 ],
             )
 
-        journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format(
-            journal_uuid=journal_uuid,
-            )
+        if get_udev_version() >= 172:
+            journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format(
+                journal_uuid=journal_uuid,
+                )
+        else:
+            # udev prior to version 172 doesn't create by-partuuid directory
+            # use by-path symlink instead.  This is the third symlink returned
+            # by udevadm when queried.
+            LOG.debug('Using alternate persistant name for journal symlink')
+            symlinks = _check_output(
+                args=[
+                    'udevadm',
+                    'info',
+                    '--query=symlink',
+                    '--name={name}'.format(name=os.path.basename(journal)),
+                    ],
+                )   
+            journal_symlink='/dev/{symlink}-part{num}'.format(symlink=symlinks.split()[2], num=num)
 
         journal_dmcrypt = None
         if journal_dm_keypath:
@@ -1035,6 +1062,7 @@ def main_prepare(args):
     osd_dm_keypath = None
 
     try:
+        lock.acquire()
         if not os.path.exists(args.data):
             raise Error('data path does not exist', args.data)
 
@@ -1163,12 +1191,14 @@ def main_prepare(args):
                 )
         else:
             raise Error('not a dir or block device', args.data)
+        lock.release()
 
     except Error as e:
         if journal_dm_keypath:
             os.unlink(journal_dm_keypath)
         if osd_dm_keypath:
             os.unlink(osd_dm_keypath)
+        lock.release()
         raise e
 
 
@@ -1559,26 +1589,32 @@ def main_activate(args):
     if not os.path.exists(args.path):
         raise Error('%s does not exist', args.path)
 
-    mode = os.stat(args.path).st_mode
-    if stat.S_ISBLK(mode):
-        (cluster, osd_id) = mount_activate(
-            dev=args.path,
-            activate_key_template=args.activate_key_template,
-            init=args.mark_init,
-            )
-    elif stat.S_ISDIR(mode):
-        (cluster, osd_id) = activate_dir(
-            path=args.path,
-            activate_key_template=args.activate_key_template,
-            init=args.mark_init,
+    lock.acquire()
+    try:
+        mode = os.stat(args.path).st_mode
+        if stat.S_ISBLK(mode):
+            (cluster, osd_id) = mount_activate(
+                dev=args.path,
+                activate_key_template=args.activate_key_template,
+                init=args.mark_init,
+                )
+        elif stat.S_ISDIR(mode):
+            (cluster, osd_id) = activate_dir(
+                path=args.path,
+                activate_key_template=args.activate_key_template,
+                init=args.mark_init,
+                )
+        else:
+            raise Error('%s is not a directory or block device', args.path)
+    
+        start_daemon(
+            cluster=cluster,
+            osd_id=osd_id,
             )
-    else:
-        raise Error('%s is not a directory or block device', args.path)
+        lock.release()
 
-    start_daemon(
-        cluster=cluster,
-        osd_id=osd_id,
-        )
+    except:
+        lock.release()
 
 
 
diff --git a/src/ceph-disk-udev b/src/ceph-disk-udev
new file mode 100755
index 00000000000..a96c21652aa
--- /dev/null
+++ b/src/ceph-disk-udev
@@ -0,0 +1,56 @@
+#! /bin/sh
+
+# Wrapper for the ceph udev rules.  Since older versions of udev do not support gpt label fields, this shell
+# script is invoked from the udev rule to read the needed gpt label fields and call the appropriate ceph
+# OSD functions.
+
+PARTNO=$1
+NAME=$2
+PARENT_NAME=$3
+
+# Get GPT partition type guid
+ID_PART_ENTRY_TYPE=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition GUID code" | awk '{print $4}' | tr '[:upper:]' '[:lower:]')
+case $ID_PART_ENTRY_TYPE in
+
+45b0969e-9b03-4f30-b4c6-b4b80ceff106)
+    # JOURNAL_UUID
+    ;;
+
+45b0969e-9b03-4f30-b4c6-5ec00ceff106)
+    # DMCRYPT_JOURNAL_UUID
+    # Map journal if using dm-crypt
+    ID_PART_ENTRY_UUID=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition unique GUID" | awk '{print $4}' | tr '[:upper:]' '[:lower:]')
+    /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME}
+    ;;
+
+4fbd7e29-9d25-41b8-afd0-062c0ceff05d)
+    # OSD_UUID
+    # activate ceph-tagged partitions.
+    /usr/sbin/ceph-disk -v activate --mount /dev/${NAME}
+    ;;
+
+4fbd7e29-9d25-41b8-afd0-5ec00ceff05d)
+    # DMCRYPT_OSD_UUID
+    # Map data device and activate ceph-tagged partitions
+    # for dm-crypted data devices
+    ID_PART_ENTRY_UUID=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition unique GUID" | awk '{print $4}' | tr '[:upper:]' '[:lower:]')
+    /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME}
+    bash -c 'while [ ! -e /dev/mapper/${ID_PART_ENTRY_UUID} ];do sleep 1; done'
+    /usr/sbin/ceph-disk-activate --mount /dev/mapper/${ID_PART_ENTRY_UUID}
+    ;;
+
+89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be)
+    # TOBE_UUID
+    ;;
+
+89c57f98-2fe5-4dc0-89c1-5ec00ceff2be)
+    # DMCRYPT_TOBE_UUID
+    ;;
+
+*)
+    # Not a Ceph device
+    ;;
+
+esac
+
+exit
diff --git a/udev/95-ceph-osd-alt.rules b/udev/95-ceph-osd-alt.rules
new file mode 100644
index 00000000000..702ceea5d39
--- /dev/null
+++ b/udev/95-ceph-osd-alt.rules
@@ -0,0 +1,5 @@
+#  Check gpt partion for ceph tags and activate
+ACTION=="add", SUBSYSTEM=="block", \
+  ENV{DEVTYPE}=="partition", \
+  ENV{ID_PART_TABLE_TYPE}=="gpt", \
+  RUN+="/usr/sbin/ceph-disk-udev $number $name"