From: Andrew Bartlett Date: Thu, 30 Oct 2014 21:29:36 +0000 (+1300) Subject: Rework ceph-disk to allow LUKS for encrypted partitions X-Git-Tag: v0.93~62^2^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c83a288ab84bf06cd9b09e823626a2ccdf2852f9;p=ceph.git Rework ceph-disk to allow LUKS for encrypted partitions LUKS allows for validation of the key at mount time (rather than simply mounting a random partition), specification of the encryption parameters in the header and key rollover of the slot key (the one that needs to be stored). New parameters 'osd cryptsetup parameters' and 'osd dmcrypt key size' are added. These allow these important policy choices to be overridden or kept consistent per-site. The previous default plain mode (rather than using LUKS) remains, select LUKS by setting 'osd dmcrypt type = luks' Signed-off-by: Andrew Bartlett --- diff --git a/src/ceph-disk b/src/ceph-disk index 57663133a731..6280856ab335 100755 --- a/src/ceph-disk +++ b/src/ceph-disk @@ -2,6 +2,7 @@ # # Copyright (C) 2014 Inktank # Copyright (C) 2014 Cloudwatt +# Copyright (C) 2014 Catalyst.net Ltd # # Author: Loic Dachary # @@ -30,6 +31,8 @@ import sys import tempfile import uuid import time +import shlex +import stat """ Prepare: @@ -74,12 +77,15 @@ knew the GPT partition type. CEPH_OSD_ONDISK_MAGIC = 'ceph osd volume v026' -JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-b4b80ceff106' -DMCRYPT_JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-5ec00ceff106' -OSD_UUID = '4fbd7e29-9d25-41b8-afd0-062c0ceff05d' -DMCRYPT_OSD_UUID = '4fbd7e29-9d25-41b8-afd0-5ec00ceff05d' -TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be' -DMCRYPT_TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be' +JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-b4b80ceff106' +DMCRYPT_JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-5ec00ceff106' +DMCRYPT_LUKS_JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-35865ceff106' +OSD_UUID = '4fbd7e29-9d25-41b8-afd0-062c0ceff05d' +DMCRYPT_OSD_UUID = '4fbd7e29-9d25-41b8-afd0-5ec00ceff05d' +DMCRYPT_LUKS_OSD_UUID = '4fbd7e29-9d25-41b8-afd0-35865ceff05d' +TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be' +DMCRYPT_TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be' +DMCRYPT_JOURNAL_TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-35865ceff2be' DEFAULT_FS_TYPE = 'xfs' @@ -777,13 +783,18 @@ def get_fsid(cluster): def get_or_create_dmcrypt_key( _uuid, key_dir, + key_size, + luks ): """ Get path to dmcrypt key or create a new key file. :return: Path to the dmcrypt key file. """ - path = os.path.join(key_dir, _uuid) + if luks: + path = os.path.join(key_dir, _uuid + ".luks.key") + else: + path = os.path.join(key_dir, _uuid) # already have it? if os.path.exists(path): @@ -794,7 +805,7 @@ def get_or_create_dmcrypt_key( if not os.path.exists(key_dir): os.makedirs(key_dir, stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR) with file('/dev/urandom', 'rb') as i: - key = i.read(256 / 8) + key = i.read(key_size / 8) fd = os.open(path, os.O_WRONLY|os.O_CREAT, stat.S_IRUSR|stat.S_IWUSR) assert os.write(fd, key) == len(key) @@ -808,6 +819,8 @@ def dmcrypt_map( rawdev, keypath, _uuid, + cryptsetup_parameters, + luks ): """ Maps a device to a dmcrypt device. @@ -815,17 +828,40 @@ def dmcrypt_map( :return: Path to the dmcrypt device. """ dev = '/dev/mapper/' + _uuid - args = [ + luksFormat_args = [ + 'cryptsetup', + '--batch-mode', + '--key-file', + keypath, + 'luksFormat', + rawdev, + ] + cryptsetup_parameters + + luksOpen_args = [ + 'cryptsetup', + '--key-file', + keypath, + 'luksOpen', + rawdev, + _uuid, + ] + + create_args = [ 'cryptsetup', '--key-file', keypath, - '--key-size', '256', 'create', _uuid, rawdev, - ] + ] + cryptsetup_parameters + try: - command_check_call(args) + if luks: + command_check_call(luksFormat_args) + command_check_call(luksOpen_args) + else: + # Plain mode has no format function, nor any validation that the key is correct. + command_check_call(create_args) return dev except subprocess.CalledProcessError as e: @@ -1046,7 +1082,9 @@ def prepare_journal_dev( journal_size, journal_uuid, journal_dm_keypath, - ): + cryptsetup_parameters, + luks + ): reusing_partition = False @@ -1087,8 +1125,13 @@ def prepare_journal_dev( # From here on we are creating a new journal device, not reusing. ptype = JOURNAL_UUID + ptype_tobe = JOURNAL_UUID if journal_dm_keypath: - ptype = DMCRYPT_JOURNAL_UUID + if luks: + ptype = DMCRYPT_LUKS_JOURNAL_UUID + else: + ptype = DMCRYPT_JOURNAL_UUID + ptype_tobe = DMCRYPT_JOURNAL_TOBE_UUID # it is a whole disk. create a partition! num = None @@ -1136,7 +1179,7 @@ def prepare_journal_dev( ), '--typecode={num}:{uuid}'.format( num=num, - uuid=ptype, + uuid=ptype_tobe, ), '--mbrtogpt', '--', @@ -1156,9 +1199,38 @@ def prepare_journal_dev( LOG.debug('Journal is GPT partition %s', journal_symlink) - # udev should have created the symlink by now. If not, abort. - assert os.path.exists(journal_symlink) + if journal_dm_keypath: + if luks: + luksFormat_args = [ + 'cryptsetup', + '--batch-mode', + '--key-file', + journal_dm_keypath, + 'luksFormat', + journal_dmcrypt, + ] + cryptsetup_parameters + try: + command_check_call(luksFormat_args) + except subprocess.CalledProcessError as e: + raise Error('unable to format device for LUKS', journal_symlink, e) + + try: + command_check_call( + [ + 'sgdisk', + '--typecode={num}:{uuid}'.format( + num=num, + uuid=ptype, + ), + '--', + journal, + ], + ) + except subprocess.CalledProcessError as e: + raise Error('unable to mark device as formatted for LUKS', journal_symlink, e) + + LOG.debug('Journal is GPT partition %s', journal_symlink) return (journal_symlink, journal_dmcrypt, journal_uuid) except subprocess.CalledProcessError as e: @@ -1185,6 +1257,8 @@ def prepare_journal( force_file, force_dev, journal_dm_keypath, + cryptsetup_parameters, + luks ): if journal is None: @@ -1206,7 +1280,7 @@ def prepare_journal( if stat.S_ISBLK(jmode): if force_file: raise Error('Journal is not a regular file', journal) - return prepare_journal_dev(data, journal, journal_size, journal_uuid, journal_dm_keypath) + return prepare_journal_dev(data, journal, journal_size, journal_uuid, journal_dm_keypath, cryptsetup_parameters, luks) raise Error('Journal %s is neither a block device nor regular file' % journal) @@ -1287,6 +1361,8 @@ def prepare_dev( journal_uuid, journal_dmcrypt, osd_dm_keypath, + cryptsetup_parameters, + luks ): """ Prepare a data/journal combination to be used for an OSD. @@ -1302,7 +1378,10 @@ def prepare_dev( ptype_osd = OSD_UUID if osd_dm_keypath: ptype_tobe = DMCRYPT_TOBE_UUID - ptype_osd = DMCRYPT_OSD_UUID + if luks: + ptype_osd = DMCRYPT_LUKS_OSD_UUID + else: + ptype_osd = DMCRYPT_OSD_UUID rawdev = None if is_partition(data): @@ -1339,7 +1418,7 @@ def prepare_dev( dev = None if osd_dm_keypath: - dev = dmcrypt_map(rawdev, osd_dm_keypath, osd_uuid) + dev = dmcrypt_map(rawdev, osd_dm_keypath, osd_uuid, cryptsetup_parameters, luks) else: dev = rawdev @@ -1483,6 +1562,62 @@ def main_prepare(args): ) journal_size = int(journal_size) + cryptsetup_parameters_str = get_conf( + cluster=args.cluster, + variable='osd_cryptsetup_parameters', + ) + if cryptsetup_parameters_str is None: + cryptsetup_parameters = [] + else: + cryptsetup_parameters = shlex.split(cryptsetup_parameters_str) + + dmcrypt_keysize_str = get_conf( + cluster=args.cluster, + variable='osd_dmcrypt_key_size', + ) + + dmcrypt_type = get_conf( + cluster=args.cluster, + variable='osd_dmcrypt_type', + ) + + if dmcrypt_type is None: + dmcrypt_type = "plain" + + if dmcrypt_type == "plain": + if dmcrypt_keysize_str is None: + # This value is hard-coded in the udev script + dmcrypt_keysize = 256 + else: + dmcrypt_keysize = int(dmcrypt_keysize_str) + LOG.warning('''ensure the 95-ceph-osd.rules file has been copied to /etc/udev/rules.d + and modified to call cryptsetup with --key-size=%s''' + % dmcrypt_keysize_str) + + if len (cryptsetup_parameters) > 0: + LOG.warning('''ensure the 95-ceph-osd.rules file has been copied to /etc/udev/rules.d + and modified to call cryptsetup with %s''' + % cryptsetup_parameters_str) + + cryptsetup_parameters = ['--key-size', str(dmcrypt_keysize)] + cryptsetup_parameters + luks = False + elif dmcrypt_type == "luks": + if dmcrypt_keysize_str is None: + # As LUKS will hash the 'passphrase' in .luks.key into a key, set a large default + # so if not updated for some time, it is still a + # reasonable value. + # + # We don't force this into the cryptsetup_parameters, as we want the cryptsetup defaults + # to prevail for the actual LUKS key lengths. + dmcrypt_keysize = 1024 + else: + dmcrypt_keysize = int(dmcrypt_keysize_str) + cryptsetup_parameters = ['--key-size', str(dmcrypt_keysize)] + cryptsetup_parameters + + luks = True + else: + raise Error('invalid osd_dmcrypt_type parameter (must be luks or plain): ', dmcrypt_type) + # colocate journal with data? if stat.S_ISBLK(dmode) and not is_partition(args.data) and args.journal is None and args.journal_file is None: LOG.info('Will colocate journal with data on %s', args.data) @@ -1495,8 +1630,8 @@ def main_prepare(args): # dm-crypt keys? if args.dmcrypt: - journal_dm_keypath = get_or_create_dmcrypt_key(args.journal_uuid, args.dmcrypt_key_dir) - osd_dm_keypath = get_or_create_dmcrypt_key(args.osd_uuid, args.dmcrypt_key_dir) + journal_dm_keypath = get_or_create_dmcrypt_key(args.journal_uuid, args.dmcrypt_key_dir, dmcrypt_keysize, luks) + osd_dm_keypath = get_or_create_dmcrypt_key(args.osd_uuid, args.dmcrypt_key_dir, dmcrypt_keysize, luks) # prepare journal (journal_symlink, journal_dmcrypt, journal_uuid) = prepare_journal( @@ -1507,6 +1642,8 @@ def main_prepare(args): force_file=args.journal_file, force_dev=args.journal_dev, journal_dm_keypath=journal_dm_keypath, + cryptsetup_parameters=cryptsetup_parameters, + luks=luks ) # prepare data @@ -1535,6 +1672,8 @@ def main_prepare(args): journal_uuid=journal_uuid, journal_dmcrypt=journal_dmcrypt, osd_dm_keypath=osd_dm_keypath, + cryptsetup_parameters=cryptsetup_parameters, + luks=luks ) else: raise Error('not a dir or block device', args.data) @@ -1545,9 +1684,17 @@ def main_prepare(args): except Error as e: if journal_dm_keypath: - os.unlink(journal_dm_keypath) + try: + os.unlink(journal_dm_keypath) + except OSError as e2: + if e2.errno != errno.ENOENT: # errno.ENOENT = no such file or directory + raise # re-raise exception if a different error occured if osd_dm_keypath: - os.unlink(osd_dm_keypath) + try: + os.unlink(osd_dm_keypath) + except OSError as e2: + if e2.errno != errno.ENOENT: # errno.ENOENT = no such file or directory + raise # re-raise exception if a different error occured prepare_lock.release() # noqa raise e @@ -2139,9 +2286,9 @@ def main_activate_all(args): continue (tag, uuid) = name.split('.') - if tag == OSD_UUID or tag == DMCRYPT_OSD_UUID: + if tag == OSD_UUID or tag == DMCRYPT_OSD_UUID or tag == DMCRYPT_LUKS_OSD_UUID: - if tag == DMCRYPT_OSD_UUID: + if tag == DMCRYPT_OSD_UUID or tag == DMCRYPT_LUKS_OSD_UUID: path = os.path.join('/dev/mapper', uuid) else: path = os.path.join(dir, name) @@ -2361,13 +2508,23 @@ def list_dev(dev, uuid_map, journal_map): elif ptype == DMCRYPT_OSD_UUID: holders = is_held(dev) if not holders: - desc = ['ceph data (dmcrypt)', 'not currently mapped'] + desc = ['ceph data (dmcrypt plain)', 'not currently mapped'] elif len(holders) == 1: holder = '/dev/' + holders[0] fs_desc = list_dev_osd(holder, uuid_map) - desc = ['ceph data (dmcrypt %s)' % holder] + fs_desc + desc = ['ceph data (dmcrypt plain %s)' % holder] + fs_desc else: - desc = ['ceph data (dmcrypt)', 'holders: ' + ','.join(holders)] + desc = ['ceph data (dmcrypt plain)', 'holders: ' + ','.join(holders)] + elif ptype == DMCRYPT_LUKS_OSD_UUID: + holders = is_held(dev) + if not holders: + desc = ['ceph data (dmcrypt LUKS)', 'not currently mapped'] + elif len(holders) == 1: + holder = '/dev/' + holders[0] + fs_desc = list_dev_osd(holder, uuid_map) + desc = ['ceph data (dmcrypt LUKS %s)' % holder] + fs_desc + else: + desc = ['ceph data (dmcrypt LUKS)', 'holders: ' + ','.join(holders)] elif ptype == JOURNAL_UUID: desc.append('ceph journal') part_uuid = get_partition_uuid(dev) @@ -2376,9 +2533,18 @@ def list_dev(dev, uuid_map, journal_map): elif ptype == DMCRYPT_JOURNAL_UUID: holders = is_held(dev) if len(holders) == 1: - desc = ['ceph journal (dmcrypt /dev/%s)' % holders[0]] + desc = ['ceph journal (dmcrypt plain /dev/%s)' % holders[0]] + else: + desc = ['ceph journal (dmcrypt plain)'] + part_uuid = get_partition_uuid(dev) + if part_uuid and part_uuid in journal_map: + desc.append('for %s' % journal_map[part_uuid]) + elif ptype == DMCRYPT_LUKS_JOURNAL_UUID: + holders = is_held(dev) + if len(holders) == 1: + desc = ['ceph journal (dmcrypt LUKS /dev/%s)' % holders[0]] else: - desc = ['ceph journal (dmcrypt)'] + desc = ['ceph journal (dmcrypt LUKS)'] part_uuid = get_partition_uuid(dev) if part_uuid and part_uuid in journal_map: desc.append('for %s' % journal_map[part_uuid]) @@ -2424,7 +2590,7 @@ def main_list(args): unmount(tpath) except MountError: pass - if ptype == DMCRYPT_OSD_UUID: + if ptype == DMCRYPT_OSD_UUID or ptype == DMCRYPT_LUKS_OSD_UUID: holders = is_held(dev) if len(holders) == 1: holder = '/dev/' + holders[0] diff --git a/src/ceph-disk-udev b/src/ceph-disk-udev index bdf524e6aea2..dd2ac0864594 100755 --- a/src/ceph-disk-udev +++ b/src/ceph-disk-udev @@ -38,6 +38,12 @@ case $ID_PART_ENTRY_TYPE in /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME} ;; +45b0969e-9b03-4f30-b4c6-35865ceff106) + # DMCRYPT_LUKS_JOURNAL_UUID + # Map journal if using dm-crypt + /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} luksOpen /dev/${NAME} ${ID_PART_ENTRY_UUID} + ;; + 4fbd7e29-9d25-41b8-afd0-062c0ceff05d) # OSD_UUID # activate ceph-tagged partitions. @@ -53,6 +59,15 @@ case $ID_PART_ENTRY_TYPE in /usr/sbin/ceph-disk-activate /dev/mapper/${ID_PART_ENTRY_UUID} ;; +4fbd7e29-9d25-41b8-afd0-35865ceff05d) + # DMCRYPT_LUKS_OSD_UUID + # Map data device and activate ceph-tagged partitions + # for dm-crypted data devices + /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} luksOpen /dev/${NAME} ${ID_PART_ENTRY_UUID} + bash -c 'while [ ! -e /dev/mapper/${ID_PART_ENTRY_UUID} ];do sleep 1; done' + /usr/sbin/ceph-disk-activate /dev/mapper/${ID_PART_ENTRY_UUID} + ;; + 89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be) # TOBE_UUID ;; diff --git a/udev/95-ceph-osd.rules b/udev/95-ceph-osd.rules index ec2c4a463aa4..6498cfeab4ac 100644 --- a/udev/95-ceph-osd.rules +++ b/udev/95-ceph-osd.rules @@ -10,18 +10,34 @@ ACTION=="add", SUBSYSTEM=="block", \ ENV{ID_PART_ENTRY_TYPE}=="45b0969e-9b03-4f30-b4c6-b4b80ceff106", \ RUN+="/usr/sbin/ceph-disk activate-journal /dev/$name" -# Map journal if using dm-crypt +# Map journal if using dm-crypt and plain ACTION=="add" SUBSYSTEM=="block", \ ENV{DEVTYPE}=="partition", \ ENV{ID_PART_ENTRY_TYPE}=="45b0969e-9b03-4f30-b4c6-5ec00ceff106", \ RUN+="/sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/$env{ID_PART_ENTRY_UUID} --key-size 256 create $env{ID_PART_ENTRY_UUID} /dev/$name" +# Map journal if using dm-crypt and luks +ACTION=="add" SUBSYSTEM=="block", \ + ENV{DEVTYPE}=="partition", \ + ENV{ID_PART_ENTRY_TYPE}=="45b0969e-9b03-4f30-b4c6-35865ceff106", \ + RUN+="/sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/$env{ID_PART_ENTRY_UUID}.luks.key luksOpen /dev/$name $env{ID_PART_ENTRY_UUID}" + # Map data device and # activate ceph-tagged partitions -# for dm-crypted data devices +# for dm-crypted data devices and plain ACTION=="add" SUBSYSTEM=="block", \ ENV{DEVTYPE}=="partition", \ ENV{ID_PART_ENTRY_TYPE}=="4fbd7e29-9d25-41b8-afd0-5ec00ceff05d", \ RUN+="/sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/$env{ID_PART_ENTRY_UUID} --key-size 256 create $env{ID_PART_ENTRY_UUID} /dev/$name", \ RUN+="/bin/bash -c 'while [ ! -e /dev/mapper/$env{ID_PART_ENTRY_UUID} ];do sleep 1; done'", \ RUN+="/usr/sbin/ceph-disk-activate /dev/mapper/$env{ID_PART_ENTRY_UUID}" + +# Map data device and +# activate ceph-tagged partitions +# for dm-crypted data devices and luks +ACTION=="add" SUBSYSTEM=="block", \ + ENV{DEVTYPE}=="partition", \ + ENV{ID_PART_ENTRY_TYPE}=="4fbd7e29-9d25-41b8-afd0-35865ceff05d", \ + RUN+="/sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/$env{ID_PART_ENTRY_UUID}.luks.key luksOpen /dev/$name $env{ID_PART_ENTRY_UUID}", \ + RUN+="/bin/bash -c 'while [ ! -e /dev/mapper/$env{ID_PART_ENTRY_UUID} ];do sleep 1; done'", \ + RUN+="/usr/sbin/ceph-disk-activate /dev/mapper/$env{ID_PART_ENTRY_UUID}"