From: Sage Weil Date: Wed, 13 Feb 2013 05:35:56 +0000 (-0800) Subject: ceph-disk-prepare: add initial support for dm-crypt X-Git-Tag: v0.58~51^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c6ac0ddf91915ba2aeae46d21367f017e18e82cd;p=ceph.git ceph-disk-prepare: add initial support for dm-crypt Keep keys in /etc/ceph/dmcrypt-keys. Identify partition instances by the partition UUID. Identify encrypted partitions by a parallel set of type UUIDs. Signed-off-by: Alexandre Marangone Signed-off-by: Sage Weil --- diff --git a/src/ceph-disk-prepare b/src/ceph-disk-prepare index 6670a5281031..7dd1cba07173 100755 --- a/src/ceph-disk-prepare +++ b/src/ceph-disk-prepare @@ -12,9 +12,12 @@ import uuid CEPH_OSD_ONDISK_MAGIC = 'ceph osd volume v026' -JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-b4b80ceff106' -OSD_UUID = '4fbd7e29-9d25-41b8-afd0-062c0ceff05d' -TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be' +JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-b4b80ceff106' +DMCRYPT_JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-5ec00ceff106' +OSD_UUID = '4fbd7e29-9d25-41b8-afd0-062c0ceff05d' +DMCRYPT_OSD_UUID = '4fbd7e29-9d25-41b8-afd0-5ec00ceff05d' +TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be' +DMCRYPT_TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be' DEFAULT_FS_TYPE = 'xfs' @@ -187,6 +190,68 @@ def get_fsid(cluster): return fsid +def get_or_create_dmcrypt_key( + uuid, + key_dir, + ): + path = os.path.join(key_dir, uuid) + + # already have it? + if os.path.exists(path): + return path + + # make a new key + try: + if not os.path.exists(key_dir): + os.makedirs(key_dir) + with file('/dev/urandom', 'rb') as i: + key = i.read(256) + with file(path, 'wb') as f: + f.write(key) + return path + except: + raise PrepareError('unable to read or create dm-crypt key', path) + + +def dmcrypt_map( + rawdev, + keypath, + uuid, + ): + dev = '/dev/mapper/'+ uuid + args = [ + 'cryptsetup', + '--key-file', + keypath, + '--key-size', '256', + 'create', + uuid, + rawdev, + ] + try: + subprocess.check_call(args) + return dev + + except subprocess.CalledProcessError as e: + raise PrepareError('unable to map device', rawdev) + + +def dmcrypt_unmap( + uuid + ): + args = [ + 'cryptsetup', + 'remove', + uuid + ] + + try: + subprocess.check_call(args) + + except subprocess.CalledProcessError as e: + raise PrepareError('unable to unmap device', uuid) + + def mount( dev, fstype, @@ -313,12 +378,18 @@ def prepare_journal_dev( journal, journal_size, journal_uuid, + journal_dm_keypath, ): if is_partition(journal): log.debug('Journal %s is a partition', journal) log.warning('OSD will not be hot-swappable if journal is not the same device as the osd data') - return journal + return (journal, None, None) + + key = None + ptype = JOURNAL_UUID + if journal_dm_keypath: + ptype = DMCRYPT_JOURNAL_UUID # it is a whole disk. create a partition! num = None @@ -359,7 +430,7 @@ def prepare_journal_dev( ), '--typecode={num}:{uuid}'.format( num=num, - uuid=JOURNAL_UUID, + uuid=ptype, ), '--', journal, @@ -376,8 +447,14 @@ def prepare_journal_dev( journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format( journal_uuid=journal_uuid, ) + + journal_dmcrypt = None + if journal_dm_keypath: + journal_dmcrypt = journal_symlink + journal_symlink = '/dev/mapper/{uuid}'.format(uuid=journal_uuid) + log.debug('Journal is GPT partition %s', journal_symlink) - return journal_symlink + return (journal_symlink, journal_dmcrypt, journal_uuid) except subprocess.CalledProcessError as e: raise PrepareError(e) @@ -396,7 +473,7 @@ def prepare_journal_file( log.debug('Journal is file %s', journal) log.warning('OSD will not be hot-swappable if journal is not the same device as the osd data') - return journal + return (journal, None, None) def prepare_journal( @@ -406,12 +483,13 @@ def prepare_journal( journal_uuid, force_file, force_dev, + journal_dm_keypath, ): if journal is None: if force_dev: raise PrepareError('Journal is unspecified; not a block device') - return None + return (None, None, None) if not os.path.exists(journal): if force_dev: @@ -427,16 +505,42 @@ def prepare_journal( if stat.S_ISBLK(jmode): if force_file: raise PrepareError('Journal is not a regular file', journal) - return prepare_journal_dev(data, journal, journal_size, journal_uuid) + return prepare_journal_dev(data, journal, journal_size, journal_uuid, journal_dm_keypath) raise PrepareError('Journal %s is neither a block device nor regular file', journal) +def adjust_symlink(target, path): + create = True + if os.path.lexists(path): + try: + mode = os.path.lstat(canonical).st_mode + if stat.S_ISREG(mode): + log.debug('Removing old file %s', canonical) + os.unlink(canonical) + elif stat.S_ISLNK(mode): + old = os.readlink(canonical) + if old != journal: + log.debug('Removing old symlink %s -> %s', canonical, old) + os.unlink(canonical) + else: + create = False + except: + raise PrepareError('unable to remove (or adjust) old file (symlink)', canonical) + if create: + log.debug('Creating symlink %s -> %s', path, target) + try: + os.symlink(target, path) + except: + raise PrepareError('unable to create symlink %s -> %s' % (path, target)) + def prepare_dir( path, journal, cluster_uuid, - osd_uuid=None, + osd_uuid, + journal_uuid, + journal_dmcrypt = None, ): log.debug('Preparing osd data dir %s', path) @@ -445,34 +549,23 @@ def prepare_dir( if journal is not None: # we're using an external journal; point to it here - create = True - canonical = os.path.join(path, 'journal') - if os.path.lexists(canonical): - try: - mode = os.path.lstat(canonical).st_mode - if stat.S_ISREG(mode): - log.debug('Removing old journal file %s', canonical) - os.unlink(canonical) - elif stat.S_ISLNK(mode): - old = os.readlink(canonical) - if old != journal: - log.debug('Removing old journal symlink %s -> %s', canonical, old) - os.unlink(canonical) - else: - create = False - except: - raise PrepareError('unable to remove (or adjust) old journal (symlink)', canonical) - if create: - log.debug('Creating journal symlink %s -> %s', canonical, journal) - try: - os.symlink(journal, canonical) - except: - raise PrepareError('unable to create symlink %s -> %s' % (canonical, journal)) + adjust_symlink(journal, os.path.join(path, 'journal')) + + if journal_dmcrypt is not None: + adjust_symlink(journal_dmcrypt, os.path.join(path, 'journal_dmcrypt')) + else: + try: + os.unlink(os.path.join(path, 'journal_dmcrypt')) + except: + pass write_one_line(path, 'ceph_fsid', cluster_uuid) write_one_line(path, 'fsid', osd_uuid) write_one_line(path, 'magic', CEPH_OSD_ONDISK_MAGIC) + if journal_uuid is not None: + # i.e., journal is a tagged partition + write_one_line(path, 'journal_uuid', journal_uuid) def prepare_dev( data, @@ -482,6 +575,9 @@ def prepare_dev( mount_options, cluster_uuid, osd_uuid, + journal_uuid, + journal_dmcrypt, + osd_dm_keypath, ): """ Prepare a data/journal combination to be used for an OSD. @@ -493,10 +589,16 @@ def prepare_dev( it. """ - dev = None + ptype_tobe = TOBE_UUID + ptype_osd = OSD_UUID + if osd_dm_keypath: + ptype_tobe = DMCRYPT_TOBE_UUID + ptype_osd = DMCRYPT_OSD_UUID + + rawdev = None if is_partition(data): log.debug('OSD data device %s is a partition', data) - dev = data + rawdev = data else: log.debug('Creating osd partition on %s', data) try: @@ -508,7 +610,7 @@ def prepare_dev( '--partition-guid=1:{osd_uuid}'.format( osd_uuid=osd_uuid, ), - '--typecode=1:%s' % TOBE_UUID, + '--typecode=1:%s' % ptype_tobe, '--', data, ], @@ -523,45 +625,56 @@ def prepare_dev( except subprocess.CalledProcessError as e: raise PrepareError(e) - dev = '{data}1'.format(data=data) + rawdev = '{data}1'.format(data=data) - args = [ - 'mkfs', - '--type={fstype}'.format(fstype=fstype), - ] - if mkfs_args is not None: - args.extend(mkfs_args.split()) + dev = None + if osd_dm_keypath: + dev = dmcrypt_map(rawdev, osd_dm_keypath, osd_uuid) else: - args.extend(MKFS_ARGS.get(fstype, [])) - args.extend - args.extend([ - '--', - dev, - ]) + dev = rawdev + try: - log.debug('Creating %s fs on %s', fstype, dev) - subprocess.check_call(args=args) - except subprocess.CalledProcessError as e: - raise PrepareError(e) + args = [ + 'mkfs', + '--type={fstype}'.format(fstype=fstype), + ] + if mkfs_args is not None: + args.extend(mkfs_args.split()) + else: + args.extend(MKFS_ARGS.get(fstype, [])) + args.extend([ + '--', + dev, + ]) + try: + log.debug('Creating %s fs on %s', fstype, dev) + subprocess.check_call(args=args) + except subprocess.CalledProcessError as e: + raise PrepareError(e) - path = mount(dev=dev, fstype=fstype, options=mount_options) + path = mount(dev=dev, fstype=fstype, options=mount_options) - try: - prepare_dir( - path=path, - journal=journal, - cluster_uuid=cluster_uuid, - osd_uuid=osd_uuid, - ) + try: + prepare_dir( + path=path, + journal=journal, + cluster_uuid=cluster_uuid, + osd_uuid=osd_uuid, + journal_uuid=journal_uuid, + journal_dmcrypt=journal_dmcrypt, + ) + finally: + unmount(path) finally: - unmount(path) + if rawdev != dev: + dmcrypt_unmap(osd_uuid) if not is_partition(data): try: subprocess.check_call( args=[ 'sgdisk', - '--typecode=1:%s' % OSD_UUID, + '--typecode=1:%s' % ptype_osd, '--', data, ], @@ -635,6 +748,17 @@ def parse_args(): action='store_true', default=None, help='verify that JOURNAL is a block device', ) + parser.add_argument( + '--dmcrypt', + action='store_true', default=None, + help='encrypt DATA and/or JOURNAL devices with dm-crypt', + ) + parser.add_argument( + '--dmcrypt-key-dir', + metavar='KEYDIR', + default='/etc/ceph/dmcrypt-keys', + help='directory where dm-crypt keys are stored', + ) parser.add_argument( 'data', metavar='DATA', @@ -667,6 +791,9 @@ def main(): level=loglevel, ) + journal_dm_keypath = None + osd_dm_keypath = None + try: if not os.path.exists(args.data): raise PrepareError('data path does not exist', args.data) @@ -741,31 +868,38 @@ def main(): log.info('Will colocate journal with data on %s', args.data) args.journal = args.data - # first set up the journal if args.journal_uuid is None: args.journal_uuid = str(uuid.uuid4()) + if args.osd_uuid is None: + args.osd_uuid = str(uuid.uuid4()) + + # dm-crypt keys? + if args.dmcrypt: + journal_dm_keypath = get_or_create_dmcrypt_key(args.journal_uuid, args.dmcrypt_key_dir) + osd_dm_keypath = get_or_create_dmcrypt_key(args.osd_uuid, args.dmcrypt_key_dir) - journal_symlink = prepare_journal( + # prepare journal + (journal_symlink, journal_dmcrypt, journal_uuid) = prepare_journal( data=args.data, journal=args.journal, journal_size=journal_size, journal_uuid=args.journal_uuid, force_file=args.journal_file, force_dev=args.journal_dev, + journal_dm_keypath=journal_dm_keypath, ) - if args.osd_uuid is None: - args.osd_uuid = str(uuid.uuid4()) - # prepare data if stat.S_ISDIR(dmode): if args.data_dev: raise PrepareError('data path is not a block device', args.data) prepare_dir( - data=args.data, + path=args.data, journal=journal_symlink, cluster_uuid=args.cluster_uuid, osd_uuid=args.osd_uuid, + journal_uuid=journal_uuid, + journal_dmcrypt=journal_dmcrypt, ) elif stat.S_ISBLK(dmode): if args.data_dir: @@ -778,11 +912,18 @@ def main(): mount_options=mount_options, cluster_uuid=args.cluster_uuid, osd_uuid=args.osd_uuid, + journal_uuid=journal_uuid, + journal_dmcrypt=journal_dmcrypt, + osd_dm_keypath=osd_dm_keypath, ) else: raise PrepareError('not a dir or block device', args.data) except PrepareError as e: + if journal_dm_keypath: + os.unlink(journal_dm_keypath) + if osd_dm_keypath: + os.unlink(osd_dm_keypath) print >>sys.stderr, '{prog}: {msg}'.format( prog=args.prog, msg=e,