CEPH_OSD_ONDISK_MAGIC = 'ceph osd volume v026'
-JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-b4b80ceff106'
-OSD_UUID = '4fbd7e29-9d25-41b8-afd0-062c0ceff05d'
-TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be'
+JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-b4b80ceff106'
+DMCRYPT_JOURNAL_UUID = '45b0969e-9b03-4f30-b4c6-5ec00ceff106'
+OSD_UUID = '4fbd7e29-9d25-41b8-afd0-062c0ceff05d'
+DMCRYPT_OSD_UUID = '4fbd7e29-9d25-41b8-afd0-5ec00ceff05d'
+TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be'
+DMCRYPT_TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be'
DEFAULT_FS_TYPE = 'xfs'
return fsid
+def get_or_create_dmcrypt_key(
+ uuid,
+ key_dir,
+ ):
+ path = os.path.join(key_dir, uuid)
+
+ # already have it?
+ if os.path.exists(path):
+ return path
+
+ # make a new key
+ try:
+ if not os.path.exists(key_dir):
+ os.makedirs(key_dir)
+ with file('/dev/urandom', 'rb') as i:
+ key = i.read(256)
+ with file(path, 'wb') as f:
+ f.write(key)
+ return path
+ except:
+ raise PrepareError('unable to read or create dm-crypt key', path)
+
+
+def dmcrypt_map(
+ rawdev,
+ keypath,
+ uuid,
+ ):
+ dev = '/dev/mapper/'+ uuid
+ args = [
+ 'cryptsetup',
+ '--key-file',
+ keypath,
+ '--key-size', '256',
+ 'create',
+ uuid,
+ rawdev,
+ ]
+ try:
+ subprocess.check_call(args)
+ return dev
+
+ except subprocess.CalledProcessError as e:
+ raise PrepareError('unable to map device', rawdev)
+
+
+def dmcrypt_unmap(
+ uuid
+ ):
+ args = [
+ 'cryptsetup',
+ 'remove',
+ uuid
+ ]
+
+ try:
+ subprocess.check_call(args)
+
+ except subprocess.CalledProcessError as e:
+ raise PrepareError('unable to unmap device', uuid)
+
+
def mount(
dev,
fstype,
journal,
journal_size,
journal_uuid,
+ journal_dm_keypath,
):
if is_partition(journal):
log.debug('Journal %s is a partition', journal)
log.warning('OSD will not be hot-swappable if journal is not the same device as the osd data')
- return journal
+ return (journal, None, None)
+
+ key = None
+ ptype = JOURNAL_UUID
+ if journal_dm_keypath:
+ ptype = DMCRYPT_JOURNAL_UUID
# it is a whole disk. create a partition!
num = None
),
'--typecode={num}:{uuid}'.format(
num=num,
- uuid=JOURNAL_UUID,
+ uuid=ptype,
),
'--',
journal,
journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format(
journal_uuid=journal_uuid,
)
+
+ journal_dmcrypt = None
+ if journal_dm_keypath:
+ journal_dmcrypt = journal_symlink
+ journal_symlink = '/dev/mapper/{uuid}'.format(uuid=journal_uuid)
+
log.debug('Journal is GPT partition %s', journal_symlink)
- return journal_symlink
+ return (journal_symlink, journal_dmcrypt, journal_uuid)
except subprocess.CalledProcessError as e:
raise PrepareError(e)
log.debug('Journal is file %s', journal)
log.warning('OSD will not be hot-swappable if journal is not the same device as the osd data')
- return journal
+ return (journal, None, None)
def prepare_journal(
journal_uuid,
force_file,
force_dev,
+ journal_dm_keypath,
):
if journal is None:
if force_dev:
raise PrepareError('Journal is unspecified; not a block device')
- return None
+ return (None, None, None)
if not os.path.exists(journal):
if force_dev:
if stat.S_ISBLK(jmode):
if force_file:
raise PrepareError('Journal is not a regular file', journal)
- return prepare_journal_dev(data, journal, journal_size, journal_uuid)
+ return prepare_journal_dev(data, journal, journal_size, journal_uuid, journal_dm_keypath)
raise PrepareError('Journal %s is neither a block device nor regular file', journal)
+def adjust_symlink(target, path):
+ create = True
+ if os.path.lexists(path):
+ try:
+ mode = os.path.lstat(canonical).st_mode
+ if stat.S_ISREG(mode):
+ log.debug('Removing old file %s', canonical)
+ os.unlink(canonical)
+ elif stat.S_ISLNK(mode):
+ old = os.readlink(canonical)
+ if old != journal:
+ log.debug('Removing old symlink %s -> %s', canonical, old)
+ os.unlink(canonical)
+ else:
+ create = False
+ except:
+ raise PrepareError('unable to remove (or adjust) old file (symlink)', canonical)
+ if create:
+ log.debug('Creating symlink %s -> %s', path, target)
+ try:
+ os.symlink(target, path)
+ except:
+ raise PrepareError('unable to create symlink %s -> %s' % (path, target))
+
def prepare_dir(
path,
journal,
cluster_uuid,
- osd_uuid=None,
+ osd_uuid,
+ journal_uuid,
+ journal_dmcrypt = None,
):
log.debug('Preparing osd data dir %s', path)
if journal is not None:
# we're using an external journal; point to it here
- create = True
- canonical = os.path.join(path, 'journal')
- if os.path.lexists(canonical):
- try:
- mode = os.path.lstat(canonical).st_mode
- if stat.S_ISREG(mode):
- log.debug('Removing old journal file %s', canonical)
- os.unlink(canonical)
- elif stat.S_ISLNK(mode):
- old = os.readlink(canonical)
- if old != journal:
- log.debug('Removing old journal symlink %s -> %s', canonical, old)
- os.unlink(canonical)
- else:
- create = False
- except:
- raise PrepareError('unable to remove (or adjust) old journal (symlink)', canonical)
- if create:
- log.debug('Creating journal symlink %s -> %s', canonical, journal)
- try:
- os.symlink(journal, canonical)
- except:
- raise PrepareError('unable to create symlink %s -> %s' % (canonical, journal))
+ adjust_symlink(journal, os.path.join(path, 'journal'))
+
+ if journal_dmcrypt is not None:
+ adjust_symlink(journal_dmcrypt, os.path.join(path, 'journal_dmcrypt'))
+ else:
+ try:
+ os.unlink(os.path.join(path, 'journal_dmcrypt'))
+ except:
+ pass
write_one_line(path, 'ceph_fsid', cluster_uuid)
write_one_line(path, 'fsid', osd_uuid)
write_one_line(path, 'magic', CEPH_OSD_ONDISK_MAGIC)
+ if journal_uuid is not None:
+ # i.e., journal is a tagged partition
+ write_one_line(path, 'journal_uuid', journal_uuid)
def prepare_dev(
data,
mount_options,
cluster_uuid,
osd_uuid,
+ journal_uuid,
+ journal_dmcrypt,
+ osd_dm_keypath,
):
"""
Prepare a data/journal combination to be used for an OSD.
it.
"""
- dev = None
+ ptype_tobe = TOBE_UUID
+ ptype_osd = OSD_UUID
+ if osd_dm_keypath:
+ ptype_tobe = DMCRYPT_TOBE_UUID
+ ptype_osd = DMCRYPT_OSD_UUID
+
+ rawdev = None
if is_partition(data):
log.debug('OSD data device %s is a partition', data)
- dev = data
+ rawdev = data
else:
log.debug('Creating osd partition on %s', data)
try:
'--partition-guid=1:{osd_uuid}'.format(
osd_uuid=osd_uuid,
),
- '--typecode=1:%s' % TOBE_UUID,
+ '--typecode=1:%s' % ptype_tobe,
'--',
data,
],
except subprocess.CalledProcessError as e:
raise PrepareError(e)
- dev = '{data}1'.format(data=data)
+ rawdev = '{data}1'.format(data=data)
- args = [
- 'mkfs',
- '--type={fstype}'.format(fstype=fstype),
- ]
- if mkfs_args is not None:
- args.extend(mkfs_args.split())
+ dev = None
+ if osd_dm_keypath:
+ dev = dmcrypt_map(rawdev, osd_dm_keypath, osd_uuid)
else:
- args.extend(MKFS_ARGS.get(fstype, []))
- args.extend
- args.extend([
- '--',
- dev,
- ])
+ dev = rawdev
+
try:
- log.debug('Creating %s fs on %s', fstype, dev)
- subprocess.check_call(args=args)
- except subprocess.CalledProcessError as e:
- raise PrepareError(e)
+ args = [
+ 'mkfs',
+ '--type={fstype}'.format(fstype=fstype),
+ ]
+ if mkfs_args is not None:
+ args.extend(mkfs_args.split())
+ else:
+ args.extend(MKFS_ARGS.get(fstype, []))
+ args.extend([
+ '--',
+ dev,
+ ])
+ try:
+ log.debug('Creating %s fs on %s', fstype, dev)
+ subprocess.check_call(args=args)
+ except subprocess.CalledProcessError as e:
+ raise PrepareError(e)
- path = mount(dev=dev, fstype=fstype, options=mount_options)
+ path = mount(dev=dev, fstype=fstype, options=mount_options)
- try:
- prepare_dir(
- path=path,
- journal=journal,
- cluster_uuid=cluster_uuid,
- osd_uuid=osd_uuid,
- )
+ try:
+ prepare_dir(
+ path=path,
+ journal=journal,
+ cluster_uuid=cluster_uuid,
+ osd_uuid=osd_uuid,
+ journal_uuid=journal_uuid,
+ journal_dmcrypt=journal_dmcrypt,
+ )
+ finally:
+ unmount(path)
finally:
- unmount(path)
+ if rawdev != dev:
+ dmcrypt_unmap(osd_uuid)
if not is_partition(data):
try:
subprocess.check_call(
args=[
'sgdisk',
- '--typecode=1:%s' % OSD_UUID,
+ '--typecode=1:%s' % ptype_osd,
'--',
data,
],
action='store_true', default=None,
help='verify that JOURNAL is a block device',
)
+ parser.add_argument(
+ '--dmcrypt',
+ action='store_true', default=None,
+ help='encrypt DATA and/or JOURNAL devices with dm-crypt',
+ )
+ parser.add_argument(
+ '--dmcrypt-key-dir',
+ metavar='KEYDIR',
+ default='/etc/ceph/dmcrypt-keys',
+ help='directory where dm-crypt keys are stored',
+ )
parser.add_argument(
'data',
metavar='DATA',
level=loglevel,
)
+ journal_dm_keypath = None
+ osd_dm_keypath = None
+
try:
if not os.path.exists(args.data):
raise PrepareError('data path does not exist', args.data)
log.info('Will colocate journal with data on %s', args.data)
args.journal = args.data
- # first set up the journal
if args.journal_uuid is None:
args.journal_uuid = str(uuid.uuid4())
+ if args.osd_uuid is None:
+ args.osd_uuid = str(uuid.uuid4())
+
+ # dm-crypt keys?
+ if args.dmcrypt:
+ journal_dm_keypath = get_or_create_dmcrypt_key(args.journal_uuid, args.dmcrypt_key_dir)
+ osd_dm_keypath = get_or_create_dmcrypt_key(args.osd_uuid, args.dmcrypt_key_dir)
- journal_symlink = prepare_journal(
+ # prepare journal
+ (journal_symlink, journal_dmcrypt, journal_uuid) = prepare_journal(
data=args.data,
journal=args.journal,
journal_size=journal_size,
journal_uuid=args.journal_uuid,
force_file=args.journal_file,
force_dev=args.journal_dev,
+ journal_dm_keypath=journal_dm_keypath,
)
- if args.osd_uuid is None:
- args.osd_uuid = str(uuid.uuid4())
-
# prepare data
if stat.S_ISDIR(dmode):
if args.data_dev:
raise PrepareError('data path is not a block device', args.data)
prepare_dir(
- data=args.data,
+ path=args.data,
journal=journal_symlink,
cluster_uuid=args.cluster_uuid,
osd_uuid=args.osd_uuid,
+ journal_uuid=journal_uuid,
+ journal_dmcrypt=journal_dmcrypt,
)
elif stat.S_ISBLK(dmode):
if args.data_dir:
mount_options=mount_options,
cluster_uuid=args.cluster_uuid,
osd_uuid=args.osd_uuid,
+ journal_uuid=journal_uuid,
+ journal_dmcrypt=journal_dmcrypt,
+ osd_dm_keypath=osd_dm_keypath,
)
else:
raise PrepareError('not a dir or block device', args.data)
except PrepareError as e:
+ if journal_dm_keypath:
+ os.unlink(journal_dm_keypath)
+ if osd_dm_keypath:
+ os.unlink(osd_dm_keypath)
print >>sys.stderr, '{prog}: {msg}'.format(
prog=args.prog,
msg=e,
projects / ceph.git / commitdiff