From: Sébastien Han Date: Fri, 24 Jan 2020 15:29:54 +0000 (+0100) Subject: ceph-volume: add db and wal support to raw mode X-Git-Tag: v14.2.8~102^2^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F32979%2Fhead;p=ceph.git ceph-volume: add db and wal support to raw mode Using the raw mode, the OSD preparation can now be called with extra arguments to specify a block device for either rocksdb db or wal. The code expects the device to be available and no validation will be performed. Also, the entire block device will be consumed. The goal is run this on Kubernetes with Rook and OSD are running on PVC. Users will request PVC from a certain size, the request will be acknowledged and a block will be created, then attached to the machine. Later on consumed by Rook's preparation job. Signed-off-by: Sébastien Han (cherry picked from commit c8f23c7182e2681f2bc756776a890995aca1efd2) --- diff --git a/src/ceph-volume/ceph_volume/devices/raw/activate.py b/src/ceph-volume/ceph_volume/devices/raw/activate.py index df6606093c5d7..94f54f1a20b31 100644 --- a/src/ceph-volume/ceph_volume/devices/raw/activate.py +++ b/src/ceph-volume/ceph_volume/devices/raw/activate.py @@ -11,9 +11,10 @@ from .list import direct_report logger = logging.getLogger(__name__) -def activate_bluestore(meta, tmpfs, systemd): +def activate_bluestore(meta, tmpfs, systemd, block_wal=None, block_db=None): # find the osd osd_id = meta['osd_id'] + osd_uuid = meta['osd_uuid'] # mount on tmpfs the osd directory osd_path = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id) @@ -42,10 +43,15 @@ def activate_bluestore(meta, tmpfs, systemd): # always re-do the symlink regardless if it exists, so that the block, # block.wal, and block.db devices that may have changed can be mapped # correctly every time - process.run(['ln', '-snf', meta['device'], os.path.join(osd_path, 'block')]) - system.chown(os.path.join(osd_path, 'block')) - system.chown(osd_path) + prepare_utils.link_block( meta['device'], osd_id) + + if block_wal: + prepare_utils.link_wal(block_wal, osd_id, osd_uuid) + + if block_db: + prepare_utils.link_db(block_db, osd_id, osd_uuid) + system.chown(osd_path) terminal.success("ceph-volume raw activate successful for osd ID: %s" % osd_id) @@ -58,7 +64,7 @@ class Activate(object): self.args = None @decorators.needs_root - def activate(self, devices, tmpfs, systemd): + def activate(self, devices, tmpfs, systemd, block_wal, block_db): """ :param args: The parsed arguments coming from the CLI """ @@ -67,10 +73,12 @@ class Activate(object): for osd_id, meta in found.items(): logger.info('Activating osd.%s uuid %s cluster %s' % ( - osd_id, meta['osd_uuid'], meta['ceph_fsid'])) + osd_id, meta['osd_uuid'], meta['ceph_fsid'])) activate_bluestore(meta, tmpfs=tmpfs, - systemd=systemd) + systemd=systemd, + block_wal=block_wal, + block_db=block_db) def main(self): sub_command_help = dedent(""" @@ -87,22 +95,34 @@ class Activate(object): formatter_class=argparse.RawDescriptionHelpFormatter, description=sub_command_help, ) - parser.add_argument( '--device', nargs='+', - help='The device(s) for the OSD to start') + help='The device for the OSD to start' + ) parser.add_argument( '--no-systemd', dest='no_systemd', action='store_true', - help='Skip creating and enabling systemd units and starting OSD services', + help='Skip creating and enabling systemd units and starting OSD services' + ) + parser.add_argument( + '--block.db', + dest='block_db', + help='Path to bluestore block.db block device' + ) + parser.add_argument( + '--block.wal', + dest='block_wal', + help='Path to bluestore block.wal block device' ) parser.add_argument( '--no-tmpfs', action='store_true', - help='Do not use a tmpfs mount for OSD data dir') - if len(self.argv) == 0: + help='Do not use a tmpfs mount for OSD data dir' + ) + + if not self.argv: print(sub_command_help) return args = parser.parse_args(self.argv) @@ -112,4 +132,6 @@ class Activate(object): raise SystemExit(1) self.activate(args.device, tmpfs=not args.no_tmpfs, - systemd=not self.args.no_systemd) + systemd=not self.args.no_systemd, + block_wal=self.args.block_wal, + block_db=self.args.block_db) diff --git a/src/ceph-volume/ceph_volume/devices/raw/common.py b/src/ceph-volume/ceph_volume/devices/raw/common.py index ba1dae00f16ef..d34a2941d1684 100644 --- a/src/ceph-volume/ceph_volume/devices/raw/common.py +++ b/src/ceph-volume/ceph_volume/devices/raw/common.py @@ -33,5 +33,16 @@ def create_parser(prog, description): parser.add_argument( '--no-tmpfs', action='store_true', - help='Do not use a tmpfs mount for OSD data dir') + help='Do not use a tmpfs mount for OSD data dir' + ) + parser.add_argument( + '--block.db', + dest='block_db', + help='Path to bluestore block.db block device' + ) + parser.add_argument( + '--block.wal', + dest='block_wal', + help='Path to bluestore block.wal block device' + ) return parser diff --git a/src/ceph-volume/ceph_volume/devices/raw/prepare.py b/src/ceph-volume/ceph_volume/devices/raw/prepare.py index 54e82c99202b3..cb5c59ce40f53 100644 --- a/src/ceph-volume/ceph_volume/devices/raw/prepare.py +++ b/src/ceph-volume/ceph_volume/devices/raw/prepare.py @@ -11,7 +11,7 @@ from .common import create_parser logger = logging.getLogger(__name__) -def prepare_bluestore(block, secrets, tmpfs): +def prepare_bluestore(block, wal, db, secrets, osd_id, fsid, tmpfs): """ :param block: The name of the logical volume for the bluestore data :param wal: a regular/plain disk or logical volume, to be used for block.wal @@ -34,6 +34,8 @@ def prepare_bluestore(block, secrets, tmpfs): prepare_utils.osd_mkfs_bluestore( osd_id, fsid, keyring=cephx_secret, + wal=wal, + db=db ) @@ -71,8 +73,8 @@ class Prepare(object): """ if self.args.cluster_fsid: return self.args.cluster_fsid - else: - return conf.ceph.get('global', 'fsid') + + return conf.ceph.get('global', 'fsid') @decorators.needs_root def prepare(self): @@ -82,11 +84,20 @@ class Prepare(object): if crush_device_class: secrets['crush_device_class'] = crush_device_class tmpfs = not self.args.no_tmpfs + wal = "" + db = "" + if self.args.block_wal: + wal = self.args.block_wal + if self.args.block_db: + db = self.args.block_db + # reuse a given ID if it exists, otherwise create a new ID self.osd_id = prepare_utils.create_id( - osd_fsid, json.dumps(secrets), osd_id=self.args.osd_id) + osd_fsid, json.dumps(secrets)) prepare_bluestore( self.args.data, + wal, + db, secrets, self.osd_id, osd_fsid, @@ -103,16 +114,18 @@ class Prepare(object): Encryption is not supported. - DB and WAL devices are not supported. - ceph-volume raw prepare --bluestore --data {device} + DB and WAL devices are supported. + + ceph-volume raw prepare --bluestore --data {device} --block.db {device} --block.wal {device} + """) parser = create_parser( prog='ceph-volume raw prepare', description=sub_command_help, ) - if len(self.argv) == 0: + if not self.argv: print(sub_command_help) return self.args = parser.parse_args(self.argv) diff --git a/src/ceph-volume/ceph_volume/util/prepare.py b/src/ceph-volume/ceph_volume/util/prepare.py index c258dc4a359af..867e2e2ec1072 100644 --- a/src/ceph-volume/ceph_volume/util/prepare.py +++ b/src/ceph-volume/ceph_volume/util/prepare.py @@ -334,6 +334,30 @@ def _link_device(device, device_type, osd_id): process.run(command) +def _validate_bluestore_device(device, excepted_device_type, osd_uuid): + """ + Validate whether the given device is truly what it is supposed to be + """ + + out, err, ret = process.call(['ceph-bluestore-tool', 'show-label', '--dev', device]) + if err: + terminal.error('ceph-bluestore-tool failed to run. %s'% err) + raise SystemExit(1) + if ret: + terminal.error('no label on %s'% device) + raise SystemExit(1) + oj = json.loads(''.join(out)) + if device not in oj: + terminal.error('%s not in the output of ceph-bluestore-tool, buggy?'% device) + raise SystemExit(1) + current_device_type = oj[device]['description'] + if current_device_type != excepted_device_type: + terminal.error('%s is not a %s device but %s'% (device, excepted_device_type, current_device_type)) + raise SystemExit(1) + current_osd_uuid = oj[device]['osd_uuid'] + if current_osd_uuid != osd_uuid: + terminal.error('device %s is used by another osd %s as %s, should be %s'% (device, current_osd_uuid, current_device_type, osd_uuid)) + raise SystemExit(1) def link_journal(journal_device, osd_id): _link_device(journal_device, 'journal', osd_id) @@ -343,11 +367,13 @@ def link_block(block_device, osd_id): _link_device(block_device, 'block', osd_id) -def link_wal(wal_device, osd_id): +def link_wal(wal_device, osd_id, osd_uuid=None): + _validate_bluestore_device(wal_device, 'bluefs wal', osd_uuid) _link_device(wal_device, 'block.wal', osd_id) -def link_db(db_device, osd_id): +def link_db(db_device, osd_id, osd_uuid=None): + _validate_bluestore_device(db_device, 'bluefs db', osd_uuid) _link_device(db_device, 'block.db', osd_id)