ceph-volume: add db and wal support to raw mode

Using the raw mode, the OSD preparation can now be called with extra
arguments to specify a block device for either rocksdb db or wal.

The code expects the device to be available and no validation will be
performed. Also, the entire block device will be consumed.

The goal is run this on Kubernetes with Rook and OSD are running on PVC.
Users will request PVC from a certain size, the request will be
acknowledged and a block will be created, then attached to the machine.
Later on consumed by Rook's preparation job.

Signed-off-by: Sébastien Han <seb@redhat.com>
(cherry picked from commit c8f23c7182e2681f2bc756776a890995aca1efd2)

diff --git a/src/ceph-volume/ceph_volume/devices/raw/activate.py b/src/ceph-volume/ceph_volume/devices/raw/activate.py
index df6606093c5d7ec99564ba423686b5ffe134e0d2..94f54f1a20b31070c8bfab64485740ebaba69e2b 100644 (file)
--- a/src/ceph-volume/ceph_volume/devices/raw/activate.py
+++ b/src/ceph-volume/ceph_volume/devices/raw/activate.py
@@ -11,9 +11,10 @@ from .list import direct_report
 
 logger = logging.getLogger(__name__)
 
-def activate_bluestore(meta, tmpfs, systemd):
+def activate_bluestore(meta, tmpfs, systemd, block_wal=None, block_db=None):
     # find the osd
     osd_id = meta['osd_id']
+    osd_uuid = meta['osd_uuid']
 
     # mount on tmpfs the osd directory
     osd_path = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id)
@@ -42,10 +43,15 @@ def activate_bluestore(meta, tmpfs, systemd):
     # always re-do the symlink regardless if it exists, so that the block,
     # block.wal, and block.db devices that may have changed can be mapped
     # correctly every time
-    process.run(['ln', '-snf', meta['device'], os.path.join(osd_path, 'block')])
-    system.chown(os.path.join(osd_path, 'block'))
-    system.chown(osd_path)
+    prepare_utils.link_block( meta['device'], osd_id)
+
+    if block_wal:
+        prepare_utils.link_wal(block_wal, osd_id, osd_uuid)
+
+    if block_db:
+        prepare_utils.link_db(block_db, osd_id, osd_uuid)
 
+    system.chown(osd_path)
     terminal.success("ceph-volume raw activate successful for osd ID: %s" % osd_id)
 
 
@@ -58,7 +64,7 @@ class Activate(object):
         self.args = None
 
     @decorators.needs_root
-    def activate(self, devices, tmpfs, systemd):
+    def activate(self, devices, tmpfs, systemd, block_wal, block_db):
         """
         :param args: The parsed arguments coming from the CLI
         """
@@ -67,10 +73,12 @@ class Activate(object):
 
         for osd_id, meta in found.items():
             logger.info('Activating osd.%s uuid %s cluster %s' % (
-                    osd_id, meta['osd_uuid'], meta['ceph_fsid']))
+                        osd_id, meta['osd_uuid'], meta['ceph_fsid']))
             activate_bluestore(meta,
                                tmpfs=tmpfs,
-                               systemd=systemd)
+                               systemd=systemd,
+                               block_wal=block_wal,
+                               block_db=block_db)
 
     def main(self):
         sub_command_help = dedent("""
@@ -87,22 +95,34 @@ class Activate(object):
             formatter_class=argparse.RawDescriptionHelpFormatter,
             description=sub_command_help,
         )
-
         parser.add_argument(
             '--device',
             nargs='+',
-            help='The device(s) for the OSD to start')
+            help='The device for the OSD to start'
+            )
         parser.add_argument(
             '--no-systemd',
             dest='no_systemd',
             action='store_true',
-            help='Skip creating and enabling systemd units and starting OSD services',
+            help='Skip creating and enabling systemd units and starting OSD services'
+        )
+        parser.add_argument(
+            '--block.db',
+            dest='block_db',
+            help='Path to bluestore block.db block device'
+        )
+        parser.add_argument(
+            '--block.wal',
+            dest='block_wal',
+            help='Path to bluestore block.wal block device'
         )
         parser.add_argument(
             '--no-tmpfs',
             action='store_true',
-            help='Do not use a tmpfs mount for OSD data dir')
-        if len(self.argv) == 0:
+            help='Do not use a tmpfs mount for OSD data dir'
+            )
+
+        if not self.argv:
             print(sub_command_help)
             return
         args = parser.parse_args(self.argv)
@@ -112,4 +132,6 @@ class Activate(object):
             raise SystemExit(1)
         self.activate(args.device,
                       tmpfs=not args.no_tmpfs,
-                      systemd=not self.args.no_systemd)
+                      systemd=not self.args.no_systemd,
+                      block_wal=self.args.block_wal,
+                      block_db=self.args.block_db)

diff --git a/src/ceph-volume/ceph_volume/devices/raw/common.py b/src/ceph-volume/ceph_volume/devices/raw/common.py
index ba1dae00f16efedddd632f849d131d8462d4cc34..d34a2941d1684cc70ec1e5ab9ebf5dc4588a30ad 100644 (file)
--- a/src/ceph-volume/ceph_volume/devices/raw/common.py
+++ b/src/ceph-volume/ceph_volume/devices/raw/common.py
@@ -33,5 +33,16 @@ def create_parser(prog, description):
     parser.add_argument(
         '--no-tmpfs',
         action='store_true',
-        help='Do not use a tmpfs mount for OSD data dir')
+        help='Do not use a tmpfs mount for OSD data dir'
+    )
+    parser.add_argument(
+        '--block.db',
+        dest='block_db',
+        help='Path to bluestore block.db block device'
+    )
+    parser.add_argument(
+        '--block.wal',
+        dest='block_wal',
+        help='Path to bluestore block.wal block device'
+    )
     return parser

diff --git a/src/ceph-volume/ceph_volume/devices/raw/prepare.py b/src/ceph-volume/ceph_volume/devices/raw/prepare.py
index 54e82c99202b3794acca7c5d1abc2c563968fa29..cb5c59ce40f5315946a9b1d0d2e3686793a7ec07 100644 (file)
--- a/src/ceph-volume/ceph_volume/devices/raw/prepare.py
+++ b/src/ceph-volume/ceph_volume/devices/raw/prepare.py
@@ -11,7 +11,7 @@ from .common import create_parser
 logger = logging.getLogger(__name__)
 
 
-def prepare_bluestore(block, secrets, tmpfs):
+def prepare_bluestore(block, wal, db, secrets, osd_id, fsid, tmpfs):
     """
     :param block: The name of the logical volume for the bluestore data
     :param wal: a regular/plain disk or logical volume, to be used for block.wal
@@ -34,6 +34,8 @@ def prepare_bluestore(block, secrets, tmpfs):
     prepare_utils.osd_mkfs_bluestore(
         osd_id, fsid,
         keyring=cephx_secret,
+        wal=wal,
+        db=db
     )
 
 
@@ -71,8 +73,8 @@ class Prepare(object):
         """
         if self.args.cluster_fsid:
             return self.args.cluster_fsid
-        else:
-            return conf.ceph.get('global', 'fsid')
+
+        return conf.ceph.get('global', 'fsid')
 
     @decorators.needs_root
     def prepare(self):
@@ -82,11 +84,20 @@ class Prepare(object):
         if crush_device_class:
             secrets['crush_device_class'] = crush_device_class
         tmpfs = not self.args.no_tmpfs
+        wal = ""
+        db = ""
+        if self.args.block_wal:
+            wal = self.args.block_wal
+        if self.args.block_db:
+            db = self.args.block_db
+
         # reuse a given ID if it exists, otherwise create a new ID
         self.osd_id = prepare_utils.create_id(
-            osd_fsid, json.dumps(secrets), osd_id=self.args.osd_id)
+            osd_fsid, json.dumps(secrets))
         prepare_bluestore(
             self.args.data,
+            wal,
+            db,
             secrets,
             self.osd_id,
             osd_fsid,
@@ -103,16 +114,18 @@ class Prepare(object):
 
         Encryption is not supported.
 
-        DB and WAL devices are not supported.
-
             ceph-volume raw prepare --bluestore --data {device}
 
+        DB and WAL devices are supported.
+
+            ceph-volume raw prepare --bluestore --data {device} --block.db {device} --block.wal {device}
+
         """)
         parser = create_parser(
             prog='ceph-volume raw prepare',
             description=sub_command_help,
         )
-        if len(self.argv) == 0:
+        if not self.argv:
             print(sub_command_help)
             return
         self.args = parser.parse_args(self.argv)

diff --git a/src/ceph-volume/ceph_volume/util/prepare.py b/src/ceph-volume/ceph_volume/util/prepare.py
index 027b4c081842bf29d78316ac1c31ba0b78f798e9..3afd1763fdf23de89bfe77a3d21b65bc4efa68b6 100644 (file)
--- a/src/ceph-volume/ceph_volume/util/prepare.py
+++ b/src/ceph-volume/ceph_volume/util/prepare.py
@@ -334,6 +334,30 @@ def _link_device(device, device_type, osd_id):
 
     process.run(command)
 
+def _validate_bluestore_device(device, excepted_device_type, osd_uuid):
+    """
+    Validate whether the given device is truly what it is supposed to be
+    """
+
+    out, err, ret = process.call(['ceph-bluestore-tool', 'show-label', '--dev', device])
+    if err:
+        terminal.error('ceph-bluestore-tool failed to run. %s'% err)
+        raise SystemExit(1)
+    if ret:
+        terminal.error('no label on %s'% device)
+        raise SystemExit(1)
+    oj = json.loads(''.join(out))
+    if device not in oj:
+        terminal.error('%s not in the output of ceph-bluestore-tool, buggy?'% device)
+        raise SystemExit(1)
+    current_device_type = oj[device]['description']
+    if current_device_type != excepted_device_type:
+        terminal.error('%s is not a %s device but %s'% (device, excepted_device_type, current_device_type))
+        raise SystemExit(1)
+    current_osd_uuid = oj[device]['osd_uuid']
+    if current_osd_uuid != osd_uuid:
+        terminal.error('device %s is used by another osd %s as %s, should be %s'% (device, current_osd_uuid, current_device_type, osd_uuid))
+        raise SystemExit(1)
 
 def link_journal(journal_device, osd_id):
     _link_device(journal_device, 'journal', osd_id)
@@ -343,11 +367,13 @@ def link_block(block_device, osd_id):
     _link_device(block_device, 'block', osd_id)
 
 
-def link_wal(wal_device, osd_id):
+def link_wal(wal_device, osd_id, osd_uuid=None):
+    _validate_bluestore_device(wal_device, 'bluefs wal', osd_uuid)
     _link_device(wal_device, 'block.wal', osd_id)
 
 
-def link_db(db_device, osd_id):
+def link_db(db_device, osd_id, osd_uuid=None):
+    _validate_bluestore_device(db_device, 'bluefs db', osd_uuid)
     _link_device(db_device, 'block.db', osd_id)