From: Jan Fajerski Date: Wed, 31 Oct 2018 13:59:05 +0000 (+0100) Subject: ceph-volume: add inventory command X-Git-Tag: v12.2.10~10^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=ffd145e4fe1c6fc81e14948b6808d274844b1c2f;p=ceph.git ceph-volume: add inventory command The inventory command provides information about a nodes disk inventory. Existing logical volumes on a disk or one of its partitions are scanned and reported. The output can be formatted as plain text or json. Signed-off-by: Jan Fajerski (cherry picked from commit 57adfc6bb8abd9fc84b1378d5f69a16994d19e11) --- diff --git a/src/ceph-volume/ceph_volume/api/lvm.py b/src/ceph-volume/ceph_volume/api/lvm.py index 2f442a1e779c..f6527e33f595 100644 --- a/src/ceph-volume/ceph_volume/api/lvm.py +++ b/src/ceph-volume/ceph_volume/api/lvm.py @@ -1081,6 +1081,7 @@ class Volume(object): self.name = kw['lv_name'] self.tags = parse_tags(kw['lv_tags']) self.encrypted = self.tags.get('ceph.encrypted', '0') == '1' + self.used_by_ceph = 'ceph.osd_id' in self.tags def __str__(self): return '<%s>' % self.lv_api['lv_path'] @@ -1097,6 +1098,26 @@ class Volume(object): obj['path'] = self.lv_path return obj + def report(self): + if not self.used_by_ceph: + return { + 'name': self.lv_name, + 'comment': 'not used by ceph' + } + else: + type_ = self.tags['ceph.type'] + report = { + 'name': self.lv_name, + 'osd_id': self.tags['ceph.osd_id'], + 'cluster_name': self.tags['ceph.cluster_name'], + 'type': type_, + 'osd_fsid': self.tags['ceph.osd_fsid'], + 'cluster_fsid': self.tags['ceph.cluster_fsid'], + } + type_uuid = '{}_uuid'.format(type_) + report[type_uuid] = self.tags['ceph.{}'.format(type_uuid)] + return report + def clear_tags(self): """ Removes all tags from the Logical Volume. diff --git a/src/ceph-volume/ceph_volume/inventory/__init__.py b/src/ceph-volume/ceph_volume/inventory/__init__.py new file mode 100644 index 000000000000..c9e0c0ccc3ad --- /dev/null +++ b/src/ceph-volume/ceph_volume/inventory/__init__.py @@ -0,0 +1 @@ +from .main import Inventory # noqa diff --git a/src/ceph-volume/ceph_volume/inventory/main.py b/src/ceph-volume/ceph_volume/inventory/main.py new file mode 100644 index 000000000000..f4c732cab72f --- /dev/null +++ b/src/ceph-volume/ceph_volume/inventory/main.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +import argparse +import pprint + +from ceph_volume.util.device import Devices, Device + + +class Inventory(object): + + help = "Get this nodes available disk inventory" + + def __init__(self, argv): + self.argv = argv + + def main(self): + parser = argparse.ArgumentParser( + prog='ceph-volume inventory', + formatter_class=argparse.RawDescriptionHelpFormatter, + description=self.help, + ) + parser.add_argument( + 'path', + nargs='?', + default=None, + help=('Report on specific disk'), + ) + parser.add_argument( + '--format', + choices=['plain', 'json', 'json-pretty'], + default='plain', + help='Output format', + ) + self.args = parser.parse_args(self.argv) + if self.args.path: + self.format_report(Device(self.args.path)) + else: + self.format_report(Devices()) + + def format_report(self, inventory): + if self.args.format == 'json': + print(inventory.json_report()) + elif self.args.format == 'json-pretty': + pprint.pprint(inventory.json_report()) + else: + print(inventory.pretty_report()) diff --git a/src/ceph-volume/ceph_volume/main.py b/src/ceph-volume/ceph_volume/main.py index 47358f2dd341..edd9db37a336 100644 --- a/src/ceph-volume/ceph_volume/main.py +++ b/src/ceph-volume/ceph_volume/main.py @@ -6,7 +6,7 @@ import sys import logging from ceph_volume.decorators import catches -from ceph_volume import log, devices, configuration, conf, exceptions, terminal +from ceph_volume import log, devices, configuration, conf, exceptions, terminal, inventory class Volume(object): @@ -27,6 +27,7 @@ Ceph Conf: {ceph_path} self.mapper = { 'lvm': devices.lvm.LVM, 'simple': devices.simple.Simple, + 'inventory': inventory.Inventory, } self.plugin_help = "No plugins found/loaded" if argv is None: diff --git a/src/ceph-volume/ceph_volume/tests/test_inventory.py b/src/ceph-volume/ceph_volume/tests/test_inventory.py new file mode 100644 index 000000000000..655d613ac1c3 --- /dev/null +++ b/src/ceph-volume/ceph_volume/tests/test_inventory.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- + +import pytest +from ceph_volume.util.device import Devices +from ceph_volume import sys_info + +@pytest.fixture +def device_report_keys(): + report = Devices().json_report()[0] + return list(report.keys()) + +@pytest.fixture +def device_sys_api_keys(): + report = Devices().json_report()[0] + return list(report['sys_api'].keys()) + + +class TestInventory(object): + + # populate sys_info with something; creating a Device instance will use + # this data + sys_info.devices = { + # example output of disk.get_devices() + '/dev/sdb': {'human_readable_size': '1.82 TB', + 'locked': 0, + 'model': 'PERC H700', + 'nr_requests': '128', + 'partitions': {}, + 'path': '/dev/sdb', + 'removable': '0', + 'rev': '2.10', + 'ro': '0', + 'rotational': '1', + 'sas_address': '', + 'sas_device_handle': '', + 'scheduler_mode': 'cfq', + 'sectors': 0, + 'sectorsize': '512', + 'size': 1999844147200.0, + 'support_discard': '', + 'vendor': 'DELL'} + } + + expected_keys = [ + 'path', + 'rejected_reasons', + 'sys_api', + 'valid', + 'lvs', + ] + + expected_sys_api_keys = [ + 'human_readable_size', + 'locked', + 'model', + 'nr_requests', + 'partitions', + 'path', + 'removable', + 'rev', + 'ro', + 'rotational', + 'sas_address', + 'sas_device_handle', + 'scheduler_mode', + 'sectors', + 'sectorsize', + 'size', + 'support_discard', + 'vendor', + ] + + def test_json_inventory_keys_unexpected(self, device_report_keys): + for k in device_report_keys: + assert k in self.expected_keys, "unexpected key {} in report".format(k) + + def test_json_inventory_keys_missing(self, device_report_keys): + for k in self.expected_keys: + assert k in device_report_keys, "expected key {} in report".format(k) + + def test_sys_api_keys_unexpected(self, device_sys_api_keys): + for k in device_sys_api_keys: + assert k in self.expected_sys_api_keys, "unexpected key {} in sys_api field".format(k) + + def test_sys_api_keys_missing(self, device_sys_api_keys): + for k in self.expected_sys_api_keys: + assert k in device_sys_api_keys, "expected key {} in sys_api field".format(k) + diff --git a/src/ceph-volume/ceph_volume/tests/util/test_device.py b/src/ceph-volume/ceph_volume/tests/util/test_device.py index a44be7697dcb..3d646a22c782 100644 --- a/src/ceph-volume/ceph_volume/tests/util/test_device.py +++ b/src/ceph-volume/ceph_volume/tests/util/test_device.py @@ -117,6 +117,21 @@ class TestDevice(object): disk = device.Device("/dev/sda") assert not disk.used_by_ceph + disk1 = device.Device("/dev/sda") + disk2 = device.Device("/dev/sdb") + disk2._valid = False + disk3 = device.Device("/dev/sdc") + disk4 = device.Device("/dev/sdd") + disk4._valid = False + + @pytest.mark.parametrize("diska, diskb", [ + pytest.param(disk1, disk2, id="(_, valid) < (_, invalid)"), + pytest.param(disk1, disk3, id="(sda, valid) < (sdc, valid)"), + pytest.param(disk3, disk2, id="(sdc, valid) < (sdb, invalid)"), + pytest.param(disk2, disk4, id="(sdb, invalid) < (sdd, invalid)"), + ]) + def test_ordering(self, diska, diskb): + assert diska < diskb and diskb > diska ceph_partlabels = [ 'ceph data', 'ceph journal', 'ceph block', diff --git a/src/ceph-volume/ceph_volume/util/device.py b/src/ceph-volume/ceph_volume/util/device.py index 96cf2b12a7c1..d6e83be41a9d 100644 --- a/src/ceph-volume/ceph_volume/util/device.py +++ b/src/ceph-volume/ceph_volume/util/device.py @@ -1,11 +1,68 @@ +# -*- coding: utf-8 -*- + import os +from functools import total_ordering from ceph_volume import sys_info from ceph_volume.api import lvm from ceph_volume.util import disk +report_template = """ +{dev:<25} {size:<12} {rot!s:<7} {valid!s:<7} {model}""" + + +class Devices(object): + """ + A container for Device instances with reporting + """ + + def __init__(self, devices=None): + if not sys_info.devices: + sys_info.devices = disk.get_devices() + self.devices = [Device(k) for k in + sys_info.devices.keys()] + + def pretty_report(self, all=True): + output = [ + report_template.format( + dev='Device Path', + size='Size', + rot='rotates', + model='Model name', + valid='valid', + )] + for device in sorted(self.devices): + output.append(device.report()) + return ''.join(output) + + def json_report(self): + output = [] + for device in sorted(self.devices): + output.append(device.json_report()) + return output +@total_ordering class Device(object): + pretty_template = """ + {attr:<25} {value}""" + + report_fields = [ + '_rejected_reasons', + '_valid', + 'path', + 'sys_api', + ] + pretty_report_sys_fields = [ + 'human_readable_size', + 'model', + 'removable', + 'ro', + 'rotational', + 'sas_address', + 'scheduler_mode', + 'vendor', + ] + def __init__(self, path): self.path = path # LVs can have a vg/lv path, while disks will have /dev/sda @@ -24,7 +81,26 @@ class Device(object): self._parse() self.is_valid + def __lt__(self, other): + ''' + Implementing this method and __eq__ allows the @total_ordering + decorator to turn the Device class into a totally ordered type. + This can slower then implementing all comparison operations. + This sorting should put valid devices before invalid devices and sort + on the path otherwise (str sorting). + ''' + if self._valid == other._valid: + return self.path < other.path + return self._valid and not other._valid + + def __eq__(self, other): + return self.path == other.path + def _parse(self): + if not sys_info.devices: + sys_info.devices = disk.get_devices() + self.sys_api = sys_info.devices.get(self.abspath, {}) + # start with lvm since it can use an absolute or relative path lv = lvm.get_lv_from_argument(self.path) if lv: @@ -41,10 +117,6 @@ class Device(object): if device_type in ['part', 'disk']: self._set_lvm_membership() - if not sys_info.devices: - sys_info.devices = disk.get_devices() - self.sys_api = sys_info.devices.get(self.abspath, {}) - self.ceph_disk = CephDiskDevice(self) def __repr__(self): @@ -57,43 +129,103 @@ class Device(object): prefix = 'Raw Device' return '<%s: %s>' % (prefix, self.abspath) - def _set_lvm_membership(self): - if self._is_lvm_member is None: - # check if there was a pv created with the - # name of device - pvs = lvm.PVolumes() - pvs.filter(pv_name=self.abspath) - if not pvs: - self._is_lvm_member = False - return self._is_lvm_member - has_vgs = [pv.vg_name for pv in pvs if pv.vg_name] - if has_vgs: - # a pv can only be in one vg, so this should be safe - self.vg_name = has_vgs[0] - self._is_lvm_member = True - self.pvs_api = pvs - for pv in pvs: - if pv.vg_name and pv.lv_uuid: - lv = lvm.get_lv(vg_name=pv.vg_name, lv_uuid=pv.lv_uuid) - if lv: - self.lvs.append(lv) + def pretty_report(self): + def format_value(v): + if isinstance(v, list): + return ', '.join(v) else: - # this is contentious, if a PV is recognized by LVM but has no - # VGs, should we consider it as part of LVM? We choose not to - # here, because most likely, we need to use VGs from this PV. - self._is_lvm_member = False + return v + def format_key(k): + return k.strip('_').replace('_', ' ') + output = ['\n====== Device report {} ======\n'.format(self.path)] + output.extend( + [self.pretty_template.format( + attr=format_key(k), + value=format_value(v)) for k, v in vars(self).items() if k in + self.report_fields and k != 'disk_api' and k != 'sys_api'] ) + output.extend( + [self.pretty_template.format( + attr=format_key(k), + value=format_value(v)) for k, v in self.sys_api.items() if k in + self.pretty_report_sys_fields]) + for lv in self.lvs: + output.append(""" + --- Logical Volume ---""") + output.extend( + [self.pretty_template.format( + attr=format_key(k), + value=format_value(v)) for k, v in lv.report().items()]) + return ''.join(output) + def report(self): + return report_template.format( + dev=self.abspath, + size=self.size_human, + rot=self.rotational, + valid=self.is_valid, + model=self.model, + ) + + def json_report(self): + output = {k.strip('_'): v for k, v in vars(self).items() if k in + self.report_fields} + output['lvs'] = [lv.report() for lv in self.lvs] + return output + + def _set_lvm_membership(self): + if self._is_lvm_member is None: + # this is contentious, if a PV is recognized by LVM but has no + # VGs, should we consider it as part of LVM? We choose not to + # here, because most likely, we need to use VGs from this PV. + self._is_lvm_member = False + for path in self._get_pv_paths(): + # check if there was a pv created with the + # name of device + pvs = lvm.PVolumes() + pvs.filter(pv_name=path) + has_vgs = [pv.vg_name for pv in pvs if pv.vg_name] + if has_vgs: + # a pv can only be in one vg, so this should be safe + self.vg_name = has_vgs[0] + self._is_lvm_member = True + self.pvs_api = pvs + for pv in pvs: + if pv.vg_name and pv.lv_uuid: + lv = lvm.get_lv(vg_name=pv.vg_name, lv_uuid=pv.lv_uuid) + if lv: + self.lvs.append(lv) return self._is_lvm_member + def _get_pv_paths(self): + """ + For block devices LVM can reside on the raw block device or on a + partition. Return a list of paths to be checked for a pv. + """ + paths = [self.abspath] + path_dir = os.path.dirname(self.abspath) + for part in self.sys_api.get('partitions', {}).keys(): + paths.append(os.path.join(path_dir, part)) + return paths + @property def exists(self): return os.path.exists(self.abspath) @property def rotational(self): - if self.sys_api['rotational'] == '1': - return True - return False + return self.sys_api['rotational'] == '1' + + @property + def model(self): + return self.sys_api['model'] + + @property + def size_human(self): + return self.sys_api['human_readable_size'] + + @property + def size(self): + return self.sys_api['size'] @property def is_lvm_member(self):