From: Alfredo Deza Date: Tue, 3 Oct 2017 20:14:16 +0000 (-0400) Subject: ceph-volume move lvm/api.py to api/lvm.py so disk can consume it X-Git-Tag: v13.0.1~679^2~6 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=bb72480d2bf44ff04ea93c98f4a4e59032a28896;p=ceph-ci.git ceph-volume move lvm/api.py to api/lvm.py so disk can consume it Signed-off-by: Alfredo Deza --- diff --git a/src/ceph-volume/ceph_volume/api/lvm.py b/src/ceph-volume/ceph_volume/api/lvm.py new file mode 100644 index 00000000000..3a2187ae521 --- /dev/null +++ b/src/ceph-volume/ceph_volume/api/lvm.py @@ -0,0 +1,708 @@ +""" +API for CRUD lvm tag operations. Follows the Ceph LVM tag naming convention +that prefixes tags with ``ceph.`` and uses ``=`` for assignment, and provides +set of utilities for interacting with LVM. +""" +from ceph_volume import process +from ceph_volume.exceptions import MultipleLVsError, MultipleVGsError, MultiplePVsError + + +def _output_parser(output, fields): + """ + Newer versions of LVM allow ``--reportformat=json``, but older versions, + like the one included in Xenial do not. LVM has the ability to filter and + format its output so we assume the output will be in a format this parser + can handle (using ',' as a delimiter) + + :param fields: A string, possibly using ',' to group many items, as it + would be used on the CLI + :param output: The CLI output from the LVM call + """ + field_items = fields.split(',') + report = [] + for line in output: + # clear the leading/trailing whitespace + line = line.strip() + + # remove the extra '"' in each field + line = line.replace('"', '') + + # prevent moving forward with empty contents + if not line: + continue + + # spliting on ';' because that is what the lvm call uses as + # '--separator' + output_items = [i.strip() for i in line.split(';')] + # map the output to the fiels + report.append( + dict(zip(field_items, output_items)) + ) + + return report + + +def parse_tags(lv_tags): + """ + Return a dictionary mapping of all the tags associated with + a Volume from the comma-separated tags coming from the LVM API + + Input look like:: + + "ceph.osd_fsid=aaa-fff-bbbb,ceph.osd_id=0" + + For the above example, the expected return value would be:: + + { + "ceph.osd_fsid": "aaa-fff-bbbb", + "ceph.osd_id": "0" + } + """ + if not lv_tags: + return {} + tag_mapping = {} + tags = lv_tags.split(',') + for tag_assignment in tags: + key, value = tag_assignment.split('=', 1) + tag_mapping[key] = value + + return tag_mapping + + +def get_api_vgs(): + """ + Return the list of group volumes available in the system using flags to + include common metadata associated with them + + Command and sample delimeted output, should look like:: + + $ sudo vgs --noheadings --separator=';' \ + -o vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free + ubuntubox-vg;1;2;0;wz--n-;299.52g;12.00m + osd_vg;3;1;0;wz--n-;29.21g;9.21g + + """ + fields = 'vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free' + stdout, stderr, returncode = process.call( + ['sudo', 'vgs', '--noheadings', '--separator=";"', '-o', fields] + ) + return _output_parser(stdout, fields) + + +def get_api_lvs(): + """ + Return the list of logical volumes available in the system using flags to include common + metadata associated with them + + Command and delimeted output, should look like:: + + $ sudo lvs --noheadings --separator=';' -o lv_tags,lv_path,lv_name,vg_name + ;/dev/ubuntubox-vg/root;root;ubuntubox-vg + ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg + + """ + fields = 'lv_tags,lv_path,lv_name,vg_name,lv_uuid' + stdout, stderr, returncode = process.call( + ['sudo', 'lvs', '--noheadings', '--separator=";"', '-o', fields] + ) + return _output_parser(stdout, fields) + + +def get_api_pvs(): + """ + Return the list of physical volumes configured for lvm and available in the + system using flags to include common metadata associated with them like the uuid + + Command and delimeted output, should look like:: + + $ sudo pvs --noheadings --separator=';' -o pv_name,pv_tags,pv_uuid + /dev/sda1;; + /dev/sdv;;07A4F654-4162-4600-8EB3-88D1E42F368D + + """ + fields = 'pv_name,pv_tags,pv_uuid' + + # note the use of `pvs -a` which will return every physical volume including + # ones that have not been initialized as "pv" by LVM + stdout, stderr, returncode = process.call( + ['sudo', 'pvs', '-a', '--no-heading', '--separator=";"', '-o', fields] + ) + + return _output_parser(stdout, fields) + + +def get_lv_from_argument(argument): + """ + Helper proxy function that consumes a possible logical volume passed in from the CLI + in the form of `vg/lv`, but with some validation so that an argument that is a full + path to a device can be ignored + """ + try: + vg_name, lv_name = argument.split('/') + except (ValueError, AttributeError): + return None + return get_lv(lv_name=lv_name, vg_name=vg_name) + + +def get_lv(lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): + """ + Return a matching lv for the current system, requiring ``lv_name``, + ``vg_name``, ``lv_path`` or ``tags``. Raises an error if more than one lv + is found. + + It is useful to use ``tags`` when trying to find a specific logical volume, + but it can also lead to multiple lvs being found, since a lot of metadata + is shared between lvs of a distinct OSD. + """ + if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): + return None + lvs = Volumes() + return lvs.get( + lv_name=lv_name, vg_name=vg_name, lv_path=lv_path, lv_uuid=lv_uuid, + lv_tags=lv_tags + ) + + +def get_pv(pv_name=None, pv_uuid=None, pv_tags=None): + """ + Return a matching pv (physical volume) for the current system, requiring + ``pv_name``, ``pv_uuid``, or ``pv_tags``. Raises an error if more than one + pv is found. + """ + if not any([pv_name, pv_uuid, pv_tags]): + return None + pvs = PVolumes() + return pvs.get(pv_name=pv_name, pv_uuid=pv_uuid, pv_tags=pv_tags) + + +def create_pv(device): + """ + Create a physical volume from a device, useful when devices need to be later mapped + to journals. + """ + process.run([ + 'sudo', + 'pvcreate', + '-v', # verbose + '-f', # force it + '--yes', # answer yes to any prompts + device + ]) + + +def create_lv(name, group, size=None, **tags): + """ + Create a Logical Volume in a Volume Group. Command looks like:: + + lvcreate -L 50G -n gfslv vg0 + + ``name``, ``group``, and ``size`` are required. Tags are optional and are "translated" to include + the prefixes for the Ceph LVM tag API. + + """ + # XXX add CEPH_VOLUME_LVM_DEBUG to enable -vvvv on lv operations + type_path_tag = { + 'journal': 'ceph.journal_device', + 'data': 'ceph.data_device', + 'block': 'ceph.block', + 'wal': 'ceph.wal', + 'db': 'ceph.db', + 'lockbox': 'ceph.lockbox_device', + } + if size: + process.run([ + 'sudo', + 'lvcreate', + '--yes', + '-L', + '%sG' % size, + '-n', name, group + ]) + # create the lv with all the space available, this is needed because the + # system call is different for LVM + else: + process.run([ + 'sudo', + 'lvcreate', + '--yes', + '-l', + '100%FREE', + '-n', name, group + ]) + + lv = get_lv(lv_name=name, vg_name=group) + ceph_tags = {} + for k, v in tags.items(): + ceph_tags['ceph.%s' % k] = v + lv.set_tags(ceph_tags) + + # when creating a distinct type, the caller doesn't know what the path will + # be so this function will set it after creation using the mapping + path_tag = type_path_tag[tags['type']] + lv.set_tags( + {path_tag: lv.lv_path} + ) + return lv + + +def get_vg(vg_name=None, vg_tags=None): + """ + Return a matching vg for the current system, requires ``vg_name`` or + ``tags``. Raises an error if more than one vg is found. + + It is useful to use ``tags`` when trying to find a specific volume group, + but it can also lead to multiple vgs being found. + """ + if not any([vg_name, vg_tags]): + return None + vgs = VolumeGroups() + return vgs.get(vg_name=vg_name, vg_tags=vg_tags) + + +class VolumeGroups(list): + """ + A list of all known volume groups for the current system, with the ability + to filter them via keyword arguments. + """ + + def __init__(self): + self._populate() + + def _populate(self): + # get all the vgs in the current system + for vg_item in get_api_vgs(): + self.append(VolumeGroup(**vg_item)) + + def _purge(self): + """ + Deplete all the items in the list, used internally only so that we can + dynamically allocate the items when filtering without the concern of + messing up the contents + """ + self[:] = [] + + def _filter(self, vg_name=None, vg_tags=None): + """ + The actual method that filters using a new list. Useful so that other + methods that do not want to alter the contents of the list (e.g. + ``self.find``) can operate safely. + + .. note:: ``vg_tags`` is not yet implemented + """ + filtered = [i for i in self] + if vg_name: + filtered = [i for i in filtered if i.vg_name == vg_name] + + # at this point, `filtered` has either all the volumes in self or is an + # actual filtered list if any filters were applied + if vg_tags: + tag_filtered = [] + for volume in filtered: + matches = all(volume.tags.get(k) == str(v) for k, v in vg_tags.items()) + if matches: + tag_filtered.append(volume) + return tag_filtered + + return filtered + + def filter(self, vg_name=None, vg_tags=None): + """ + Filter out groups on top level attributes like ``vg_name`` or by + ``vg_tags`` where a dict is required. For example, to find a Ceph group + with dmcache as the type, the filter would look like:: + + vg_tags={'ceph.type': 'dmcache'} + + .. warning:: These tags are not documented because they are currently + unused, but are here to maintain API consistency + """ + if not any([vg_name, vg_tags]): + raise TypeError('.filter() requires vg_name or vg_tags (none given)') + # first find the filtered volumes with the values in self + filtered_groups = self._filter( + vg_name=vg_name, + vg_tags=vg_tags + ) + # then purge everything + self._purge() + # and add the filtered items + self.extend(filtered_groups) + + def get(self, vg_name=None, vg_tags=None): + """ + This is a bit expensive, since it will try to filter out all the + matching items in the list, filter them out applying anything that was + added and return the matching item. + + This method does *not* alter the list, and it will raise an error if + multiple VGs are matched + + It is useful to use ``tags`` when trying to find a specific volume group, + but it can also lead to multiple vgs being found (although unlikely) + """ + if not any([vg_name, vg_tags]): + return None + vgs = self._filter( + vg_name=vg_name, + vg_tags=vg_tags + ) + if not vgs: + return None + if len(vgs) > 1: + # this is probably never going to happen, but it is here to keep + # the API code consistent + raise MultipleVGsError(vg_name) + return vgs[0] + + +class Volumes(list): + """ + A list of all known (logical) volumes for the current system, with the ability + to filter them via keyword arguments. + """ + + def __init__(self): + self._populate() + + def _populate(self): + # get all the lvs in the current system + for lv_item in get_api_lvs(): + self.append(Volume(**lv_item)) + + def _purge(self): + """ + Deplete all the items in the list, used internally only so that we can + dynamically allocate the items when filtering without the concern of + messing up the contents + """ + self[:] = [] + + def _filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): + """ + The actual method that filters using a new list. Useful so that other + methods that do not want to alter the contents of the list (e.g. + ``self.find``) can operate safely. + """ + filtered = [i for i in self] + if lv_name: + filtered = [i for i in filtered if i.lv_name == lv_name] + + if vg_name: + filtered = [i for i in filtered if i.vg_name == vg_name] + + if lv_uuid: + filtered = [i for i in filtered if i.lv_uuid == lv_uuid] + + if lv_path: + filtered = [i for i in filtered if i.lv_path == lv_path] + + # at this point, `filtered` has either all the volumes in self or is an + # actual filtered list if any filters were applied + if lv_tags: + tag_filtered = [] + for volume in filtered: + # all the tags we got need to match on the volume + matches = all(volume.tags.get(k) == str(v) for k, v in lv_tags.items()) + if matches: + tag_filtered.append(volume) + return tag_filtered + + return filtered + + def filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): + """ + Filter out volumes on top level attributes like ``lv_name`` or by + ``lv_tags`` where a dict is required. For example, to find a volume + that has an OSD ID of 0, the filter would look like:: + + lv_tags={'ceph.osd_id': '0'} + + """ + if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): + raise TypeError('.filter() requires lv_name, vg_name, lv_path, lv_uuid, or tags (none given)') + # first find the filtered volumes with the values in self + filtered_volumes = self._filter( + lv_name=lv_name, + vg_name=vg_name, + lv_path=lv_path, + lv_uuid=lv_uuid, + lv_tags=lv_tags + ) + # then purge everything + self._purge() + # and add the filtered items + self.extend(filtered_volumes) + + def get(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): + """ + This is a bit expensive, since it will try to filter out all the + matching items in the list, filter them out applying anything that was + added and return the matching item. + + This method does *not* alter the list, and it will raise an error if + multiple LVs are matched + + It is useful to use ``tags`` when trying to find a specific logical volume, + but it can also lead to multiple lvs being found, since a lot of metadata + is shared between lvs of a distinct OSD. + """ + if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): + return None + lvs = self._filter( + lv_name=lv_name, + vg_name=vg_name, + lv_path=lv_path, + lv_uuid=lv_uuid, + lv_tags=lv_tags + ) + if not lvs: + return None + if len(lvs) > 1: + raise MultipleLVsError(lv_name, lv_path) + return lvs[0] + + +class PVolumes(list): + """ + A list of all known (physical) volumes for the current system, with the ability + to filter them via keyword arguments. + """ + + def __init__(self): + self._populate() + + def _populate(self): + # get all the pvs in the current system + for pv_item in get_api_pvs(): + self.append(PVolume(**pv_item)) + + def _purge(self): + """ + Deplete all the items in the list, used internally only so that we can + dynamically allocate the items when filtering without the concern of + messing up the contents + """ + self[:] = [] + + def _filter(self, pv_name=None, pv_uuid=None, pv_tags=None): + """ + The actual method that filters using a new list. Useful so that other + methods that do not want to alter the contents of the list (e.g. + ``self.find``) can operate safely. + """ + filtered = [i for i in self] + if pv_name: + filtered = [i for i in filtered if i.pv_name == pv_name] + + if pv_uuid: + filtered = [i for i in filtered if i.pv_uuid == pv_uuid] + + # at this point, `filtered` has either all the physical volumes in self + # or is an actual filtered list if any filters were applied + if pv_tags: + tag_filtered = [] + for pvolume in filtered: + matches = all(pvolume.tags.get(k) == str(v) for k, v in pv_tags.items()) + if matches: + tag_filtered.append(pvolume) + # return the tag_filtered pvolumes here, the `filtered` list is no + # longer useable + return tag_filtered + + return filtered + + def filter(self, pv_name=None, pv_uuid=None, pv_tags=None): + """ + Filter out volumes on top level attributes like ``pv_name`` or by + ``pv_tags`` where a dict is required. For example, to find a physical volume + that has an OSD ID of 0, the filter would look like:: + + pv_tags={'ceph.osd_id': '0'} + + """ + if not any([pv_name, pv_uuid, pv_tags]): + raise TypeError('.filter() requires pv_name, pv_uuid, or pv_tags (none given)') + # first find the filtered volumes with the values in self + filtered_volumes = self._filter( + pv_name=pv_name, + pv_uuid=pv_uuid, + pv_tags=pv_tags + ) + # then purge everything + self._purge() + # and add the filtered items + self.extend(filtered_volumes) + + def get(self, pv_name=None, pv_uuid=None, pv_tags=None): + """ + This is a bit expensive, since it will try to filter out all the + matching items in the list, filter them out applying anything that was + added and return the matching item. + + This method does *not* alter the list, and it will raise an error if + multiple pvs are matched + + It is useful to use ``tags`` when trying to find a specific logical volume, + but it can also lead to multiple pvs being found, since a lot of metadata + is shared between pvs of a distinct OSD. + """ + if not any([pv_name, pv_uuid, pv_tags]): + return None + pvs = self._filter( + pv_name=pv_name, + pv_uuid=pv_uuid, + pv_tags=pv_tags + ) + if not pvs: + return None + if len(pvs) > 1: + raise MultiplePVsError(pv_name) + return pvs[0] + + +class VolumeGroup(object): + """ + Represents an LVM group, with some top-level attributes like ``vg_name`` + """ + + def __init__(self, **kw): + for k, v in kw.items(): + setattr(self, k, v) + self.name = kw['vg_name'] + self.tags = parse_tags(kw.get('vg_tags', '')) + + def __str__(self): + return '<%s>' % self.name + + def __repr__(self): + return self.__str__() + + +class Volume(object): + """ + Represents a Logical Volume from LVM, with some top-level attributes like + ``lv_name`` and parsed tags as a dictionary of key/value pairs. + """ + + def __init__(self, **kw): + for k, v in kw.items(): + setattr(self, k, v) + self.lv_api = kw + self.name = kw['lv_name'] + self.tags = parse_tags(kw['lv_tags']) + + def __str__(self): + return '<%s>' % self.lv_api['lv_path'] + + def __repr__(self): + return self.__str__() + + def as_dict(self): + obj = {} + obj.update(self.lv_api) + obj['tags'] = self.tags + obj['name'] = self.name + obj['type'] = self.tags['ceph.type'] + obj['path'] = self.lv_path + return obj + + def set_tags(self, tags): + """ + :param tags: A dictionary of tag names and values, like:: + + { + "ceph.osd_fsid": "aaa-fff-bbbb", + "ceph.osd_id": "0" + } + + At the end of all modifications, the tags are refreshed to reflect + LVM's most current view. + """ + for k, v in tags.items(): + self.set_tag(k, v) + # after setting all the tags, refresh them for the current object, use the + # lv_* identifiers to filter because those shouldn't change + lv_object = get_lv(lv_name=self.lv_name, lv_path=self.lv_path) + self.tags = lv_object.tags + + def set_tag(self, key, value): + """ + Set the key/value pair as an LVM tag. Does not "refresh" the values of + the current object for its tags. Meant to be a "fire and forget" type + of modification. + """ + # remove it first if it exists + if self.tags.get(key): + current_value = self.tags[key] + tag = "%s=%s" % (key, current_value) + process.call(['sudo', 'lvchange', '--deltag', tag, self.lv_api['lv_path']]) + + process.call( + [ + 'sudo', 'lvchange', + '--addtag', '%s=%s' % (key, value), self.lv_path + ] + ) + + +class PVolume(object): + """ + Represents a Physical Volume from LVM, with some top-level attributes like + ``pv_name`` and parsed tags as a dictionary of key/value pairs. + """ + + def __init__(self, **kw): + for k, v in kw.items(): + setattr(self, k, v) + self.pv_api = kw + self.name = kw['pv_name'] + self.tags = parse_tags(kw['pv_tags']) + + def __str__(self): + return '<%s>' % self.pv_api['pv_name'] + + def __repr__(self): + return self.__str__() + + def set_tags(self, tags): + """ + :param tags: A dictionary of tag names and values, like:: + + { + "ceph.osd_fsid": "aaa-fff-bbbb", + "ceph.osd_id": "0" + } + + At the end of all modifications, the tags are refreshed to reflect + LVM's most current view. + """ + for k, v in tags.items(): + self.set_tag(k, v) + # after setting all the tags, refresh them for the current object, use the + # pv_* identifiers to filter because those shouldn't change + pv_object = get_pv(pv_name=self.pv_name, pv_uuid=self.pv_uuid) + self.tags = pv_object.tags + + def set_tag(self, key, value): + """ + Set the key/value pair as an LVM tag. Does not "refresh" the values of + the current object for its tags. Meant to be a "fire and forget" type + of modification. + + **warning**: Altering tags on a PV has to be done ensuring that the + device is actually the one intended. ``pv_name`` is *not* a persistent + value, only ``pv_uuid`` is. Using ``pv_uuid`` is the best way to make + sure the device getting changed is the one needed. + """ + # remove it first if it exists + if self.tags.get(key): + current_value = self.tags[key] + tag = "%s=%s" % (key, current_value) + process.call(['sudo', 'pvchange', '--deltag', tag, self.pv_name]) + + process.call( + [ + 'sudo', 'pvchange', + '--addtag', '%s=%s' % (key, value), self.pv_name + ] + ) diff --git a/src/ceph-volume/ceph_volume/devices/lvm/api.py b/src/ceph-volume/ceph_volume/devices/lvm/api.py deleted file mode 100644 index 3a2187ae521..00000000000 --- a/src/ceph-volume/ceph_volume/devices/lvm/api.py +++ /dev/null @@ -1,708 +0,0 @@ -""" -API for CRUD lvm tag operations. Follows the Ceph LVM tag naming convention -that prefixes tags with ``ceph.`` and uses ``=`` for assignment, and provides -set of utilities for interacting with LVM. -""" -from ceph_volume import process -from ceph_volume.exceptions import MultipleLVsError, MultipleVGsError, MultiplePVsError - - -def _output_parser(output, fields): - """ - Newer versions of LVM allow ``--reportformat=json``, but older versions, - like the one included in Xenial do not. LVM has the ability to filter and - format its output so we assume the output will be in a format this parser - can handle (using ',' as a delimiter) - - :param fields: A string, possibly using ',' to group many items, as it - would be used on the CLI - :param output: The CLI output from the LVM call - """ - field_items = fields.split(',') - report = [] - for line in output: - # clear the leading/trailing whitespace - line = line.strip() - - # remove the extra '"' in each field - line = line.replace('"', '') - - # prevent moving forward with empty contents - if not line: - continue - - # spliting on ';' because that is what the lvm call uses as - # '--separator' - output_items = [i.strip() for i in line.split(';')] - # map the output to the fiels - report.append( - dict(zip(field_items, output_items)) - ) - - return report - - -def parse_tags(lv_tags): - """ - Return a dictionary mapping of all the tags associated with - a Volume from the comma-separated tags coming from the LVM API - - Input look like:: - - "ceph.osd_fsid=aaa-fff-bbbb,ceph.osd_id=0" - - For the above example, the expected return value would be:: - - { - "ceph.osd_fsid": "aaa-fff-bbbb", - "ceph.osd_id": "0" - } - """ - if not lv_tags: - return {} - tag_mapping = {} - tags = lv_tags.split(',') - for tag_assignment in tags: - key, value = tag_assignment.split('=', 1) - tag_mapping[key] = value - - return tag_mapping - - -def get_api_vgs(): - """ - Return the list of group volumes available in the system using flags to - include common metadata associated with them - - Command and sample delimeted output, should look like:: - - $ sudo vgs --noheadings --separator=';' \ - -o vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free - ubuntubox-vg;1;2;0;wz--n-;299.52g;12.00m - osd_vg;3;1;0;wz--n-;29.21g;9.21g - - """ - fields = 'vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free' - stdout, stderr, returncode = process.call( - ['sudo', 'vgs', '--noheadings', '--separator=";"', '-o', fields] - ) - return _output_parser(stdout, fields) - - -def get_api_lvs(): - """ - Return the list of logical volumes available in the system using flags to include common - metadata associated with them - - Command and delimeted output, should look like:: - - $ sudo lvs --noheadings --separator=';' -o lv_tags,lv_path,lv_name,vg_name - ;/dev/ubuntubox-vg/root;root;ubuntubox-vg - ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg - - """ - fields = 'lv_tags,lv_path,lv_name,vg_name,lv_uuid' - stdout, stderr, returncode = process.call( - ['sudo', 'lvs', '--noheadings', '--separator=";"', '-o', fields] - ) - return _output_parser(stdout, fields) - - -def get_api_pvs(): - """ - Return the list of physical volumes configured for lvm and available in the - system using flags to include common metadata associated with them like the uuid - - Command and delimeted output, should look like:: - - $ sudo pvs --noheadings --separator=';' -o pv_name,pv_tags,pv_uuid - /dev/sda1;; - /dev/sdv;;07A4F654-4162-4600-8EB3-88D1E42F368D - - """ - fields = 'pv_name,pv_tags,pv_uuid' - - # note the use of `pvs -a` which will return every physical volume including - # ones that have not been initialized as "pv" by LVM - stdout, stderr, returncode = process.call( - ['sudo', 'pvs', '-a', '--no-heading', '--separator=";"', '-o', fields] - ) - - return _output_parser(stdout, fields) - - -def get_lv_from_argument(argument): - """ - Helper proxy function that consumes a possible logical volume passed in from the CLI - in the form of `vg/lv`, but with some validation so that an argument that is a full - path to a device can be ignored - """ - try: - vg_name, lv_name = argument.split('/') - except (ValueError, AttributeError): - return None - return get_lv(lv_name=lv_name, vg_name=vg_name) - - -def get_lv(lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): - """ - Return a matching lv for the current system, requiring ``lv_name``, - ``vg_name``, ``lv_path`` or ``tags``. Raises an error if more than one lv - is found. - - It is useful to use ``tags`` when trying to find a specific logical volume, - but it can also lead to multiple lvs being found, since a lot of metadata - is shared between lvs of a distinct OSD. - """ - if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): - return None - lvs = Volumes() - return lvs.get( - lv_name=lv_name, vg_name=vg_name, lv_path=lv_path, lv_uuid=lv_uuid, - lv_tags=lv_tags - ) - - -def get_pv(pv_name=None, pv_uuid=None, pv_tags=None): - """ - Return a matching pv (physical volume) for the current system, requiring - ``pv_name``, ``pv_uuid``, or ``pv_tags``. Raises an error if more than one - pv is found. - """ - if not any([pv_name, pv_uuid, pv_tags]): - return None - pvs = PVolumes() - return pvs.get(pv_name=pv_name, pv_uuid=pv_uuid, pv_tags=pv_tags) - - -def create_pv(device): - """ - Create a physical volume from a device, useful when devices need to be later mapped - to journals. - """ - process.run([ - 'sudo', - 'pvcreate', - '-v', # verbose - '-f', # force it - '--yes', # answer yes to any prompts - device - ]) - - -def create_lv(name, group, size=None, **tags): - """ - Create a Logical Volume in a Volume Group. Command looks like:: - - lvcreate -L 50G -n gfslv vg0 - - ``name``, ``group``, and ``size`` are required. Tags are optional and are "translated" to include - the prefixes for the Ceph LVM tag API. - - """ - # XXX add CEPH_VOLUME_LVM_DEBUG to enable -vvvv on lv operations - type_path_tag = { - 'journal': 'ceph.journal_device', - 'data': 'ceph.data_device', - 'block': 'ceph.block', - 'wal': 'ceph.wal', - 'db': 'ceph.db', - 'lockbox': 'ceph.lockbox_device', - } - if size: - process.run([ - 'sudo', - 'lvcreate', - '--yes', - '-L', - '%sG' % size, - '-n', name, group - ]) - # create the lv with all the space available, this is needed because the - # system call is different for LVM - else: - process.run([ - 'sudo', - 'lvcreate', - '--yes', - '-l', - '100%FREE', - '-n', name, group - ]) - - lv = get_lv(lv_name=name, vg_name=group) - ceph_tags = {} - for k, v in tags.items(): - ceph_tags['ceph.%s' % k] = v - lv.set_tags(ceph_tags) - - # when creating a distinct type, the caller doesn't know what the path will - # be so this function will set it after creation using the mapping - path_tag = type_path_tag[tags['type']] - lv.set_tags( - {path_tag: lv.lv_path} - ) - return lv - - -def get_vg(vg_name=None, vg_tags=None): - """ - Return a matching vg for the current system, requires ``vg_name`` or - ``tags``. Raises an error if more than one vg is found. - - It is useful to use ``tags`` when trying to find a specific volume group, - but it can also lead to multiple vgs being found. - """ - if not any([vg_name, vg_tags]): - return None - vgs = VolumeGroups() - return vgs.get(vg_name=vg_name, vg_tags=vg_tags) - - -class VolumeGroups(list): - """ - A list of all known volume groups for the current system, with the ability - to filter them via keyword arguments. - """ - - def __init__(self): - self._populate() - - def _populate(self): - # get all the vgs in the current system - for vg_item in get_api_vgs(): - self.append(VolumeGroup(**vg_item)) - - def _purge(self): - """ - Deplete all the items in the list, used internally only so that we can - dynamically allocate the items when filtering without the concern of - messing up the contents - """ - self[:] = [] - - def _filter(self, vg_name=None, vg_tags=None): - """ - The actual method that filters using a new list. Useful so that other - methods that do not want to alter the contents of the list (e.g. - ``self.find``) can operate safely. - - .. note:: ``vg_tags`` is not yet implemented - """ - filtered = [i for i in self] - if vg_name: - filtered = [i for i in filtered if i.vg_name == vg_name] - - # at this point, `filtered` has either all the volumes in self or is an - # actual filtered list if any filters were applied - if vg_tags: - tag_filtered = [] - for volume in filtered: - matches = all(volume.tags.get(k) == str(v) for k, v in vg_tags.items()) - if matches: - tag_filtered.append(volume) - return tag_filtered - - return filtered - - def filter(self, vg_name=None, vg_tags=None): - """ - Filter out groups on top level attributes like ``vg_name`` or by - ``vg_tags`` where a dict is required. For example, to find a Ceph group - with dmcache as the type, the filter would look like:: - - vg_tags={'ceph.type': 'dmcache'} - - .. warning:: These tags are not documented because they are currently - unused, but are here to maintain API consistency - """ - if not any([vg_name, vg_tags]): - raise TypeError('.filter() requires vg_name or vg_tags (none given)') - # first find the filtered volumes with the values in self - filtered_groups = self._filter( - vg_name=vg_name, - vg_tags=vg_tags - ) - # then purge everything - self._purge() - # and add the filtered items - self.extend(filtered_groups) - - def get(self, vg_name=None, vg_tags=None): - """ - This is a bit expensive, since it will try to filter out all the - matching items in the list, filter them out applying anything that was - added and return the matching item. - - This method does *not* alter the list, and it will raise an error if - multiple VGs are matched - - It is useful to use ``tags`` when trying to find a specific volume group, - but it can also lead to multiple vgs being found (although unlikely) - """ - if not any([vg_name, vg_tags]): - return None - vgs = self._filter( - vg_name=vg_name, - vg_tags=vg_tags - ) - if not vgs: - return None - if len(vgs) > 1: - # this is probably never going to happen, but it is here to keep - # the API code consistent - raise MultipleVGsError(vg_name) - return vgs[0] - - -class Volumes(list): - """ - A list of all known (logical) volumes for the current system, with the ability - to filter them via keyword arguments. - """ - - def __init__(self): - self._populate() - - def _populate(self): - # get all the lvs in the current system - for lv_item in get_api_lvs(): - self.append(Volume(**lv_item)) - - def _purge(self): - """ - Deplete all the items in the list, used internally only so that we can - dynamically allocate the items when filtering without the concern of - messing up the contents - """ - self[:] = [] - - def _filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): - """ - The actual method that filters using a new list. Useful so that other - methods that do not want to alter the contents of the list (e.g. - ``self.find``) can operate safely. - """ - filtered = [i for i in self] - if lv_name: - filtered = [i for i in filtered if i.lv_name == lv_name] - - if vg_name: - filtered = [i for i in filtered if i.vg_name == vg_name] - - if lv_uuid: - filtered = [i for i in filtered if i.lv_uuid == lv_uuid] - - if lv_path: - filtered = [i for i in filtered if i.lv_path == lv_path] - - # at this point, `filtered` has either all the volumes in self or is an - # actual filtered list if any filters were applied - if lv_tags: - tag_filtered = [] - for volume in filtered: - # all the tags we got need to match on the volume - matches = all(volume.tags.get(k) == str(v) for k, v in lv_tags.items()) - if matches: - tag_filtered.append(volume) - return tag_filtered - - return filtered - - def filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): - """ - Filter out volumes on top level attributes like ``lv_name`` or by - ``lv_tags`` where a dict is required. For example, to find a volume - that has an OSD ID of 0, the filter would look like:: - - lv_tags={'ceph.osd_id': '0'} - - """ - if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): - raise TypeError('.filter() requires lv_name, vg_name, lv_path, lv_uuid, or tags (none given)') - # first find the filtered volumes with the values in self - filtered_volumes = self._filter( - lv_name=lv_name, - vg_name=vg_name, - lv_path=lv_path, - lv_uuid=lv_uuid, - lv_tags=lv_tags - ) - # then purge everything - self._purge() - # and add the filtered items - self.extend(filtered_volumes) - - def get(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): - """ - This is a bit expensive, since it will try to filter out all the - matching items in the list, filter them out applying anything that was - added and return the matching item. - - This method does *not* alter the list, and it will raise an error if - multiple LVs are matched - - It is useful to use ``tags`` when trying to find a specific logical volume, - but it can also lead to multiple lvs being found, since a lot of metadata - is shared between lvs of a distinct OSD. - """ - if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): - return None - lvs = self._filter( - lv_name=lv_name, - vg_name=vg_name, - lv_path=lv_path, - lv_uuid=lv_uuid, - lv_tags=lv_tags - ) - if not lvs: - return None - if len(lvs) > 1: - raise MultipleLVsError(lv_name, lv_path) - return lvs[0] - - -class PVolumes(list): - """ - A list of all known (physical) volumes for the current system, with the ability - to filter them via keyword arguments. - """ - - def __init__(self): - self._populate() - - def _populate(self): - # get all the pvs in the current system - for pv_item in get_api_pvs(): - self.append(PVolume(**pv_item)) - - def _purge(self): - """ - Deplete all the items in the list, used internally only so that we can - dynamically allocate the items when filtering without the concern of - messing up the contents - """ - self[:] = [] - - def _filter(self, pv_name=None, pv_uuid=None, pv_tags=None): - """ - The actual method that filters using a new list. Useful so that other - methods that do not want to alter the contents of the list (e.g. - ``self.find``) can operate safely. - """ - filtered = [i for i in self] - if pv_name: - filtered = [i for i in filtered if i.pv_name == pv_name] - - if pv_uuid: - filtered = [i for i in filtered if i.pv_uuid == pv_uuid] - - # at this point, `filtered` has either all the physical volumes in self - # or is an actual filtered list if any filters were applied - if pv_tags: - tag_filtered = [] - for pvolume in filtered: - matches = all(pvolume.tags.get(k) == str(v) for k, v in pv_tags.items()) - if matches: - tag_filtered.append(pvolume) - # return the tag_filtered pvolumes here, the `filtered` list is no - # longer useable - return tag_filtered - - return filtered - - def filter(self, pv_name=None, pv_uuid=None, pv_tags=None): - """ - Filter out volumes on top level attributes like ``pv_name`` or by - ``pv_tags`` where a dict is required. For example, to find a physical volume - that has an OSD ID of 0, the filter would look like:: - - pv_tags={'ceph.osd_id': '0'} - - """ - if not any([pv_name, pv_uuid, pv_tags]): - raise TypeError('.filter() requires pv_name, pv_uuid, or pv_tags (none given)') - # first find the filtered volumes with the values in self - filtered_volumes = self._filter( - pv_name=pv_name, - pv_uuid=pv_uuid, - pv_tags=pv_tags - ) - # then purge everything - self._purge() - # and add the filtered items - self.extend(filtered_volumes) - - def get(self, pv_name=None, pv_uuid=None, pv_tags=None): - """ - This is a bit expensive, since it will try to filter out all the - matching items in the list, filter them out applying anything that was - added and return the matching item. - - This method does *not* alter the list, and it will raise an error if - multiple pvs are matched - - It is useful to use ``tags`` when trying to find a specific logical volume, - but it can also lead to multiple pvs being found, since a lot of metadata - is shared between pvs of a distinct OSD. - """ - if not any([pv_name, pv_uuid, pv_tags]): - return None - pvs = self._filter( - pv_name=pv_name, - pv_uuid=pv_uuid, - pv_tags=pv_tags - ) - if not pvs: - return None - if len(pvs) > 1: - raise MultiplePVsError(pv_name) - return pvs[0] - - -class VolumeGroup(object): - """ - Represents an LVM group, with some top-level attributes like ``vg_name`` - """ - - def __init__(self, **kw): - for k, v in kw.items(): - setattr(self, k, v) - self.name = kw['vg_name'] - self.tags = parse_tags(kw.get('vg_tags', '')) - - def __str__(self): - return '<%s>' % self.name - - def __repr__(self): - return self.__str__() - - -class Volume(object): - """ - Represents a Logical Volume from LVM, with some top-level attributes like - ``lv_name`` and parsed tags as a dictionary of key/value pairs. - """ - - def __init__(self, **kw): - for k, v in kw.items(): - setattr(self, k, v) - self.lv_api = kw - self.name = kw['lv_name'] - self.tags = parse_tags(kw['lv_tags']) - - def __str__(self): - return '<%s>' % self.lv_api['lv_path'] - - def __repr__(self): - return self.__str__() - - def as_dict(self): - obj = {} - obj.update(self.lv_api) - obj['tags'] = self.tags - obj['name'] = self.name - obj['type'] = self.tags['ceph.type'] - obj['path'] = self.lv_path - return obj - - def set_tags(self, tags): - """ - :param tags: A dictionary of tag names and values, like:: - - { - "ceph.osd_fsid": "aaa-fff-bbbb", - "ceph.osd_id": "0" - } - - At the end of all modifications, the tags are refreshed to reflect - LVM's most current view. - """ - for k, v in tags.items(): - self.set_tag(k, v) - # after setting all the tags, refresh them for the current object, use the - # lv_* identifiers to filter because those shouldn't change - lv_object = get_lv(lv_name=self.lv_name, lv_path=self.lv_path) - self.tags = lv_object.tags - - def set_tag(self, key, value): - """ - Set the key/value pair as an LVM tag. Does not "refresh" the values of - the current object for its tags. Meant to be a "fire and forget" type - of modification. - """ - # remove it first if it exists - if self.tags.get(key): - current_value = self.tags[key] - tag = "%s=%s" % (key, current_value) - process.call(['sudo', 'lvchange', '--deltag', tag, self.lv_api['lv_path']]) - - process.call( - [ - 'sudo', 'lvchange', - '--addtag', '%s=%s' % (key, value), self.lv_path - ] - ) - - -class PVolume(object): - """ - Represents a Physical Volume from LVM, with some top-level attributes like - ``pv_name`` and parsed tags as a dictionary of key/value pairs. - """ - - def __init__(self, **kw): - for k, v in kw.items(): - setattr(self, k, v) - self.pv_api = kw - self.name = kw['pv_name'] - self.tags = parse_tags(kw['pv_tags']) - - def __str__(self): - return '<%s>' % self.pv_api['pv_name'] - - def __repr__(self): - return self.__str__() - - def set_tags(self, tags): - """ - :param tags: A dictionary of tag names and values, like:: - - { - "ceph.osd_fsid": "aaa-fff-bbbb", - "ceph.osd_id": "0" - } - - At the end of all modifications, the tags are refreshed to reflect - LVM's most current view. - """ - for k, v in tags.items(): - self.set_tag(k, v) - # after setting all the tags, refresh them for the current object, use the - # pv_* identifiers to filter because those shouldn't change - pv_object = get_pv(pv_name=self.pv_name, pv_uuid=self.pv_uuid) - self.tags = pv_object.tags - - def set_tag(self, key, value): - """ - Set the key/value pair as an LVM tag. Does not "refresh" the values of - the current object for its tags. Meant to be a "fire and forget" type - of modification. - - **warning**: Altering tags on a PV has to be done ensuring that the - device is actually the one intended. ``pv_name`` is *not* a persistent - value, only ``pv_uuid`` is. Using ``pv_uuid`` is the best way to make - sure the device getting changed is the one needed. - """ - # remove it first if it exists - if self.tags.get(key): - current_value = self.tags[key] - tag = "%s=%s" % (key, current_value) - process.call(['sudo', 'pvchange', '--deltag', tag, self.pv_name]) - - process.call( - [ - 'sudo', 'pvchange', - '--addtag', '%s=%s' % (key, value), self.pv_name - ] - )