--- /dev/null
+"""
+API for CRUD lvm tag operations. Follows the Ceph LVM tag naming convention
+that prefixes tags with ``ceph.`` and uses ``=`` for assignment, and provides
+set of utilities for interacting with LVM.
+"""
+from ceph_volume import process
+from ceph_volume.exceptions import MultipleLVsError, MultipleVGsError, MultiplePVsError
+
+
+def _output_parser(output, fields):
+ """
+ Newer versions of LVM allow ``--reportformat=json``, but older versions,
+ like the one included in Xenial do not. LVM has the ability to filter and
+ format its output so we assume the output will be in a format this parser
+ can handle (using ',' as a delimiter)
+
+ :param fields: A string, possibly using ',' to group many items, as it
+ would be used on the CLI
+ :param output: The CLI output from the LVM call
+ """
+ field_items = fields.split(',')
+ report = []
+ for line in output:
+ # clear the leading/trailing whitespace
+ line = line.strip()
+
+ # remove the extra '"' in each field
+ line = line.replace('"', '')
+
+ # prevent moving forward with empty contents
+ if not line:
+ continue
+
+ # spliting on ';' because that is what the lvm call uses as
+ # '--separator'
+ output_items = [i.strip() for i in line.split(';')]
+ # map the output to the fiels
+ report.append(
+ dict(zip(field_items, output_items))
+ )
+
+ return report
+
+
+def parse_tags(lv_tags):
+ """
+ Return a dictionary mapping of all the tags associated with
+ a Volume from the comma-separated tags coming from the LVM API
+
+ Input look like::
+
+ "ceph.osd_fsid=aaa-fff-bbbb,ceph.osd_id=0"
+
+ For the above example, the expected return value would be::
+
+ {
+ "ceph.osd_fsid": "aaa-fff-bbbb",
+ "ceph.osd_id": "0"
+ }
+ """
+ if not lv_tags:
+ return {}
+ tag_mapping = {}
+ tags = lv_tags.split(',')
+ for tag_assignment in tags:
+ key, value = tag_assignment.split('=', 1)
+ tag_mapping[key] = value
+
+ return tag_mapping
+
+
+def get_api_vgs():
+ """
+ Return the list of group volumes available in the system using flags to
+ include common metadata associated with them
+
+ Command and sample delimeted output, should look like::
+
+ $ sudo vgs --noheadings --separator=';' \
+ -o vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free
+ ubuntubox-vg;1;2;0;wz--n-;299.52g;12.00m
+ osd_vg;3;1;0;wz--n-;29.21g;9.21g
+
+ """
+ fields = 'vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free'
+ stdout, stderr, returncode = process.call(
+ ['sudo', 'vgs', '--noheadings', '--separator=";"', '-o', fields]
+ )
+ return _output_parser(stdout, fields)
+
+
+def get_api_lvs():
+ """
+ Return the list of logical volumes available in the system using flags to include common
+ metadata associated with them
+
+ Command and delimeted output, should look like::
+
+ $ sudo lvs --noheadings --separator=';' -o lv_tags,lv_path,lv_name,vg_name
+ ;/dev/ubuntubox-vg/root;root;ubuntubox-vg
+ ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg
+
+ """
+ fields = 'lv_tags,lv_path,lv_name,vg_name,lv_uuid'
+ stdout, stderr, returncode = process.call(
+ ['sudo', 'lvs', '--noheadings', '--separator=";"', '-o', fields]
+ )
+ return _output_parser(stdout, fields)
+
+
+def get_api_pvs():
+ """
+ Return the list of physical volumes configured for lvm and available in the
+ system using flags to include common metadata associated with them like the uuid
+
+ Command and delimeted output, should look like::
+
+ $ sudo pvs --noheadings --separator=';' -o pv_name,pv_tags,pv_uuid
+ /dev/sda1;;
+ /dev/sdv;;07A4F654-4162-4600-8EB3-88D1E42F368D
+
+ """
+ fields = 'pv_name,pv_tags,pv_uuid'
+
+ # note the use of `pvs -a` which will return every physical volume including
+ # ones that have not been initialized as "pv" by LVM
+ stdout, stderr, returncode = process.call(
+ ['sudo', 'pvs', '-a', '--no-heading', '--separator=";"', '-o', fields]
+ )
+
+ return _output_parser(stdout, fields)
+
+
+def get_lv_from_argument(argument):
+ """
+ Helper proxy function that consumes a possible logical volume passed in from the CLI
+ in the form of `vg/lv`, but with some validation so that an argument that is a full
+ path to a device can be ignored
+ """
+ try:
+ vg_name, lv_name = argument.split('/')
+ except (ValueError, AttributeError):
+ return None
+ return get_lv(lv_name=lv_name, vg_name=vg_name)
+
+
+def get_lv(lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
+ """
+ Return a matching lv for the current system, requiring ``lv_name``,
+ ``vg_name``, ``lv_path`` or ``tags``. Raises an error if more than one lv
+ is found.
+
+ It is useful to use ``tags`` when trying to find a specific logical volume,
+ but it can also lead to multiple lvs being found, since a lot of metadata
+ is shared between lvs of a distinct OSD.
+ """
+ if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
+ return None
+ lvs = Volumes()
+ return lvs.get(
+ lv_name=lv_name, vg_name=vg_name, lv_path=lv_path, lv_uuid=lv_uuid,
+ lv_tags=lv_tags
+ )
+
+
+def get_pv(pv_name=None, pv_uuid=None, pv_tags=None):
+ """
+ Return a matching pv (physical volume) for the current system, requiring
+ ``pv_name``, ``pv_uuid``, or ``pv_tags``. Raises an error if more than one
+ pv is found.
+ """
+ if not any([pv_name, pv_uuid, pv_tags]):
+ return None
+ pvs = PVolumes()
+ return pvs.get(pv_name=pv_name, pv_uuid=pv_uuid, pv_tags=pv_tags)
+
+
+def create_pv(device):
+ """
+ Create a physical volume from a device, useful when devices need to be later mapped
+ to journals.
+ """
+ process.run([
+ 'sudo',
+ 'pvcreate',
+ '-v', # verbose
+ '-f', # force it
+ '--yes', # answer yes to any prompts
+ device
+ ])
+
+
+def create_lv(name, group, size=None, **tags):
+ """
+ Create a Logical Volume in a Volume Group. Command looks like::
+
+ lvcreate -L 50G -n gfslv vg0
+
+ ``name``, ``group``, and ``size`` are required. Tags are optional and are "translated" to include
+ the prefixes for the Ceph LVM tag API.
+
+ """
+ # XXX add CEPH_VOLUME_LVM_DEBUG to enable -vvvv on lv operations
+ type_path_tag = {
+ 'journal': 'ceph.journal_device',
+ 'data': 'ceph.data_device',
+ 'block': 'ceph.block',
+ 'wal': 'ceph.wal',
+ 'db': 'ceph.db',
+ 'lockbox': 'ceph.lockbox_device',
+ }
+ if size:
+ process.run([
+ 'sudo',
+ 'lvcreate',
+ '--yes',
+ '-L',
+ '%sG' % size,
+ '-n', name, group
+ ])
+ # create the lv with all the space available, this is needed because the
+ # system call is different for LVM
+ else:
+ process.run([
+ 'sudo',
+ 'lvcreate',
+ '--yes',
+ '-l',
+ '100%FREE',
+ '-n', name, group
+ ])
+
+ lv = get_lv(lv_name=name, vg_name=group)
+ ceph_tags = {}
+ for k, v in tags.items():
+ ceph_tags['ceph.%s' % k] = v
+ lv.set_tags(ceph_tags)
+
+ # when creating a distinct type, the caller doesn't know what the path will
+ # be so this function will set it after creation using the mapping
+ path_tag = type_path_tag[tags['type']]
+ lv.set_tags(
+ {path_tag: lv.lv_path}
+ )
+ return lv
+
+
+def get_vg(vg_name=None, vg_tags=None):
+ """
+ Return a matching vg for the current system, requires ``vg_name`` or
+ ``tags``. Raises an error if more than one vg is found.
+
+ It is useful to use ``tags`` when trying to find a specific volume group,
+ but it can also lead to multiple vgs being found.
+ """
+ if not any([vg_name, vg_tags]):
+ return None
+ vgs = VolumeGroups()
+ return vgs.get(vg_name=vg_name, vg_tags=vg_tags)
+
+
+class VolumeGroups(list):
+ """
+ A list of all known volume groups for the current system, with the ability
+ to filter them via keyword arguments.
+ """
+
+ def __init__(self):
+ self._populate()
+
+ def _populate(self):
+ # get all the vgs in the current system
+ for vg_item in get_api_vgs():
+ self.append(VolumeGroup(**vg_item))
+
+ def _purge(self):
+ """
+ Deplete all the items in the list, used internally only so that we can
+ dynamically allocate the items when filtering without the concern of
+ messing up the contents
+ """
+ self[:] = []
+
+ def _filter(self, vg_name=None, vg_tags=None):
+ """
+ The actual method that filters using a new list. Useful so that other
+ methods that do not want to alter the contents of the list (e.g.
+ ``self.find``) can operate safely.
+
+ .. note:: ``vg_tags`` is not yet implemented
+ """
+ filtered = [i for i in self]
+ if vg_name:
+ filtered = [i for i in filtered if i.vg_name == vg_name]
+
+ # at this point, `filtered` has either all the volumes in self or is an
+ # actual filtered list if any filters were applied
+ if vg_tags:
+ tag_filtered = []
+ for volume in filtered:
+ matches = all(volume.tags.get(k) == str(v) for k, v in vg_tags.items())
+ if matches:
+ tag_filtered.append(volume)
+ return tag_filtered
+
+ return filtered
+
+ def filter(self, vg_name=None, vg_tags=None):
+ """
+ Filter out groups on top level attributes like ``vg_name`` or by
+ ``vg_tags`` where a dict is required. For example, to find a Ceph group
+ with dmcache as the type, the filter would look like::
+
+ vg_tags={'ceph.type': 'dmcache'}
+
+ .. warning:: These tags are not documented because they are currently
+ unused, but are here to maintain API consistency
+ """
+ if not any([vg_name, vg_tags]):
+ raise TypeError('.filter() requires vg_name or vg_tags (none given)')
+ # first find the filtered volumes with the values in self
+ filtered_groups = self._filter(
+ vg_name=vg_name,
+ vg_tags=vg_tags
+ )
+ # then purge everything
+ self._purge()
+ # and add the filtered items
+ self.extend(filtered_groups)
+
+ def get(self, vg_name=None, vg_tags=None):
+ """
+ This is a bit expensive, since it will try to filter out all the
+ matching items in the list, filter them out applying anything that was
+ added and return the matching item.
+
+ This method does *not* alter the list, and it will raise an error if
+ multiple VGs are matched
+
+ It is useful to use ``tags`` when trying to find a specific volume group,
+ but it can also lead to multiple vgs being found (although unlikely)
+ """
+ if not any([vg_name, vg_tags]):
+ return None
+ vgs = self._filter(
+ vg_name=vg_name,
+ vg_tags=vg_tags
+ )
+ if not vgs:
+ return None
+ if len(vgs) > 1:
+ # this is probably never going to happen, but it is here to keep
+ # the API code consistent
+ raise MultipleVGsError(vg_name)
+ return vgs[0]
+
+
+class Volumes(list):
+ """
+ A list of all known (logical) volumes for the current system, with the ability
+ to filter them via keyword arguments.
+ """
+
+ def __init__(self):
+ self._populate()
+
+ def _populate(self):
+ # get all the lvs in the current system
+ for lv_item in get_api_lvs():
+ self.append(Volume(**lv_item))
+
+ def _purge(self):
+ """
+ Deplete all the items in the list, used internally only so that we can
+ dynamically allocate the items when filtering without the concern of
+ messing up the contents
+ """
+ self[:] = []
+
+ def _filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
+ """
+ The actual method that filters using a new list. Useful so that other
+ methods that do not want to alter the contents of the list (e.g.
+ ``self.find``) can operate safely.
+ """
+ filtered = [i for i in self]
+ if lv_name:
+ filtered = [i for i in filtered if i.lv_name == lv_name]
+
+ if vg_name:
+ filtered = [i for i in filtered if i.vg_name == vg_name]
+
+ if lv_uuid:
+ filtered = [i for i in filtered if i.lv_uuid == lv_uuid]
+
+ if lv_path:
+ filtered = [i for i in filtered if i.lv_path == lv_path]
+
+ # at this point, `filtered` has either all the volumes in self or is an
+ # actual filtered list if any filters were applied
+ if lv_tags:
+ tag_filtered = []
+ for volume in filtered:
+ # all the tags we got need to match on the volume
+ matches = all(volume.tags.get(k) == str(v) for k, v in lv_tags.items())
+ if matches:
+ tag_filtered.append(volume)
+ return tag_filtered
+
+ return filtered
+
+ def filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
+ """
+ Filter out volumes on top level attributes like ``lv_name`` or by
+ ``lv_tags`` where a dict is required. For example, to find a volume
+ that has an OSD ID of 0, the filter would look like::
+
+ lv_tags={'ceph.osd_id': '0'}
+
+ """
+ if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
+ raise TypeError('.filter() requires lv_name, vg_name, lv_path, lv_uuid, or tags (none given)')
+ # first find the filtered volumes with the values in self
+ filtered_volumes = self._filter(
+ lv_name=lv_name,
+ vg_name=vg_name,
+ lv_path=lv_path,
+ lv_uuid=lv_uuid,
+ lv_tags=lv_tags
+ )
+ # then purge everything
+ self._purge()
+ # and add the filtered items
+ self.extend(filtered_volumes)
+
+ def get(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
+ """
+ This is a bit expensive, since it will try to filter out all the
+ matching items in the list, filter them out applying anything that was
+ added and return the matching item.
+
+ This method does *not* alter the list, and it will raise an error if
+ multiple LVs are matched
+
+ It is useful to use ``tags`` when trying to find a specific logical volume,
+ but it can also lead to multiple lvs being found, since a lot of metadata
+ is shared between lvs of a distinct OSD.
+ """
+ if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
+ return None
+ lvs = self._filter(
+ lv_name=lv_name,
+ vg_name=vg_name,
+ lv_path=lv_path,
+ lv_uuid=lv_uuid,
+ lv_tags=lv_tags
+ )
+ if not lvs:
+ return None
+ if len(lvs) > 1:
+ raise MultipleLVsError(lv_name, lv_path)
+ return lvs[0]
+
+
+class PVolumes(list):
+ """
+ A list of all known (physical) volumes for the current system, with the ability
+ to filter them via keyword arguments.
+ """
+
+ def __init__(self):
+ self._populate()
+
+ def _populate(self):
+ # get all the pvs in the current system
+ for pv_item in get_api_pvs():
+ self.append(PVolume(**pv_item))
+
+ def _purge(self):
+ """
+ Deplete all the items in the list, used internally only so that we can
+ dynamically allocate the items when filtering without the concern of
+ messing up the contents
+ """
+ self[:] = []
+
+ def _filter(self, pv_name=None, pv_uuid=None, pv_tags=None):
+ """
+ The actual method that filters using a new list. Useful so that other
+ methods that do not want to alter the contents of the list (e.g.
+ ``self.find``) can operate safely.
+ """
+ filtered = [i for i in self]
+ if pv_name:
+ filtered = [i for i in filtered if i.pv_name == pv_name]
+
+ if pv_uuid:
+ filtered = [i for i in filtered if i.pv_uuid == pv_uuid]
+
+ # at this point, `filtered` has either all the physical volumes in self
+ # or is an actual filtered list if any filters were applied
+ if pv_tags:
+ tag_filtered = []
+ for pvolume in filtered:
+ matches = all(pvolume.tags.get(k) == str(v) for k, v in pv_tags.items())
+ if matches:
+ tag_filtered.append(pvolume)
+ # return the tag_filtered pvolumes here, the `filtered` list is no
+ # longer useable
+ return tag_filtered
+
+ return filtered
+
+ def filter(self, pv_name=None, pv_uuid=None, pv_tags=None):
+ """
+ Filter out volumes on top level attributes like ``pv_name`` or by
+ ``pv_tags`` where a dict is required. For example, to find a physical volume
+ that has an OSD ID of 0, the filter would look like::
+
+ pv_tags={'ceph.osd_id': '0'}
+
+ """
+ if not any([pv_name, pv_uuid, pv_tags]):
+ raise TypeError('.filter() requires pv_name, pv_uuid, or pv_tags (none given)')
+ # first find the filtered volumes with the values in self
+ filtered_volumes = self._filter(
+ pv_name=pv_name,
+ pv_uuid=pv_uuid,
+ pv_tags=pv_tags
+ )
+ # then purge everything
+ self._purge()
+ # and add the filtered items
+ self.extend(filtered_volumes)
+
+ def get(self, pv_name=None, pv_uuid=None, pv_tags=None):
+ """
+ This is a bit expensive, since it will try to filter out all the
+ matching items in the list, filter them out applying anything that was
+ added and return the matching item.
+
+ This method does *not* alter the list, and it will raise an error if
+ multiple pvs are matched
+
+ It is useful to use ``tags`` when trying to find a specific logical volume,
+ but it can also lead to multiple pvs being found, since a lot of metadata
+ is shared between pvs of a distinct OSD.
+ """
+ if not any([pv_name, pv_uuid, pv_tags]):
+ return None
+ pvs = self._filter(
+ pv_name=pv_name,
+ pv_uuid=pv_uuid,
+ pv_tags=pv_tags
+ )
+ if not pvs:
+ return None
+ if len(pvs) > 1:
+ raise MultiplePVsError(pv_name)
+ return pvs[0]
+
+
+class VolumeGroup(object):
+ """
+ Represents an LVM group, with some top-level attributes like ``vg_name``
+ """
+
+ def __init__(self, **kw):
+ for k, v in kw.items():
+ setattr(self, k, v)
+ self.name = kw['vg_name']
+ self.tags = parse_tags(kw.get('vg_tags', ''))
+
+ def __str__(self):
+ return '<%s>' % self.name
+
+ def __repr__(self):
+ return self.__str__()
+
+
+class Volume(object):
+ """
+ Represents a Logical Volume from LVM, with some top-level attributes like
+ ``lv_name`` and parsed tags as a dictionary of key/value pairs.
+ """
+
+ def __init__(self, **kw):
+ for k, v in kw.items():
+ setattr(self, k, v)
+ self.lv_api = kw
+ self.name = kw['lv_name']
+ self.tags = parse_tags(kw['lv_tags'])
+
+ def __str__(self):
+ return '<%s>' % self.lv_api['lv_path']
+
+ def __repr__(self):
+ return self.__str__()
+
+ def as_dict(self):
+ obj = {}
+ obj.update(self.lv_api)
+ obj['tags'] = self.tags
+ obj['name'] = self.name
+ obj['type'] = self.tags['ceph.type']
+ obj['path'] = self.lv_path
+ return obj
+
+ def set_tags(self, tags):
+ """
+ :param tags: A dictionary of tag names and values, like::
+
+ {
+ "ceph.osd_fsid": "aaa-fff-bbbb",
+ "ceph.osd_id": "0"
+ }
+
+ At the end of all modifications, the tags are refreshed to reflect
+ LVM's most current view.
+ """
+ for k, v in tags.items():
+ self.set_tag(k, v)
+ # after setting all the tags, refresh them for the current object, use the
+ # lv_* identifiers to filter because those shouldn't change
+ lv_object = get_lv(lv_name=self.lv_name, lv_path=self.lv_path)
+ self.tags = lv_object.tags
+
+ def set_tag(self, key, value):
+ """
+ Set the key/value pair as an LVM tag. Does not "refresh" the values of
+ the current object for its tags. Meant to be a "fire and forget" type
+ of modification.
+ """
+ # remove it first if it exists
+ if self.tags.get(key):
+ current_value = self.tags[key]
+ tag = "%s=%s" % (key, current_value)
+ process.call(['sudo', 'lvchange', '--deltag', tag, self.lv_api['lv_path']])
+
+ process.call(
+ [
+ 'sudo', 'lvchange',
+ '--addtag', '%s=%s' % (key, value), self.lv_path
+ ]
+ )
+
+
+class PVolume(object):
+ """
+ Represents a Physical Volume from LVM, with some top-level attributes like
+ ``pv_name`` and parsed tags as a dictionary of key/value pairs.
+ """
+
+ def __init__(self, **kw):
+ for k, v in kw.items():
+ setattr(self, k, v)
+ self.pv_api = kw
+ self.name = kw['pv_name']
+ self.tags = parse_tags(kw['pv_tags'])
+
+ def __str__(self):
+ return '<%s>' % self.pv_api['pv_name']
+
+ def __repr__(self):
+ return self.__str__()
+
+ def set_tags(self, tags):
+ """
+ :param tags: A dictionary of tag names and values, like::
+
+ {
+ "ceph.osd_fsid": "aaa-fff-bbbb",
+ "ceph.osd_id": "0"
+ }
+
+ At the end of all modifications, the tags are refreshed to reflect
+ LVM's most current view.
+ """
+ for k, v in tags.items():
+ self.set_tag(k, v)
+ # after setting all the tags, refresh them for the current object, use the
+ # pv_* identifiers to filter because those shouldn't change
+ pv_object = get_pv(pv_name=self.pv_name, pv_uuid=self.pv_uuid)
+ self.tags = pv_object.tags
+
+ def set_tag(self, key, value):
+ """
+ Set the key/value pair as an LVM tag. Does not "refresh" the values of
+ the current object for its tags. Meant to be a "fire and forget" type
+ of modification.
+
+ **warning**: Altering tags on a PV has to be done ensuring that the
+ device is actually the one intended. ``pv_name`` is *not* a persistent
+ value, only ``pv_uuid`` is. Using ``pv_uuid`` is the best way to make
+ sure the device getting changed is the one needed.
+ """
+ # remove it first if it exists
+ if self.tags.get(key):
+ current_value = self.tags[key]
+ tag = "%s=%s" % (key, current_value)
+ process.call(['sudo', 'pvchange', '--deltag', tag, self.pv_name])
+
+ process.call(
+ [
+ 'sudo', 'pvchange',
+ '--addtag', '%s=%s' % (key, value), self.pv_name
+ ]
+ )
+++ /dev/null
-"""
-API for CRUD lvm tag operations. Follows the Ceph LVM tag naming convention
-that prefixes tags with ``ceph.`` and uses ``=`` for assignment, and provides
-set of utilities for interacting with LVM.
-"""
-from ceph_volume import process
-from ceph_volume.exceptions import MultipleLVsError, MultipleVGsError, MultiplePVsError
-
-
-def _output_parser(output, fields):
- """
- Newer versions of LVM allow ``--reportformat=json``, but older versions,
- like the one included in Xenial do not. LVM has the ability to filter and
- format its output so we assume the output will be in a format this parser
- can handle (using ',' as a delimiter)
-
- :param fields: A string, possibly using ',' to group many items, as it
- would be used on the CLI
- :param output: The CLI output from the LVM call
- """
- field_items = fields.split(',')
- report = []
- for line in output:
- # clear the leading/trailing whitespace
- line = line.strip()
-
- # remove the extra '"' in each field
- line = line.replace('"', '')
-
- # prevent moving forward with empty contents
- if not line:
- continue
-
- # spliting on ';' because that is what the lvm call uses as
- # '--separator'
- output_items = [i.strip() for i in line.split(';')]
- # map the output to the fiels
- report.append(
- dict(zip(field_items, output_items))
- )
-
- return report
-
-
-def parse_tags(lv_tags):
- """
- Return a dictionary mapping of all the tags associated with
- a Volume from the comma-separated tags coming from the LVM API
-
- Input look like::
-
- "ceph.osd_fsid=aaa-fff-bbbb,ceph.osd_id=0"
-
- For the above example, the expected return value would be::
-
- {
- "ceph.osd_fsid": "aaa-fff-bbbb",
- "ceph.osd_id": "0"
- }
- """
- if not lv_tags:
- return {}
- tag_mapping = {}
- tags = lv_tags.split(',')
- for tag_assignment in tags:
- key, value = tag_assignment.split('=', 1)
- tag_mapping[key] = value
-
- return tag_mapping
-
-
-def get_api_vgs():
- """
- Return the list of group volumes available in the system using flags to
- include common metadata associated with them
-
- Command and sample delimeted output, should look like::
-
- $ sudo vgs --noheadings --separator=';' \
- -o vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free
- ubuntubox-vg;1;2;0;wz--n-;299.52g;12.00m
- osd_vg;3;1;0;wz--n-;29.21g;9.21g
-
- """
- fields = 'vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free'
- stdout, stderr, returncode = process.call(
- ['sudo', 'vgs', '--noheadings', '--separator=";"', '-o', fields]
- )
- return _output_parser(stdout, fields)
-
-
-def get_api_lvs():
- """
- Return the list of logical volumes available in the system using flags to include common
- metadata associated with them
-
- Command and delimeted output, should look like::
-
- $ sudo lvs --noheadings --separator=';' -o lv_tags,lv_path,lv_name,vg_name
- ;/dev/ubuntubox-vg/root;root;ubuntubox-vg
- ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg
-
- """
- fields = 'lv_tags,lv_path,lv_name,vg_name,lv_uuid'
- stdout, stderr, returncode = process.call(
- ['sudo', 'lvs', '--noheadings', '--separator=";"', '-o', fields]
- )
- return _output_parser(stdout, fields)
-
-
-def get_api_pvs():
- """
- Return the list of physical volumes configured for lvm and available in the
- system using flags to include common metadata associated with them like the uuid
-
- Command and delimeted output, should look like::
-
- $ sudo pvs --noheadings --separator=';' -o pv_name,pv_tags,pv_uuid
- /dev/sda1;;
- /dev/sdv;;07A4F654-4162-4600-8EB3-88D1E42F368D
-
- """
- fields = 'pv_name,pv_tags,pv_uuid'
-
- # note the use of `pvs -a` which will return every physical volume including
- # ones that have not been initialized as "pv" by LVM
- stdout, stderr, returncode = process.call(
- ['sudo', 'pvs', '-a', '--no-heading', '--separator=";"', '-o', fields]
- )
-
- return _output_parser(stdout, fields)
-
-
-def get_lv_from_argument(argument):
- """
- Helper proxy function that consumes a possible logical volume passed in from the CLI
- in the form of `vg/lv`, but with some validation so that an argument that is a full
- path to a device can be ignored
- """
- try:
- vg_name, lv_name = argument.split('/')
- except (ValueError, AttributeError):
- return None
- return get_lv(lv_name=lv_name, vg_name=vg_name)
-
-
-def get_lv(lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
- """
- Return a matching lv for the current system, requiring ``lv_name``,
- ``vg_name``, ``lv_path`` or ``tags``. Raises an error if more than one lv
- is found.
-
- It is useful to use ``tags`` when trying to find a specific logical volume,
- but it can also lead to multiple lvs being found, since a lot of metadata
- is shared between lvs of a distinct OSD.
- """
- if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
- return None
- lvs = Volumes()
- return lvs.get(
- lv_name=lv_name, vg_name=vg_name, lv_path=lv_path, lv_uuid=lv_uuid,
- lv_tags=lv_tags
- )
-
-
-def get_pv(pv_name=None, pv_uuid=None, pv_tags=None):
- """
- Return a matching pv (physical volume) for the current system, requiring
- ``pv_name``, ``pv_uuid``, or ``pv_tags``. Raises an error if more than one
- pv is found.
- """
- if not any([pv_name, pv_uuid, pv_tags]):
- return None
- pvs = PVolumes()
- return pvs.get(pv_name=pv_name, pv_uuid=pv_uuid, pv_tags=pv_tags)
-
-
-def create_pv(device):
- """
- Create a physical volume from a device, useful when devices need to be later mapped
- to journals.
- """
- process.run([
- 'sudo',
- 'pvcreate',
- '-v', # verbose
- '-f', # force it
- '--yes', # answer yes to any prompts
- device
- ])
-
-
-def create_lv(name, group, size=None, **tags):
- """
- Create a Logical Volume in a Volume Group. Command looks like::
-
- lvcreate -L 50G -n gfslv vg0
-
- ``name``, ``group``, and ``size`` are required. Tags are optional and are "translated" to include
- the prefixes for the Ceph LVM tag API.
-
- """
- # XXX add CEPH_VOLUME_LVM_DEBUG to enable -vvvv on lv operations
- type_path_tag = {
- 'journal': 'ceph.journal_device',
- 'data': 'ceph.data_device',
- 'block': 'ceph.block',
- 'wal': 'ceph.wal',
- 'db': 'ceph.db',
- 'lockbox': 'ceph.lockbox_device',
- }
- if size:
- process.run([
- 'sudo',
- 'lvcreate',
- '--yes',
- '-L',
- '%sG' % size,
- '-n', name, group
- ])
- # create the lv with all the space available, this is needed because the
- # system call is different for LVM
- else:
- process.run([
- 'sudo',
- 'lvcreate',
- '--yes',
- '-l',
- '100%FREE',
- '-n', name, group
- ])
-
- lv = get_lv(lv_name=name, vg_name=group)
- ceph_tags = {}
- for k, v in tags.items():
- ceph_tags['ceph.%s' % k] = v
- lv.set_tags(ceph_tags)
-
- # when creating a distinct type, the caller doesn't know what the path will
- # be so this function will set it after creation using the mapping
- path_tag = type_path_tag[tags['type']]
- lv.set_tags(
- {path_tag: lv.lv_path}
- )
- return lv
-
-
-def get_vg(vg_name=None, vg_tags=None):
- """
- Return a matching vg for the current system, requires ``vg_name`` or
- ``tags``. Raises an error if more than one vg is found.
-
- It is useful to use ``tags`` when trying to find a specific volume group,
- but it can also lead to multiple vgs being found.
- """
- if not any([vg_name, vg_tags]):
- return None
- vgs = VolumeGroups()
- return vgs.get(vg_name=vg_name, vg_tags=vg_tags)
-
-
-class VolumeGroups(list):
- """
- A list of all known volume groups for the current system, with the ability
- to filter them via keyword arguments.
- """
-
- def __init__(self):
- self._populate()
-
- def _populate(self):
- # get all the vgs in the current system
- for vg_item in get_api_vgs():
- self.append(VolumeGroup(**vg_item))
-
- def _purge(self):
- """
- Deplete all the items in the list, used internally only so that we can
- dynamically allocate the items when filtering without the concern of
- messing up the contents
- """
- self[:] = []
-
- def _filter(self, vg_name=None, vg_tags=None):
- """
- The actual method that filters using a new list. Useful so that other
- methods that do not want to alter the contents of the list (e.g.
- ``self.find``) can operate safely.
-
- .. note:: ``vg_tags`` is not yet implemented
- """
- filtered = [i for i in self]
- if vg_name:
- filtered = [i for i in filtered if i.vg_name == vg_name]
-
- # at this point, `filtered` has either all the volumes in self or is an
- # actual filtered list if any filters were applied
- if vg_tags:
- tag_filtered = []
- for volume in filtered:
- matches = all(volume.tags.get(k) == str(v) for k, v in vg_tags.items())
- if matches:
- tag_filtered.append(volume)
- return tag_filtered
-
- return filtered
-
- def filter(self, vg_name=None, vg_tags=None):
- """
- Filter out groups on top level attributes like ``vg_name`` or by
- ``vg_tags`` where a dict is required. For example, to find a Ceph group
- with dmcache as the type, the filter would look like::
-
- vg_tags={'ceph.type': 'dmcache'}
-
- .. warning:: These tags are not documented because they are currently
- unused, but are here to maintain API consistency
- """
- if not any([vg_name, vg_tags]):
- raise TypeError('.filter() requires vg_name or vg_tags (none given)')
- # first find the filtered volumes with the values in self
- filtered_groups = self._filter(
- vg_name=vg_name,
- vg_tags=vg_tags
- )
- # then purge everything
- self._purge()
- # and add the filtered items
- self.extend(filtered_groups)
-
- def get(self, vg_name=None, vg_tags=None):
- """
- This is a bit expensive, since it will try to filter out all the
- matching items in the list, filter them out applying anything that was
- added and return the matching item.
-
- This method does *not* alter the list, and it will raise an error if
- multiple VGs are matched
-
- It is useful to use ``tags`` when trying to find a specific volume group,
- but it can also lead to multiple vgs being found (although unlikely)
- """
- if not any([vg_name, vg_tags]):
- return None
- vgs = self._filter(
- vg_name=vg_name,
- vg_tags=vg_tags
- )
- if not vgs:
- return None
- if len(vgs) > 1:
- # this is probably never going to happen, but it is here to keep
- # the API code consistent
- raise MultipleVGsError(vg_name)
- return vgs[0]
-
-
-class Volumes(list):
- """
- A list of all known (logical) volumes for the current system, with the ability
- to filter them via keyword arguments.
- """
-
- def __init__(self):
- self._populate()
-
- def _populate(self):
- # get all the lvs in the current system
- for lv_item in get_api_lvs():
- self.append(Volume(**lv_item))
-
- def _purge(self):
- """
- Deplete all the items in the list, used internally only so that we can
- dynamically allocate the items when filtering without the concern of
- messing up the contents
- """
- self[:] = []
-
- def _filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
- """
- The actual method that filters using a new list. Useful so that other
- methods that do not want to alter the contents of the list (e.g.
- ``self.find``) can operate safely.
- """
- filtered = [i for i in self]
- if lv_name:
- filtered = [i for i in filtered if i.lv_name == lv_name]
-
- if vg_name:
- filtered = [i for i in filtered if i.vg_name == vg_name]
-
- if lv_uuid:
- filtered = [i for i in filtered if i.lv_uuid == lv_uuid]
-
- if lv_path:
- filtered = [i for i in filtered if i.lv_path == lv_path]
-
- # at this point, `filtered` has either all the volumes in self or is an
- # actual filtered list if any filters were applied
- if lv_tags:
- tag_filtered = []
- for volume in filtered:
- # all the tags we got need to match on the volume
- matches = all(volume.tags.get(k) == str(v) for k, v in lv_tags.items())
- if matches:
- tag_filtered.append(volume)
- return tag_filtered
-
- return filtered
-
- def filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
- """
- Filter out volumes on top level attributes like ``lv_name`` or by
- ``lv_tags`` where a dict is required. For example, to find a volume
- that has an OSD ID of 0, the filter would look like::
-
- lv_tags={'ceph.osd_id': '0'}
-
- """
- if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
- raise TypeError('.filter() requires lv_name, vg_name, lv_path, lv_uuid, or tags (none given)')
- # first find the filtered volumes with the values in self
- filtered_volumes = self._filter(
- lv_name=lv_name,
- vg_name=vg_name,
- lv_path=lv_path,
- lv_uuid=lv_uuid,
- lv_tags=lv_tags
- )
- # then purge everything
- self._purge()
- # and add the filtered items
- self.extend(filtered_volumes)
-
- def get(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
- """
- This is a bit expensive, since it will try to filter out all the
- matching items in the list, filter them out applying anything that was
- added and return the matching item.
-
- This method does *not* alter the list, and it will raise an error if
- multiple LVs are matched
-
- It is useful to use ``tags`` when trying to find a specific logical volume,
- but it can also lead to multiple lvs being found, since a lot of metadata
- is shared between lvs of a distinct OSD.
- """
- if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
- return None
- lvs = self._filter(
- lv_name=lv_name,
- vg_name=vg_name,
- lv_path=lv_path,
- lv_uuid=lv_uuid,
- lv_tags=lv_tags
- )
- if not lvs:
- return None
- if len(lvs) > 1:
- raise MultipleLVsError(lv_name, lv_path)
- return lvs[0]
-
-
-class PVolumes(list):
- """
- A list of all known (physical) volumes for the current system, with the ability
- to filter them via keyword arguments.
- """
-
- def __init__(self):
- self._populate()
-
- def _populate(self):
- # get all the pvs in the current system
- for pv_item in get_api_pvs():
- self.append(PVolume(**pv_item))
-
- def _purge(self):
- """
- Deplete all the items in the list, used internally only so that we can
- dynamically allocate the items when filtering without the concern of
- messing up the contents
- """
- self[:] = []
-
- def _filter(self, pv_name=None, pv_uuid=None, pv_tags=None):
- """
- The actual method that filters using a new list. Useful so that other
- methods that do not want to alter the contents of the list (e.g.
- ``self.find``) can operate safely.
- """
- filtered = [i for i in self]
- if pv_name:
- filtered = [i for i in filtered if i.pv_name == pv_name]
-
- if pv_uuid:
- filtered = [i for i in filtered if i.pv_uuid == pv_uuid]
-
- # at this point, `filtered` has either all the physical volumes in self
- # or is an actual filtered list if any filters were applied
- if pv_tags:
- tag_filtered = []
- for pvolume in filtered:
- matches = all(pvolume.tags.get(k) == str(v) for k, v in pv_tags.items())
- if matches:
- tag_filtered.append(pvolume)
- # return the tag_filtered pvolumes here, the `filtered` list is no
- # longer useable
- return tag_filtered
-
- return filtered
-
- def filter(self, pv_name=None, pv_uuid=None, pv_tags=None):
- """
- Filter out volumes on top level attributes like ``pv_name`` or by
- ``pv_tags`` where a dict is required. For example, to find a physical volume
- that has an OSD ID of 0, the filter would look like::
-
- pv_tags={'ceph.osd_id': '0'}
-
- """
- if not any([pv_name, pv_uuid, pv_tags]):
- raise TypeError('.filter() requires pv_name, pv_uuid, or pv_tags (none given)')
- # first find the filtered volumes with the values in self
- filtered_volumes = self._filter(
- pv_name=pv_name,
- pv_uuid=pv_uuid,
- pv_tags=pv_tags
- )
- # then purge everything
- self._purge()
- # and add the filtered items
- self.extend(filtered_volumes)
-
- def get(self, pv_name=None, pv_uuid=None, pv_tags=None):
- """
- This is a bit expensive, since it will try to filter out all the
- matching items in the list, filter them out applying anything that was
- added and return the matching item.
-
- This method does *not* alter the list, and it will raise an error if
- multiple pvs are matched
-
- It is useful to use ``tags`` when trying to find a specific logical volume,
- but it can also lead to multiple pvs being found, since a lot of metadata
- is shared between pvs of a distinct OSD.
- """
- if not any([pv_name, pv_uuid, pv_tags]):
- return None
- pvs = self._filter(
- pv_name=pv_name,
- pv_uuid=pv_uuid,
- pv_tags=pv_tags
- )
- if not pvs:
- return None
- if len(pvs) > 1:
- raise MultiplePVsError(pv_name)
- return pvs[0]
-
-
-class VolumeGroup(object):
- """
- Represents an LVM group, with some top-level attributes like ``vg_name``
- """
-
- def __init__(self, **kw):
- for k, v in kw.items():
- setattr(self, k, v)
- self.name = kw['vg_name']
- self.tags = parse_tags(kw.get('vg_tags', ''))
-
- def __str__(self):
- return '<%s>' % self.name
-
- def __repr__(self):
- return self.__str__()
-
-
-class Volume(object):
- """
- Represents a Logical Volume from LVM, with some top-level attributes like
- ``lv_name`` and parsed tags as a dictionary of key/value pairs.
- """
-
- def __init__(self, **kw):
- for k, v in kw.items():
- setattr(self, k, v)
- self.lv_api = kw
- self.name = kw['lv_name']
- self.tags = parse_tags(kw['lv_tags'])
-
- def __str__(self):
- return '<%s>' % self.lv_api['lv_path']
-
- def __repr__(self):
- return self.__str__()
-
- def as_dict(self):
- obj = {}
- obj.update(self.lv_api)
- obj['tags'] = self.tags
- obj['name'] = self.name
- obj['type'] = self.tags['ceph.type']
- obj['path'] = self.lv_path
- return obj
-
- def set_tags(self, tags):
- """
- :param tags: A dictionary of tag names and values, like::
-
- {
- "ceph.osd_fsid": "aaa-fff-bbbb",
- "ceph.osd_id": "0"
- }
-
- At the end of all modifications, the tags are refreshed to reflect
- LVM's most current view.
- """
- for k, v in tags.items():
- self.set_tag(k, v)
- # after setting all the tags, refresh them for the current object, use the
- # lv_* identifiers to filter because those shouldn't change
- lv_object = get_lv(lv_name=self.lv_name, lv_path=self.lv_path)
- self.tags = lv_object.tags
-
- def set_tag(self, key, value):
- """
- Set the key/value pair as an LVM tag. Does not "refresh" the values of
- the current object for its tags. Meant to be a "fire and forget" type
- of modification.
- """
- # remove it first if it exists
- if self.tags.get(key):
- current_value = self.tags[key]
- tag = "%s=%s" % (key, current_value)
- process.call(['sudo', 'lvchange', '--deltag', tag, self.lv_api['lv_path']])
-
- process.call(
- [
- 'sudo', 'lvchange',
- '--addtag', '%s=%s' % (key, value), self.lv_path
- ]
- )
-
-
-class PVolume(object):
- """
- Represents a Physical Volume from LVM, with some top-level attributes like
- ``pv_name`` and parsed tags as a dictionary of key/value pairs.
- """
-
- def __init__(self, **kw):
- for k, v in kw.items():
- setattr(self, k, v)
- self.pv_api = kw
- self.name = kw['pv_name']
- self.tags = parse_tags(kw['pv_tags'])
-
- def __str__(self):
- return '<%s>' % self.pv_api['pv_name']
-
- def __repr__(self):
- return self.__str__()
-
- def set_tags(self, tags):
- """
- :param tags: A dictionary of tag names and values, like::
-
- {
- "ceph.osd_fsid": "aaa-fff-bbbb",
- "ceph.osd_id": "0"
- }
-
- At the end of all modifications, the tags are refreshed to reflect
- LVM's most current view.
- """
- for k, v in tags.items():
- self.set_tag(k, v)
- # after setting all the tags, refresh them for the current object, use the
- # pv_* identifiers to filter because those shouldn't change
- pv_object = get_pv(pv_name=self.pv_name, pv_uuid=self.pv_uuid)
- self.tags = pv_object.tags
-
- def set_tag(self, key, value):
- """
- Set the key/value pair as an LVM tag. Does not "refresh" the values of
- the current object for its tags. Meant to be a "fire and forget" type
- of modification.
-
- **warning**: Altering tags on a PV has to be done ensuring that the
- device is actually the one intended. ``pv_name`` is *not* a persistent
- value, only ``pv_uuid`` is. Using ``pv_uuid`` is the best way to make
- sure the device getting changed is the one needed.
- """
- # remove it first if it exists
- if self.tags.get(key):
- current_value = self.tags[key]
- tag = "%s=%s" % (key, current_value)
- process.call(['sudo', 'pvchange', '--deltag', tag, self.pv_name])
-
- process.call(
- [
- 'sudo', 'pvchange',
- '--addtag', '%s=%s' % (key, value), self.pv_name
- ]
- )
projects / ceph-ci.git / commitdiff