From 74977b17b9c4f9b149bc5b6be04e571148a1d209 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 8 Jul 2019 10:25:07 +0200 Subject: [PATCH] python-common: Initialize package structure This package is supposed to contain common Python code usable by all modules and tools. It is also supposed to contain code to deploy ceph clutsers. Signed-off-by: Sebastian Wagner --- src/pybind/mgr/orchestrator.py | 140 ----------------- src/python-common/.gitignore | 1 + src/python-common/ceph/__init__.py | 0 src/python-common/ceph/deployment/__init__.py | 0 .../ceph/deployment/drive_group.py | 145 ++++++++++++++++++ .../ceph/deployment/ssh_orchestrator.py | 10 ++ src/python-common/ceph/exceptions.py | 87 +++++++++++ src/python-common/requirements.txt | 1 + src/python-common/setup.py | 31 ++++ 9 files changed, 275 insertions(+), 140 deletions(-) create mode 100644 src/python-common/.gitignore create mode 100644 src/python-common/ceph/__init__.py create mode 100644 src/python-common/ceph/deployment/__init__.py create mode 100644 src/python-common/ceph/deployment/drive_group.py create mode 100644 src/python-common/ceph/deployment/ssh_orchestrator.py create mode 100644 src/python-common/ceph/exceptions.py create mode 100644 src/python-common/requirements.txt create mode 100644 src/python-common/setup.py diff --git a/src/pybind/mgr/orchestrator.py b/src/pybind/mgr/orchestrator.py index ee9a289ecb3..1e20cce4a81 100644 --- a/src/pybind/mgr/orchestrator.py +++ b/src/pybind/mgr/orchestrator.py @@ -577,146 +577,6 @@ class ServiceDescription(object): return cls(**data) -class DeviceSelection(object): - """ - Used within :class:`myclass.DriveGroupSpec` to specify the devices - used by the Drive Group. - - Any attributes (even none) can be included in the device - specification structure. - """ - - def __init__(self, paths=None, id_model=None, size=None, rotates=None, count=None): - # type: (List[str], str, str, bool, int) -> None - """ - ephemeral drive group device specification - - TODO: translate from the user interface (Drive Groups) to an actual list of devices. - """ - if paths is None: - paths = [] - - #: List of absolute paths to the devices. - self.paths = paths # type: List[str] - - #: A wildcard string. e.g: "SDD*" - self.id_model = id_model - - #: Size specification of format LOW:HIGH. - #: Can also take the the form :HIGH, LOW: - #: or an exact value (as ceph-volume inventory reports) - self.size = size - - #: is the drive rotating or not - self.rotates = rotates - - #: if this is present limit the number of drives to this number. - self.count = count - self.validate() - - def validate(self): - props = [self.id_model, self.size, self.rotates, self.count] - if self.paths and any(p is not None for p in props): - raise DriveGroupValidationError('DeviceSelection: `paths` and other parameters are mutually exclusive') - if not any(p is not None for p in [self.paths] + props): - raise DriveGroupValidationError('DeviceSelection cannot be empty') - - @classmethod - def from_json(cls, device_spec): - return cls(**device_spec) - - -class DriveGroupValidationError(Exception): - """ - Defining an exception here is a bit problematic, cause you cannot properly catch it, - if it was raised in a different mgr module. - """ - - def __init__(self, msg): - super(DriveGroupValidationError, self).__init__('Failed to validate Drive Group: ' + msg) - -class DriveGroupSpec(object): - """ - Describe a drive group in the same form that ceph-volume - understands. - """ - def __init__(self, host_pattern, data_devices=None, db_devices=None, wal_devices=None, journal_devices=None, - data_directories=None, osds_per_device=None, objectstore='bluestore', encrypted=False, - db_slots=None, wal_slots=None): - # type: (str, Optional[DeviceSelection], Optional[DeviceSelection], Optional[DeviceSelection], Optional[DeviceSelection], Optional[List[str]], int, str, bool, int, int) -> None - - # concept of applying a drive group to a (set) of hosts is tightly - # linked to the drive group itself - # - #: An fnmatch pattern to select hosts. Can also be a single host. - self.host_pattern = host_pattern - - #: A :class:`orchestrator.DeviceSelection` - self.data_devices = data_devices - - #: A :class:`orchestrator.DeviceSelection` - self.db_devices = db_devices - - #: A :class:`orchestrator.DeviceSelection` - self.wal_devices = wal_devices - - #: A :class:`orchestrator.DeviceSelection` - self.journal_devices = journal_devices - - #: Number of osd daemons per "DATA" device. - #: To fully utilize nvme devices multiple osds are required. - self.osds_per_device = osds_per_device - - #: A list of strings, containing paths which should back OSDs - self.data_directories = data_directories - - #: ``filestore`` or ``bluestore`` - self.objectstore = objectstore - - #: ``true`` or ``false`` - self.encrypted = encrypted - - #: How many OSDs per DB device - self.db_slots = db_slots - - #: How many OSDs per WAL device - self.wal_slots = wal_slots - - # FIXME: needs ceph-volume support - #: Optional: mapping of drive to OSD ID, used when the - #: created OSDs are meant to replace previous OSDs on - #: the same node. - self.osd_id_claims = {} - - @classmethod - def from_json(self, json_drive_group): - """ - Initialize 'Drive group' structure - - :param json_drive_group: A valid json string with a Drive Group - specification - """ - args = {k: (DeviceSelection.from_json(v) if k.endswith('_devices') else v) for k, v in - json_drive_group.items()} - return DriveGroupSpec(**args) - - def hosts(self, all_hosts): - return fnmatch.filter(all_hosts, self.host_pattern) - - def validate(self, all_hosts): - if not isinstance(self.host_pattern, six.string_types): - raise DriveGroupValidationError('host_pattern must be of type string') - - specs = [self.data_devices, self.db_devices, self.wal_devices, self.journal_devices] - for s in filter(None, specs): - s.validate() - if self.objectstore not in ('filestore', 'bluestore'): - raise DriveGroupValidationError("objectstore not in ('filestore', 'bluestore')") - if not self.hosts(all_hosts): - raise DriveGroupValidationError( - "host_pattern '{}' does not match any hosts".format(self.host_pattern)) - - class StatelessServiceSpec(object): # Request to orchestrator for a group of stateless services # such as MDS, RGW, nfs gateway, iscsi gateway diff --git a/src/python-common/.gitignore b/src/python-common/.gitignore new file mode 100644 index 00000000000..727b0b6447a --- /dev/null +++ b/src/python-common/.gitignore @@ -0,0 +1 @@ +ceph.egg-info diff --git a/src/python-common/ceph/__init__.py b/src/python-common/ceph/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python-common/ceph/deployment/__init__.py b/src/python-common/ceph/deployment/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python-common/ceph/deployment/drive_group.py b/src/python-common/ceph/deployment/drive_group.py new file mode 100644 index 00000000000..74e005a5173 --- /dev/null +++ b/src/python-common/ceph/deployment/drive_group.py @@ -0,0 +1,145 @@ +import fnmatch +try: + from typing import Optional, List +except ImportError: + pass + +import six + + +class DeviceSelection(object): + """ + Used within :class:`ceph.deployment.drive_group.DriveGroupSpec` to specify the devices + used by the Drive Group. + + Any attributes (even none) can be included in the device + specification structure. + """ + + def __init__(self, paths=None, id_model=None, size=None, rotates=None, count=None): + # type: (List[str], str, str, bool, int) -> None + """ + ephemeral drive group device specification + """ + if paths is None: + paths = [] + + #: List of absolute paths to the devices. + self.paths = paths # type: List[str] + + #: A wildcard string. e.g: "SDD*" + self.id_model = id_model + + #: Size specification of format LOW:HIGH. + #: Can also take the the form :HIGH, LOW: + #: or an exact value (as ceph-volume inventory reports) + self.size = size + + #: is the drive rotating or not + self.rotates = rotates + + #: if this is present limit the number of drives to this number. + self.count = count + self.validate() + + def validate(self): + props = [self.id_model, self.size, self.rotates, self.count] + if self.paths and any(p is not None for p in props): + raise DriveGroupValidationError('DeviceSelection: `paths` and other parameters are mutually exclusive') + if not any(p is not None for p in [self.paths] + props): + raise DriveGroupValidationError('DeviceSelection cannot be empty') + + @classmethod + def from_json(cls, device_spec): + return cls(**device_spec) + + +class DriveGroupValidationError(Exception): + """ + Defining an exception here is a bit problematic, cause you cannot properly catch it, + if it was raised in a different mgr module. + """ + + def __init__(self, msg): + super(DriveGroupValidationError, self).__init__('Failed to validate Drive Group: ' + msg) + +class DriveGroupSpec(object): + """ + Describe a drive group in the same form that ceph-volume + understands. + """ + def __init__(self, host_pattern, data_devices=None, db_devices=None, wal_devices=None, journal_devices=None, + data_directories=None, osds_per_device=None, objectstore='bluestore', encrypted=False, + db_slots=None, wal_slots=None): + # type: (str, Optional[DeviceSelection], Optional[DeviceSelection], Optional[DeviceSelection], Optional[DeviceSelection], Optional[List[str]], int, str, bool, int, int) -> None + + # concept of applying a drive group to a (set) of hosts is tightly + # linked to the drive group itself + # + #: An fnmatch pattern to select hosts. Can also be a single host. + self.host_pattern = host_pattern + + #: A :class:`orchestrator.DeviceSelection` + self.data_devices = data_devices + + #: A :class:`orchestrator.DeviceSelection` + self.db_devices = db_devices + + #: A :class:`orchestrator.DeviceSelection` + self.wal_devices = wal_devices + + #: A :class:`orchestrator.DeviceSelection` + self.journal_devices = journal_devices + + #: Number of osd daemons per "DATA" device. + #: To fully utilize nvme devices multiple osds are required. + self.osds_per_device = osds_per_device + + #: A list of strings, containing paths which should back OSDs + self.data_directories = data_directories + + #: ``filestore`` or ``bluestore`` + self.objectstore = objectstore + + #: ``true`` or ``false`` + self.encrypted = encrypted + + #: How many OSDs per DB device + self.db_slots = db_slots + + #: How many OSDs per WAL device + self.wal_slots = wal_slots + + # FIXME: needs ceph-volume support + #: Optional: mapping of drive to OSD ID, used when the + #: created OSDs are meant to replace previous OSDs on + #: the same node. + self.osd_id_claims = {} + + @classmethod + def from_json(self, json_drive_group): + """ + Initialize 'Drive group' structure + + :param json_drive_group: A valid json string with a Drive Group + specification + """ + args = {k: (DeviceSelection.from_json(v) if k.endswith('_devices') else v) for k, v in + json_drive_group.items()} + return DriveGroupSpec(**args) + + def hosts(self, all_hosts): + return fnmatch.filter(all_hosts, self.host_pattern) + + def validate(self, all_hosts): + if not isinstance(self.host_pattern, six.string_types): + raise DriveGroupValidationError('host_pattern must be of type string') + + specs = [self.data_devices, self.db_devices, self.wal_devices, self.journal_devices] + for s in filter(None, specs): + s.validate() + if self.objectstore not in ('filestore', 'bluestore'): + raise DriveGroupValidationError("objectstore not in ('filestore', 'bluestore')") + if not self.hosts(all_hosts): + raise DriveGroupValidationError( + "host_pattern '{}' does not match any hosts".format(self.host_pattern)) \ No newline at end of file diff --git a/src/python-common/ceph/deployment/ssh_orchestrator.py b/src/python-common/ceph/deployment/ssh_orchestrator.py new file mode 100644 index 00000000000..2a81c527f5f --- /dev/null +++ b/src/python-common/ceph/deployment/ssh_orchestrator.py @@ -0,0 +1,10 @@ + +def bootstrap_cluster(): + create_mon() + create_mgr() + +def create_mon(): + pass + +def create_mgr(): + pass diff --git a/src/python-common/ceph/exceptions.py b/src/python-common/ceph/exceptions.py new file mode 100644 index 00000000000..872bd8bbc7e --- /dev/null +++ b/src/python-common/ceph/exceptions.py @@ -0,0 +1,87 @@ +class Error(Exception): + """ `Error` class, derived from `Exception` """ + def __init__(self, message, errno=None): + super(Exception, self).__init__(message) + self.errno = errno + + def __str__(self): + msg = super(Exception, self).__str__() + if self.errno is None: + return msg + return '[errno {0}] {1}'.format(self.errno, msg) + +class InvalidArgumentError(Error): + pass + +class OSError(Error): + """ `OSError` class, derived from `Error` """ + pass + +class InterruptedOrTimeoutError(OSError): + """ `InterruptedOrTimeoutError` class, derived from `OSError` """ + pass + + +class PermissionError(OSError): + """ `PermissionError` class, derived from `OSError` """ + pass + + +class PermissionDeniedError(OSError): + """ deal with EACCES related. """ + pass + + +class ObjectNotFound(OSError): + """ `ObjectNotFound` class, derived from `OSError` """ + pass + + +class NoData(OSError): + """ `NoData` class, derived from `OSError` """ + pass + + +class ObjectExists(OSError): + """ `ObjectExists` class, derived from `OSError` """ + pass + + +class ObjectBusy(OSError): + """ `ObjectBusy` class, derived from `IOError` """ + pass + + +class IOError(OSError): + """ `ObjectBusy` class, derived from `OSError` """ + pass + + +class NoSpace(OSError): + """ `NoSpace` class, derived from `OSError` """ + pass + + +class RadosStateError(Error): + """ `RadosStateError` class, derived from `Error` """ + pass + + +class IoctxStateError(Error): + """ `IoctxStateError` class, derived from `Error` """ + pass + + +class ObjectStateError(Error): + """ `ObjectStateError` class, derived from `Error` """ + pass + + +class LogicError(Error): + """ `` class, derived from `Error` """ + pass + + +class TimedOut(OSError): + """ `TimedOut` class, derived from `OSError` """ + pass \ No newline at end of file diff --git a/src/python-common/requirements.txt b/src/python-common/requirements.txt new file mode 100644 index 00000000000..ffe2fce4989 --- /dev/null +++ b/src/python-common/requirements.txt @@ -0,0 +1 @@ +six diff --git a/src/python-common/setup.py b/src/python-common/setup.py new file mode 100644 index 00000000000..9d357baa448 --- /dev/null +++ b/src/python-common/setup.py @@ -0,0 +1,31 @@ +from setuptools import setup, find_packages + + +setup( + name='ceph', + version='1.0.0', + packages=find_packages(), + author='', + author_email='dev@ceph.io', + description='Ceph common library', + license='LGPLv2+', + keywords='ceph', + url="https://github.com/ceph/ceph", + zip_safe = False, + install_requires=( + 'six', + ), + tests_require=[ + 'pytest >=2.1.3', + 'tox', + ], + classifiers = [ + 'Intended Audience :: Developer', + 'Operating System :: POSIX :: Linux', + 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + ] +) -- 2.39.5