From: Zack Cerza Date: Mon, 7 Aug 2017 20:24:57 +0000 (-0700) Subject: Import modules, not individual objects X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d8d85e421b6e0df2930b25f13baf7da04de10ec6;p=cephmetrics.git Import modules, not individual objects This helps with testing and also avoiding circular imports Signed-off-by: Zack Cerza --- diff --git a/cephmetrics/__init__.py b/cephmetrics/__init__.py index 5684e45..4f230d3 100644 --- a/cephmetrics/__init__.py +++ b/cephmetrics/__init__.py @@ -12,11 +12,7 @@ except ImportError: if not TEST_MODE: raise -from collectors.mon import Mon -from collectors.rgw import RGW -from collectors.osd import OSDs -from collectors.iscsi import ISCSIGateway -from collectors.common import flatten_dict, get_hostname, freadlines +from collectors import (common, iscsi, mon, osd, rgw) __author__ = 'Paul Cuzner' @@ -26,7 +22,7 @@ PLUGIN_NAME = 'cephmetrics' class Ceph(object): def __init__(self): self.cluster_name = None - self.host_name = get_hostname() + self.host_name = common.get_hostname() self.mon_socket = None self.rgw_socket = None @@ -46,7 +42,7 @@ class Ceph(object): self.host_name) if os.path.exists(mon_socket): self.mon_socket = mon_socket - self.mon = Mon(self.cluster_name, + self.mon = mon.Mon(self.cluster_name, admin_socket=mon_socket) rgw_socket_list = glob.glob('/var/run/ceph/{}-client.rgw.*.' @@ -54,32 +50,33 @@ class Ceph(object): if rgw_socket_list: rgw_socket = rgw_socket_list[0] - self.rgw = RGW(self.cluster_name, + self.rgw = rgw.RGW(self.cluster_name, admin_socket=rgw_socket) osd_socket_list = glob.glob('/var/run/ceph/{}-osd.*' '.asok'.format(self.cluster_name)) - mounted = freadlines('/proc/mounts') + mounted = common.freadlines('/proc/mounts') osds_mounted = [mnt for mnt in mounted if mnt.split()[1].startswith('/var/lib/ceph')] if osd_socket_list or osds_mounted: - self.osd = OSDs(self.cluster_name) + self.osd = osd.OSDs(self.cluster_name) if os.path.exists('/sys/kernel/config/target/iscsi'): - self.iscsi = ISCSIGateway(self.cluster_name) + self.iscsi = iscsi.ISCSIGateway(self.cluster_name) - collectd.info("{}: Roles detected - " - "mon:{} osd:{} rgw:{} " - "iscsi:{}".format(__name__, - isinstance(self.mon, Mon), - isinstance(self.osd, OSDs), - isinstance(self.rgw, RGW), - isinstance(self.iscsi, ISCSIGateway))) + collectd.info( + "{}: Roles detected - mon:{} osd:{} rgw:{} iscsi:{}".format( + __name__, + isinstance(self.mon, mon.Mon), + isinstance(self.osd, osd.OSDs), + isinstance(self.rgw, rgw.RGW), + isinstance(self.iscsi, iscsi.ISCSIGateway), + )) def write_stats(role_metrics, stats): - flat_stats = flatten_dict(stats, '.') + flat_stats = common.flatten_dict(stats, '.') for key_name in flat_stats: attr_name = key_name.split('.')[-1] @@ -152,19 +149,19 @@ def read_callback(): if CEPH.mon: mon_stats = CEPH.mon.get_stats() - write_stats(Mon.all_metrics, mon_stats) + write_stats(mon.Mon.all_metrics, mon_stats) if CEPH.rgw: rgw_stats = CEPH.rgw.get_stats() - write_stats(RGW.all_metrics, rgw_stats) + write_stats(rgw.RGW.all_metrics, rgw_stats) if CEPH.osd: osd_node_stats = CEPH.osd.get_stats() - write_stats(OSDs.all_metrics, osd_node_stats) + write_stats(osd.OSDs.all_metrics, osd_node_stats) if CEPH.iscsi: iscsi_stats = CEPH.iscsi.get_stats() - write_stats(ISCSIGateway.metrics, iscsi_stats) + write_stats(iscsi.ISCSIGateway.metrics, iscsi_stats) if TEST_MODE: diff --git a/cephmetrics/collectors/iscsi.py b/cephmetrics/collectors/iscsi.py index df14acb..ec8ab56 100644 --- a/cephmetrics/collectors/iscsi.py +++ b/cephmetrics/collectors/iscsi.py @@ -10,8 +10,7 @@ import os import sys import time -from collectors.base import BaseCollector -from collectors.common import fread +from cephmetrics.collectors import (base, common) class Client(object): @@ -58,9 +57,10 @@ class LUN(object): self._cycle = cycle_id self.size = self._so.size stats_path = os.path.join(self._path, 'statistics/scsi_lu') - self.iops = int(fread(os.path.join(stats_path, "num_cmds"))) - read_mb = float(fread(os.path.join(stats_path, "read_mbytes"))) - write_mb = float(fread(os.path.join(stats_path, "write_mbytes"))) + self.iops = int(common.fread(os.path.join(stats_path, "num_cmds"))) + read_mb = float(common.fread(os.path.join(stats_path, "read_mbytes"))) + write_mb = float( + common.fread(os.path.join(stats_path, "write_mbytes"))) self.read_bytes_per_sec = int(read_mb * 1024 ** 2) self.write_bytes_per_sec = int(write_mb * 1024 ** 2) self.total_bytes_per_sec = self.read_bytes_per_sec + \ @@ -76,7 +76,7 @@ class LUN(object): if not k.startswith("_")}} -class ISCSIGateway(BaseCollector): +class ISCSIGateway(base.BaseCollector): """ created on a host that has a /sys/kernel/config/target/iscsi dir i.e. there is an iscsi gateway here! @@ -95,7 +95,7 @@ class ISCSIGateway(BaseCollector): } def __init__(self, *args, **kwargs): - BaseCollector.__init__(self, *args, **kwargs) + base.BaseCollector.__init__(self, *args, **kwargs) # Since the module can be imported by a parent class but not # instantiated, the rtslib import is deferred until the first instance diff --git a/cephmetrics/collectors/mon.py b/cephmetrics/collectors/mon.py index e332485..a552b27 100644 --- a/cephmetrics/collectors/mon.py +++ b/cephmetrics/collectors/mon.py @@ -7,8 +7,7 @@ import threading import time import logging -from cephmetrics.collectors.base import BaseCollector -from cephmetrics.collectors.common import merge_dicts, get_hostname +from cephmetrics.collectors import (base, common) class RBDScanner(threading.Thread): @@ -35,7 +34,7 @@ class RBDScanner(threading.Thread): self.num_rbds = len(rbd_images) -class Mon(BaseCollector): +class Mon(base.BaseCollector): health = { "HEALTH_OK": 0, @@ -106,13 +105,13 @@ class Mon(BaseCollector): "mon_status": ("mon_status", "gauge") } - all_metrics = merge_dicts(pool_recovery_metrics, pool_client_metrics) - all_metrics = merge_dicts(all_metrics, cluster_metrics) - all_metrics = merge_dicts(all_metrics, osd_metrics) - all_metrics = merge_dicts(all_metrics, mon_states) + all_metrics = common.merge_dicts(pool_recovery_metrics, pool_client_metrics) + all_metrics = common.merge_dicts(all_metrics, cluster_metrics) + all_metrics = common.merge_dicts(all_metrics, osd_metrics) + all_metrics = common.merge_dicts(all_metrics, mon_states) def __init__(self, *args, **kwargs): - BaseCollector.__init__(self, *args, **kwargs) + base.BaseCollector.__init__(self, *args, **kwargs) self.version = self._get_version() if self.version < 12: self.get_mon_health = self._mon_health @@ -304,9 +303,9 @@ class Mon(BaseCollector): pool_md = Mon._seed(Mon.pool_client_metrics) if recovery: - pool_md = merge_dicts(pool_md, recovery) + pool_md = common.merge_dicts(pool_md, recovery) else: - pool_md = merge_dicts(pool_md, Mon._seed( + pool_md = common.merge_dicts(pool_md, Mon._seed( Mon.pool_recovery_metrics)) pool_stats[pool_name] = pool_md @@ -348,7 +347,7 @@ class Mon(BaseCollector): pools_to_scan = [] try: - freq = mons.index(get_hostname()) + freq = mons.index(common.get_hostname()) except ValueError: # this host's name is not in the monitor list? # twilight zone moment @@ -421,7 +420,7 @@ class Mon(BaseCollector): cluster_state['num_osd_hosts'] = num_osd_hosts cluster_state['num_rbds'] = self._get_rbds(cluster_state['mon_status']) - all_stats = merge_dicts(cluster_state, {"pools": pool_stats, + all_stats = common.merge_dicts(cluster_state, {"pools": pool_stats, "osd_state": osd_states}) end = time.time() diff --git a/cephmetrics/collectors/osd.py b/cephmetrics/collectors/osd.py index 4d394c2..2c526ff 100644 --- a/cephmetrics/collectors/osd.py +++ b/cephmetrics/collectors/osd.py @@ -4,9 +4,7 @@ import os import time import math -from cephmetrics.collectors.base import BaseCollector -from cephmetrics.collectors.common import ( - todict, fread, freadlines, merge_dicts, IOstat, Disk) +from cephmetrics.collectors import (base, common) __author__ = "Paul Cuzner" @@ -70,12 +68,12 @@ class OSDstats(object): self.osd_percent_used = math.ceil((float(self.stat_bytes_used) / self.stat_bytes) * 100) -class OSDs(BaseCollector): +class OSDs(base.BaseCollector): - all_metrics = merge_dicts(Disk.metrics, IOstat.metrics) + all_metrics = common.merge_dicts(common.Disk.metrics, common.IOstat.metrics) def __init__(self, cluster_name, **kwargs): - BaseCollector.__init__(self, cluster_name, **kwargs) + base.BaseCollector.__init__(self, cluster_name, **kwargs) self.timestamp = int(time.time()) self.osd = {} # dict of disk objects, each disk contains osd_id @@ -129,7 +127,7 @@ class OSDs(BaseCollector): osd_type_fname = os.path.join(osd_path, 'type') if os.path.exists(osd_type_fname): - return fread(osd_type_fname) + return common.fread(osd_type_fname) else: if os.path.exists(os.path.join(osd_path, 'journal')): return "filestore" @@ -148,7 +146,7 @@ class OSDs(BaseCollector): osd_indicators = {'var', 'lib', 'osd'} - for mnt in freadlines('/proc/mounts'): + for mnt in common.freadlines('/proc/mounts'): items = mnt.split(' ') dev_path, path_name = items[:2] if path_name.startswith('/var/lib'): @@ -162,7 +160,7 @@ class OSDs(BaseCollector): # though, plan 'b' is the whoami file osd_id = path_name.split('-')[-1] if not osd_id.isdigit(): - osd_id = fread(os.path.join(path_name, 'whoami')) + osd_id = common.fread(os.path.join(path_name, 'whoami')) if osd_id not in self.osd: osd_type = OSDs.get_osd_type(path_name) @@ -191,11 +189,12 @@ class OSDs(BaseCollector): # if the osd_id hasn't been seem neither has the # disk - self.osd[osd_device] = Disk(osd_device, - path_name=path_name, - osd_id=osd_id, - in_osd_type=osd_type, - encrypted=encrypted) + self.osd[osd_device] = common.Disk( + osd_device, + path_name=path_name, + osd_id=osd_id, + in_osd_type=osd_type, + encrypted=encrypted) self.dev_lookup[osd_device] = 'osd' self.osd_count += 1 @@ -217,10 +216,11 @@ class OSDs(BaseCollector): jrnl_dev = jrnl_path.split('/')[-1] if jrnl_dev not in self.osd: - self.jrnl[jrnl_dev] = Disk(jrnl_dev, - osd_id=osd_id, - in_osd_type=osd_type, - encrypted=encrypted) + self.jrnl[jrnl_dev] = common.Disk( + jrnl_dev, + osd_id=osd_id, + in_osd_type=osd_type, + encrypted=encrypted) self.dev_lookup[jrnl_dev] = 'jrnl' @@ -239,7 +239,7 @@ class OSDs(BaseCollector): self.timestamp = int(now) # Fetch diskstats from the OS - for perf_entry in freadlines('/proc/diskstats'): + for perf_entry in common.freadlines('/proc/diskstats'): field = perf_entry.split() dev_name = field[2] @@ -293,7 +293,7 @@ class OSDs(BaseCollector): for dev_name in sorted(device_dict): device = device_dict[dev_name] - dumped[dev_name] = todict(device) + dumped[dev_name] = common.todict(device) return dumped diff --git a/cephmetrics/collectors/rgw.py b/cephmetrics/collectors/rgw.py index 8fbbcf5..5dbdab3 100644 --- a/cephmetrics/collectors/rgw.py +++ b/cephmetrics/collectors/rgw.py @@ -2,13 +2,12 @@ import time -from cephmetrics.collectors.base import BaseCollector -from cephmetrics.collectors.common import get_hostname, merge_dicts +from cephmetrics.collectors import (base, common) __author__ = "paul.cuzner@redhat.com" -class RGW(BaseCollector): +class RGW(base.BaseCollector): simple_metrics = { "req": ("requests", "derive"), @@ -33,11 +32,11 @@ class RGW(BaseCollector): "put_initial_lat_avgcount": ("put_initial_lat_avgcount", "derive") } - all_metrics = merge_dicts(simple_metrics, latencies) + all_metrics = common.merge_dicts(simple_metrics, latencies) def __init__(self, cluster_name, admin_socket, **kwargs): - BaseCollector.__init__(self, cluster_name, admin_socket, **kwargs) - self.host_name = get_hostname() + base.BaseCollector.__init__(self, cluster_name, admin_socket, **kwargs) + self.host_name = common.get_hostname() def _get_rgw_data(self):