From: Babu Shanmugam <anbu@enovance.com>
Date: Sat, 8 Mar 2014 05:17:13 +0000 (+0000)
Subject: Broke down sysinfo's format into a histogram with a value and count
X-Git-Tag: v0.78~41^2^2~2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=9bf39e2cc167cbc4756fba45589390e44cc8f652;p=ceph.git

Broke down sysinfo's format into a histogram with a value and count
so that we just see how many of each version/distro/kernel/os/arch/cpu/etc are running

Signed-off-by: Babu Shanmugam <anbu@enovance.com>
---

diff --git a/README.md b/README.md
index 9791eeea9b50..f0224f505fe6 100644
--- a/README.md
+++ b/README.md
@@ -73,24 +73,50 @@ Run 'ceph-brag -h' to get the usage information of this tool.
           "name": "replicated"
         }
       ],
-      "sysinfo": [
-        {
-          "hw_info": {
-            "swap_kb": 0,
-            "arch": "x86_64",
-            "cpu": "Intel Xeon E312xx (Sandy Bridge)",
-            "mem_kb": 2051648
-          },
-          "id": 0,
-          "os_info": {
-            "version": "3.2.0-23-virtual",
-            "os": "Linux",
-            "description": "#36-Ubuntu SMP Tue Apr 10 22:29:03 UTC 2012",
+      "sysinfo": {
+        "kernel_types": [
+          {
+            "count": 1,
+            "type": "#36-Ubuntu SMP Tue Apr 10 22:29:03 UTC 2012"
+          }
+        ],
+        "cpu_archs": [
+          {
+            "count": 1,
+            "arch": "x86_64"
+          }
+        ],
+        "cpus": [
+          {
+            "count": 1,
+            "cpu": "Intel Xeon E312xx (Sandy Bridge)"
+          }
+        ],
+        "kernel_versions": [
+          {
+            "count": 1,
+            "version": "3.2.0-23-virtual"
+          }
+        ],
+        "ceph_versions": [
+          {
+            "count": 1,
+            "version": "0.75-229-g4050eae(4050eae32cd77a1c210ca11d0f12c74daecb1bd3)"
+          }
+        ],
+        "os_info": [
+          {
+            "count": 1,
+            "os": "Linux"
+          }
+        ],
+        "distros": [
+          {
+            "count": 1,
             "distro": "Ubuntu 12.04 precise (Ubuntu 12.04 LTS)"
-          },
-          "ceph_version": "ceph version 0.75-229-g4050eae (4050eae32cd77a1c210ca11d0f12c74daecb1bd3)"
-        }
-      ]
+          }
+        ]
+      }
     }
 
 
diff --git a/client/ceph-brag b/client/ceph-brag
index c4dc41cd7911..bec86da0a2c6 100755
--- a/client/ceph-brag
+++ b/client/ceph-brag
@@ -153,52 +153,71 @@ def get_pool_metadata():
   return pool_meta
 
 def get_sysinfo(max_osds):
-  sysinfo = []
   count = 0
   osd_metadata_available = False
-
+  
+  os = {}
+  kern_version = {}
+  kern_description = {}
+  distro = {}
+  cpu = {}
+  arch = {}
+  ceph_version = {}
+
+  incr = lambda a,k: 1 if k not in a else a[k]+1
   while count < max_osds:
     meta = {'id':count}
     (rc, o, e) = run_command(['ceph', 'osd', 'metadata', str(count)])
     if rc is 0:
       if osd_metadata_available is False:
         osd_metadata_available = True
-      os_info = {}
-      hw_info = {}
 
       jmeta = json.loads(o)
 
       version = jmeta['ceph_version'].split()
-      meta['ceph_version'] = version[2]
+      cv = version[2]
       if (len(version) > 3):
-        meta['ceph_version'] += version[3]
+        cv += version[3]
 
-      os_info['os'] = jmeta['os']
-      os_info['version'] = jmeta['kernel_version']
-      os_info['description'] = jmeta['kernel_description']
+      ceph_version[cv] = incr(ceph_version, cv)
+      os[jmeta['os']] = incr(os, jmeta['os'])
+      kern_version[jmeta['kernel_version']] = \
+            incr(kern_version, jmeta['kernel_version'])
+      kern_description[jmeta['kernel_description']] = \
+            incr(kern_description, jmeta['kernel_description'])
 
       try:
-        distro = jmeta['distro'] + ' '
-        distro += jmeta['distro_version'] + ' '
-        distro += jmeta['distro_codename'] + ' ('
-        distro += jmeta['distro_description'] + ')'
-        os_info['distro'] = distro
+        dstr = jmeta['distro'] + ' '
+        dstr += jmeta['distro_version'] + ' '
+        dstr += jmeta['distro_codename'] + ' ('
+        dstr += jmeta['distro_description'] + ')'
+        distro[dstr] = incr(distro, dstr)
       except KeyError as ke:
         pass
-      meta['os_info'] = os_info
   
-      hw_info['cpu'] = jmeta['cpu']
-      hw_info['arch'] = jmeta['arch']
-      hw_info['mem_kb'] = int(jmeta['mem_total_kb'])
-      hw_info['swap_kb'] = int(jmeta['mem_swap_kb'])
-      meta['hw_info'] = hw_info
+      cpu[jmeta['cpu']] = incr(cpu, jmeta['cpu'])
+      arch[jmeta['arch']] = incr(arch, jmeta['arch'])
   
-    sysinfo.append(meta)
     count = count + 1
 
+  sysinfo = {}
   if osd_metadata_available is False:
     print >> sys.stderr, "'ceph osd metadata' is not available at all"
-
+    return sysinfo
+
+  def jsonify(type_count, name, type_name):
+    tmp = []
+    for k, v in type_count.items():
+      tmp.append({type_name:k, 'count':v})
+    sysinfo[name] = tmp
+
+  jsonify(os, 'os_info', 'os')
+  jsonify(kern_version, 'kernel_versions', 'version')
+  jsonify(kern_description, 'kernel_types', 'type')
+  jsonify(distro, 'distros', 'distro')
+  jsonify(cpu, 'cpus', 'cpu')
+  jsonify(arch, 'cpu_archs', 'arch')
+  jsonify(ceph_version, 'ceph_versions', 'version')
   return sysinfo
 
 def get_ownership_info():
diff --git a/server/ceph_brag/json.py b/server/ceph_brag/json.py
index 07ba7c4256bb..57d106fa6a06 100644
--- a/server/ceph_brag/json.py
+++ b/server/ceph_brag/json.py
@@ -28,8 +28,7 @@ def jsonify_components_info(comps):
             num_pgs=comps.num_pgs,
             num_pools=comps.num_pools,
             num_mdss=comps.num_mdss,
-            num_mons=comps.num_mons
-            )
+            num_mons=comps.num_mons)
 
 @jsonify.register(db.crush_types)
 def jsonify_crush_types(crush):
@@ -40,18 +39,63 @@ def jsonify_crush_types(crush):
 def jsonify_pools_info(pool):
     return dict(size=pool.pool_rep_size,
                 type=pool.pool_type,
-                id=pool.pool_id
-               )
+                id=pool.pool_id)
+
+@jsonify.register(db.os_info)
+def jsonify_os_info(value):
+    return dict(os=value.os,
+                count=value.count)
+
+@jsonify.register(db.kernel_versions)
+def jsonify_kernel_versions(value):
+    return dict(version=value.version,
+                count=value.count)
+
+@jsonify.register(db.kernel_types)
+def jsonify_kernel_types(value):
+    return dict(type=value.type,
+                count=value.count)
+
+@jsonify.register(db.distros)
+def jsonify_distros(value):
+    return dict(distro=value.distro,
+                count=value.count)
+
+@jsonify.register(db.cpus)
+def jsonify_cpus(value):
+    return dict(cpu=value.cpu,
+                count=value.count)
+
+@jsonify.register(db.cpu_archs)
+def jsonify_cpu_archs(value):
+    return dict(arch=value.arch,
+                count=value.count)
+
+@jsonify.register(db.ceph_versions)
+def jsonify_ceph_versions(value):
+    return dict(version=value.version,
+                count=value.count)
+
+@jsonify.register(db.sysinfo)
+def jsonify_sysinfo(value):
+    retval = {}
+    
+    if value.os:
+      retval['os_info'] = value.os
+    if value.kern_vers:
+      retval['kernel_versions'] = value.kern_vers
+    if value.kern_types:
+      retval['kernel_types'] = value.kern_types
+    if value.distros:
+      retval['distros'] = value.distros
+    if value.cpus:
+      retval['cpus'] = value.cpus
+    if value.cpu_archs:
+      retval['cpu_archs'] = value.cpu_archs
+    if value.ceph_vers:
+      retval['ceph_versions'] = value.ceph_vers
 
-@jsonify.register(db.osds_info)
-def jsonify_osds_info(osd):
-    return dict(hw_info={'swap_kb':osd.swap_kb,'mem_kb':osd.mem_kb,
-                         'arch':osd.arch, 'cpu':osd.cpu},
-                id=osd.osd_id,
-                os_info={'os':osd.os,'version':osd.os_version,
-                         'description':osd.os_desc, 'distro':osd.distro},
-                ceph_version=osd.ceph_version
-               )
+    return retval
 
 @jsonify.register(db.brag)
 def jsonify_brag(b):
@@ -66,5 +110,5 @@ def jsonify_brag(b):
                 crush_types=b.crush,
                 ownership=ownership,
                 pool_metadata=b.pools,
-                sysinfo=b.osds
+                sysinfo=b.sysinfo
                 )
diff --git a/server/ceph_brag/model/db.py b/server/ceph_brag/model/db.py
index 66b8d6118150..14c009aa3b57 100644
--- a/server/ceph_brag/model/db.py
+++ b/server/ceph_brag/model/db.py
@@ -61,21 +61,71 @@ class pools_info(Base):
   pool_type = Column(String(16))
   pool_rep_size = Column(Integer)
 
-class osds_info(Base):
-  __tablename__ = 'osds_info'
+class os_info(Base):
+  __tablename__ = 'os_info'
 
   index = Column(Integer, primary_key=True)
   vid = Column(ForeignKey('version_info.index'))
-  osd_id = Column(String(8))
-  swap_kb = Column(Integer)
-  mem_kb = Column(Integer)
-  arch = Column(String(16))
-  cpu = Column(String(16))
   os = Column(String(16))
-  os_version = Column(String(16))
-  os_desc = Column(String(64))
+  count = Column(Integer)
+
+class kernel_versions(Base):
+  __tablename__ = 'kernel_versions'
+
+  index = Column(Integer, primary_key=True)
+  vid = Column(ForeignKey('version_info.index'))
+  version = Column(String(16))
+  count = Column(Integer)
+
+class kernel_types(Base):
+  __tablename__ = 'kernel_types'
+
+  index = Column(Integer, primary_key=True)
+  vid = Column(ForeignKey('version_info.index'))
+  type = Column(String(64))
+  count = Column(Integer)
+
+class distros(Base):
+  __tablename__ = 'distros'
+
+  index = Column(Integer, primary_key=True)
+  vid = Column(ForeignKey('version_info.index'))
   distro = Column(String(64))
-  ceph_version = Column(String(64))
+  count = Column(Integer)
+
+class cpus(Base):
+  __tablename__ = 'cpus'
+
+  index = Column(Integer, primary_key=True)
+  vid = Column(ForeignKey('version_info.index'))
+  cpu = Column(String(16))
+  count = Column(Integer)
+
+class cpu_archs(Base):
+  __tablename__ = 'cpu_archs'
+
+  index = Column(Integer, primary_key=True)
+  vid = Column(ForeignKey('version_info.index'))
+  arch = Column(String(16))
+  count = Column(Integer)
+
+class ceph_versions(Base):
+  __tablename__ = 'ceph_versions'
+
+  index = Column(Integer, primary_key=True)
+  vid = Column(ForeignKey('version_info.index'))
+  version = Column(String(16))
+  count = Column(Integer)
+
+class sysinfo(object):
+  def __init__(self, vindex):
+    self.os = Session.query(os_info).filter_by(vid=vindex).all()
+    self.kern_vers = Session.query(kernel_versions).filter_by(vid=vindex).all()
+    self.kern_types = Session.query(kernel_types).filter_by(vid=vindex).all()
+    self.distros = Session.query(distros).filter_by(vid=vindex).all()
+    self.cpus = Session.query(cpus).filter_by(vid=vindex).all()
+    self.cpu_archs = Session.query(cpu_archs).filter_by(vid=vindex).all()
+    self.ceph_vers = Session.query(ceph_versions).filter_by(vid=vindex).all()
 
 class brag(object):
   def __init__(self, uuid, version_number):
@@ -87,7 +137,7 @@ class brag(object):
       self.comps = Session.query(components_info).filter_by(vid=self.vi.index).first()
       self.crush = Session.query(crush_types).filter_by(vid=self.vi.index).all()
       self.pools = Session.query(pools_info).filter_by(vid=self.vi.index).all()
-      self.osds = Session.query(osds_info).filter_by(vid=self.vi.index).all()
+      self.sysinfo = sysinfo(self.vi.index)
 
 def bytes_pretty_to_raw(pretty):
   mo = re.search("(\d+)\ (\S+)", pretty)
@@ -183,22 +233,46 @@ def put_new_version(data):
                              pool_type=p['type'],
                              pool_rep_size=p['size']))
 
-  def add_osds_info(vi):
-    osds = info['sysinfo']
-    for o in osds:
-      osd = osds_info(vid=vi.index,
-                      osd_id=o['id'],
-                      swap_kb=o['hw_info']['swap_kb'],
-                      mem_kb=o['hw_info']['mem_kb'],
-                      arch=o['hw_info']['arch'],
-                      cpu=o['hw_info']['cpu'],
-                      os=o['os_info']['os'],
-                      os_version=o['os_info']['version'],
-                      os_desc=o['os_info']['description'],
-                      distro=o['os_info']['distro'],
-                      ceph_version=o['ceph_version'])
-      Session.add(osd)
-                    
+  def add_sys_info(vi):
+    si = info['sysinfo']
+    while si:
+      k,v = si.popitem()
+      if k == 'os_info':
+        for o in v:
+          Session.add(os_info(vid=vi.index, 
+                              os=o['os'],
+                              count=o['count']))
+      elif k == 'kernel_versions':
+        for k in v:
+          Session.add(kernel_versions(vid=vi.index,
+                                      version=k['version'],
+                                      count=k['count']))
+      elif k == 'kernel_types':
+        for k in v:
+          Session.add(kernel_types(vid=vi.index,
+                                   type=k['type'],
+                                   count=k['count']))
+      elif k == 'distros':
+        for d in v:
+          Session.add(distros(vid=vi.index,
+                              distro=d['distro'],
+                              count=d['count']))
+      elif k == 'cpus':
+        for c in v:
+          Session.add(cpus(vid=vi.index,
+                           cpu=c['cpu'],
+                           count=c['count']))
+      elif k == 'cpu_archs':
+        for c in v:
+          Session.add(cpu_archs(vid=vi.index,
+                                arch=c['arch'],
+                                count=c['count']))
+      elif k == 'ceph_versions':
+        for c in v:
+          Session.add(ceph_versions(vid=vi.index,
+                                    version=c['version'],
+                                    count=c['count']))
+
   ci = add_cluster_info()
   add_version_info(ci)
   vi = Session.query(version_info).filter_by(cluster_id=ci.index, 
@@ -206,7 +280,7 @@ def put_new_version(data):
   add_components_info(vi)
   add_crush_types(vi)
   add_pools_info(vi)
-  add_osds_info(vi)
+  add_sys_info(vi)
  
 def delete_uuid(uuid):
   ci = Session.query(cluster_info).filter_by(uuid=uuid).first()
@@ -217,7 +291,14 @@ def delete_uuid(uuid):
     Session.query(components_info).filter_by(vid=v.index).delete()
     Session.query(crush_types).filter_by(vid=v.index).delete()
     Session.query(pools_info).filter_by(vid=v.index).delete()
-    Session.query(osds_info).filter_by(vid=v.index).delete()
+    Session.query(os_info).filter_by(vid=v.index).delete()
+    Session.query(kernel_versions).filter_by(vid=v.index).delete()
+    Session.query(kernel_types).filter_by(vid=v.index).delete()
+    Session.query(distros).filter_by(vid=v.index).delete()
+    Session.query(cpus).filter_by(vid=v.index).delete()
+    Session.query(cpu_archs).filter_by(vid=v.index).delete()
+    Session.query(ceph_versions).filter_by(vid=v.index).delete()
+
     Session.flush()
     Session.delete(v)
     Session.flush()
diff --git a/server/sample.json b/server/sample.json
index 5218b65b0d02..61d9534282fd 100644
--- a/server/sample.json
+++ b/server/sample.json
@@ -51,22 +51,48 @@
       "size": 3
     }
   ],
-  "sysinfo": [
-    {
-      "hw_info": {
-        "swap_kb": 0,
-        "arch": "x86_64",
-        "cpu": "Intel Xeon E312xx (Sandy Bridge)",
-        "mem_kb": 2051648
-      },
-      "id": 0,
-      "os_info": {
-        "version": "3.2.0-23-virtual",
-        "os": "Linux",
-        "description": "#36-Ubuntu SMP Tue Apr 10 22:29:03 UTC 2012",
+  "sysinfo": {
+    "kernel_types": [
+      {
+        "count": 1,
+        "type": "#36-Ubuntu SMP Tue Apr 10 22:29:03 UTC 2012"
+      }
+    ],
+    "cpu_archs": [
+      {
+        "count": 1,
+        "arch": "x86_64"
+      }
+    ],
+    "cpus": [
+      {
+        "count": 1,
+        "cpu": "Intel Xeon E312xx (Sandy Bridge)"
+      }
+    ],
+    "kernel_versions": [
+      {
+        "count": 1,
+        "version": "3.2.0-23-virtual"
+      }
+    ],
+    "ceph_versions": [
+      {
+        "count": 1,
+        "version": "0.75-229-g4050eae(4050eae32cd77a1c210ca11d0f12c74daecb1bd3)"
+      }
+    ],
+    "os_info": [
+      {
+        "count": 1,
+        "os": "Linux"
+      }
+    ],
+    "distros": [
+      {
+        "count": 1,
         "distro": "Ubuntu 12.04 precise (Ubuntu 12.04 LTS)"
-      },
-      "ceph_version": "0.75-229-g4050eae(4050eae32cd77a1c210ca11d0f12c74daecb1bd3)"
-    }
-  ]
+      }
+    ]
+  }
 }