Make teuthology.config *far* more robust

In addition to parsing its configuration from a yaml file, it can now be
created from a dict. It can also be dumped as a dict, or as a yaml
stream. It is also now split into multiple classes so that the same
implementation can be used for job configs, and not just as a proxy for
~/.teuthology.yaml.

Signed-off-by: Zack Cerza <zack.cerza@inktank.com>

diff --git a/teuthology/config.py b/teuthology/config.py
index 2fd6ff565a29ad9d15fe025989c4793fddc76bd2..3e6c14d996fe7aacbc18dd3b89189a0b69b05696 100644 (file)
--- a/teuthology/config.py
+++ b/teuthology/config.py
@@ -2,16 +2,101 @@ import os
 import yaml
 import logging
 
-log = logging.getLogger(__name__)
 
+def init_logging():
+    log = logging.getLogger(__name__)
+    return log
 
-class Config(object):
+log = init_logging()
+
+
+class YamlConfig(object):
+    defaults = dict()
+
+    def __init__(self, yaml_path=None):
+        self.yaml_path = yaml_path
+        if self.yaml_path:
+            self.load()
+        else:
+            self.__conf = dict()
+
+    def load(self):
+        if os.path.exists(self.yaml_path):
+            self.__conf = yaml.safe_load(file(self.yaml_path))
+        else:
+            log.debug("%s not found", self.yaml_path)
+            self.__conf = dict()
+
+    def update(self, in_dict):
+        """
+        Update an existing configuration using dict.update()
+
+        :param in_dict: The dict to use to update
+        """
+        self.__conf.update(in_dict)
+
+    @classmethod
+    def from_dict(cls, in_dict):
+        """
+        Build a config object from a dict.
+
+        :param in_dict: The dict to use
+        :returns:       The config object
+        """
+        conf_obj = cls()
+        conf_obj.__conf = in_dict
+        return conf_obj
+
+    def to_dict(self):
+        """
+        :returns: A shallow copy of the configuration as a dict
+        """
+        return dict(self.__conf)
+
+    @classmethod
+    def from_str(cls, in_str):
+        """
+        Build a config object from a string or yaml stream.
+
+        :param in_str: The stream or string
+        :returns:      The config object
+        """
+        conf_obj = cls()
+        conf_obj.__conf = yaml.safe_load(in_str)
+        return conf_obj
+
+    def to_str(self):
+        """
+        :returns: str(self)
+        """
+        return str(self)
+
+    def __str__(self):
+        return yaml.safe_dump(self.__conf, default_flow_style=False).strip()
+
+    def __getitem__(self, name):
+        return self.__conf.__getitem__(name)
+
+    def __getattr__(self, name):
+        return self.__conf.get(name, self.defaults.get(name))
+
+    def __setattr__(self, name, value):
+        if name.endswith('__conf') or name in ('yaml_path'):
+            object.__setattr__(self, name, value)
+        else:
+            self.__conf[name] = value
+
+    def __delattr__(self, name):
+        del self.__conf[name]
+
+
+class TeuthologyConfig(YamlConfig):
     """
     This class is intended to unify teuthology's many configuration files and
     objects. Currently it serves as a convenient interface to
     ~/.teuthology.yaml and nothing else.
     """
-    teuthology_yaml = os.path.join(os.environ['HOME'], '.teuthology.yaml')
+    yaml_path = os.path.join(os.environ['HOME'], '.teuthology.yaml')
     defaults = {
         'archive_base': '/var/lib/teuthworker/archive',
         'automated_scheduling': False,
@@ -24,22 +109,11 @@ class Config(object):
     }
 
     def __init__(self):
-        self.load_files()
+        super(TeuthologyConfig, self).__init__(self.yaml_path)
 
-    def load_files(self):
-        if os.path.exists(self.teuthology_yaml):
-            self.__conf = yaml.safe_load(file(self.teuthology_yaml))
-        else:
-            log.debug("%s not found", self.teuthology_yaml)
-            self.__conf = {}
 
-    def __getattr__(self, name):
-        return self.__conf.get(name, self.defaults.get(name))
+class JobConfig(YamlConfig):
+    pass
 
-    def __setattribute__(self, name, value):
-        if name.endswith('__conf'):
-            setattr(self, name, value)
-        else:
-            self.__conf[name] = value
 
-config = Config()
+config = TeuthologyConfig()

diff --git a/teuthology/kill.py b/teuthology/kill.py
index 3eef680ab1cea2b173db325d49fa70b7a15db500..bbdfc5bc4e7b2f8c36da9909e9ed9c737122a959 100755 (executable)
--- a/teuthology/kill.py
+++ b/teuthology/kill.py
@@ -107,7 +107,7 @@ def remove_beanstalk_jobs(run_name, tube_name):
     if qhost is None or qport is None:
         raise RuntimeError(
             'Beanstalk queue information not found in {conf_path}'.format(
-                conf_path=config.teuthology_yaml))
+                conf_path=config.yaml_path))
     log.info("Checking Beanstalk Queue...")
     beanstalk_conn = beanstalk.connect()
     real_tube_name = beanstalk.watch_tube(beanstalk_conn, tube_name)

diff --git a/teuthology/orchestra/test/test_connection.py b/teuthology/orchestra/test/test_connection.py
index 5dd3861860f941a63a94fc0369fd6c4356fa195c..ef13c1edff3f31741682336b6e1fb05d2be99676 100644 (file)
--- a/teuthology/orchestra/test/test_connection.py
+++ b/teuthology/orchestra/test/test_connection.py
@@ -12,7 +12,7 @@ class TestConnection(object):
 
     def clear_config(self):
         config.config.teuthology_yaml = ''
-        config.config.load_files()
+        config.config.load()
 
     def test_split_user_just_host(self):
         got = connection.split_user('somehost.example.com')

diff --git a/teuthology/report.py b/teuthology/report.py
index cb0489e09db5be0c80306c297da37930641dc075..95ed0b34d25ea7a9b88db0bed98c5c4fdee4067c 100644 (file)
--- a/teuthology/report.py
+++ b/teuthology/report.py
@@ -190,7 +190,7 @@ class ResultsReporter(object):
 
         if not self.base_uri:
             msg = "No results_server set in {yaml}; cannot report results"
-            self.log.warn(msg.format(yaml=config.teuthology_yaml))
+            self.log.warn(msg.format(yaml=config.yaml_path))
 
     def _make_session(self, max_retries=10):
         session = requests.Session()

diff --git a/teuthology/test/test_config.py b/teuthology/test/test_config.py
index f94ff35fdaeba9313c2a29b22808283ff8a8ac86..5b443a2a30186eb91b0e8a29ef9bcd0cc0de946e 100644 (file)
--- a/teuthology/test/test_config.py
+++ b/teuthology/test/test_config.py
@@ -1,19 +1,86 @@
 from .. import config
 
 
-class TestConfig(object):
+class TestYamlConfig(object):
+    def test_set_multiple(self):
+        conf_obj = config.YamlConfig()
+        conf_obj.foo = 'foo'
+        conf_obj.bar = 'bar'
+        assert conf_obj.foo == 'foo'
+        assert conf_obj.bar == 'bar'
+        assert conf_obj.to_dict()['foo'] == 'foo'
+
+    def test_from_dict(self):
+        in_dict = dict(foo='bar')
+        conf_obj = config.YamlConfig.from_dict(in_dict)
+        assert conf_obj.foo == 'bar'
+
+    def test_to_dict(self):
+        in_dict = dict(foo='bar')
+        conf_obj = config.YamlConfig.from_dict(in_dict)
+        assert conf_obj.to_dict() == in_dict
+
+    def test_from_str(self):
+        in_str = "foo: bar"
+        conf_obj = config.YamlConfig.from_str(in_str)
+        assert conf_obj.foo == 'bar'
+
+    def test_to_str(self):
+        in_str = "foo: bar"
+        conf_obj = config.YamlConfig.from_str(in_str)
+        assert conf_obj.to_str() == in_str
+
+    def test_update(self):
+        conf_obj = config.YamlConfig()
+        conf_obj.foo = 'foo'
+        conf_obj.bar = 'bar'
+        conf_obj.update(dict(bar='baz'))
+        assert conf_obj.to_dict() == dict(foo='foo', bar='baz')
+
+    def test_defaults(self):
+        conf_obj = config.YamlConfig()
+        # Save a copy of the original defaults so we can restore them later.
+        # Not doing so would break other tests.
+        old_defaults = dict(conf_obj.defaults)
+        conf_obj.defaults['foo'] = 'bar'
+        assert conf_obj.foo == 'bar'
+        # restore defaults
+        conf_obj.__class__.defaults = old_defaults
+
+    def test_delattr(self):
+        conf_obj = config.YamlConfig()
+        conf_obj.foo = 'bar'
+        assert conf_obj.foo == 'bar'
+        del conf_obj.foo
+        assert conf_obj.foo is None
+
+
+class TestTeuthologyConfig(object):
+    def test_defaults(self):
+        conf_obj = config.TeuthologyConfig()
+        conf_obj.defaults['archive_base'] = 'bar'
+        assert conf_obj.archive_base == 'bar'
+
     def test_get_ceph_git_base_default(self):
-        conf_obj = config.Config()
-        conf_obj.teuthology_yaml = ''
-        conf_obj.load_files()
+        conf_obj = config.TeuthologyConfig()
+        conf_obj.yaml_path = ''
+        conf_obj.load()
         assert conf_obj.ceph_git_base_url == "https://github.com/ceph/"
 
     def test_set_ceph_git_base_via_private(self):
-        conf_obj = config.Config()
-        conf_obj._Config__conf['ceph_git_base_url'] = "git://ceph.com/"
+        conf_obj = config.TeuthologyConfig()
+        conf_obj._YamlConfig__conf['ceph_git_base_url'] = \
+            "git://ceph.com/"
         assert conf_obj.ceph_git_base_url == "git://ceph.com/"
 
     def test_set_nonstandard(self):
-        conf_obj = config.Config()
+        conf_obj = config.TeuthologyConfig()
         conf_obj.something = 'something else'
         assert conf_obj.something == 'something else'
+
+
+class TestJobConfig(object):
+    def test_to_str(self):
+        in_str = "foo: bar"
+        conf_obj = config.JobConfig.from_str(in_str)
+        assert conf_obj.to_str() == in_str