rgw_sync_agent: fix a bunch of issues

- Read ceph.conf from stored copy that includes overrides
- Get system users and keys from cluster instead of reading other
  tasks' yaml, which may not be complete.
- Put zone info extraction from the cluster into utility functions,
  since it'll be useful for other tests later.
- Work with more than one agent on a single host
- Accept more than one client to run, like almost every other task
- Rename target to dest for consistency with radosgw-agent
- Don't make everything one large function

Signed-off-by: Josh Durgin <josh.durgin@inktank.com>

diff --git a/teuthology/task/rgw_sync_agent.py b/teuthology/task/rgw_sync_agent.py
index ebd985d3ed1ba215987ebaef00f176a54ca9a661..b33e5c483b7c22e10465a0ac1fa3d1f48cc8cde6 100644 (file)
--- a/teuthology/task/rgw_sync_agent.py
+++ b/teuthology/task/rgw_sync_agent.py
@@ -1,127 +1,138 @@
 import contextlib
-import json
+from copy import deepcopy
 import logging
 
 from ..orchestra import run
 from teuthology import misc as teuthology
-from teuthology.task_util.rgw import rgwadmin
+import teuthology.task_util.rgw as rgw_utils
 
 log = logging.getLogger(__name__)
 
-def find_task_config(ctx, name):
-    for entry in ctx.config['tasks']:
-        log.info("checking entry: %s", entry)
-        if name in entry:
-            return entry[name]
-    return dict()
-
-@contextlib.contextmanager
-def task(ctx, config):
+def run_radosgw_agent(ctx, client, config):
     """
-    Turn on a radosgw sync agent in test mode. Specify:
-    host: to run on, or leave it blank for client.0
-    source: the source region and zone
-    target: the target region and zone
-
-    tasks:
-    - ceph:
-    - rgw: <insert rgw region stuff here>
-    - rgw_sync_agent:
-        host: client.0
-        source: client.0
-        target: client.1
+    Run a single radosgw-agent. See task() for config format.
     """
+    src_client = config['src']
+    dest_client = config['dest']
 
-    assert config is not None, "rgw_sync_agent requires a config"
-    if not 'host' in config:
-        log.info("setting blank host to be client.0")
-        config['host'] = "client.0"
-
-    log.info("config is %s", config)
-
-    ceph_conf = find_task_config(ctx, 'ceph')['conf']
-    log.info("ceph_conf is %s", ceph_conf)
-
-    source = config['source']
-    target = config['target']
-
-    log.info("source is %s", source)
-    log.info("target is %s", target)
-    source_region_name = ceph_conf[source]['rgw region']
-    target_region_name = ceph_conf[target]['rgw region']
-    
-    rgw_conf = find_task_config(ctx, 'rgw')
-    log.info("rgw_conf is %s", rgw_conf)
-    source_system_user = rgw_conf[source]['system user']['name']
-    source_access_key = rgw_conf[source]['system user']['access key']
-    source_secret_key = rgw_conf[source]['system user']['secret key']
-    target_system_user = rgw_conf[target]['system user']['name']
-    target_access_key = rgw_conf[target]['system user']['access key']
-    target_secret_key = rgw_conf[target]['system user']['secret key']
-
-    (sync_host,) = ctx.cluster.only(config['host']).remotes
-    (source_host,) = ctx.cluster.only(source).remotes
-    
-    (error, source_region_json) = rgwadmin(ctx, source,
-                                           cmd=['-n', source, 'region', 'get',
-                                                '--rgw-region', source_region_name])
-    log.info("got source_region_json %s", source_region_json)
-
-    (source_host,) = source_region_json['endpoints']
-    source_zone = source_region_json['master_zone']
+    ceph_config = ctx.ceph.conf.get('global', {})
+    ceph_config.update(ctx.ceph.conf.get('client', {}))
+    src_ceph_config = deepcopy(ceph_config)
+    src_ceph_config.update(ctx.ceph.conf.get(src_client, {}))
+    dest_ceph_config = deepcopy(ceph_config)
+    dest_ceph_config.update(ctx.ceph.conf.get(dest_client, {}))
 
-    (error, target_region_json) = rgwadmin(ctx, target,
-                                           cmd=['-n', target, 'region', 'get',
-                                                '--rgw-region', target_region_name])
-
-    (target_host,) = target_region_json['endpoints']
-    target_zone = target_region_json['master_zone']
-    
-    log.info("got target_region_json %s", target_region_json)
+    src_zone = src_ceph_config['rgw zone']
+    dest_zone = dest_ceph_config['rgw zone']
 
+    log.info("source is %s", src_zone)
+    log.info("dest is %s", dest_zone)
 
     testdir = teuthology.get_testdir(ctx)
-
-    sync_host.run(
+    (remote,) = ctx.cluster.only(client).remotes.keys()
+    remote.run(
         args=[
             'cd', testdir, run.Raw('&&'),
-            'git', 'clone', 'https://github.com/ceph/radosgw-agent.git', run.Raw('&&'),
-            'cd', "radosgw-agent", run.Raw('&&'),
-            './bootstrap'
+            'git', 'clone', 'https://github.com/ceph/radosgw-agent.git',
+            'radosgw-agent.{client}'.format(client=client),
+            run.Raw('&&'),
+            'cd', 'radosgw-agent.{client}'.format(client=client),
+            run.Raw('&&'),
+            './bootstrap',
             ]
         )
 
-    sync_proc = sync_host.run(
+    src_host, src_port = rgw_utils.get_zone_host_and_port(ctx, src_client,
+                                                          src_zone)
+    dest_host, dest_port = rgw_utils.get_zone_host_and_port(ctx, dest_client,
+                                                            dest_zone)
+    src_access, src_secret = rgw_utils.get_zone_system_keys(ctx, src_client,
+                                                            src_zone)
+    dest_access, dest_secret = rgw_utils.get_zone_system_keys(ctx, dest_client,
+                                                              dest_zone)
+    port = config.get('port', 8000)
+    daemon_name = '{host}.syncdaemon'.format(host=remote.name)
+
+    return remote.run(
         args=[
             '{tdir}/daemon-helper'.format(tdir=testdir), 'kill',
-            '{tdir}/radosgw-agent/radosgw-agent'.format(tdir=testdir),
-            '--src-access-key', source_access_key,
-            '--src-secret-key', source_secret_key,
-            '--src-host', source_host, '--src-zone', source_zone,
-            '--dest-access-key', target_access_key,
-            '--dest-secret-key', target_secret_key,
-            '--dest-host', target_host, '--dest-zone', target_zone,
-            '--daemon-id', '{host}.syncdaemon'.format(host=sync_host.name),
-            '--test-server-host', 'localhost', '--test-server-port', '8181',
-            '--log-file', '{tdir}/archive/rgw_sync_agent.log'.format(tdir=testdir)
+            '{tdir}/radosgw-agent.{client}/radosgw-agent'.format(tdir=testdir,
+                                                                 client=client),
+            '--src-access-key', src_access,
+            '--src-secret-key', src_secret,
+            '--src-host', src_host,
+            '--src-port', str(src_port),
+            '--src-zone', src_zone,
+            '--dest-access-key', dest_access,
+            '--dest-secret-key', dest_secret,
+            '--dest-host', dest_host,
+            '--dest-port', str(dest_port),
+            '--dest-zone', dest_zone,
+            '--daemon-id', daemon_name,
+            '--test-server-host', '0.0.0.0', '--test-server-port', str(port),
+            '--log-file', '{tdir}/archive/rgw_sync_agent.{client}.log'.format(
+                tdir=testdir,
+                client=client),
             ],
         wait=False,
         stdin=run.PIPE,
-        logger=log.getChild(config['host'])
+        logger=log.getChild(daemon_name)
         )
-    
-    yield
-    
+
+
+@contextlib.contextmanager
+def task(ctx, config):
+    """
+    Run radosgw-agents in test mode.
+
+    Configuration is clients to run the agents on, with settings for
+    source client, destination client, and port to listen on.  Binds
+    to 0.0.0.0. Port defaults to 8000. This must be run on clients
+    that have the correct zone root pools and rgw zone set in
+    ceph.conf, or the task cannot read the region information from the
+    cluster. An example::
+
+      tasks:
+      - ceph:
+          conf:
+            client.0:
+              rgw zone = foo
+              rgw zone root pool = .root.pool
+            client.1:
+              rgw zone = bar
+              rgw zone root pool = .root.pool2
+      - rgw: # region configuration omitted for brevity
+      - rgw_sync_agent:
+          client.0:
+            src: client.0
+            dest: client.1
+            # port: 8000 (default)
+          client.1:
+            src: client.1
+            dest: client.0
+            port: 8001
+    """
+    assert isinstance(config, dict), 'rgw_sync_agent requires a dictionary config'
+    log.debug("config is %s", config)
+
+    procs = [(client, run_radosgw_agent(ctx, client, c_config)) for
+             client, c_config in config.iteritems()]
+
     try:
-        log.info("shutting down sync agent")
-        sync_proc.stdin.close()
-        log.info("waiting on sync agent")
-        sync_proc.exitstatus.get()
+        yield
     finally:
-        log.info("cleaning up sync agent directory")
-        sync_host.run(
-            args=[
-                'rm', '-r', '{tdir}/radosgw-agent'.format(tdir=testdir)
-                ]
-            )
-    
+        testdir = teuthology.get_testdir(ctx)
+        try:
+            for client, proc in procs:
+                log.info("shutting down sync agent on %s", client)
+                proc.stdin.close()
+                proc.exitstatus.get()
+        finally:
+            for client, proc in procs:
+                ctx.cluster.only(client).run(
+                    args=[
+                        'rm', '-rf',
+                        '{tdir}/radosgw-agent.{client}'.format(tdir=testdir,
+                                                               client=client)
+                        ]
+                    )

diff --git a/teuthology/task_util/rgw.py b/teuthology/task_util/rgw.py
index 43bbdcf9ff99049a2b136b17d2350b73ddedd1f3..8a0c87e54b7101883545658cdc51a57ee787bd85 100644 (file)
--- a/teuthology/task_util/rgw.py
+++ b/teuthology/task_util/rgw.py
@@ -1,6 +1,7 @@
 from cStringIO import StringIO
 import logging
 import json
+from urlparse import urlparse
 
 from teuthology import misc as teuthology
 
@@ -37,3 +38,24 @@ def rgwadmin(ctx, client, cmd, stdin=StringIO(), check_status=False):
             j = out
             log.info(' raw result: %s' % j)
     return (r, j)
+
+def get_zone_host_and_port(ctx, client, zone):
+    _, region_map = rgwadmin(ctx, client, check_status=True,
+                             cmd=['-n', client, 'region-map', 'get'])
+    regions = region_map['regions']
+    for region in regions:
+        for zone_info in region['val']['zones']:
+            if zone_info['name'] == zone:
+                endpoint = urlparse(zone_info['endpoints'][0])
+                host, port = endpoint.hostname, endpoint.port
+                if port is None:
+                    port = 80
+                return host, port
+    assert False, 'no endpoint for zone {zone} found'.format(zone=zone)
+
+def get_zone_system_keys(ctx, client, zone):
+    _, zone_info = rgwadmin(ctx, client, check_status=True,
+                            cmd=['-n', client,
+                                 'zone', 'get', '--rgw-zone', zone])
+    system_key = zone_info['system_key']
+    return system_key['access_key'], system_key['secret_key']