radosgw-admin: various multiregion related fixes and changes

Go to the master zone in the master region for radosgw-admin
operations. Trigger metadata sync. Other fixes.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

diff --git a/teuthology/task/radosgw-admin.py b/teuthology/task/radosgw-admin.py
index 6b6e5cbf91b6eda04b88703aa473f2e284c57fef..090c4fec39c72aa53e69ca7a12134bd9c9ac79d0 100644 (file)
--- a/teuthology/task/radosgw-admin.py
+++ b/teuthology/task/radosgw-admin.py
@@ -12,6 +12,8 @@ import boto.exception
 import boto.s3.connection
 import boto.s3.acl
 
+import teuthology.task_util.rgw as rgw_utils
+
 import time
 
 from teuthology import misc as teuthology
@@ -41,8 +43,11 @@ def task(ctx, config):
         config = dict.fromkeys(config)
     clients = config.keys()
 
-    # just use the first client...
-    client = clients[0];
+    client = rgw_utils.get_master_client(ctx, clients)
+
+    if not client:
+        # oh, well, just use the first client... multiregion stuff might not work correctly
+        client = clients[0];
 
     ##
     user1='foo'
@@ -215,6 +220,8 @@ def task(ctx, config):
     assert not err
     assert len(out) == 0
 
+    rgw_utils.radosgw_agent_sync_all(ctx)
+
     # connect to rgw
     (remote,) = ctx.cluster.only(client).remotes.iterkeys()
     (remote_user, remote_host) = remote.name.split('@')
@@ -367,9 +374,13 @@ def task(ctx, config):
 
     for obj in out:
         # TESTCASE 'log-show','log','show','after activity','returns expected info'
+        if obj[:4] == 'meta' or obj[:4] == 'data':
+            continue
+
         (err, log) = rgwadmin(ctx, client, ['log', 'show', '--object', obj])
         assert not err
         assert len(log) > 0
+
         assert log['bucket'].find(bucket_name) == 0
         assert log['bucket'] != bucket_name or log['bucket_id'] == bucket_id 
         assert log['bucket_owner'] == user1 or log['bucket'] == bucket_name + '5'
@@ -563,12 +574,19 @@ def task(ctx, config):
     assert err
 
     # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
+    # 
+
     (err, out) = rgwadmin(ctx, client, ['zone', 'get'])
-    assert len(out) > 0
-    assert len(out['placement_pools']) == 1
+    orig_placement_pools = len(out['placement_pools'])
+
+    # removed this test, it is not correct to assume that zone has default placement, it really
+    # depends on how we set it up before
+    #
+    # assert len(out) > 0
+    # assert len(out['placement_pools']) == 1
 
-    default_rule = out['placement_pools'][0]
-    assert default_rule['key'] == 'default-placement'
+    # default_rule = out['placement_pools'][0]
+    # assert default_rule['key'] == 'default-placement'
 
     rule={'key': 'new-placement', 'val': {'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2'}}
 
@@ -579,4 +597,4 @@ def task(ctx, config):
 
     (err, out) = rgwadmin(ctx, client, ['zone', 'get'])
     assert len(out) > 0
-    assert len(out['placement_pools']) == 2
+    assert len(out['placement_pools']) == orig_placement_pools + 1

diff --git a/teuthology/task/radosgw-agent.py b/teuthology/task/radosgw-agent.py
index 62e6025672ff6a9fdd16dcbd44275cd546b97089..40920ff6af7790f9c400b74c5a83422a7a05d084 100644 (file)
--- a/teuthology/task/radosgw-agent.py
+++ b/teuthology/task/radosgw-agent.py
@@ -1,5 +1,6 @@
 import contextlib
 import logging
+import argparse
 
 from ..orchestra import run
 from teuthology import misc as teuthology
@@ -108,9 +109,15 @@ def task(ctx, config):
     assert isinstance(config, dict), 'rgw_sync_agent requires a dictionary config'
     log.debug("config is %s", config)
 
+
+    ctx.radosgw_agent = argparse.Namespace()
+    ctx.radosgw_agent.config = config
+
     procs = [(client, run_radosgw_agent(ctx, client, c_config)) for
              client, c_config in config.iteritems()]
 
+    ctx.radosgw_agent.procs = procs
+
     try:
         yield
     finally:

diff --git a/teuthology/task_util/rgw.py b/teuthology/task_util/rgw.py
index c737e29795f7ef769246975cb8cb275b9b7dd12a..3dca2ccd2ba53d41cda2856d32351967fd82a358 100644 (file)
--- a/teuthology/task_util/rgw.py
+++ b/teuthology/task_util/rgw.py
@@ -1,6 +1,7 @@
 from cStringIO import StringIO
 import logging
 import json
+import requests
 from urlparse import urlparse
 
 from teuthology import misc as teuthology
@@ -17,8 +18,10 @@ def rgwadmin(ctx, client, cmd, stdin=StringIO(), check_status=False):
         'radosgw-admin'.format(tdir=testdir),
         '--log-to-stderr',
         '--format', 'json',
+        '-n',  client,
         ]
     pre.extend(cmd)
+    log.info('radosgw-admin: cmd=%s' % pre)
     (remote,) = ctx.cluster.only(client).remotes.iterkeys()
     proc = remote.run(
         args=pre,
@@ -53,6 +56,35 @@ def get_zone_host_and_port(ctx, client, zone):
                 return host, port
     assert False, 'no endpoint for zone {zone} found'.format(zone=zone)
 
+def get_master_zone(ctx, client):
+    _, region_map = rgwadmin(ctx, client, check_status=True,
+                             cmd=['-n', client, 'region-map', 'get'])
+    regions = region_map['regions']
+    for region in regions:
+        is_master = (region['val']['is_master'] == "true")
+        log.info('region={r} is_master={ism}'.format(r=region, ism=is_master))
+        if not is_master:
+          continue
+        master_zone = region['val']['master_zone']
+        log.info('master_zone=%s' % master_zone)
+        for zone_info in region['val']['zones']:
+            if zone_info['name'] == master_zone:
+                return master_zone
+    log.info('couldn\'t find master zone')
+    return None
+
+def get_master_client(ctx, clients):
+    master_zone = get_master_zone(ctx, clients[0]) # can use any client for this as long as system configured correctly
+    if not master_zone:
+        return None
+
+    for client in clients:
+        zone = zone_for_client(ctx, client)
+        if zone == master_zone:
+            return client
+
+    return None
+
 def get_zone_system_keys(ctx, client, zone):
     _, zone_info = rgwadmin(ctx, client, check_status=True,
                             cmd=['-n', client,
@@ -65,3 +97,28 @@ def zone_for_client(ctx, client):
     ceph_config.update(ctx.ceph.conf.get('client', {}))
     ceph_config.update(ctx.ceph.conf.get(client, {}))
     return ceph_config.get('rgw zone')
+
+
+def radosgw_agent_sync(ctx, agent_host, agent_port):
+    print 'agent_host', agent_host, 'port', agent_port
+    log.info('sync agent {h}:{p}'.format(h=agent_host, p=agent_port))
+    return requests.post('http://{addr}:{port}/metadata/incremental'.format(addr = agent_host, port = agent_port))
+
+def radosgw_agent_sync_all(ctx):
+    if ctx.radosgw_agent.procs:
+        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
+            dest_zone = zone_for_client(ctx, agent_client)
+            port = c_config.get('port', 8000)
+            dest_host, dest_port = get_zone_host_and_port(ctx, agent_client, dest_zone)
+            radosgw_agent_sync(ctx, dest_host, port)
+
+def radosgw_agent_sync_all(ctx):
+    if ctx.radosgw_agent.procs:
+        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
+            dest_zone = zone_for_client(ctx, agent_client)
+            port = c_config.get('port', 8000)
+            dest_host, dest_port = get_zone_host_and_port(ctx, agent_client, dest_zone)
+            radosgw_agent_sync(ctx, dest_host, port)
+
+
+