From e5b1ead4a438a79427eb210787f0d98a90d14531 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Thu, 11 Aug 2016 12:53:58 +0200 Subject: [PATCH] openstack: cache auth tokens Fixes: http://tracker.ceph.com/issues/16893 Signed-off-by: Loic Dachary --- teuthology/nuke/__init__.py | 6 +- teuthology/openstack/__init__.py | 94 +++++++++++++------ .../openstack/test/openstack-integration.py | 8 +- teuthology/openstack/test/test_openstack.py | 82 +++++++++++++++- teuthology/provision/openstack.py | 25 +++-- 5 files changed, 163 insertions(+), 52 deletions(-) diff --git a/teuthology/nuke/__init__.py b/teuthology/nuke/__init__.py index 8a3623e46c..7a5019ce9a 100644 --- a/teuthology/nuke/__init__.py +++ b/teuthology/nuke/__init__.py @@ -86,7 +86,7 @@ def stale_openstack_instances(ctx, instances, locked_nodes): def openstack_delete_volume(id): - sh("openstack volume delete " + id + " || true") + OpenStack().run("volume delete " + id + " || true") def stale_openstack_volumes(ctx, volumes): @@ -94,8 +94,8 @@ def stale_openstack_volumes(ctx, volumes): for volume in volumes: volume_id = volume.get('ID') or volume['id'] try: - volume = json.loads(sh("openstack -q volume show -f json " + - volume_id)) + volume = json.loads(OpenStack().run("volume show -f json " + + volume_id)) except subprocess.CalledProcessError: log.debug("stale-openstack: {id} disappeared, ignored" .format(id=volume_id)) diff --git a/teuthology/openstack/__init__.py b/teuthology/openstack/__init__.py index 1bc8fa6676..fc2a548347 100644 --- a/teuthology/openstack/__init__.py +++ b/teuthology/openstack/__init__.py @@ -32,6 +32,7 @@ import socket import subprocess import tempfile import teuthology +import time import types from subprocess import CalledProcessError @@ -66,7 +67,7 @@ class OpenStackInstance(object): def set_info(self): try: self.info = json.loads( - misc.sh("openstack -q server show -f json " + self.name_or_id)) + OpenStack().run("server show -f json " + self.name_or_id)) enforce_json_dictionary(self.info) except CalledProcessError: self.info = None @@ -142,7 +143,7 @@ class OpenStackInstance(object): self.get_addresses())[0] def get_floating_ip(self): - ips = json.loads(misc.sh("openstack -q ip floating list -f json")) + ips = json.loads(OpenStack().run("ip floating list -f json")) for ip in ips: if ip['Instance ID'] == self['id']: return ip['IP'] @@ -162,13 +163,13 @@ class OpenStackInstance(object): if not self.exists(): return True volumes = self.get_volumes() - misc.sh("openstack -q server set --name REMOVE-ME-" + self.name_or_id + - " " + self['id']) - misc.sh("openstack -q server delete --wait " + self['id'] + - " || true") + OpenStack().run("server set --name REMOVE-ME-" + self.name_or_id + + " " + self['id']) + OpenStack().run("server delete --wait " + self['id'] + + " || true") for volume in volumes: - misc.sh("openstack -q volume set --name REMOVE-ME " + volume + " || true") - misc.sh("openstack -q volume delete " + volume + " || true") + OpenStack().run("volume set --name REMOVE-ME " + volume + " || true") + OpenStack().run("volume delete " + volume + " || true") return True @@ -205,6 +206,40 @@ class OpenStack(object): self.up_string = "UNKNOWN" self.teuthology_suite = 'teuthology-suite' + token = None + token_expires = None + token_cache_duration = 3600 + + def cache_token(self): + if self.provider != 'ovh': + return False + if (OpenStack.token is None and + os.environ.get('OS_AUTH_TYPE') == 'v2token' and + 'OS_TOKEN' in os.environ and + 'OS_TOKEN_EXPIRES' in os.environ): + log.debug("get token from the environment of the parent process") + OpenStack.token = os.environ['OS_TOKEN'] + OpenStack.token_expires = int(os.environ['OS_TOKEN_EXPIRES']) + if (OpenStack.token_expires is not None and + OpenStack.token_expires < time.time()): + log.debug("token discarded because it has expired") + OpenStack.token = None + if OpenStack.token is None: + if os.environ.get('OS_AUTH_TYPE') == 'v2token': + del os.environ['OS_AUTH_TYPE'] + OpenStack.token = misc.sh("openstack -q token issue -c id -f value").strip() + os.environ['OS_AUTH_TYPE'] = 'v2token' + os.environ['OS_TOKEN'] = OpenStack.token + OpenStack.token_expires = int(time.time() + OpenStack.token_cache_duration) + os.environ['OS_TOKEN_EXPIRES'] = str(OpenStack.token_expires) + log.info("caching OS_TOKEN and setting OS_AUTH_TYPE=v2token " + "during %s seconds" % OpenStack.token_cache_duration) + return True + + def run(self, cmd, *args, **kwargs): + self.cache_token() + return misc.sh("openstack --quiet " + cmd, *args, **kwargs) + def set_provider(self): if 'OS_AUTH_URL' not in os.environ: raise Exception('no OS_AUTH_URL environment variable') @@ -242,7 +277,7 @@ class OpenStack(object): """ Return true if the image exists in OpenStack. """ - found = misc.sh("openstack -q image list -f json --property name='" + + found = self.run("image list -f json --property name='" + self.image_name(image) + "'") return len(json.loads(found)) > 0 @@ -250,7 +285,7 @@ class OpenStack(object): """ Return the uuid of the network in OpenStack. """ - r = json.loads(misc.sh("openstack -q network show -f json " + + r = json.loads(self.run("network show -f json " + network)) return self.get_value(r, 'id') @@ -298,7 +333,7 @@ class OpenStack(object): """ Return the smallest flavor that satisfies the desired size. """ - flavors_string = misc.sh("openstack -q flavor list -f json") + flavors_string = self.run("flavor list -f json") flavors = json.loads(flavors_string) found = [] for flavor in flavors: @@ -343,15 +378,14 @@ class OpenStack(object): @staticmethod def list_instances(): ownedby = "ownedby='" + teuth_config.openstack['ip'] + "'" - all = json.loads(misc.sh( - "openstack -q server list -f json --long --name 'target'")) + all = json.loads(OpenStack().run( + "server list -f json --long --name 'target'")) return filter(lambda instance: ownedby in instance['Properties'], all) @staticmethod def list_volumes(): ownedby = "ownedby='" + teuth_config.openstack['ip'] + "'" - all = json.loads(misc.sh( - "openstack -q volume list -f json --long")) + all = json.loads(OpenStack().run("volume list -f json --long")) def select(volume): return (ownedby in volume['Properties'] and volume['Display Name'].startswith('target')) @@ -592,9 +626,9 @@ ssh access : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/ know already. """ try: - misc.sh("openstack -q flavor list | tail -2") + self.run("flavor list | tail -2") except subprocess.CalledProcessError: - log.exception("openstack -q flavor list") + log.exception("flavor list") raise Exception("verify openrc.sh has been sourced") def flavor(self): @@ -680,7 +714,7 @@ ssh access : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/ among instances created within the same tenant. """ try: - misc.sh("openstack -q security group show teuthology") + self.run("security group show teuthology") return except subprocess.CalledProcessError: pass @@ -702,7 +736,7 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo """ Return a floating IP address not associated with an instance or None. """ - ips = json.loads(misc.sh("openstack -q ip floating list -f json")) + ips = json.loads(OpenStack().run("ip floating list -f json")) for ip in ips: if not ip['Instance ID']: return ip['IP'] @@ -710,13 +744,13 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo @staticmethod def create_floating_ip(): - pools = json.loads(misc.sh("openstack -q ip floating pool list -f json")) + pools = json.loads(OpenStack().run("ip floating pool list -f json")) if not pools: return None pool = pools[0]['Name'] try: - ip = json.loads(misc.sh( - "openstack -q ip floating create -f json '" + pool + "'")) + ip = json.loads(OpenStack().run( + "ip floating create -f json '" + pool + "'")) return TeuthologyOpenStack.get_value(ip, 'ip') except subprocess.CalledProcessError: log.debug("create_floating_ip: not creating a floating ip") @@ -733,14 +767,14 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo if not ip: ip = TeuthologyOpenStack.create_floating_ip() if ip: - misc.sh("openstack -q ip floating add " + ip + " " + name_or_id) + OpenStack().run("ip floating add " + ip + " " + name_or_id) @staticmethod def get_floating_ip_id(ip): """ Return the id of a floating IP """ - results = json.loads(misc.sh("openstack -q ip floating list -f json")) + results = json.loads(OpenStack().run("ip floating list -f json")) for result in results: if result['IP'] == ip: return str(result['ID']) @@ -758,9 +792,9 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo ip = OpenStackInstance(instance_id).get_floating_ip() if not ip: return - misc.sh("openstack -q ip floating remove " + ip + " " + instance_id) + OpenStack().run("ip floating remove " + ip + " " + instance_id) ip_id = TeuthologyOpenStack.get_floating_ip_id(ip) - misc.sh("openstack -q ip floating delete " + ip_id) + OpenStack().run("ip floating delete " + ip_id) def create_cluster(self): user_data = self.get_user_data() @@ -768,8 +802,8 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo security_group = '' else: security_group = " --security-group teuthology" - misc.sh( - "openstack server create " + + self.run( + "server create " + " --image '" + self.image('ubuntu', '14.04') + "' " + " --flavor '" + self.flavor() + "' " + " " + self.net() + @@ -791,8 +825,8 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo self.ssh("sudo /etc/init.d/teuthology stop || true") instance_id = self.get_instance_id(self.args.name) self.delete_floating_ip(instance_id) - misc.sh("openstack -q server delete packages-repository || true") - misc.sh("openstack -q server delete --wait " + self.args.name) + self.run("server delete packages-repository || true") + self.run("server delete --wait " + self.args.name) def main(ctx, argv): return TeuthologyOpenStack(ctx, teuth_config, argv).main() diff --git a/teuthology/openstack/test/openstack-integration.py b/teuthology/openstack/test/openstack-integration.py index e679967b07..25695a1f7e 100644 --- a/teuthology/openstack/test/openstack-integration.py +++ b/teuthology/openstack/test/openstack-integration.py @@ -58,10 +58,12 @@ class Integration(object): # move that to def tearDown for debug and when it works move it # back in tearDownClass so it is not called on every test ownedby = "ownedby='" + teuth_config.openstack['ip'] - all_instances = teuthology.misc.sh("openstack -q server list -f json --long") + all_instances = teuthology.openstack.OpenStack().run( + "server list -f json --long") for instance in json.loads(all_instances): if ownedby in instance['Properties']: - teuthology.misc.sh("openstack -q server delete --wait " + instance['ID']) + teuthology.openstack.OpenStack().run( + "server delete --wait " + instance['ID']) def setup_worker(self): self.logs = self.d + "/log" @@ -192,7 +194,7 @@ class TestSchedule(Integration): account if the instance has less than 4GB RAM. """ try: - teuthology.misc.sh("openstack -q volume list") + teuthology.openstack.OpenStack().run("volume list") job = 'teuthology/openstack/test/resources_hint.yaml' has_cinder = True except subprocess.CalledProcessError: diff --git a/teuthology/openstack/test/test_openstack.py b/teuthology/openstack/test/test_openstack.py index fcb10496a1..df778ba195 100644 --- a/teuthology/openstack/test/test_openstack.py +++ b/teuthology/openstack/test/test_openstack.py @@ -27,6 +27,7 @@ import os import pytest import subprocess import tempfile +import time from mock import patch import teuthology @@ -273,6 +274,83 @@ class TestOpenStack(object): else: del os.environ['OS_AUTH_URL'] + @patch('teuthology.misc.sh') + def test_cache_token(self, m_sh): + token = 'TOKEN VALUE' + m_sh.return_value = token + OpenStack.token = None + o = OpenStack() + # + # Only for OVH + # + o.provider = 'something' + assert False == o.cache_token() + o.provider = 'ovh' + # + # Set the environment with the token + # + assert 'OS_AUTH_TYPE' not in os.environ + assert 'OS_TOKEN' not in os.environ + assert 'OS_TOKEN_EXPIRES' not in os.environ + assert True == o.cache_token() + m_sh.assert_called_with('openstack token issue -c id -f value') + assert 'v2token' == os.environ['OS_AUTH_TYPE'] + assert token == os.environ['OS_TOKEN'] + assert token == OpenStack.token + assert time.time() < int(os.environ['OS_TOKEN_EXPIRES']) + assert time.time() < OpenStack.token_expires + # + # Reset after it expires + # + token_expires = int(time.time()) - 2000 + OpenStack.token_expires = token_expires + assert True == o.cache_token() + assert time.time() < int(os.environ['OS_TOKEN_EXPIRES']) + assert time.time() < OpenStack.token_expires + del os.environ['OS_AUTH_TYPE'] + del os.environ['OS_TOKEN'] + del os.environ['OS_TOKEN_EXPIRES'] + + @patch('teuthology.misc.sh') + def test_cache_token_from_environment(self, m_sh): + OpenStack.token = None + o = OpenStack() + o.provider = 'ovh' + token = 'TOKEN VALUE' + os.environ['OS_AUTH_TYPE'] = 'v2token' + os.environ['OS_TOKEN'] = token + token_expires = int(time.time()) + OpenStack.token_cache_duration + os.environ['OS_TOKEN_EXPIRES'] = str(token_expires) + assert True == o.cache_token() + assert token == OpenStack.token + assert token_expires == OpenStack.token_expires + m_sh.assert_not_called() + del os.environ['OS_AUTH_TYPE'] + del os.environ['OS_TOKEN'] + del os.environ['OS_TOKEN_EXPIRES'] + + @patch('teuthology.misc.sh') + def test_cache_token_expired_environment(self, m_sh): + token = 'TOKEN VALUE' + m_sh.return_value = token + OpenStack.token = None + o = OpenStack() + o.provider = 'ovh' + os.environ['OS_AUTH_TYPE'] = 'v2token' + os.environ['OS_TOKEN'] = token + token_expires = int(time.time()) - 2000 + os.environ['OS_TOKEN_EXPIRES'] = str(token_expires) + assert True == o.cache_token() + m_sh.assert_called_with('openstack token issue -c id -f value') + assert 'v2token' == os.environ['OS_AUTH_TYPE'] + assert token == os.environ['OS_TOKEN'] + assert token == OpenStack.token + assert time.time() < int(os.environ['OS_TOKEN_EXPIRES']) + assert time.time() < OpenStack.token_expires + del os.environ['OS_AUTH_TYPE'] + del os.environ['OS_TOKEN'] + del os.environ['OS_TOKEN_EXPIRES'] + class TestTeuthologyOpenStack(object): @classmethod @@ -286,7 +364,7 @@ class TestTeuthologyOpenStack(object): ip = TeuthologyOpenStack.create_floating_ip() if ip: ip_id = TeuthologyOpenStack.get_floating_ip_id(ip) - misc.sh("openstack -q ip floating delete " + ip_id) + OpenStack().run("ip floating delete " + ip_id) self.can_create_floating_ips = True else: self.can_create_floating_ips = False @@ -378,4 +456,4 @@ openstack keypair delete {key_name} || true ip = TeuthologyOpenStack.get_unassociated_floating_ip() assert expected == ip ip_id = TeuthologyOpenStack.get_floating_ip_id(ip) - misc.sh("openstack -q ip floating delete " + ip_id) + OpenStack().run("ip floating delete " + ip_id) diff --git a/teuthology/provision/openstack.py b/teuthology/provision/openstack.py index 2080fde6a3..27f5e6529b 100644 --- a/teuthology/provision/openstack.py +++ b/teuthology/provision/openstack.py @@ -62,29 +62,26 @@ class ProvisionOpenStack(OpenStack): for i in range(volumes['count']): volume_name = name + '-' + str(i) try: - misc.sh("openstack volume show -f json " + - volume_name) + self.run("volume show -f json " + volume_name) except subprocess.CalledProcessError as e: if 'No volume with a name or ID' not in e.output: raise e - misc.sh("openstack volume create -f json " + - config['openstack'].get('volume-create', '') + " " + - " --property ownedby=" + config.openstack['ip'] + - " --size " + str(volumes['size']) + " " + - volume_name) + self.run("volume create -f json " + + config['openstack'].get('volume-create', '') + " " + + " --property ownedby=" + config.openstack['ip'] + + " --size " + str(volumes['size']) + " " + + volume_name) with safe_while(sleep=2, tries=100, action="volume " + volume_name) as proceed: while proceed(): - r = misc.sh("openstack -q volume show -f json " + - volume_name) + r = self.run("volume show -f json " + volume_name) status = self.get_value(json.loads(r), 'status') if status == 'available': break else: log.info("volume " + volume_name + " not available yet") - misc.sh("openstack server add volume " + - name + " " + volume_name) + self.run("server add volume " + name + " " + volume_name) @staticmethod def ip2name(prefix, ip): @@ -145,9 +142,9 @@ class ProvisionOpenStack(OpenStack): for instance in instances: ip = instance.get_ip(network) name = self.ip2name(self.basename, ip) - misc.sh("openstack server set " + - "--name " + name + " " + - instance['ID']) + self.run("server set " + + "--name " + name + " " + + instance['ID']) fqdn = name + '.' + config.lab_domain if not misc.ssh_keyscan_wait(fqdn): raise ValueError('ssh_keyscan_wait failed for ' + fqdn) -- 2.39.5