openstack: cache auth tokens

Fixes: http://tracker.ceph.com/issues/16893
Signed-off-by: Loic Dachary <loic@dachary.org>

diff --git a/teuthology/nuke/__init__.py b/teuthology/nuke/__init__.py
index 8a3623e46c62084810d164c7ee74ed6e2d5303ee..7a5019ce9ae3ee7b747e7e43eaf86c146129fca7 100644 (file)
--- a/teuthology/nuke/__init__.py
+++ b/teuthology/nuke/__init__.py
@@ -86,7 +86,7 @@ def stale_openstack_instances(ctx, instances, locked_nodes):
 
 
 def openstack_delete_volume(id):
-    sh("openstack volume delete " + id + " || true")
+    OpenStack().run("volume delete " + id + " || true")
 
 
 def stale_openstack_volumes(ctx, volumes):
@@ -94,8 +94,8 @@ def stale_openstack_volumes(ctx, volumes):
     for volume in volumes:
         volume_id = volume.get('ID') or volume['id']
         try:
-            volume = json.loads(sh("openstack -q volume show -f json " +
-                                   volume_id))
+            volume = json.loads(OpenStack().run("volume show -f json " +
+                                                volume_id))
         except subprocess.CalledProcessError:
             log.debug("stale-openstack: {id} disappeared, ignored"
                       .format(id=volume_id))

diff --git a/teuthology/openstack/__init__.py b/teuthology/openstack/__init__.py
index 1bc8fa667619cec90c0b3376a6654216589cc971..fc2a548347c986bf11f91a48f6bfa4e3cbd3d44c 100644 (file)
--- a/teuthology/openstack/__init__.py
+++ b/teuthology/openstack/__init__.py
@@ -32,6 +32,7 @@ import socket
 import subprocess
 import tempfile
 import teuthology
+import time
 import types
 
 from subprocess import CalledProcessError
@@ -66,7 +67,7 @@ class OpenStackInstance(object):
     def set_info(self):
         try:
             self.info = json.loads(
-                misc.sh("openstack -q server show -f json " + self.name_or_id))
+                OpenStack().run("server show -f json " + self.name_or_id))
             enforce_json_dictionary(self.info)
         except CalledProcessError:
             self.info = None
@@ -142,7 +143,7 @@ class OpenStackInstance(object):
                               self.get_addresses())[0]
 
     def get_floating_ip(self):
-        ips = json.loads(misc.sh("openstack -q ip floating list -f json"))
+        ips = json.loads(OpenStack().run("ip floating list -f json"))
         for ip in ips:
             if ip['Instance ID'] == self['id']:
                 return ip['IP']
@@ -162,13 +163,13 @@ class OpenStackInstance(object):
         if not self.exists():
             return True
         volumes = self.get_volumes()
-        misc.sh("openstack -q server set --name REMOVE-ME-" + self.name_or_id +
-                " " + self['id'])
-        misc.sh("openstack -q server delete --wait " + self['id'] +
-                " || true")
+        OpenStack().run("server set --name REMOVE-ME-" + self.name_or_id +
+                        " " + self['id'])
+        OpenStack().run("server delete --wait " + self['id'] +
+                        " || true")
         for volume in volumes:
-            misc.sh("openstack -q volume set --name REMOVE-ME " + volume + " || true")
-            misc.sh("openstack -q volume delete " + volume + " || true")
+            OpenStack().run("volume set --name REMOVE-ME " + volume + " || true")
+            OpenStack().run("volume delete " + volume + " || true")
         return True
 
 
@@ -205,6 +206,40 @@ class OpenStack(object):
         self.up_string = "UNKNOWN"
         self.teuthology_suite = 'teuthology-suite'
 
+    token = None
+    token_expires = None
+    token_cache_duration = 3600
+
+    def cache_token(self):
+        if self.provider != 'ovh':
+            return False
+        if (OpenStack.token is None and
+            os.environ.get('OS_AUTH_TYPE') == 'v2token' and
+            'OS_TOKEN' in os.environ and
+            'OS_TOKEN_EXPIRES' in os.environ):
+            log.debug("get token from the environment of the parent process")
+            OpenStack.token = os.environ['OS_TOKEN']
+            OpenStack.token_expires = int(os.environ['OS_TOKEN_EXPIRES'])
+        if (OpenStack.token_expires is not None and
+            OpenStack.token_expires < time.time()):
+            log.debug("token discarded because it has expired")
+            OpenStack.token = None
+        if OpenStack.token is None:
+            if os.environ.get('OS_AUTH_TYPE') == 'v2token':
+                del os.environ['OS_AUTH_TYPE']
+            OpenStack.token = misc.sh("openstack -q token issue -c id -f value").strip()
+            os.environ['OS_AUTH_TYPE'] = 'v2token'
+            os.environ['OS_TOKEN'] = OpenStack.token
+            OpenStack.token_expires = int(time.time() + OpenStack.token_cache_duration)
+            os.environ['OS_TOKEN_EXPIRES'] = str(OpenStack.token_expires)
+            log.info("caching OS_TOKEN and setting OS_AUTH_TYPE=v2token "
+                     "during %s seconds" % OpenStack.token_cache_duration)
+        return True
+        
+    def run(self, cmd, *args, **kwargs):
+        self.cache_token()
+        return misc.sh("openstack --quiet " + cmd, *args, **kwargs)
+    
     def set_provider(self):
         if 'OS_AUTH_URL' not in os.environ:
             raise Exception('no OS_AUTH_URL environment variable')
@@ -242,7 +277,7 @@ class OpenStack(object):
         """
         Return true if the image exists in OpenStack.
         """
-        found = misc.sh("openstack -q image list -f json --property name='" +
+        found = self.run("image list -f json --property name='" +
                         self.image_name(image) + "'")
         return len(json.loads(found)) > 0
 
@@ -250,7 +285,7 @@ class OpenStack(object):
         """
         Return the uuid of the network in OpenStack.
         """
-        r = json.loads(misc.sh("openstack -q network show -f json " +
+        r = json.loads(self.run("network show -f json " +
                                network))
         return self.get_value(r, 'id')
 
@@ -298,7 +333,7 @@ class OpenStack(object):
         """
         Return the smallest flavor that satisfies the desired size.
         """
-        flavors_string = misc.sh("openstack -q flavor list -f json")
+        flavors_string = self.run("flavor list -f json")
         flavors = json.loads(flavors_string)
         found = []
         for flavor in flavors:
@@ -343,15 +378,14 @@ class OpenStack(object):
     @staticmethod
     def list_instances():
         ownedby = "ownedby='" + teuth_config.openstack['ip'] + "'"
-        all = json.loads(misc.sh(
-            "openstack -q server list -f json --long --name 'target'"))
+        all = json.loads(OpenStack().run(
+            "server list -f json --long --name 'target'"))
         return filter(lambda instance: ownedby in instance['Properties'], all)
 
     @staticmethod
     def list_volumes():
         ownedby = "ownedby='" + teuth_config.openstack['ip'] + "'"
-        all = json.loads(misc.sh(
-            "openstack -q volume list -f json --long"))
+        all = json.loads(OpenStack().run("volume list -f json --long"))
         def select(volume):
             return (ownedby in volume['Properties'] and
                     volume['Display Name'].startswith('target'))
@@ -592,9 +626,9 @@ ssh access           : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/
         know already.
         """
         try:
-            misc.sh("openstack -q flavor list | tail -2")
+            self.run("flavor list | tail -2")
         except subprocess.CalledProcessError:
-            log.exception("openstack -q flavor list")
+            log.exception("flavor list")
             raise Exception("verify openrc.sh has been sourced")
 
     def flavor(self):
@@ -680,7 +714,7 @@ ssh access           : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/
         among instances created within the same tenant.
         """
         try:
-            misc.sh("openstack -q security group show teuthology")
+            self.run("security group show teuthology")
             return
         except subprocess.CalledProcessError:
             pass
@@ -702,7 +736,7 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo
         """
         Return a floating IP address not associated with an instance or None.
         """
-        ips = json.loads(misc.sh("openstack -q ip floating list -f json"))
+        ips = json.loads(OpenStack().run("ip floating list -f json"))
         for ip in ips:
             if not ip['Instance ID']:
                 return ip['IP']
@@ -710,13 +744,13 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo
 
     @staticmethod
     def create_floating_ip():
-        pools = json.loads(misc.sh("openstack -q ip floating pool list -f json"))
+        pools = json.loads(OpenStack().run("ip floating pool list -f json"))
         if not pools:
             return None
         pool = pools[0]['Name']
         try:
-            ip = json.loads(misc.sh(
-                "openstack -q ip floating create -f json '" + pool + "'"))
+            ip = json.loads(OpenStack().run(
+                "ip floating create -f json '" + pool + "'"))
             return TeuthologyOpenStack.get_value(ip, 'ip')
         except subprocess.CalledProcessError:
             log.debug("create_floating_ip: not creating a floating ip")
@@ -733,14 +767,14 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo
         if not ip:
             ip = TeuthologyOpenStack.create_floating_ip()
         if ip:
-            misc.sh("openstack -q ip floating add " + ip + " " + name_or_id)
+            OpenStack().run("ip floating add " + ip + " " + name_or_id)
 
     @staticmethod
     def get_floating_ip_id(ip):
         """
         Return the id of a floating IP
         """
-        results = json.loads(misc.sh("openstack -q ip floating list -f json"))
+        results = json.loads(OpenStack().run("ip floating list -f json"))
         for result in results:
             if result['IP'] == ip:
                 return str(result['ID'])
@@ -758,9 +792,9 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo
         ip = OpenStackInstance(instance_id).get_floating_ip()
         if not ip:
             return
-        misc.sh("openstack -q ip floating remove " + ip + " " + instance_id)
+        OpenStack().run("ip floating remove " + ip + " " + instance_id)
         ip_id = TeuthologyOpenStack.get_floating_ip_id(ip)
-        misc.sh("openstack -q ip floating delete " + ip_id)
+        OpenStack().run("ip floating delete " + ip_id)
 
     def create_cluster(self):
         user_data = self.get_user_data()
@@ -768,8 +802,8 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo
             security_group = ''
         else:
             security_group = " --security-group teuthology"
-        misc.sh(
-            "openstack server create " +
+        self.run(
+            "server create " +
             " --image '" + self.image('ubuntu', '14.04') + "' " +
             " --flavor '" + self.flavor() + "' " +
             " " + self.net() +
@@ -791,8 +825,8 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo
         self.ssh("sudo /etc/init.d/teuthology stop || true")
         instance_id = self.get_instance_id(self.args.name)
         self.delete_floating_ip(instance_id)
-        misc.sh("openstack -q server delete packages-repository || true")
-        misc.sh("openstack -q server delete --wait " + self.args.name)
+        self.run("server delete packages-repository || true")
+        self.run("server delete --wait " + self.args.name)
 
 def main(ctx, argv):
     return TeuthologyOpenStack(ctx, teuth_config, argv).main()

diff --git a/teuthology/openstack/test/openstack-integration.py b/teuthology/openstack/test/openstack-integration.py
index e679967b070a23f53b7d63d4b8b34b93c6e4c589..25695a1f7ef73b3ffd78ed78121ae04169b1b3dd 100644 (file)
--- a/teuthology/openstack/test/openstack-integration.py
+++ b/teuthology/openstack/test/openstack-integration.py
@@ -58,10 +58,12 @@ class Integration(object):
         # move that to def tearDown for debug and when it works move it
         # back in tearDownClass so it is not called on every test
         ownedby = "ownedby='" + teuth_config.openstack['ip']
-        all_instances = teuthology.misc.sh("openstack -q server list -f json --long")
+        all_instances = teuthology.openstack.OpenStack().run(
+            "server list -f json --long")
         for instance in json.loads(all_instances):
             if ownedby in instance['Properties']:
-                teuthology.misc.sh("openstack -q server delete --wait " + instance['ID'])
+                teuthology.openstack.OpenStack().run(
+                    "server delete --wait " + instance['ID'])
 
     def setup_worker(self):
         self.logs = self.d + "/log"
@@ -192,7 +194,7 @@ class TestSchedule(Integration):
         account if the instance has less than 4GB RAM.
         """
         try:
-            teuthology.misc.sh("openstack -q volume list")
+            teuthology.openstack.OpenStack().run("volume list")
             job = 'teuthology/openstack/test/resources_hint.yaml'
             has_cinder = True
         except subprocess.CalledProcessError:

diff --git a/teuthology/openstack/test/test_openstack.py b/teuthology/openstack/test/test_openstack.py
index fcb10496a1c3cde88954055dd3a43148cc87bcab..df778ba1955518e8c84bfd2cf9b81ef1369e8836 100644 (file)
--- a/teuthology/openstack/test/test_openstack.py
+++ b/teuthology/openstack/test/test_openstack.py
@@ -27,6 +27,7 @@ import os
 import pytest
 import subprocess
 import tempfile
+import time
 from mock import patch
 
 import teuthology
@@ -273,6 +274,83 @@ class TestOpenStack(object):
         else:
             del os.environ['OS_AUTH_URL']
 
+    @patch('teuthology.misc.sh')
+    def test_cache_token(self, m_sh):
+        token = 'TOKEN VALUE'
+        m_sh.return_value = token
+        OpenStack.token = None
+        o = OpenStack()
+        #
+        # Only for OVH
+        #
+        o.provider = 'something'
+        assert False == o.cache_token()
+        o.provider = 'ovh'
+        #
+        # Set the environment with the token
+        #
+        assert 'OS_AUTH_TYPE' not in os.environ
+        assert 'OS_TOKEN' not in os.environ
+        assert 'OS_TOKEN_EXPIRES' not in os.environ
+        assert True == o.cache_token()
+        m_sh.assert_called_with('openstack token issue -c id -f value')
+        assert 'v2token' == os.environ['OS_AUTH_TYPE']
+        assert token == os.environ['OS_TOKEN']
+        assert token == OpenStack.token
+        assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
+        assert time.time() < OpenStack.token_expires
+        #
+        # Reset after it expires
+        #
+        token_expires = int(time.time()) - 2000
+        OpenStack.token_expires = token_expires
+        assert True == o.cache_token()
+        assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
+        assert time.time() < OpenStack.token_expires
+        del os.environ['OS_AUTH_TYPE']
+        del os.environ['OS_TOKEN']
+        del os.environ['OS_TOKEN_EXPIRES']
+
+    @patch('teuthology.misc.sh')
+    def test_cache_token_from_environment(self, m_sh):
+        OpenStack.token = None
+        o = OpenStack()
+        o.provider = 'ovh'
+        token = 'TOKEN VALUE'
+        os.environ['OS_AUTH_TYPE'] = 'v2token'
+        os.environ['OS_TOKEN'] = token
+        token_expires = int(time.time()) + OpenStack.token_cache_duration
+        os.environ['OS_TOKEN_EXPIRES'] = str(token_expires)
+        assert True == o.cache_token()
+        assert token == OpenStack.token
+        assert token_expires == OpenStack.token_expires
+        m_sh.assert_not_called()
+        del os.environ['OS_AUTH_TYPE']
+        del os.environ['OS_TOKEN']
+        del os.environ['OS_TOKEN_EXPIRES']
+        
+    @patch('teuthology.misc.sh')
+    def test_cache_token_expired_environment(self, m_sh):
+        token = 'TOKEN VALUE'
+        m_sh.return_value = token
+        OpenStack.token = None
+        o = OpenStack()
+        o.provider = 'ovh'
+        os.environ['OS_AUTH_TYPE'] = 'v2token'
+        os.environ['OS_TOKEN'] = token
+        token_expires = int(time.time()) - 2000
+        os.environ['OS_TOKEN_EXPIRES'] = str(token_expires)
+        assert True == o.cache_token()
+        m_sh.assert_called_with('openstack token issue -c id -f value')
+        assert 'v2token' == os.environ['OS_AUTH_TYPE']
+        assert token == os.environ['OS_TOKEN']
+        assert token == OpenStack.token
+        assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
+        assert time.time() < OpenStack.token_expires
+        del os.environ['OS_AUTH_TYPE']
+        del os.environ['OS_TOKEN']
+        del os.environ['OS_TOKEN_EXPIRES']
+
 class TestTeuthologyOpenStack(object):
 
     @classmethod
@@ -286,7 +364,7 @@ class TestTeuthologyOpenStack(object):
         ip = TeuthologyOpenStack.create_floating_ip()
         if ip:
             ip_id = TeuthologyOpenStack.get_floating_ip_id(ip)
-            misc.sh("openstack -q ip floating delete " + ip_id)
+            OpenStack().run("ip floating delete " + ip_id)
             self.can_create_floating_ips = True
         else:
             self.can_create_floating_ips = False
@@ -378,4 +456,4 @@ openstack keypair delete {key_name} || true
         ip = TeuthologyOpenStack.get_unassociated_floating_ip()
         assert expected == ip
         ip_id = TeuthologyOpenStack.get_floating_ip_id(ip)
-        misc.sh("openstack -q ip floating delete " + ip_id)
+        OpenStack().run("ip floating delete " + ip_id)

diff --git a/teuthology/provision/openstack.py b/teuthology/provision/openstack.py
index 2080fde6a37fcd7cff5c65ba404a5f1e0e7980d1..27f5e6529b5cba3bf902a834e9570ece0c891914 100644 (file)
--- a/teuthology/provision/openstack.py
+++ b/teuthology/provision/openstack.py
@@ -62,29 +62,26 @@ class ProvisionOpenStack(OpenStack):
         for i in range(volumes['count']):
             volume_name = name + '-' + str(i)
             try:
-                misc.sh("openstack volume show -f json " +
-                         volume_name)
+                self.run("volume show -f json " + volume_name)
             except subprocess.CalledProcessError as e:
                 if 'No volume with a name or ID' not in e.output:
                     raise e
-                misc.sh("openstack volume create -f json " +
-                        config['openstack'].get('volume-create', '') + " " +
-                        " --property ownedby=" + config.openstack['ip'] +
-                        " --size " + str(volumes['size']) + " " +
-                        volume_name)
+                self.run("volume create -f json " +
+                         config['openstack'].get('volume-create', '') + " " +
+                         " --property ownedby=" + config.openstack['ip'] +
+                         " --size " + str(volumes['size']) + " " +
+                         volume_name)
             with safe_while(sleep=2, tries=100,
                             action="volume " + volume_name) as proceed:
                 while proceed():
-                    r = misc.sh("openstack -q volume show  -f json " +
-                                volume_name)
+                    r = self.run("volume show  -f json " + volume_name)
                     status = self.get_value(json.loads(r), 'status')
                     if status == 'available':
                         break
                     else:
                         log.info("volume " + volume_name +
                                  " not available yet")
-            misc.sh("openstack server add volume " +
-                    name + " " + volume_name)
+            self.run("server add volume " + name + " " + volume_name)
 
     @staticmethod
     def ip2name(prefix, ip):
@@ -145,9 +142,9 @@ class ProvisionOpenStack(OpenStack):
             for instance in instances:
                 ip = instance.get_ip(network)
                 name = self.ip2name(self.basename, ip)
-                misc.sh("openstack server set " +
-                        "--name " + name + " " +
-                        instance['ID'])
+                self.run("server set " +
+                         "--name " + name + " " +
+                         instance['ID'])
                 fqdn = name + '.' + config.lab_domain
                 if not misc.ssh_keyscan_wait(fqdn):
                     raise ValueError('ssh_keyscan_wait failed for ' + fqdn)