openstack: TeuthologyOpenStack.cloud_init_wait continuous output

Instead of tailing the files every 30 seconds, wait for the
string that proves the teuthology installation completed with a tail -f
and display the result continously.

The number of attempts is reduced from 100 to 30 because there is no
need for repeated calls to tail to check for the desired string.

Signed-off-by: Loic Dachary <loic@dachary.org>

diff --git a/teuthology/openstack/__init__.py b/teuthology/openstack/__init__.py
index 1eb5066d7b899a47043d281b5e7236f11ec4a91a..666e57004fc5f7ea59cdd1725d82e3b32f5840aa 100644 (file)
--- a/teuthology/openstack/__init__.py
+++ b/teuthology/openstack/__init__.py
@@ -341,11 +341,11 @@ class OpenStack(object):
                     volume['Display Name'].startswith('target'))
         return filter(select, all)
 
-    def cloud_init_wait(self):
+    def cloud_init_wait(self, instance):
         """
         Wait for cloud-init to complete on the name_or_ip OpenStack instance.
         """
-        ip = self.instance.get_floating_ip_or_ip()
+        ip = instance.get_floating_ip_or_ip()
         log.debug('cloud_init_wait ' + ip)
         client_args = {
             'user_at_host': '@'.join((self.username, ip)),
@@ -355,15 +355,13 @@ class OpenStack(object):
         if self.key_filename:
             log.debug("using key " + self.key_filename)
             client_args['key_filename'] = self.key_filename
-        with safe_while(sleep=30, tries=100,
+        with safe_while(sleep=30, tries=30,
                         action="cloud_init_wait " + ip) as proceed:
             success = False
             # CentOS 6.6 logs in /var/log/clout-init-output.log
             # CentOS 7.0 logs in /var/log/clout-init.log
-            all_done = ("tail /var/log/cloud-init*.log ; " +
-                        " test -f /tmp/init.out && tail /tmp/init.out ; " +
-                        " grep '" + self.up_string + "' " +
-                        "/var/log/cloud-init*.log")
+            tail = ("tail --follow=name --retry"
+                        " /var/log/cloud-init*.log /tmp/init.out")
             while proceed():
                 try:
                     client = connection.connect(**client_args)
@@ -391,24 +389,28 @@ class OpenStack(object):
                             continue
                     log.exception('cloud_init_wait ' + ip)
                     raise
-                log.debug('cloud_init_wait ' + all_done)
+                log.debug('cloud_init_wait ' + tail)
                 try:
-                    stdin, stdout, stderr = client.exec_command(all_done)
-                    stdout.channel.settimeout(5)
-                    out = stdout.read()
-                    log.debug('cloud_init_wait stdout ' + all_done + ' ' + out)
+                    # get the I/O channel to iterate line by line
+                    transport = client.get_transport()
+                    channel = transport.open_session()
+                    channel.get_pty()
+                    channel.settimeout(240)
+                    output = channel.makefile('r', 1)
+                    channel.exec_command(tail)
+                    for line in iter(output.readline, b''):
+                        log.info(line.strip())
+                        if self.up_string in line:
+                            success = True
+                            break
                 except socket.timeout as e:
                     client.close()
-                    log.debug('cloud_init_wait socket.timeout ' + all_done)
+                    log.debug('cloud_init_wait socket.timeout ' + tail)
                     continue
                 except socket.error as e:
                     client.close()
-                    log.debug('cloud_init_wait socket.error ' + str(e) + ' ' + all_done)
+                    log.debug('cloud_init_wait socket.error ' + str(e) + ' ' + tail)
                     continue
-                log.debug('cloud_init_wait stderr ' + all_done +
-                          ' ' + stderr.read())
-                if stdout.channel.recv_exit_status() == 0:
-                    success = True
                 client.close()
                 if success:
                     break
@@ -489,7 +491,7 @@ class TeuthologyOpenStack(OpenStack):
         else:
             identity = ''
         if self.args.upload:
-            upload = 'upload to    : ' + self.args.archive_upload
+            upload = 'upload to            : ' + self.args.archive_upload
         else:
             upload = ''
         log.info("""
@@ -708,7 +710,7 @@ openstack security group rule create --proto udp --dst-port 53 teuthology # dns
         """
         Remove the floating ip from instance_id and delete it.
         """
-        ip = TeuthologyOpenStack.get_floating_ip(instance_id)
+        ip = OpenStackInstance(instance_id).get_floating_ip()
         if not ip:
             return
         misc.sh("openstack ip floating remove " + ip + " " + instance_id)
@@ -734,7 +736,7 @@ openstack security group rule create --proto udp --dst-port 53 teuthology # dns
         os.unlink(user_data)
         self.instance = OpenStackInstance(self.args.name)
         self.associate_floating_ip(self.instance['id'])
-        return self.cloud_init_wait()
+        return self.cloud_init_wait(self.instance)
 
     def teardown(self):
         """

diff --git a/teuthology/provision.py b/teuthology/provision.py
index 33fafa8ad470867107ab925384e6415dd9500951..8c028b48b389c21bf995938570386931bb182300 100644 (file)
--- a/teuthology/provision.py
+++ b/teuthology/provision.py
@@ -312,12 +312,13 @@ class ProvisionOpenStack(OpenStack):
         instances = filter(
             lambda instance: self.property in instance['Properties'],
             self.list_instances())
-        instances = [OpenStackInstance(i['ID'], i) for i in instances]
+        instances = [OpenStackInstance(i['ID']) for i in instances]
         fqdns = []
         try:
             network = config['openstack'].get('network', '')
             for instance in instances:
-                name = self.ip2name(self.basename, instance.get_ip(network))
+                ip = instance.get_ip(network)
+                name = self.ip2name(self.basename, ip)
                 misc.sh("openstack server set " +
                         "--name " + name + " " +
                         instance['ID'])
@@ -325,8 +326,8 @@ class ProvisionOpenStack(OpenStack):
                 if not misc.ssh_keyscan_wait(fqdn):
                     raise ValueError('ssh_keyscan_wait failed for ' + fqdn)
                 time.sleep(15)
-                if not self.cloud_init_wait(fqdn):
-                    raise ValueError('clound_init_wait failed for ' + fqdn)
+                if not self.cloud_init_wait(instance):
+                    raise ValueError('cloud_init_wait failed for ' + fqdn)
                 self.attach_volumes(name, resources_hint['volumes'])
                 fqdns.append(fqdn)
         except Exception as e: