From 935a749a9250cb961b51fe2c89379f62c86edd69 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sun, 29 Nov 2015 19:54:08 +0100 Subject: [PATCH] openstack: TeuthologyOpenStack.cloud_init_wait continuous output Instead of tailing the files every 30 seconds, wait for the string that proves the teuthology installation completed with a tail -f and display the result continously. The number of attempts is reduced from 100 to 30 because there is no need for repeated calls to tail to check for the desired string. Signed-off-by: Loic Dachary --- teuthology/openstack/__init__.py | 44 +++++++++++++++++--------------- teuthology/provision.py | 9 ++++--- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/teuthology/openstack/__init__.py b/teuthology/openstack/__init__.py index 1eb5066d7..666e57004 100644 --- a/teuthology/openstack/__init__.py +++ b/teuthology/openstack/__init__.py @@ -341,11 +341,11 @@ class OpenStack(object): volume['Display Name'].startswith('target')) return filter(select, all) - def cloud_init_wait(self): + def cloud_init_wait(self, instance): """ Wait for cloud-init to complete on the name_or_ip OpenStack instance. """ - ip = self.instance.get_floating_ip_or_ip() + ip = instance.get_floating_ip_or_ip() log.debug('cloud_init_wait ' + ip) client_args = { 'user_at_host': '@'.join((self.username, ip)), @@ -355,15 +355,13 @@ class OpenStack(object): if self.key_filename: log.debug("using key " + self.key_filename) client_args['key_filename'] = self.key_filename - with safe_while(sleep=30, tries=100, + with safe_while(sleep=30, tries=30, action="cloud_init_wait " + ip) as proceed: success = False # CentOS 6.6 logs in /var/log/clout-init-output.log # CentOS 7.0 logs in /var/log/clout-init.log - all_done = ("tail /var/log/cloud-init*.log ; " + - " test -f /tmp/init.out && tail /tmp/init.out ; " + - " grep '" + self.up_string + "' " + - "/var/log/cloud-init*.log") + tail = ("tail --follow=name --retry" + " /var/log/cloud-init*.log /tmp/init.out") while proceed(): try: client = connection.connect(**client_args) @@ -391,24 +389,28 @@ class OpenStack(object): continue log.exception('cloud_init_wait ' + ip) raise - log.debug('cloud_init_wait ' + all_done) + log.debug('cloud_init_wait ' + tail) try: - stdin, stdout, stderr = client.exec_command(all_done) - stdout.channel.settimeout(5) - out = stdout.read() - log.debug('cloud_init_wait stdout ' + all_done + ' ' + out) + # get the I/O channel to iterate line by line + transport = client.get_transport() + channel = transport.open_session() + channel.get_pty() + channel.settimeout(240) + output = channel.makefile('r', 1) + channel.exec_command(tail) + for line in iter(output.readline, b''): + log.info(line.strip()) + if self.up_string in line: + success = True + break except socket.timeout as e: client.close() - log.debug('cloud_init_wait socket.timeout ' + all_done) + log.debug('cloud_init_wait socket.timeout ' + tail) continue except socket.error as e: client.close() - log.debug('cloud_init_wait socket.error ' + str(e) + ' ' + all_done) + log.debug('cloud_init_wait socket.error ' + str(e) + ' ' + tail) continue - log.debug('cloud_init_wait stderr ' + all_done + - ' ' + stderr.read()) - if stdout.channel.recv_exit_status() == 0: - success = True client.close() if success: break @@ -489,7 +491,7 @@ class TeuthologyOpenStack(OpenStack): else: identity = '' if self.args.upload: - upload = 'upload to : ' + self.args.archive_upload + upload = 'upload to : ' + self.args.archive_upload else: upload = '' log.info(""" @@ -708,7 +710,7 @@ openstack security group rule create --proto udp --dst-port 53 teuthology # dns """ Remove the floating ip from instance_id and delete it. """ - ip = TeuthologyOpenStack.get_floating_ip(instance_id) + ip = OpenStackInstance(instance_id).get_floating_ip() if not ip: return misc.sh("openstack ip floating remove " + ip + " " + instance_id) @@ -734,7 +736,7 @@ openstack security group rule create --proto udp --dst-port 53 teuthology # dns os.unlink(user_data) self.instance = OpenStackInstance(self.args.name) self.associate_floating_ip(self.instance['id']) - return self.cloud_init_wait() + return self.cloud_init_wait(self.instance) def teardown(self): """ diff --git a/teuthology/provision.py b/teuthology/provision.py index 33fafa8ad..8c028b48b 100644 --- a/teuthology/provision.py +++ b/teuthology/provision.py @@ -312,12 +312,13 @@ class ProvisionOpenStack(OpenStack): instances = filter( lambda instance: self.property in instance['Properties'], self.list_instances()) - instances = [OpenStackInstance(i['ID'], i) for i in instances] + instances = [OpenStackInstance(i['ID']) for i in instances] fqdns = [] try: network = config['openstack'].get('network', '') for instance in instances: - name = self.ip2name(self.basename, instance.get_ip(network)) + ip = instance.get_ip(network) + name = self.ip2name(self.basename, ip) misc.sh("openstack server set " + "--name " + name + " " + instance['ID']) @@ -325,8 +326,8 @@ class ProvisionOpenStack(OpenStack): if not misc.ssh_keyscan_wait(fqdn): raise ValueError('ssh_keyscan_wait failed for ' + fqdn) time.sleep(15) - if not self.cloud_init_wait(fqdn): - raise ValueError('clound_init_wait failed for ' + fqdn) + if not self.cloud_init_wait(instance): + raise ValueError('cloud_init_wait failed for ' + fqdn) self.attach_volumes(name, resources_hint['volumes']) fqdns.append(fqdn) except Exception as e: -- 2.47.3