From: Loic Dachary Date: Sun, 29 Nov 2015 18:54:08 +0000 (+0100) Subject: openstack: TeuthologyOpenStack.cloud_init_wait continuous output X-Git-Tag: 1.1.0~722^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=935a749a9250cb961b51fe2c89379f62c86edd69;p=teuthology.git openstack: TeuthologyOpenStack.cloud_init_wait continuous output Instead of tailing the files every 30 seconds, wait for the string that proves the teuthology installation completed with a tail -f and display the result continously. The number of attempts is reduced from 100 to 30 because there is no need for repeated calls to tail to check for the desired string. Signed-off-by: Loic Dachary --- diff --git a/teuthology/openstack/__init__.py b/teuthology/openstack/__init__.py index 1eb5066d7b..666e57004f 100644 --- a/teuthology/openstack/__init__.py +++ b/teuthology/openstack/__init__.py @@ -341,11 +341,11 @@ class OpenStack(object): volume['Display Name'].startswith('target')) return filter(select, all) - def cloud_init_wait(self): + def cloud_init_wait(self, instance): """ Wait for cloud-init to complete on the name_or_ip OpenStack instance. """ - ip = self.instance.get_floating_ip_or_ip() + ip = instance.get_floating_ip_or_ip() log.debug('cloud_init_wait ' + ip) client_args = { 'user_at_host': '@'.join((self.username, ip)), @@ -355,15 +355,13 @@ class OpenStack(object): if self.key_filename: log.debug("using key " + self.key_filename) client_args['key_filename'] = self.key_filename - with safe_while(sleep=30, tries=100, + with safe_while(sleep=30, tries=30, action="cloud_init_wait " + ip) as proceed: success = False # CentOS 6.6 logs in /var/log/clout-init-output.log # CentOS 7.0 logs in /var/log/clout-init.log - all_done = ("tail /var/log/cloud-init*.log ; " + - " test -f /tmp/init.out && tail /tmp/init.out ; " + - " grep '" + self.up_string + "' " + - "/var/log/cloud-init*.log") + tail = ("tail --follow=name --retry" + " /var/log/cloud-init*.log /tmp/init.out") while proceed(): try: client = connection.connect(**client_args) @@ -391,24 +389,28 @@ class OpenStack(object): continue log.exception('cloud_init_wait ' + ip) raise - log.debug('cloud_init_wait ' + all_done) + log.debug('cloud_init_wait ' + tail) try: - stdin, stdout, stderr = client.exec_command(all_done) - stdout.channel.settimeout(5) - out = stdout.read() - log.debug('cloud_init_wait stdout ' + all_done + ' ' + out) + # get the I/O channel to iterate line by line + transport = client.get_transport() + channel = transport.open_session() + channel.get_pty() + channel.settimeout(240) + output = channel.makefile('r', 1) + channel.exec_command(tail) + for line in iter(output.readline, b''): + log.info(line.strip()) + if self.up_string in line: + success = True + break except socket.timeout as e: client.close() - log.debug('cloud_init_wait socket.timeout ' + all_done) + log.debug('cloud_init_wait socket.timeout ' + tail) continue except socket.error as e: client.close() - log.debug('cloud_init_wait socket.error ' + str(e) + ' ' + all_done) + log.debug('cloud_init_wait socket.error ' + str(e) + ' ' + tail) continue - log.debug('cloud_init_wait stderr ' + all_done + - ' ' + stderr.read()) - if stdout.channel.recv_exit_status() == 0: - success = True client.close() if success: break @@ -489,7 +491,7 @@ class TeuthologyOpenStack(OpenStack): else: identity = '' if self.args.upload: - upload = 'upload to : ' + self.args.archive_upload + upload = 'upload to : ' + self.args.archive_upload else: upload = '' log.info(""" @@ -708,7 +710,7 @@ openstack security group rule create --proto udp --dst-port 53 teuthology # dns """ Remove the floating ip from instance_id and delete it. """ - ip = TeuthologyOpenStack.get_floating_ip(instance_id) + ip = OpenStackInstance(instance_id).get_floating_ip() if not ip: return misc.sh("openstack ip floating remove " + ip + " " + instance_id) @@ -734,7 +736,7 @@ openstack security group rule create --proto udp --dst-port 53 teuthology # dns os.unlink(user_data) self.instance = OpenStackInstance(self.args.name) self.associate_floating_ip(self.instance['id']) - return self.cloud_init_wait() + return self.cloud_init_wait(self.instance) def teardown(self): """ diff --git a/teuthology/provision.py b/teuthology/provision.py index 33fafa8ad4..8c028b48b3 100644 --- a/teuthology/provision.py +++ b/teuthology/provision.py @@ -312,12 +312,13 @@ class ProvisionOpenStack(OpenStack): instances = filter( lambda instance: self.property in instance['Properties'], self.list_instances()) - instances = [OpenStackInstance(i['ID'], i) for i in instances] + instances = [OpenStackInstance(i['ID']) for i in instances] fqdns = [] try: network = config['openstack'].get('network', '') for instance in instances: - name = self.ip2name(self.basename, instance.get_ip(network)) + ip = instance.get_ip(network) + name = self.ip2name(self.basename, ip) misc.sh("openstack server set " + "--name " + name + " " + instance['ID']) @@ -325,8 +326,8 @@ class ProvisionOpenStack(OpenStack): if not misc.ssh_keyscan_wait(fqdn): raise ValueError('ssh_keyscan_wait failed for ' + fqdn) time.sleep(15) - if not self.cloud_init_wait(fqdn): - raise ValueError('clound_init_wait failed for ' + fqdn) + if not self.cloud_init_wait(instance): + raise ValueError('cloud_init_wait failed for ' + fqdn) self.attach_volumes(name, resources_hint['volumes']) fqdns.append(fqdn) except Exception as e: