From 76fa017aec7b1ee90a1f8451ebc1ed2e2a09dc5e Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sun, 9 Aug 2015 21:59:31 +0200 Subject: [PATCH] upload teuthology archive on completion Signed-off-by: Loic Dachary --- README.rst | 32 ++++++++++++++----- scripts/openstack.py | 10 ++++++ scripts/suite.py | 1 + teuthology/config.py | 2 ++ teuthology/openstack/__init__.py | 28 +++++++++++++--- teuthology/openstack/archive-key | 27 ++++++++++++++++ teuthology/openstack/archive-key.pub | 1 + teuthology/openstack/openstack-user-data.txt | 3 +- teuthology/openstack/setup-openstack.sh | 21 +++++++++++- .../openstack/test/openstack-integration.py | 10 ++++++ teuthology/openstack/test/test_openstack.py | 8 +++-- teuthology/openstack/test/user-data-test1.txt | 2 +- teuthology/run.py | 1 + teuthology/suite.py | 7 ++++ teuthology/task/internal.py | 25 +++++++++++++++ teuthology/test/test_suite.py | 4 +++ 16 files changed, 164 insertions(+), 18 deletions(-) create mode 100644 teuthology/openstack/archive-key create mode 100644 teuthology/openstack/archive-key.pub diff --git a/README.rst b/README.rst index d987bdb641..be041a4b45 100644 --- a/README.rst +++ b/README.rst @@ -380,9 +380,9 @@ Get OpenStack credentials and test it | ID | Name | Status | Task State | Power State | Networks | +----+------------+--------+------------+-------------+-------------------------+ +----+------------+--------+------------+-------------+-------------------------+ -* upload your ssh public key with:: +* create a passwordless ssh public key with:: - $ openstack keypair create --public-key ~/.ssh/id_rsa.pub myself + $ openstack keypair create myself > myself.pem +-------------+-------------------------------------------------+ | Field | Value | +-------------+-------------------------------------------------+ @@ -390,14 +390,20 @@ Get OpenStack credentials and test it | name | myself | | user_id | 5cf9fa21b2e9406b9c4108c42aec6262 | +-------------+-------------------------------------------------+ + $ chmod 600 myself.pem Usage ----- -* Run the dummy suite as a test (``myself`` is a keypair created as - explained in the previous section):: +* Create a passwordless ssh public key:: - $ teuthology-openstack --key-name myself --suite dummy + $ openstack keypair create myself > myself.pem + $ chmod 600 myself.pem + +* Run the dummy suite (it does nothing useful but shows all works as + expected):: + + $ teuthology-openstack --key-filename myself.pem --key-name myself --suite dummy Job scheduled with name ubuntu-2015-07-24_09:03:29-dummy-master---basic-openstack and ID 1 2015-07-24 09:03:30,520.520 INFO:teuthology.suite:ceph sha1: dedda6245ce8db8828fdf2d1a2bfe6163f1216a1 2015-07-24 09:03:31,620.620 INFO:teuthology.suite:ceph version: v9.0.2-829.gdedda62 @@ -420,7 +426,14 @@ Usage * The virtual machine running the suite will persist for forensic analysis purposes. To destroy it run:: - $ teuthology-openstack --key-name myself --teardown + $ teuthology-openstack --key-filename myself.pem --key-name myself --teardown + +* The test results can be uploaded to a publicly accessible location + with the ``--upload`` flag:: + + $ teuthology-openstack --key-filename myself.pem --key-name myself \ + --suite dummy --upload + Running the OpenStack backend integration tests ----------------------------------------------- @@ -428,9 +441,12 @@ Running the OpenStack backend integration tests The easiest way to run the integration tests is to first run a dummy suite:: $ teuthology-openstack --key-name myself --suite dummy + ... + ssh access : ssh ubuntu@167.114.242.13 -This will create a virtual machine suitable for running the -integration tests. Once logged in the virtual machine: +This will create a virtual machine suitable for the integration +test. Login wih the ssh access displayed at the end of the +``teuthology-openstack`` command and run the following:: $ pkill -f teuthology-worker $ cd teuthology ; pip install "tox>=1.9" diff --git a/scripts/openstack.py b/scripts/openstack.py index 523180e7e4..7581bcbcab 100644 --- a/scripts/openstack.py +++ b/scripts/openstack.py @@ -58,6 +58,16 @@ and analyze results. action='store_true', default=None, help='destroy the cluster, if it exists', ) + parser.add_argument( + '--upload', + action='store_true', default=False, + help='upload archives to an rsync server', + ) + parser.add_argument( + '--archive-upload', + help='rsync destination to upload archives', + default='ubuntu@integration.ceph.dachary.org:./', + ) # copy/pasted from scripts/suite.py parser.add_argument( 'config_yaml', diff --git a/scripts/suite.py b/scripts/suite.py index f9f8796ece..5bcf9fdf24 100644 --- a/scripts/suite.py +++ b/scripts/suite.py @@ -80,6 +80,7 @@ Scheduler arguments: --filter-out KEYWORDS Do not run jobs whose description contains any of the keywords in the comma separated keyword string specified. + --archive-upload RSYNC_DEST Rsync destination to upload archives. """.format(default_machine_type=config.default_machine_type, default_results_timeout=config.results_timeout) diff --git a/teuthology/config.py b/teuthology/config.py index afb740b5c3..5944f3e0e8 100644 --- a/teuthology/config.py +++ b/teuthology/config.py @@ -126,6 +126,8 @@ class TeuthologyConfig(YamlConfig): yaml_path = os.path.join(os.path.expanduser('~/.teuthology.yaml')) _defaults = { 'archive_base': '/var/lib/teuthworker/archive', + 'archive_upload': None, + 'archive_upload_key': None, 'automated_scheduling': False, 'reserve_machines': 5, 'ceph_git_base_url': 'https://github.com/ceph/', diff --git a/teuthology/openstack/__init__.py b/teuthology/openstack/__init__.py index 3933561577..01082e8432 100644 --- a/teuthology/openstack/__init__.py +++ b/teuthology/openstack/__init__.py @@ -144,7 +144,7 @@ class OpenStack(object): a['RAM'] - b['RAM'] or a['Disk'] - b['Disk']) sorted_flavor = sorted(found, cmp=sort_flavor) - log.info("sorted flavor = " + str(sorted_flavor)) + log.debug("sorted flavor = " + str(sorted_flavor)) return sorted_flavor[0]['Name'] def cloud_init_wait(self, name_or_ip): @@ -283,11 +283,21 @@ class TeuthologyOpenStack(OpenStack): ip = self.setup() if self.args.suite: self.run_suite() + if self.args.key_filename: + identity = '-i ' + self.args.key_filename + ' ' + else: + identity = '' + if self.args.upload: + upload = 'upload to : ' + self.args.archive_upload + else: + upload = '' log.info(""" web interface: http://{ip}:8081/ -ssh access : ssh {username}@{ip} # logs in /usr/share/nginx/html - """.format(ip=ip, - username=self.username)) +ssh access : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/html +{upload}""".format(ip=ip, + username=self.username, + identity=identity, + upload=upload)) if self.args.teardown: self.teardown() @@ -300,11 +310,13 @@ ssh access : ssh {username}@{ip} # logs in /usr/share/nginx/html argv = [] while len(original_argv) > 0: if original_argv[0] in ('--name', + '--archive-upload', '--key-name', '--key-filename', '--simultaneous-jobs'): del original_argv[0:2] - elif original_argv[0] in ('--teardown'): + elif original_argv[0] in ('--teardown', + '--upload'): del original_argv[0] else: argv.append(original_argv.pop(0)) @@ -435,12 +447,18 @@ ssh access : ssh {username}@{ip} # logs in /usr/share/nginx/html for (var, value) in os.environ.iteritems(): if var.startswith('OS_'): openrc += ' ' + var + '=' + value + if self.args.upload: + upload = '--archive-upload ' + self.args.archive_upload + else: + upload = '' log.debug("OPENRC = " + openrc + " " + "TEUTHOLOGY_USERNAME = " + self.username + " " + + "UPLOAD = " + upload + " " + "NWORKERS = " + str(self.args.simultaneous_jobs)) content = (template. replace('OPENRC', openrc). replace('TEUTHOLOGY_USERNAME', self.username). + replace('UPLOAD', upload). replace('NWORKERS', str(self.args.simultaneous_jobs))) open(path, 'w').write(content) log.debug("get_user_data: " + content + " written to " + path) diff --git a/teuthology/openstack/archive-key b/teuthology/openstack/archive-key new file mode 100644 index 0000000000..a8861441db --- /dev/null +++ b/teuthology/openstack/archive-key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAvLz+sao32JL/yMgwTFDTnQVZK3jyXlhQJpHLsgwgHWHQ/27L +fwEbGFVYsJNBGntZwCZvH/K4c0IevbnX/Y69qgmAc9ZpZQLIcIF0A8hmwVYRU+Ap +TAK2qAvadThWfiRBA6+SGoRy6VV5MWeq+hqlGf9axRKqhECNhHuGBuBeosUOZOOH +NVzvFIbp/4842yYrZUDnDzW7JX2kYGi6kaEAYeR8qYJgT/95Pm4Bgu1V7MI36rx1 +O/5BSPF3LvDSnnaZyHCDZtwzC50lBnS2nx8kKPmmdKBSEJoTdNRPIXZ/lMq5pzIW +QPDjI8O5pbX1BJcxfFlZ/h+bI6u8IX3vfTGHWwIDAQABAoIBAG5yLp0rHfkXtKT7 +OQA/wEW/znmZEkPRbD3VzZyIafanuhTv8heFPyTTNM5Hra5ghpniI99PO07/X1vp +OBMCB81MOCYRT6WzpjXoG0rnZ/I1enhZ0fDQGbFnFlTIPh0c/Aq7IEVyQoh24y/d +GXm4Q+tdufFfRfeUivv/CORXQin/Iugbklj8erjx+fdVKPUXilmDIEVleUncer5/ +K5Fxy0lWbm6ZX1fE+rfJvCwNjAaIJgrN8TWUTE8G72F9Y0YU9hRtqOZe6MMbSufy +5+/yj2Vgp+B8Id7Ass2ylDQKsjBett/M2bNKt/DUVIiaxKi0usNSerLvtbkWEw9s +tgUI6ukCgYEA6qqnZwkbgV0lpj1MrQ3BRnFxNR42z2MyEY5xRGaYp22ByxS207z8 +mM3EuLH8k2u6jzsGoPpBWhBbs97MuGDHwsMEO5rBpytnTE4Hxrgec/13Arzk4Bme +eqg1Ji+lNkoLzEHkuihskcZwnQ8uaOdqrnH/NRGuUhA9hjeh+lQzBy8CgYEAzeV1 +zYsw8xIBFtbmFhBQ8imHr0SQalTiQU2Qn46LORK0worsf4sZV5ZF3VBRdnCUwwbm +0XaMb3kE2UBlU8qPqLgxXPNjcEKuqtVlp76dT/lrXIhYUq+Famrf20Lm01kC5itz +QF247hnUfo2uzxpatuEr2ggs2NjuODn57tVw95UCgYEAv0s+C5AxC9OSzWFLEAcW +dwYi8toedBC4z/b9/nRkHJf4JkRMhW6ZuzaCFs2Ax+wZuIi1bqSSgYi0OHx3BhZe +wTWYTb5p/owzONCjJisRKByG14SETuqTdgmIyggs9YSG+Yr9mYM6fdr2EhI+EuYS +4QGsuOYg5GS4wqC3OglJT6ECgYA8y28QRPQsIXnO259OjnzINDkLKGyX6P5xl8yH +QFidfod/FfQk6NaPxSBV67xSA4X5XBVVbfKji5FB8MC6kAoBIHn63ybSY+4dJSuB +70eV8KihxuSFbawwMuRsYoGzkAnKGrRKIiJTs67Ju14NatO0QiJnm5haYxtb4MqK +md1kTQKBgDmTxtSBVOV8eMhl076OoOvdnpb3sy/obI/XUvurS0CaAcqmkVSNJ6c+ +g1O041ocTbuW5d3fbzo9Jyle6qsvUQd7fuoUfAMrd0inKsuYPPM0IZOExbt8QqLI +KFJ+r/nQYoJkmiNO8PssxcP3CMFB6TpUx0BgFcrhH//TtKKNrGTl +-----END RSA PRIVATE KEY----- diff --git a/teuthology/openstack/archive-key.pub b/teuthology/openstack/archive-key.pub new file mode 100644 index 0000000000..57513806d4 --- /dev/null +++ b/teuthology/openstack/archive-key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8vP6xqjfYkv/IyDBMUNOdBVkrePJeWFAmkcuyDCAdYdD/bst/ARsYVViwk0Eae1nAJm8f8rhzQh69udf9jr2qCYBz1mllAshwgXQDyGbBVhFT4ClMAraoC9p1OFZ+JEEDr5IahHLpVXkxZ6r6GqUZ/1rFEqqEQI2Ee4YG4F6ixQ5k44c1XO8Uhun/jzjbJitlQOcPNbslfaRgaLqRoQBh5HypgmBP/3k+bgGC7VXswjfqvHU7/kFI8Xcu8NKedpnIcINm3DMLnSUGdLafHyQo+aZ0oFIQmhN01E8hdn+UyrmnMhZA8OMjw7mltfUElzF8WVn+H5sjq7whfe99MYdb loic@fold diff --git a/teuthology/openstack/openstack-user-data.txt b/teuthology/openstack/openstack-user-data.txt index 6b48a5dafe..437b10c5ce 100644 --- a/teuthology/openstack/openstack-user-data.txt +++ b/teuthology/openstack/openstack-user-data.txt @@ -7,9 +7,10 @@ system_info: packages: - python-virtualenv - git + - rsync runcmd: - su - -c '(set -x ; git clone -b wip-6502-openstack-v3 http://github.com/dachary/teuthology && cd teuthology && ./bootstrap install)' TEUTHOLOGY_USERNAME >> /tmp/init.out 2>&1 - echo 'export OPENRC' | tee /home/TEUTHOLOGY_USERNAME/openrc.sh - - su - -c '(set -x ; source openrc.sh ; cd teuthology ; source virtualenv/bin/activate ; openstack keypair delete teuthology || true ; teuthology/openstack/setup-openstack.sh --nworkers NWORKERS --setup-all)' TEUTHOLOGY_USERNAME >> /tmp/init.out 2>&1 + - su - -c '(set -x ; source openrc.sh ; cd teuthology ; source virtualenv/bin/activate ; openstack keypair delete teuthology || true ; teuthology/openstack/setup-openstack.sh --nworkers NWORKERS UPLOAD --setup-all)' TEUTHOLOGY_USERNAME >> /tmp/init.out 2>&1 - /etc/init.d/teuthology restart final_message: "teuthology is up and running after $UPTIME seconds" diff --git a/teuthology/openstack/setup-openstack.sh b/teuthology/openstack/setup-openstack.sh index ba794094c9..93ac375a0b 100755 --- a/teuthology/openstack/setup-openstack.sh +++ b/teuthology/openstack/setup-openstack.sh @@ -34,6 +34,7 @@ function create_config() { local labdomain="$4" local ip="$5" local flavor_select="$6" + local archive_upload="$7" if test "$flavor_select" ; then flavor_select="flavor-select-regexp: $flavor_select" @@ -43,7 +44,13 @@ function create_config() { network="network: $network" fi + if test "$archive_upload" ; then + archive_upload="archive_upload: $archive_upload" + fi + cat > ~/.teuthology.yaml <> ~/.ssh/authorized_keys + chmod 600 teuthology/openstack/archive-key + echo "APPEND to ~/.ssh/authorized_keys" +} + function setup_bootscript() { local nworkers=$1 @@ -425,6 +438,7 @@ function main() { local nworkers=2 local flavor_select local keypair=teuthology + local archive_upload local do_setup_keypair=false local do_create_config=false @@ -459,6 +473,10 @@ function main() { shift nworkers=$1 ;; + --archive-upload) + shift + archive_upload=$1 + ;; --install) do_install_packages=true ;; @@ -528,9 +546,10 @@ function main() { : ${nameserver:=$ip} if $do_create_config ; then - create_config "$network" "$subnet" "$nameserver" "$labdomain" "$ip" "$flavor_select" || return 1 + create_config "$network" "$subnet" "$nameserver" "$labdomain" "$ip" "$flavor_select" "$archive_upload" || return 1 setup_ansible $subnet $labdomain || return 1 setup_ssh_config || return 1 + setup_authorized_keys || return 1 setup_bashrc || return 1 setup_bootscript $nworkers || return 1 fi diff --git a/teuthology/openstack/test/openstack-integration.py b/teuthology/openstack/test/openstack-integration.py index f3d19f5024..8fe0412959 100644 --- a/teuthology/openstack/test/openstack-integration.py +++ b/teuthology/openstack/test/openstack-integration.py @@ -38,6 +38,7 @@ import teuthology.openstack import scripts.schedule import scripts.lock import scripts.suite +from teuthology.config import config as teuth_config class Integration(object): @@ -111,9 +112,12 @@ class TestSuite(Integration): def test_suite_noop(self): cwd = os.getcwd() + os.mkdir(self.d + '/upload', 0o755) + upload = 'localhost:' + self.d + '/upload' args = ['--suite', 'noop', '--suite-dir', cwd + '/teuthology/openstack/test', '--machine-type', 'openstack', + '--archive-upload', upload, '--verbose'] logging.info("TestSuite:test_suite_noop") scripts.suite.main(args) @@ -121,6 +125,12 @@ class TestSuite(Integration): log = self.get_teuthology_log() assert "teuthology.run:pass" in log assert "Well done" in log + upload_key = teuth_config.archive_upload_key + if upload_key: + ssh = "RSYNC_RSH='ssh -i " + upload_key + "'" + else: + ssh = '' + assert 'teuthology.log' in teuthology.misc.sh(ssh + " rsync -av " + upload) def test_suite_nuke(self): cwd = os.getcwd() diff --git a/teuthology/openstack/test/test_openstack.py b/teuthology/openstack/test/test_openstack.py index 47c60d8584..0884fb3109 100644 --- a/teuthology/openstack/test/test_openstack.py +++ b/teuthology/openstack/test/test_openstack.py @@ -48,7 +48,7 @@ class TestTeuthologyOpenStack(object): misc.sh("openstack ip floating delete " + ip_id) self.can_create_floating_ips = True else: - self.can_create_floating_ips = True + self.can_create_floating_ips = False def setup(self): self.key_filename = tempfile.mktemp() @@ -94,7 +94,10 @@ openstack keypair delete {key_name} || true '--filter', 'trasher', '--filter-out', 'erasure-code', ] - argv = self.options + teuthology_argv + argv = (self.options + + ['--upload', + '--archive-upload', 'user@archive:/tmp'] + + teuthology_argv) args = scripts.openstack.parse_args(argv) teuthology = TeuthologyOpenStack(args, None, argv) teuthology.user_data = 'teuthology/openstack/test/user-data-test1.txt' @@ -105,6 +108,7 @@ openstack keypair delete {key_name} || true variables = teuthology.ssh("grep 'substituded variables' /var/log/cloud-init.log") assert "nworkers=" + str(args.simultaneous_jobs) in variables assert "username=" + teuthology.username in variables + assert "upload=--archive-upload user@archive:/tmp" in variables assert os.environ['OS_AUTH_URL'] in variables out, err = capsys.readouterr() diff --git a/teuthology/openstack/test/user-data-test1.txt b/teuthology/openstack/test/user-data-test1.txt index bdeabfa017..b7d47de861 100644 --- a/teuthology/openstack/test/user-data-test1.txt +++ b/teuthology/openstack/test/user-data-test1.txt @@ -2,4 +2,4 @@ system_info: default_user: name: ubuntu -final_message: "teuthology is up and running after $UPTIME seconds, substituded variables nworkers=NWORKERS openrc=OPENRC username=TEUTHOLOGY_USERNAME" +final_message: "teuthology is up and running after $UPTIME seconds, substituded variables nworkers=NWORKERS openrc=OPENRC username=TEUTHOLOGY_USERNAME upload=UPLOAD" diff --git a/teuthology/run.py b/teuthology/run.py index 48a69593e1..430ed80d97 100644 --- a/teuthology/run.py +++ b/teuthology/run.py @@ -204,6 +204,7 @@ def get_initial_tasks(lock, config, machine_type): {'internal.base': None}, {'internal.archive': None}, {'internal.coredump': None}, + {'internal.archive_upload': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, diff --git a/teuthology/suite.py b/teuthology/suite.py index bf2df8a441..80d1dfc4e5 100644 --- a/teuthology/suite.py +++ b/teuthology/suite.py @@ -58,6 +58,9 @@ def main(args): email = args['--email'] if email: config.results_email = email + if args['--archive-upload']: + config.archive_upload = args['--archive-upload'] + log.info('Will upload archives to ' + args['--archive-upload']) timeout = args['--timeout'] filter_in = args['--filter'] filter_out = args['--filter-out'] @@ -244,6 +247,8 @@ def create_initial_config(suite, suite_branch, ceph_branch, teuthology_branch, teuthology_branch=teuthology_branch, machine_type=machine_type, distro=distro, + archive_upload=config.archive_upload, + archive_upload_key=config.archive_upload_key, ) conf_dict = substitute_placeholders(dict_templ, config_input) conf_dict.update(kernel_dict) @@ -1009,6 +1014,8 @@ dict_templ = { 'branch': Placeholder('ceph_branch'), 'sha1': Placeholder('ceph_hash'), 'teuthology_branch': Placeholder('teuthology_branch'), + 'archive_upload': Placeholder('archive_upload'), + 'archive_upload_key': Placeholder('archive_upload_key'), 'machine_type': Placeholder('machine_type'), 'nuke-on-error': True, 'os_type': Placeholder('distro'), diff --git a/teuthology/task/internal.py b/teuthology/task/internal.py index ef7023e409..567621eea6 100644 --- a/teuthology/task/internal.py +++ b/teuthology/task/internal.py @@ -566,6 +566,31 @@ def coredump(ctx, config): 'Found coredumps on {rem}'.format(rem=rem) +@contextlib.contextmanager +def archive_upload(ctx, config): + """ + Upload the archive directory to a designated location + """ + try: + yield + finally: + upload = ctx.config.get('archive_upload') + archive_path = ctx.config.get('archive_path') + if upload and archive_path: + log.info('Uploading archives ...') + upload_key = ctx.config.get('archive_upload_key') + if upload_key: + ssh = "RSYNC_RSH='ssh -i " + upload_key + "'" + else: + ssh = '' + split_path = archive_path.split('/') + split_path.insert(-2, '.') + misc.sh(ssh + " rsync -avz --relative /" + + os.path.join(*split_path) + " " + + upload) + else: + log.info('Not uploading archives.') + @contextlib.contextmanager def syslog(ctx, config): """ diff --git a/teuthology/test/test_suite.py b/teuthology/test/test_suite.py index 1ef535bcd5..0b25b598bd 100644 --- a/teuthology/test/test_suite.py +++ b/teuthology/test/test_suite.py @@ -41,6 +41,8 @@ class TestSuiteOffline(object): teuthology_branch='teuthology_branch', machine_type='machine_type', distro='distro', + archive_upload='archive_upload', + archive_upload_key='archive_upload_key', ) output_dict = suite.substitute_placeholders(suite.dict_templ, input_dict) @@ -58,6 +60,8 @@ class TestSuiteOffline(object): ceph_hash='ceph_hash', teuthology_branch='teuthology_branch', machine_type='machine_type', + archive_upload='archive_upload', + archive_upload_key='archive_upload_key', distro=None, ) output_dict = suite.substitute_placeholders(suite.dict_templ, -- 2.39.5