From: Kyr Shatskyy Date: Thu, 11 Jan 2018 12:26:52 +0000 (+0200) Subject: Add multi-user support for openstack X-Git-Tag: 1.1.0~211^2~46 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=93e9276995130ba3c0da283ff38e751689b31344;p=teuthology.git Add multi-user support for openstack Conflicts: requirements.txt teuthology/misc.py teuthology/openstack/__init__.py teuthology/provision/openstack.py --- diff --git a/teuthology/openstack/__init__.py b/teuthology/openstack/__init__.py index 18ecc90e2..c22285a70 100644 --- a/teuthology/openstack/__init__.py +++ b/teuthology/openstack/__init__.py @@ -858,7 +858,7 @@ ssh access : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/ upload=upload)) def setup(self): - self.instance = OpenStackInstance(self.args.name) + self.instance = OpenStackInstance(self.server_name()) if not self.instance.exists(): if self.get_provider() != 'rackspace': self.create_security_group() @@ -987,6 +987,37 @@ ssh access : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/ ceph_workbench += (" --ceph-workbench-git-url " + self.args.ceph_workbench_git_url) canonical_tags = "--no-canonical-tags" if self.args.no_canonical_tags else "" + setup_options = [ + '--keypair %s' % self.key_pair(), + '--selfname %s' % self.args.name, + '--server-name %s' % self.server_name(), + '--server-group %s' % self.server_group(), + '--worker-group %s' % self.worker_group(), + '--package-repo %s' % self.packages_repository(), + #'--setup-all', + ] + all_options = [ + '--install', #do_install_packages=true + #'--setup-ceph-workbench', #do_ceph_workbench=true + '--config', #do_create_config=true + '--setup-keypair', #do_setup_keypair=true + #'', #do_apt_get_update=true + '--setup-docker', #do_setup_docker=true + '--setup-salt-master', #do_setup_salt_master=true + '--setup-dnsmasq', #do_setup_dnsmasq=true + '--setup-fail2ban', #do_setup_fail2ban=true + '--setup-paddles', #do_setup_paddles=true + '--setup-pulpito', #do_setup_pulpito=true + '--populate-paddles', #do_populate_paddles=true + ] + ceph_workbench = '' + if self.args.ceph_workbench_git_url: + all_options += [ + '--setup-ceph-workbench', + '--ceph-workbench-branch %s' % self.args.ceph_workbench_branch, + '--ceph-workbench-git-url %s' % self.args.ceph_workbench_git_url, + ] + log.debug("OPENRC = " + openrc + " " + "TEUTHOLOGY_USERNAME = " + self.username + " " + "CLONE_OPENSTACK = " + clone + " " + @@ -996,6 +1027,7 @@ ssh access : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/ "CANONICAL_TAGS = " + ("(empty string)" if canonical_tags == "" else canonical_tags)) content = (template. + replace('SETUP_OPTIONS', ' '.join(setup_options + all_options)). replace('OPENRC', openrc). replace('TEUTHOLOGY_USERNAME', self.username). replace('CLONE_OPENSTACK', clone). @@ -1007,6 +1039,18 @@ ssh access : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/ log.debug("get_user_data: " + content + " written to " + path) return path + def key_pair(self): + return "teuth-%s" % self.args.name + + def server_name(self): + return "teuth-%s" % self.args.name + + def server_group(self): + return "teuth-%s" % self.args.name + + def worker_group(self): + return "teuth-%s-worker" % self.args.name + def create_security_group(self): """ Create a security group that will be used by all teuthology @@ -1014,23 +1058,30 @@ ssh access : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/ but some OpenStack providers enforce firewall restrictions even among instances created within the same tenant. """ - try: - self.run("security group show teuthology") + groups = misc.sh('openstack security group list -c Name -f value').split('\n') + if all(g in groups for g in [self.server_group(), self.worker_group()]): return - except subprocess.CalledProcessError: - pass - # TODO(loic): this leaves the teuthology vm very exposed - # it would be better to be very liberal for 192.168.0.0/16 - # and 172.16.0.0/12 and 10.0.0.0/8 and only allow 80/8081/22 - # for the rest. misc.sh(""" -openstack security group create teuthology -openstack security group rule create --dst-port 1:65535 teuthology -openstack security group rule create --proto udp --dst-port 53 teuthology # dns -openstack security group rule create --proto udp --dst-port 111 teuthology # for nfs -openstack security group rule create --proto udp --dst-port 2049 teuthology # for nfs -openstack security group rule create --proto udp --dst-port 16000:65535 teuthology # for nfs - """) +openstack security group delete {server} || true +openstack security group delete {worker} || true +openstack security group create {server} +openstack security group create {worker} +# access to teuthology VM from the outside +openstack security group rule create --proto tcp --dst-port 22 {server} # ssh +openstack security group rule create --proto tcp --dst-port 80 {server} # for log access +openstack security group rule create --proto tcp --dst-port 8080 {server} # pulpito +openstack security group rule create --proto tcp --dst-port 8081 {server} # paddles +# access between teuthology and workers +openstack security group rule create --src-group {worker} --dst-port 1:65535 {server} +openstack security group rule create --protocol udp --src-group {worker} --dst-port 1:65535 {server} +openstack security group rule create --src-group {server} --dst-port 1:65535 {worker} +openstack security group rule create --protocol udp --src-group {server} --dst-port 1:65535 {worker} +# access between members of one group +openstack security group rule create --src-group {worker} --dst-port 1:65535 {worker} +openstack security group rule create --protocol udp --src-group {worker} --dst-port 1:65535 {worker} +openstack security group rule create --src-group {server} --dst-port 1:65535 {server} +openstack security group rule create --protocol udp --src-group {server} --dst-port 1:65535 {server} + """.format(server=self.server_group(), worker=self.worker_group())) @staticmethod def get_unassociated_floating_ip(): @@ -1087,7 +1138,11 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo @staticmethod def get_instance_id(name): - return OpenStackInstance(name)['id'] + instance = OpenStackInstance(name) + if instance.info: + return instance['id'] + else: + return None @staticmethod def delete_floating_ip(instance_id): @@ -1103,10 +1158,10 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo def create_cluster(self): user_data = self.get_user_data() + security_group = \ + " --security-group {teuthology}".format(teuthology=self.server_group()) if self.get_provider() == 'rackspace': security_group = '' - else: - security_group = " --security-group teuthology" arch = self.get_default_arch() self.run( "server create " + @@ -1116,23 +1171,30 @@ openstack security group rule create --proto udp --dst-port 16000:65535 teutholo " --key-name " + self.args.key_name + " --user-data " + user_data + security_group + - " --wait " + self.args.name + + " --wait " + self.server_name() + " -f json") os.unlink(user_data) - self.instance = OpenStackInstance(self.args.name) + self.instance = OpenStackInstance(self.server_name()) self.associate_floating_ip(self.instance['id']) return self.cloud_init_wait(self.instance) + def packages_repository(self): + return 'teuth-%s-repo' % self.args.name #packages-repository + def teardown(self): """ Delete all instances run by the teuthology cluster and delete the instance running the teuthology cluster. """ - self.ssh("sudo /etc/init.d/teuthology stop || true") - instance_id = self.get_instance_id(self.args.name) - self.delete_floating_ip(instance_id) - self.run("server delete packages-repository || true") - self.run("server delete --wait " + self.args.name) + instance_id = self.get_instance_id(self.server_name()) + if instance_id: + self.ssh("sudo /etc/init.d/teuthology stop || true") + self.delete_floating_ip(instance_id) + self.run("server delete %s || true" % self.packages_repository()) + self.run("server delete --wait %s || true" % self.server_name()) + self.run("keypair delete %s || true" % self.key_pair()) + self.run("security group delete %s || true" % self.worker_group()) + self.run("security group delete %s || true" % self.server_group()) def main(ctx, argv): return TeuthologyOpenStack(ctx, teuth_config, argv).main() diff --git a/teuthology/openstack/openstack-user-data.txt b/teuthology/openstack/openstack-user-data.txt index 1a6b72b8c..14fa88015 100644 --- a/teuthology/openstack/openstack-user-data.txt +++ b/teuthology/openstack/openstack-user-data.txt @@ -12,6 +12,6 @@ packages: runcmd: - su - -c '(set -x ; CLONE_OPENSTACK && cd teuthology && ./bootstrap install)' TEUTHOLOGY_USERNAME >> /tmp/init.out 2>&1 - echo 'export OPENRC' | tee /home/TEUTHOLOGY_USERNAME/openrc.sh - - su - -c '(set -x ; source openrc.sh ; cd teuthology ; source virtualenv/bin/activate ; openstack keypair delete teuthology || true ; teuthology/openstack/setup-openstack.sh --nworkers NWORKERS UPLOAD CEPH_WORKBENCH CANONICAL_TAGS --setup-all)' TEUTHOLOGY_USERNAME >> /tmp/init.out 2>&1 + - su - -c '(set -x ; source openrc.sh ; cd teuthology ; source virtualenv/bin/activate ; teuthology/openstack/setup-openstack.sh --nworkers NWORKERS UPLOAD CEPH_WORKBENCH CANONICAL_TAGS SETUP_OPTIONS)' TEUTHOLOGY_USERNAME >> /tmp/init.out 2>&1 - /etc/init.d/teuthology restart final_message: "teuthology is up and running after $UPTIME seconds" diff --git a/teuthology/openstack/setup-openstack.sh b/teuthology/openstack/setup-openstack.sh index 0f6972d82..176ddb3f5 100755 --- a/teuthology/openstack/setup-openstack.sh +++ b/teuthology/openstack/setup-openstack.sh @@ -35,6 +35,12 @@ function create_config() { local ip="$5" local archive_upload="$6" local canonical_tags="$7" + local selfname="$8" + local keypair="$9" + local server_name="${10}" + local server_group="${11}" + local worker_group="${12}" + local package_repo="${13}" if test "$network" ; then network="network: $network" @@ -65,6 +71,12 @@ openstack: user-data: teuthology/openstack/openstack-{os_type}-{os_version}-user-data.txt ip: $ip nameserver: $nameserver + keypair: $keypair + selfname: $selfname + server_name: $server_name + server_group: $server_group + worker_group: $worker_group + package_repo: $package_repo # # OpenStack has predefined machine sizes (called flavors) # For a given job requiring N machines, the following will select @@ -248,6 +260,8 @@ function setup_pulpito() { sudo apt-get -qq install -y --force-yes nginx local nginx_conf=/etc/nginx/sites-available/default + sudo sed -i '/text\/plain/a\ text\/plain log;' \ + /etc/nginx/mime.types sudo perl -pi -e 's|root /var/www/html|root /usr/share/nginx/html|' $nginx_conf if ! grep -qq 'autoindex on' $nginx_conf ; then sudo perl -pi -e 's|location / {|location / { autoindex on;|' $nginx_conf @@ -525,6 +539,11 @@ function main() { local labdomain=teuthology local nworkers=2 local keypair=teuthology + local selfname=teuthology + local server_name=teuthology + local server_group=teuthology + local worker_group=teuthology + local package_repo=packages-repository local archive_upload local ceph_workbench_git_url local ceph_workbench_branch @@ -593,6 +612,30 @@ function main() { do_apt_get_update=true do_setup_salt_master=true ;; + --server-name) + shift + server_name=$1 + ;; + --server-group) + shift + server_group=$1 + ;; + --worker-group) + shift + worker_group=$1 + ;; + --package-repo) + shift + package_repo=$1 + ;; + --selfname) + shift + selfname=$1 + ;; + --keypair) + shift + keypair=$1 + ;; --setup-keypair) do_setup_keypair=true ;; @@ -677,7 +720,9 @@ function main() { : ${nameserver:=$ip} if $do_create_config ; then - create_config "$network" "$subnets" "$nameserver" "$labdomain" "$ip" "$archive_upload" "$canonical_tags" || return 1 + create_config "$network" "$subnets" "$nameserver" "$labdomain" "$ip" \ + "$archive_upload" "$canonical_tags" "$selfname" "$keypair" \ + "$server_name" "$server_group" "$worker_group" "$package_repo" || return 1 setup_ansible "$subnets" $labdomain || return 1 setup_ssh_config || return 1 setup_authorized_keys || return 1 diff --git a/teuthology/provision/openstack.py b/teuthology/provision/openstack.py index 937bdf85c..3ff7849e6 100644 --- a/teuthology/provision/openstack.py +++ b/teuthology/provision/openstack.py @@ -122,18 +122,20 @@ class ProvisionOpenStack(OpenStack): else: net = '' flavor = self.flavor(resources_hint['machine'], arch) + keypair = config['openstack']['keypair'] or 'teuthology' + worker_group = config['openstack']['worker_group'] or 'teuthology-worker' cmd = ("flock --close --timeout 28800 /tmp/teuthology-server-create.lock" + " openstack server create" + " " + config['openstack'].get('server-create', '') + " -f json " + " --image '" + str(image) + "'" + " --flavor '" + str(flavor) + "'" + - " --key-name teuthology " + + " --key-name %s " % keypair + " --user-data " + str(self.user_data) + " " + net + " --min " + str(num) + " --max " + str(num) + - " --security-group teuthology" + + " --security-group %s" % worker_group + " --property teuthology=" + self.property + " --property ownedby=" + config.openstack['ip'] + " --wait " + diff --git a/teuthology/task/buildpackages.py b/teuthology/task/buildpackages.py index b41971e1d..f4baddca3 100644 --- a/teuthology/task/buildpackages.py +++ b/teuthology/task/buildpackages.py @@ -173,7 +173,19 @@ def task(ctx, config): " tag = " + tag + "," + " branch = " + branch + "," + " sha1 = " + sha1) - target = ('ceph-' + + self_name = 'teuthology' + key_name = 'teuthology' + pkg_repo = 'packages-repository' + security_group = 'teuthology' + if teuth_config.openstack.has_key('selfname'): + self_name = teuth_config.openstack['selfname'] + if teuth_config.openstack.has_key('keypair'): + key_name = teuth_config.openstack['keypair'] + if teuth_config.openstack.has_key('package_repo'): + pkg_repo = teuth_config.openstack['package_repo'] + if teuth_config.openstack.has_key('server_group'): + security_group = teuth_config.openstack['server_group'] + target = (self_name + '-ceph-' + pkg_type + '-' + dist + '-' + arch + '-' + @@ -193,11 +205,16 @@ def task(ctx, config): 'ram': 1024, # MB 'cpus': 1, }, default_arch) + lock = "/tmp/buildpackages-" + sha1 + "-" + os_type + "-" + os_version cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " + " flock --close " + lock + " make -C " + d + network + + " SELFNAME=" + self_name + + " KEY_NAME=" + key_name + + " PKG_REPO=" + pkg_repo + + " SEC_GROUP=" + security_group + " CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() + " CEPH_PKG_TYPE=" + pkg_type + " CEPH_OS_TYPE=" + os_type + @@ -224,5 +241,5 @@ def task(ctx, config): "instance and start again from scratch.".format(pkg_type)) log.info("buildpackages make command: " + cmd) misc.sh(cmd) - teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '') + teuth_config.gitbuilder_host = openstack.get_ip(pkg_repo, '') log.info('Finished buildpackages') diff --git a/teuthology/task/buildpackages/Makefile b/teuthology/task/buildpackages/Makefile index d0d4da1a2..8ccc323eb 100644 --- a/teuthology/task/buildpackages/Makefile +++ b/teuthology/task/buildpackages/Makefile @@ -3,6 +3,9 @@ D=/tmp/stampsdir VPATH=${D} TIMEOUT_SERVER_CREATE = 30m TIMEOUT_BUILD = 220m # 20 minutes short of 4 hours +SEC_GROUP=teuthology +KEY_NAME=teuthology +SELFNAME=teuthology PKG_REPO=packages-repository PKG_REPO_OS_TYPE=ubuntu PKG_REPO_OS_VERSION=14.04 @@ -32,7 +35,7 @@ ${HOME}/.ssh_agent: grep -q ssh_agent ~/.bashrc_teuthology || echo 'source ${HOME}/.ssh_agent' >> ~/.bashrc_teuthology flock-${PKG_REPO}: - timeout $(TIMEOUT_SERVER_CREATE) openstack server create --image 'teuthology-ubuntu-14.04-${HTTP_ARCH}' ${OPENSTACK_NETWORK} --flavor ${HTTP_FLAVOR} --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --user-data ${PKG_REPO_USER_DATA} --wait ${PKG_REPO} + timeout $(TIMEOUT_SERVER_CREATE) openstack server create --image 'teuthology-ubuntu-14.04-${HTTP_ARCH}' ${OPENSTACK_NETWORK} --flavor ${HTTP_FLAVOR} --key-name ${KEY_NAME} --security-group ${SEC_GROUP} --property ownedby=${MY_IP} --user-data ${PKG_REPO_USER_DATA} --wait ${PKG_REPO} sleep 30 set -ex ; \ ip=$(call get_ip,${PKG_REPO}) ; \ @@ -55,8 +58,8 @@ ${PKG_REPO}: # Check the server status before we proceed. # If it's a weird status, bail out and let the delete fire # eg: ERROR status can happen if there is no VM host without enough capacity for the request. -ceph-${CEPH_PKG_TYPE}-${CEPH_DIST}-${CEPH_ARCH}-${CEPH_FLAVOR}-${CEPH_SHA1}: ${PKG_REPO} - timeout $(TIMEOUT_SERVER_CREATE) openstack server create --image 'teuthology-${CEPH_OS_TYPE}-${CEPH_OS_VERSION}-${CEPH_ARCH}' ${OPENSTACK_NETWORK} --flavor ${BUILD_FLAVOR} --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --user-data ${CEPH_OS_TYPE}-${CEPH_OS_VERSION}-user-data.txt --wait $@ +${SELFNAME}-ceph-${CEPH_PKG_TYPE}-${CEPH_DIST}-${CEPH_ARCH}-${CEPH_FLAVOR}-${CEPH_SHA1}: ${PKG_REPO} + timeout $(TIMEOUT_SERVER_CREATE) openstack server create --image 'teuthology-${CEPH_OS_TYPE}-${CEPH_OS_VERSION}-${CEPH_ARCH}' ${OPENSTACK_NETWORK} --flavor ${BUILD_FLAVOR} --key-name ${KEY_NAME} --security-group ${SEC_GROUP} --property ownedby=${MY_IP} --user-data ${CEPH_OS_TYPE}-${CEPH_OS_VERSION}-user-data.txt --wait $@ set -ex ; \ trap "openstack server delete --wait $@" EXIT ; \ for delay in 30 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 ; do \