From 6557bdde10c48f1e17133b2580503b207d423ba1 Mon Sep 17 00:00:00 2001 From: David Galloway Date: Thu, 14 Apr 2022 11:19:46 -0400 Subject: [PATCH] Revert "Merge pull request #1737 from ceph/build" This reverts commit 43327234bc2c8066c173e4c6d602c8a0a05db642, reversing changes made to a347b2ac5400a43ba76a678399b1deda7dae1a6c. --- MANIFEST.in | 2 - bootstrap | 66 ++++++--- docs/docker-compose/teuthology/Dockerfile | 2 +- pyproject.toml | 10 -- requirements.in | 44 ++++++ requirements.txt | 119 ++++++++++----- scripts/dispatcher.py | 3 +- setup.cfg | 133 ----------------- setup.py | 138 ++++++++++++++++++ teuthology/__init__.py | 40 +++-- teuthology/dispatcher/__init__.py | 5 +- teuthology/dispatcher/supervisor.py | 10 +- .../provision/cloud/test/test_openstack.py | 4 +- teuthology/run_tasks.py | 12 -- teuthology/task/tests/__init__.py | 8 +- teuthology/test/test_worker.py | 11 +- teuthology/worker.py | 22 ++- tox.ini | 15 +- update-requirements.sh | 3 +- 19 files changed, 388 insertions(+), 259 deletions(-) delete mode 100644 pyproject.toml create mode 100644 requirements.in delete mode 100644 setup.cfg create mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in index 2683cd6549..bb73d96269 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,2 @@ include *.rst include requirements.txt -include tox.ini -include pytest.ini diff --git a/bootstrap b/bootstrap index 394dbb0b9d..3d7f4a6250 100755 --- a/bootstrap +++ b/bootstrap @@ -11,19 +11,19 @@ else fi fi -if [[ "$PYTHON" =~ "python2" ]]; then - echo "python2 is not supported." >&2 - exit 1 -fi - PYTHON=${PYTHON:-"python3"} -VENV=${VENV:-"./virtualenv"} +VENV=${VENV:-"virtualenv"} case "$(uname -s)" in Linux) case "$(lsb_release --id --short)" in Ubuntu|Debian|LinuxMint) - deps=(qemu-utils python3-dev libssl-dev python3-pip python3-venv libev-dev libvirt-dev libffi-dev libyaml-dev) + # Ensure setuptools is installed + if [[ "$PYTHON" =~ "python2" ]]; then + deps=(qemu-utils python-dev libssl-dev python-pip python-virtualenv libev-dev libvirt-dev libffi-dev libyaml-dev) + else + deps=(qemu-utils python3-dev libssl-dev python3-pip python3-virtualenv libev-dev libvirt-dev libffi-dev libyaml-dev) + fi for package in ${deps[@]}; do if [ "$(dpkg --status -- $package|sed -n 's/^Status: //p')" != "install ok installed" ]; then # add a space after old values @@ -44,7 +44,11 @@ Linux) fi ;; RedHatEnterpriseWorkstation|RedHatEnterpriseServer|RedHatEnterprise|CentOS) - deps=(python3-pip python3-devel mariadb-devel libev-devel libvirt-devel libffi-devel) + if [[ "$PYTHON" =~ "python2" ]]; then + deps=(python-pip python-devel python-virtualenv mariadb-devel libev-devel libvirt-devel libffi-devel) + else + deps=(python3-pip python3-devel python3-virtualenv mariadb-devel libev-devel libvirt-devel libffi-devel) + fi for package in ${deps[@]}; do if [ "$(rpm -q $package)" == "package $package is not installed" ]; then missing="${missing:+$missing }$package" @@ -64,7 +68,11 @@ Linux) fi ;; Fedora) - deps=(python3-pip python3-devel libev-devel libvirt-devel libffi-devel) + if [[ "$PYTHON" =~ "python2" ]]; then + deps=(python-pip python-devel python-virtualenv libev-devel libvirt-devel libffi-devel) + else + deps=(python3-pip python3-devel python3-virtualenv libev-devel libvirt-devel libffi-devel) + fi for package in ${deps[@]}; do if [ "$(rpm -q $package)" == "package $package is not installed" ]; then missing="${missing:+$missing }$package" @@ -89,7 +97,12 @@ Linux) fi ;; "openSUSE project"|"SUSE LINUX"|"openSUSE") - deps=(python3-pip python3-devel python3 libev-devel libvirt-devel libffi-devel) + + if [[ "$PYTHON" =~ "python2" ]]; then + deps=(python-pip python-devel python-virtualenv libev-devel libvirt-devel libffi-devel) + else + deps=(python3-pip python3-devel python3-virtualenv libev-devel libvirt-devel libffi-devel) + fi for package in ${deps[@]}; do if [ "$(rpm -q $package)" == "package $package is not installed" ]; then if [ "$(rpm -q --whatprovides $package)" == "no package provides $package" ]; then @@ -135,22 +148,35 @@ Darwin) ;; esac -# If the venv was set to use system site-packages, fix that -if [ -d "$VENV" ]; then - sed -i'' -e 's/\(include-system-site-packages\s*=\s*\)true/\1false/g' $VENV/pyvenv.cfg +# Forcibly remove old virtualenvs which used system site-packages +if [ -e ./$VENV ] && [ ! -e ./$VENV/lib/python*/no-global-site-packages.txt ]; then + echo "Removing old virtualenv because it uses system site-packages" + rm -rf ./$VENV fi export LC_ALL=en_US.UTF-8 -if [ -z "$NO_CLOBBER" ] || [ ! -e $VENV ]; then - python3 -m venv $VENV +if [ -z "$NO_CLOBBER" ] || [ ! -e ./$VENV ]; then + if ! virtualenv --version &>/dev/null; then + pip install virtualenv + fi + virtualenv --python=$PYTHON $VENV fi -# First, upgrade pip -$VENV/bin/pip install --upgrade pip +# Upgrade pip first - then we have a new pip version with the --use-feature flag +./$VENV/bin/pip install --upgrade pip -# By default, install teuthology in editable mode -$VENV/bin/pip install ${PIP_INSTALL_FLAGS:---editable '.[test]'} +# Ensure setuptools is installed +./$VENV/bin/pip install --use-feature=2020-resolver setuptools --upgrade + +# Install all requirements +./$VENV/bin/pip install --use-feature=2020-resolver --upgrade -r requirements.txt # Check to make sure requirements are met -$VENV/bin/pip check +./$VENV/bin/pip check + +# Remove leftover .pyc files +find teuthology -name '*.pyc' -exec rm {} \; + +# Install teuthology +./$VENV/bin/python setup.py develop diff --git a/docs/docker-compose/teuthology/Dockerfile b/docs/docker-compose/teuthology/Dockerfile index 149e8aca56..5c9285f5f4 100644 --- a/docs/docker-compose/teuthology/Dockerfile +++ b/docs/docker-compose/teuthology/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && \ libssl-dev \ ipmitool \ python3-pip \ - python3-venv \ + python3-virtualenv \ vim \ libev-dev \ libvirt-dev \ diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index ece6fe5f51..0000000000 --- a/pyproject.toml +++ /dev/null @@ -1,10 +0,0 @@ -[build-system] -build-backend = "setuptools.build_meta" -requires = [ - "setuptools>=45", - "wheel", - "setuptools_scm>=6.2", -] - -[tool.setuptools_scm] -version_scheme = "python-simplified-semver" \ No newline at end of file diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000000..dd5bce9e38 --- /dev/null +++ b/requirements.in @@ -0,0 +1,44 @@ +# install requires +apache-libcloud +gevent +PyYAML +argparse>=1.2.1 +configobj +six>=1.9 +pexpect +docopt +netaddr +paramiko +pynacl >= 1.5.0 # for universal2 wheel +psutil >= 2.1.0 +configparser +ansible>=2.8,<2.10 +prettytable +manhole +humanfriendly +# orchestra requires +backports.ssl-match-hostname +beanstalkc3>=0.4.0 +httplib2 +ndg-httpsclient +pyasn1 +pyopenssl>=0.13 +python-dateutil +python-novaclient +python-openstackclient +requests!=2.13.0 +sentry-sdk +# test requires +boto>=2.0b4 +boto3 +cryptography>=2.7 +nose +pip-tools +pytest +mock +requests +tox +xmltodict +PyJWT +ipy +toml diff --git a/requirements.txt b/requirements.txt index 1b175b701e..8276a9ceb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,28 +2,32 @@ # This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# pip-compile pyproject.toml +# pip-compile requirements.in # ansible==2.9.27 - # via teuthology (pyproject.toml) + # via -r requirements.in apache-libcloud==3.3.1 - # via teuthology (pyproject.toml) + # via -r requirements.in appdirs==1.4.4 # via openstacksdk argparse==1.4.0 - # via teuthology (pyproject.toml) + # via -r requirements.in attrs==21.2.0 - # via cmd2 + # via + # cmd2 + # pytest +backports-entry-points-selectable==1.1.0 + # via virtualenv backports-ssl-match-hostname==3.7.0.1 - # via teuthology (pyproject.toml) + # via -r requirements.in bcrypt==3.2.0 # via paramiko beanstalkc3==0.4.0 - # via teuthology (pyproject.toml) + # via -r requirements.in boto==2.49.0 - # via teuthology (pyproject.toml) + # via -r requirements.in boto3==1.21.20 - # via teuthology (pyproject.toml) + # via -r requirements.in botocore==1.24.20 # via # boto3 @@ -50,16 +54,16 @@ cmd2==2.1.2 colorama==0.4.4 # via cmd2 configobj==5.0.6 - # via teuthology (pyproject.toml) + # via -r requirements.in configparser==5.0.2 - # via teuthology (pyproject.toml) + # via -r requirements.in cryptography==3.4.7 # via + # -r requirements.in # ansible # openstacksdk # paramiko # pyopenssl - # teuthology (pyproject.toml) debtcollector==2.2.0 # via # oslo-config @@ -69,22 +73,30 @@ decorator==5.0.9 # via # dogpile-cache # openstacksdk +distlib==0.3.2 + # via virtualenv docopt==0.6.2 - # via teuthology (pyproject.toml) + # via -r requirements.in dogpile-cache==1.1.3 # via openstacksdk +filelock==3.0.12 + # via + # tox + # virtualenv gevent==21.8.0 - # via teuthology (pyproject.toml) + # via -r requirements.in greenlet==1.1.0 # via gevent httplib2==0.19.1 - # via teuthology (pyproject.toml) + # via -r requirements.in humanfriendly==9.2 - # via teuthology (pyproject.toml) + # via -r requirements.in idna==3.2 # via requests +iniconfig==1.1.1 + # via pytest ipy==1.1 - # via teuthology (pyproject.toml) + # via -r requirements.in iso8601==0.1.16 # via # keystoneauth1 @@ -110,23 +122,29 @@ keystoneauth1==4.3.1 # python-cinderclient # python-keystoneclient # python-novaclient +manhole==1.8.0 + # via -r requirements.in markupsafe==2.0.1 # via jinja2 +mock==4.0.3 + # via -r requirements.in msgpack==1.0.2 # via oslo-serialization munch==2.5.0 # via openstacksdk ndg-httpsclient==0.5.1 - # via teuthology (pyproject.toml) + # via -r requirements.in netaddr==0.8.0 # via + # -r requirements.in # oslo-config # oslo-utils - # teuthology (pyproject.toml) netifaces==0.11.0 # via # openstacksdk # oslo-utils +nose==1.3.7 + # via -r requirements.in openstacksdk==0.58.0 # via # osc-lib @@ -161,9 +179,12 @@ oslo-utils==4.9.2 # python-novaclient # python-openstackclient packaging==21.0 - # via oslo-utils + # via + # oslo-utils + # pytest + # tox paramiko==2.10.1 - # via teuthology (pyproject.toml) + # via -r requirements.in pbr==5.6.0 # via # cliff @@ -183,33 +204,45 @@ pbr==5.6.0 pep517==0.11.0 # via pip-tools pexpect==4.8.0 - # via teuthology (pyproject.toml) + # via -r requirements.in pip-tools==6.2.0 - # via teuthology (pyproject.toml) + # via -r requirements.in +platformdirs==2.2.0 + # via virtualenv +pluggy==0.13.1 + # via + # pytest + # tox prettytable==2.1.0 # via + # -r requirements.in # cliff # python-cinderclient # python-novaclient - # teuthology (pyproject.toml) psutil==5.8.0 - # via teuthology (pyproject.toml) + # via -r requirements.in ptyprocess==0.7.0 # via pexpect +py==1.10.0 + # via + # pytest + # tox pyasn1==0.4.8 # via + # -r requirements.in # ndg-httpsclient - # teuthology (pyproject.toml) pycparser==2.20 # via cffi +pyjwt==2.1.0 + # via -r requirements.in pynacl==1.5.0 # via + # -r requirements.in # paramiko - # teuthology (pyproject.toml) pyopenssl==20.0.1 # via + # -r requirements.in # ndg-httpsclient - # teuthology (pyproject.toml) pyparsing==2.4.7 # via # cliff @@ -218,39 +251,41 @@ pyparsing==2.4.7 # packaging pyperclip==1.8.2 # via cmd2 +pytest==6.2.5 + # via -r requirements.in python-cinderclient==8.0.0 # via python-openstackclient python-dateutil==2.8.2 # via + # -r requirements.in # botocore - # teuthology (pyproject.toml) python-keystoneclient==4.2.0 # via python-openstackclient python-novaclient==17.5.0 # via + # -r requirements.in # python-openstackclient - # teuthology (pyproject.toml) python-openstackclient==5.5.0 - # via teuthology (pyproject.toml) + # via -r requirements.in pytz==2021.1 # via # oslo-serialization # oslo-utils pyyaml==5.4.1 # via + # -r requirements.in # ansible # cliff # openstacksdk # oslo-config - # teuthology (pyproject.toml) requests==2.27.1 # via + # -r requirements.in # apache-libcloud # keystoneauth1 # oslo-config # python-cinderclient # python-keystoneclient - # teuthology (pyproject.toml) requestsexceptions==1.4.0 # via openstacksdk rfc3986==1.5.0 @@ -258,23 +293,26 @@ rfc3986==1.5.0 s3transfer==0.5.2 # via boto3 sentry-sdk==1.3.1 - # via teuthology (pyproject.toml) + # via -r requirements.in simplejson==3.17.3 # via # osc-lib # python-cinderclient six==1.16.0 # via + # -r requirements.in # bcrypt # configobj # debtcollector # keystoneauth1 # munch # oslo-i18n + # paramiko # pyopenssl # python-dateutil # python-keystoneclient - # teuthology (pyproject.toml) + # tox + # virtualenv stevedore==3.3.0 # via # cliff @@ -287,14 +325,21 @@ stevedore==3.3.0 # python-novaclient # python-openstackclient toml==0.10.2 - # via teuthology (pyproject.toml) + # via + # -r requirements.in + # pytest + # tox tomli==1.2.1 # via pep517 +tox==3.24.1 + # via -r requirements.in urllib3==1.26.6 # via # botocore # requests # sentry-sdk +virtualenv==20.7.0 + # via tox wcwidth==0.2.5 # via # cmd2 @@ -304,7 +349,7 @@ wheel==0.36.2 wrapt==1.12.1 # via debtcollector xmltodict==0.12.0 - # via teuthology (pyproject.toml) + # via -r requirements.in zope-event==4.5.0 # via gevent zope-interface==5.4.0 diff --git a/scripts/dispatcher.py b/scripts/dispatcher.py index 15df103fd3..4cb1abdea6 100644 --- a/scripts/dispatcher.py +++ b/scripts/dispatcher.py @@ -1,6 +1,6 @@ """ usage: teuthology-dispatcher --help - teuthology-dispatcher --supervisor [-v] --job-config COFNFIG --archive-dir DIR + teuthology-dispatcher --supervisor [-v] --bin-path BIN_PATH --job-config COFNFIG --archive-dir DIR teuthology-dispatcher [-v] [--archive-dir DIR] [--exit-on-empty-queue] --log-dir LOG_DIR --tube TUBE Start a dispatcher for the specified tube. Grab jobs from a beanstalk @@ -19,6 +19,7 @@ standard arguments: -l, --log-dir LOG_DIR path in which to store logs -a DIR, --archive-dir DIR path to archive results in --supervisor run dispatcher in job supervisor mode + --bin-path BIN_PATH teuthology bin path --job-config CONFIG file descriptor of job's config file --exit-on-empty-queue if the queue is empty, exit """ diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 4bfdfab5dc..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,133 +0,0 @@ -[metadata] -name = teuthology -long_description = file: README.rst -long_description_content_type = text/x-rst -url = https://github.com/ceph/teuthology -author = Red Hat, Inc. -license = MIT -license_file = LICENSE -classifiers = - Intended Audience :: Developers - License :: OSI Approved :: MIT License - Natural Language :: English - Operating System :: POSIX :: Linux - Programming Language :: Python :: 3 - Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: Implementation :: CPython - Topic :: Software Development :: Quality Assurance - Topic :: Software Development :: Testing - Topic :: System :: Distributed Computing - Topic :: System :: Filesystems -description_content_type = text/x-rst; charset=UTF-8 -description_file = README.rst -keywords = teuthology, test, ceph, cluster -summary = Ceph test framework - -[options] -packages = find: -install_requires = - PyYAML - ansible>=2.8,<2.10 - apache-libcloud - argparse>=1.2.1 - backports.ssl-match-hostname - beanstalkc3>=0.4.0 - boto>=2.0b4 - boto3 - configobj - configparser - cryptography>=2.7 - docopt - gevent - httplib2 - humanfriendly - ipy - ndg-httpsclient - netaddr - paramiko - pexpect - pip-tools - prettytable - psutil>=2.1.0 - pyasn1 - pynacl>=1.5.0 - pyopenssl>=0.13 - python-dateutil - python-novaclient - python-openstackclient - requests>2.13.0 - sentry-sdk - six>=1.9 - toml - xmltodict -python_requires = >=3.6 - -[options.entry_points] -console_scripts = - teuthology = scripts.run:main - teuthology-openstack = scripts.openstack:main - teuthology-nuke = scripts.nuke:main - teuthology-suite = scripts.suite:main - teuthology-ls = scripts.ls:main - teuthology-worker = scripts.worker:main - teuthology-lock = scripts.lock:main - teuthology-schedule = scripts.schedule:main - teuthology-updatekeys = scripts.updatekeys:main - teuthology-update-inventory = scripts.update_inventory:main - teuthology-results = scripts.results:main - teuthology-report = scripts.report:main - teuthology-kill = scripts.kill:main - teuthology-queue = scripts.queue:main - teuthology-prune-logs = scripts.prune_logs:main - teuthology-describe = scripts.describe:main - teuthology-reimage = scripts.reimage:main - teuthology-dispatcher = scripts.dispatcher:main - teuthology-wait = scripts.wait:main - -[options.extras_require] -manhole = - manhole -rocketchat = - rocket-python>=1.2.15 -sentry = - sentry-sdk -test = - PyJWT - boto>=2.0b4 - boto3 - cryptography>=2.7 - ipy - mock - pip-tools - pytest - toml - tox - xmltodict - -[options.package_data] -teuthology.openstack = - archive-key - archive-key.pub - openstack-centos-6.5-user-data.txt - openstack-centos-7.0-user-data.txt - openstack-centos-7.1-user-data.txt - openstack-centos-7.2-user-data.txt - openstack-debian-8.0-user-data.txt - openstack-opensuse-42.1-user-data.txt - openstack-teuthology.cron - openstack-teuthology.init - openstack-ubuntu-12.04-user-data.txt - openstack-ubuntu-14.04-user-data.txt - openstack-user-data.txt - openstack.yaml - setup-openstack.sh -teuthology.task.install = - adjust-ulimits - daemon-helper -teuthology.task.internal = - edit_sudoers.sh diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000..87c573a4e5 --- /dev/null +++ b/setup.py @@ -0,0 +1,138 @@ +from setuptools import setup, find_packages +import re + +module_file = open("teuthology/__init__.py").read() +metadata = dict(re.findall(r"__([a-z]+)__\s*=\s*['\"]([^'\"]*)['\"]", module_file)) +long_description = open('README.rst').read() + +setup( + name='teuthology', + version=metadata['version'], + packages=find_packages(), + package_data={ + 'teuthology.task': ['adjust-ulimits', 'edit_sudoers.sh', 'daemon-helper'], + 'teuthology.task': ['adjust-ulimits', 'edit_sudoers.sh', 'daemon-helper'], + 'teuthology.openstack': [ + 'archive-key', + 'archive-key.pub', + 'openstack-centos-6.5-user-data.txt', + 'openstack-centos-7.0-user-data.txt', + 'openstack-centos-7.1-user-data.txt', + 'openstack-centos-7.2-user-data.txt', + 'openstack-debian-8.0-user-data.txt', + 'openstack-opensuse-42.1-user-data.txt', + 'openstack-teuthology.cron', + 'openstack-teuthology.init', + 'openstack-ubuntu-12.04-user-data.txt', + 'openstack-ubuntu-14.04-user-data.txt', + 'openstack-user-data.txt', + 'openstack.yaml', + 'setup-openstack.sh' + ], + }, + author='Inktank Storage, Inc.', + author_email='ceph-qa@ceph.com', + description='Ceph test framework', + license='MIT', + keywords='teuthology test ceph cluster', + url='https://github.com/ceph/teuthology', + long_description=long_description, + classifiers=[ + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Natural Language :: English', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 3.6', + 'Topic :: Software Development :: Quality Assurance', + 'Topic :: Software Development :: Testing', + 'Topic :: System :: Distributed Computing', + 'Topic :: System :: Filesystems', + ], + install_requires=['apache-libcloud', + 'gevent', + 'PyYAML', + 'argparse >= 1.2.1', + 'configobj', + 'six >= 1.9', # python-openstackclient won't work properly with less + 'pexpect', + 'docopt', + 'netaddr', # teuthology/misc.py + # only used by orchestra, but we monkey-patch it in + # teuthology/__init__.py + 'paramiko', + 'psutil >= 2.1.0', + 'configparser', + 'ansible>=2.0', + 'prettytable', + 'rocket-python >= 1.2.15', + 'manhole', + 'humanfriendly', + ], + extras_require = { + 'orchestra': [ + # For apache-libcloud when using python < 2.7.9 + 'backports.ssl_match_hostname', + 'beanstalkc3 >= 0.4.0', + 'httplib2', + 'ndg-httpsclient', # for requests, urllib3 + 'pyasn1', # for requests, urllib3 + 'pyopenssl>=0.13', # for requests, urllib3 + 'python-dateutil', + # python-novaclient is specified here, even though it is + # redundant, because python-openstackclient requires + # Babel, and installs 2.3.3, which is forbidden by + # python-novaclient 4.0.0 + 'python-novaclient', + 'python-openstackclient', + # with openstacklient >= 2.1.0, neutronclient no longer is + # a dependency but we need it anyway. + 'python-neutronclient', + 'requests != 2.13.0', + 'sentry-sdk', + ], + 'test': [ + 'boto >= 2.0b4', # for qa/tasks/radosgw_*.py + 'cryptography >= 2.7', # for qa/tasks/mgr/dashboard/test_rgw.py + 'nose', # for qa/tasks/rgw_multisite_tests.py', + 'pip-tools', + 'pytest', # for tox.ini + 'requests', # for qa/tasks/mgr/dashboard/helper.py + 'tox', + # For bucket notification testing in multisite + 'xmltodict', + 'boto3', + 'PyJWT', # for qa/tasks/mgr/dashboard/test_auth.py + 'ipy', # for qa/tasks/cephfs/mount.py + 'toml', # for qa/tasks/cephadm.py + ] + }, + + + # to find the code associated with entry point + # A.B:foo first cd into directory A, open file B + # and find sub foo + entry_points={ + 'console_scripts': [ + 'teuthology = scripts.run:main', + 'teuthology-openstack = scripts.openstack:main', + 'teuthology-nuke = scripts.nuke:main', + 'teuthology-suite = scripts.suite:main', + 'teuthology-ls = scripts.ls:main', + 'teuthology-worker = scripts.worker:main', + 'teuthology-lock = scripts.lock:main', + 'teuthology-schedule = scripts.schedule:main', + 'teuthology-updatekeys = scripts.updatekeys:main', + 'teuthology-update-inventory = scripts.update_inventory:main', + 'teuthology-results = scripts.results:main', + 'teuthology-report = scripts.report:main', + 'teuthology-kill = scripts.kill:main', + 'teuthology-queue = scripts.queue:main', + 'teuthology-prune-logs = scripts.prune_logs:main', + 'teuthology-describe = scripts.describe:main', + 'teuthology-reimage = scripts.reimage:main', + 'teuthology-dispatcher = scripts.dispatcher:main', + 'teuthology-wait = scripts.wait:main', + ], + }, + + ) diff --git a/teuthology/__init__.py b/teuthology/__init__.py index 4781f59bc1..de15f251a4 100644 --- a/teuthology/__init__.py +++ b/teuthology/__init__.py @@ -1,11 +1,5 @@ from __future__ import print_function import os -try: - import importlib.metadata as importlib_metadata -except ImportError: - import importlib_metadata - -__version__ = importlib_metadata.version("teuthology") # Tell gevent not to patch os.waitpid() since it is susceptible to race # conditions. See: @@ -14,15 +8,12 @@ os.environ['GEVENT_NOWAITPID'] = 'true' # Use manhole to give us a way to debug hung processes # https://pypi.python.org/pypi/manhole -try: - import manhole - manhole.install( - verbose=False, - # Listen for SIGUSR1 - oneshot_on="USR1" - ) -except ImportError: - pass +import manhole +manhole.install( + verbose=False, + # Listen for SIGUSR1 + oneshot_on="USR1" +) from gevent import monkey monkey.patch_all( dns=False, @@ -39,6 +30,25 @@ from teuthology.orchestra import monkey monkey.patch_all() import logging +import subprocess + +__version__ = '1.1.0' + +# do our best, but if it fails, continue with above + +try: + teuthology_dir = os.path.dirname(os.path.realpath(__file__)) + site_dir = os.path.dirname(teuthology_dir) + git_dir = os.path.join(site_dir, '.git') + # make sure we use git repo otherwise it is a released version + if os.path.exists(git_dir): + __version__ += '-' + str(subprocess.check_output( + 'git rev-parse --short HEAD'.split(), + cwd=site_dir + ).decode()).strip() +except Exception as e: + # before logging; should be unusual + print("Can't get version from git rev-parse %s" % e, file=sys.stderr) # If we are running inside a virtualenv, ensure we have its 'bin' directory in # our PATH. This doesn't happen automatically if scripts are called without diff --git a/teuthology/dispatcher/__init__.py b/teuthology/dispatcher/__init__.py index 9e513a72e9..9f7fa257e4 100644 --- a/teuthology/dispatcher/__init__.py +++ b/teuthology/dispatcher/__init__.py @@ -124,7 +124,7 @@ def main(args): keep_running = False try: - job_config = prep_job( + job_config, teuth_bin_path = prep_job( job_config, log_file_path, archive_dir, @@ -137,9 +137,10 @@ def main(args): job_config = lock_machines(job_config) run_args = [ - 'teuthology-dispatcher', + os.path.join(teuth_bin_path, 'teuthology-dispatcher'), '--supervisor', '-v', + '--bin-path', teuth_bin_path, '--archive-dir', archive_dir, ] diff --git a/teuthology/dispatcher/supervisor.py b/teuthology/dispatcher/supervisor.py index fb7cdf2fff..f52e37938f 100644 --- a/teuthology/dispatcher/supervisor.py +++ b/teuthology/dispatcher/supervisor.py @@ -31,6 +31,7 @@ def main(args): verbose = args["--verbose"] archive_dir = args["--archive-dir"] + teuth_bin_path = args["--bin-path"] config_file_path = args["--job-config"] with open(config_file_path, 'r') as config_file: @@ -55,6 +56,7 @@ def main(args): try: return run_job( job_config, + teuth_bin_path, archive_dir, verbose ) @@ -62,12 +64,12 @@ def main(args): return 0 -def run_job(job_config, archive_dir, verbose): +def run_job(job_config, teuth_bin_path, archive_dir, verbose): safe_archive = safepath.munge(job_config['name']) if job_config.get('first_in_suite') or job_config.get('last_in_suite'): job_archive = os.path.join(archive_dir, safe_archive) args = [ - 'teuthology-results', + os.path.join(teuth_bin_path, 'teuthology-results'), '--archive-dir', job_archive, '--name', job_config['name'], ] @@ -98,7 +100,9 @@ def run_job(job_config, archive_dir, verbose): log.info('Running job %s', job_config['job_id']) - arg = ['teuthology'] + arg = [ + os.path.join(teuth_bin_path, 'teuthology'), + ] # The following is for compatibility with older schedulers, from before we # started merging the contents of job_config['config'] into job_config # itself. diff --git a/teuthology/provision/cloud/test/test_openstack.py b/teuthology/provision/cloud/test/test_openstack.py index 108532ed56..08f5b9c08a 100644 --- a/teuthology/provision/cloud/test/test_openstack.py +++ b/teuthology/provision/cloud/test/test_openstack.py @@ -76,8 +76,8 @@ def get_fake_obj(mock_args=None, attributes=None): class TestOpenStackBase(object): - def setup(self): - config.load(dict(libcloud=deepcopy(test_config))) + def setup(self, conf=dict(), test_config=test_config): + config.load(conf or dict(libcloud=deepcopy(test_config))) self.start_patchers() def start_patchers(self): diff --git a/teuthology/run_tasks.py b/teuthology/run_tasks.py index 598947c807..602559f7fb 100644 --- a/teuthology/run_tasks.py +++ b/teuthology/run_tasks.py @@ -1,4 +1,3 @@ -import importlib import jinja2 import logging import os @@ -61,17 +60,6 @@ def _import(from_package, module_name, task_name, fail_on_import_error=False): if fail_on_import_error: raise else: - if ( - importlib.util.find_spec(from_package) is not None and - importlib.util.find_spec(full_module_name) is not None - ): - # If we get here, it means we could _find_ both the module and - # the package that contains it, but still got an ImportError. - # Typically that means the module failed to import because it - # could not find one of its dependencies; if we don't raise - # here it will look like we just could't find the module, - # making the dependency issue difficult to discover. - raise return None return module diff --git a/teuthology/task/tests/__init__.py b/teuthology/task/tests/__init__.py index 43c6c11699..c066f66600 100644 --- a/teuthology/task/tests/__init__.py +++ b/teuthology/task/tests/__init__.py @@ -44,12 +44,10 @@ class TeuthologyContextPlugin(object): metafunc.parametrize(["ctx", "config"], [(self.ctx, self.config),]) # log the outcome of each test - @pytest.hookimpl(hookwrapper=True) - def pytest_runtest_makereport(self, item, call): - outcome = yield - report = outcome.get_result() + def pytest_runtest_makereport(self, __multicall__, item, call): + report = __multicall__.execute() - # after the test has been called, get its report and log it + # after the test has been called, get it's report and log it if call.when == 'call': # item.location[0] is a slash delimeted path to the test file # being ran. We only want the portion after teuthology.task.tests diff --git a/teuthology/test/test_worker.py b/teuthology/test/test_worker.py index e32cdf6b8b..ae92e2441a 100644 --- a/teuthology/test/test_worker.py +++ b/teuthology/test/test_worker.py @@ -85,10 +85,10 @@ class TestWorker(object): m_p.returncode = 0 m_popen.return_value = m_p m_t_config.results_server = True - worker.run_job(config, "archive/dir", verbose=False) + worker.run_job(config, "teuth/bin/path", "archive/dir", verbose=False) m_run_watchdog.assert_called_with(m_p, config) expected_args = [ - 'teuthology', + 'teuth/bin/path/teuthology', '-v', '--lock', '--block', @@ -135,7 +135,7 @@ class TestWorker(object): m_p.returncode = 1 m_popen.return_value = m_p m_t_config.results_server = False - worker.run_job(config, "archive/dir", verbose=False) + worker.run_job(config, "teuth/bin/path", "archive/dir", verbose=False) m_symlink_log.assert_called_with(config["worker_log"], config["archive_path"]) @patch("teuthology.worker.report.try_push_job_info") @@ -195,7 +195,7 @@ class TestWorker(object): m_fetch_qa_suite.return_value = '/suite/path' m_isdir.return_value = True m_teuth_config.teuthology_path = None - got_config = worker.prep_job( + got_config, teuth_bin_path = worker.prep_job( config, log_file_path, archive_dir, @@ -208,6 +208,7 @@ class TestWorker(object): ) assert got_config['teuthology_branch'] == 'master' assert m_fetch_teuthology.called_once_with_args(branch='master') + assert teuth_bin_path == '/teuth/path/virtualenv/bin' assert m_fetch_qa_suite.called_once_with_args(branch='master') assert got_config['suite_path'] == '/suite/path' @@ -254,7 +255,7 @@ class TestWorker(object): ) m_connection.reserve.side_effect = jobs m_connect.return_value = m_connection - m_prep_job.return_value = dict() + m_prep_job.return_value = (dict(), '/bin/path') worker.main(self.ctx) # There should be one reserve call per item in the jobs list expected_reserve_calls = [ diff --git a/teuthology/worker.py b/teuthology/worker.py index 70cb72f621..be5d6c6501 100644 --- a/teuthology/worker.py +++ b/teuthology/worker.py @@ -114,13 +114,14 @@ def main(ctx): keep_running = False try: - job_config = prep_job( + job_config, teuth_bin_path = prep_job( job_config, log_file_path, ctx.archive_dir, ) run_job( job_config, + teuth_bin_path, ctx.archive_dir, ctx.verbose, ) @@ -162,6 +163,11 @@ def prep_job(job_config, log_file_path, archive_dir): log.info('Using teuthology sha1 %s', teuthology_sha1) try: + if teuth_config.teuthology_path is not None: + teuth_path = teuth_config.teuthology_path + else: + teuth_path = fetch_teuthology(branch=teuthology_branch, + commit=teuthology_sha1) # For the teuthology tasks, we look for suite_branch, and if we # don't get that, we look for branch, and fall back to 'master'. # last-in-suite jobs don't have suite_branch or branch set. @@ -190,10 +196,14 @@ def prep_job(job_config, log_file_path, archive_dir): ) raise SkipJob() - return job_config + teuth_bin_path = os.path.join(teuth_path, 'virtualenv', 'bin') + if not os.path.isdir(teuth_bin_path): + raise RuntimeError("teuthology branch %s at %s not bootstrapped!" % + (teuthology_branch, teuth_bin_path)) + return job_config, teuth_bin_path -def run_job(job_config, archive_dir, verbose): +def run_job(job_config, teuth_bin_path, archive_dir, verbose): safe_archive = safepath.munge(job_config['name']) if job_config.get('first_in_suite') or job_config.get('last_in_suite'): if teuth_config.results_server: @@ -205,7 +215,7 @@ def run_job(job_config, archive_dir, verbose): suite_archive_dir = os.path.join(archive_dir, safe_archive) safepath.makedirs('/', suite_archive_dir) args = [ - 'teuthology-results', + os.path.join(teuth_bin_path, 'teuthology-results'), '--archive-dir', suite_archive_dir, '--name', job_config['name'], ] @@ -234,7 +244,9 @@ def run_job(job_config, archive_dir, verbose): log.info('Running job %s', job_config['job_id']) suite_path = job_config['suite_path'] - arg = ['teuthology'] + arg = [ + os.path.join(teuth_bin_path, 'teuthology'), + ] # The following is for compatibility with older schedulers, from before we # started merging the contents of job_config['config'] into job_config # itself. diff --git a/tox.ini b/tox.ini index 9193865d07..3e77fd4012 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,5 @@ [tox] envlist = docs, py3, flake8, openstack -isolated_build = True [testenv] setenv = @@ -8,25 +7,26 @@ setenv = LANG=en_US [testenv:py3] +basepython=python3 install_command = pip install --upgrade {opts} {packages} passenv = HOME deps= -r{toxinidir}/requirements.txt pytest-cov==2.8.1 coverage==4.5.4 - mock==4.0.3 -extras = test log_format = %(asctime)s %(levelname)s %(message)s commands= python -m pytest --cov=teuthology --cov-report=term -v {posargs:teuthology scripts} [testenv:flake8] +basepython=python3 install_command = pip install --upgrade {opts} {packages} deps= flake8 commands=flake8 --select=F,E9 {posargs:teuthology scripts} [testenv:docs] +basepython=python3 install_command = pip install --upgrade {opts} {packages} changedir=docs deps= @@ -40,22 +40,27 @@ commands= [testenv:openstack] install_command = pip install --upgrade {opts} {packages} passenv = HOME OS_REGION_NAME OS_AUTH_URL OS_TENANT_ID OS_TENANT_NAME OS_PASSWORD OS_USERNAME +sitepackages=True deps= -r{toxinidir}/requirements.txt -extras = test + commands=py.test -v {posargs:teuthology/openstack/test/test_openstack.py} +basepython=python3 [testenv:openstack-integration] passenv = HOME OS_REGION_NAME OS_AUTH_URL OS_TENANT_ID OS_TENANT_NAME OS_PASSWORD OS_USERNAME +basepython=python3 deps= -r{toxinidir}/requirements.txt -extras = test + commands= py.test -v {posargs} teuthology/openstack/test/openstack-integration.py [testenv:openstack-delegate] passenv = HOME OS_REGION_NAME OS_AUTH_URL OS_TENANT_ID OS_TENANT_NAME OS_PASSWORD OS_USERNAME +basepython=python3 sitepackages=True deps= -r{toxinidir}/requirements.txt + commands={toxinidir}/openstack-delegate.sh diff --git a/update-requirements.sh b/update-requirements.sh index 8040c0d1e3..1334538441 100755 --- a/update-requirements.sh +++ b/update-requirements.sh @@ -1,3 +1,4 @@ #!/bin/bash -pip-compile $@ pyproject.toml +pip-compile $@ requirements.in +sed -i'' -e '/^-e / d' -e 's/-r requirements.in/teuthology/g' requirements.txt \ No newline at end of file -- 2.39.5