From 9cb9c8fa1e9b14ffcc7362b0f57e0394d60cf4bd Mon Sep 17 00:00:00 2001 From: Kyr Shatskyy Date: Sat, 12 Oct 2019 01:42:00 +0200 Subject: [PATCH] python3: use compatible urllib packages Use py2/py3 compatible functions url packages. Python3 from urllib.parse import parse_qs, urljoin, urlparse, urlencode from urllib.request import urlopen, Request from urllib.error import HTTPError Python2 from urlparse import parse_qs, urljoin, urlparse from urllib import urlencode from urllib2 import urlopen, Request, HTTPError Signed-off-by: Kyr Shatskyy --- teuthology/lock/query.py | 8 ++++++-- teuthology/misc.py | 18 ++++++++++++------ teuthology/packaging.py | 14 +++++++++----- teuthology/provision/cloud/openstack.py | 8 ++++++-- .../provision/cloud/test/test_openstack.py | 9 +++++++-- teuthology/task/kernel.py | 10 +++++++--- teuthology/task/pcp.py | 14 +++++++++----- teuthology/test/task/test_pcp.py | 12 ++++++++---- 8 files changed, 64 insertions(+), 29 deletions(-) diff --git a/teuthology/lock/query.py b/teuthology/lock/query.py index 9dcdd77015..99e5deafb5 100644 --- a/teuthology/lock/query.py +++ b/teuthology/lock/query.py @@ -1,6 +1,10 @@ import logging import os -import urllib + +try: + from urllib.parse import urlencode +except ImportError: + from urllib import urlencode import requests @@ -58,7 +62,7 @@ def list_locks(keyed_by_name=False, **kwargs): if kwargs: if 'machine_type' in kwargs: kwargs['machine_type'] = kwargs['machine_type'].replace(',','|') - uri += '?' + urllib.urlencode(kwargs) + uri += '?' + urlencode(kwargs) try: response = requests.get(uri) except requests.ConnectionError: diff --git a/teuthology/misc.py b/teuthology/misc.py index 037755e5eb..f7fba973cd 100644 --- a/teuthology/misc.py +++ b/teuthology/misc.py @@ -13,13 +13,19 @@ import subprocess import sys import tarfile import time -import urllib2 -import urlparse import yaml import json import re import pprint +try: + from urllib.parse import urljoin + from urllib.request import urlopen + from urllib.error import HTTPError +except ImportError: + from urlparse import urljoin + from urllib2 import urlopen, HTTPError + from netaddr.strategy.ipv4 import valid_str as _is_ipv4 from netaddr.strategy.ipv6 import valid_str as _is_ipv6 from teuthology import safepath @@ -230,19 +236,19 @@ def get_ceph_binary_url(package=None, branch = 'master' ref = branch - sha1_url = urlparse.urljoin(BASE, 'ref/{ref}/sha1'.format(ref=ref)) + sha1_url = urljoin(BASE, 'ref/{ref}/sha1'.format(ref=ref)) log.debug('Translating ref to sha1 using url %s', sha1_url) try: - sha1_fp = urllib2.urlopen(sha1_url) + sha1_fp = urlopen(sha1_url) sha1 = sha1_fp.read().rstrip('\n') sha1_fp.close() - except urllib2.HTTPError as e: + except HTTPError as e: log.error('Failed to get url %s', sha1_url) raise e log.debug('Using %s %s sha1 %s', package, format, sha1) - bindir_url = urlparse.urljoin(BASE, 'sha1/{sha1}/'.format(sha1=sha1)) + bindir_url = urljoin(BASE, 'sha1/{sha1}/'.format(sha1=sha1)) return (sha1, bindir_url) diff --git a/teuthology/packaging.py b/teuthology/packaging.py index 6477c97087..78e94338d5 100644 --- a/teuthology/packaging.py +++ b/teuthology/packaging.py @@ -2,8 +2,12 @@ import logging import ast import re import requests -import urllib -import urlparse + +try: + from urllib.parse import urljoin, urlencode +except ImportError: + from urlparse import urljoin + from urllib import urlencode from collections import OrderedDict from cStringIO import StringIO @@ -887,8 +891,8 @@ class ShamanProject(GitbuilderProject): req_obj['sha1'] = ref_val else: req_obj['ref'] = ref_val - req_str = urllib.urlencode(req_obj) - uri = urlparse.urljoin( + req_str = urlencode(req_obj) + uri = urljoin( self.query_url, 'search', ) + '?%s' % req_str @@ -962,7 +966,7 @@ class ShamanProject(GitbuilderProject): @property def repo_url(self): self.assert_result() - return urlparse.urljoin( + return urljoin( self._result.json()[0]['chacra_url'], 'repo', ) diff --git a/teuthology/provision/cloud/openstack.py b/teuthology/provision/cloud/openstack.py index bdfbbf2793..5882d57ad4 100644 --- a/teuthology/provision/cloud/openstack.py +++ b/teuthology/provision/cloud/openstack.py @@ -3,9 +3,13 @@ import re import requests import socket import time -import urllib import yaml +try: + from urllib.parse import urlencode +except ImportError: + from urllib import urlencode + from copy import deepcopy from libcloud.common.exceptions import RateLimitReachedError, BaseHTTPError @@ -289,7 +293,7 @@ class OpenStackProvisioner(base.Provisioner): log.exception("Could not destroy volume %s", vol) def _update_dns(self): - query = urllib.urlencode(dict( + query = urlencode(dict( name=self.name, ip=self.ips[0], )) diff --git a/teuthology/provision/cloud/test/test_openstack.py b/teuthology/provision/cloud/test/test_openstack.py index 1a99011bfc..cc3f26c256 100644 --- a/teuthology/provision/cloud/test/test_openstack.py +++ b/teuthology/provision/cloud/test/test_openstack.py @@ -1,8 +1,13 @@ import socket -import urlparse import yaml import os +try: + from urllib.parse import parse_qs +except ImportError: + from urlparse import parse_qs + + from copy import deepcopy from libcloud.compute.providers import get_driver from mock import patch, Mock, DEFAULT @@ -657,7 +662,7 @@ class TestOpenStackProvisioner(TestOpenStackBase): assert len(call_args) == 1 url_base, query_string = call_args[0][0][0].split('?') assert url_base == 'nsupdate_url' - parsed_query = urlparse.parse_qs(query_string) + parsed_query = parse_qs(query_string) assert parsed_query == dict(name=['x'], ip=['y']) @mark.parametrize( diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py index ffb236fdb7..833fcc0798 100644 --- a/teuthology/task/kernel.py +++ b/teuthology/task/kernel.py @@ -7,7 +7,11 @@ import logging import os import re import shlex -import urlparse + +try: + from urllib.parse import urljoin +except ImportError: + from urlparse import urljoin from teuthology import misc as teuthology from teuthology.parallel import parallel @@ -370,7 +374,7 @@ def download_kernel(ctx, config): if teuth_config.use_shaman: if role_remote.os.package_type == 'rpm': arch = builder.arch - baseurl = urlparse.urljoin( + baseurl = urljoin( builder.base_url, '/'.join([arch, '']) ) @@ -380,7 +384,7 @@ def download_kernel(ctx, config): ) elif role_remote.os.package_type == 'deb': arch = 'amd64' # FIXME - baseurl = urlparse.urljoin( + baseurl = urljoin( builder.base_url, '/'.join([ 'pool', 'main', 'l', diff --git a/teuthology/task/pcp.py b/teuthology/task/pcp.py index a09c56c365..d016327f08 100644 --- a/teuthology/task/pcp.py +++ b/teuthology/task/pcp.py @@ -6,8 +6,12 @@ import logging import os import requests import time -import urllib -import urlparse + +try: + from urllib.parse import urljoin, urlencode +except ImportError: + from urlparse import urljoin + from urllib import urlencode from teuthology.config import config as teuth_config from teuthology.orchestra import run @@ -64,7 +68,7 @@ class PCPGrapher(PCPDataSource): def __init__(self, hosts, time_from, time_until='now'): super(PCPGrapher, self).__init__(hosts, time_from, time_until) - self.base_url = urlparse.urljoin( + self.base_url = urljoin( teuth_config.pcp_host, self._endpoint) @@ -83,7 +87,7 @@ class GrafanaGrapher(PCPGrapher): ) if self.time_until: config['time_to'] = self._format_time(self.time_until) - args = urllib.urlencode(config) + args = urlencode(config) template = "{base_url}?{args}" return template.format(base_url=self.base_url, args=args) @@ -183,7 +187,7 @@ class GraphiteGrapher(PCPGrapher): # 'target=' arg 'target': self.get_target_globs(metric), }) - args = urllib.urlencode(config, doseq=True) + args = urlencode(config, doseq=True) template = "{base_url}?{args}" return template.format(base_url=self.base_url, args=args) diff --git a/teuthology/test/task/test_pcp.py b/teuthology/test/task/test_pcp.py index 1331be044b..68b9093e24 100644 --- a/teuthology/test/task/test_pcp.py +++ b/teuthology/test/task/test_pcp.py @@ -1,6 +1,10 @@ import os import requests -import urlparse + +try: + from urllib.parse import parse_qs, urljoin +except ImportError: + from urlparse import parse_qs, urljoin from mock import patch, DEFAULT, Mock, mock_open, call from pytest import raises @@ -87,7 +91,7 @@ class TestPCPGrapher(TestPCPDataSource): assert obj.hosts == hosts assert obj.time_from == time_from assert obj.time_until == time_until - expected_url = urlparse.urljoin(config.pcp_host, self.klass._endpoint) + expected_url = urljoin(config.pcp_host, self.klass._endpoint) assert obj.base_url == expected_url @@ -103,13 +107,13 @@ class TestGrafanaGrapher(TestPCPGrapher): time_from=time_from, time_until=time_until, ) - base_url = urlparse.urljoin( + base_url = urljoin( config.pcp_host, 'grafana/index.html#/dashboard/script/index.js', ) assert obj.base_url == base_url got_url = obj.build_graph_url() - parsed_query = urlparse.parse_qs(got_url.split('?')[1]) + parsed_query = parse_qs(got_url.split('?')[1]) assert parsed_query['hosts'] == hosts assert len(parsed_query['time_from']) == 1 assert parsed_query['time_from'][0] == time_from -- 2.39.5