From 5c758f63aaee00fcc5d104ed0bfebb93e90d896c Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Thu, 3 Sep 2020 13:03:51 +0800 Subject: [PATCH] qa/standalone: always decode output from check_output() we could pass `text=True` for better readability, but that's introduced in python3.7, or pass `error="ignore"` but it's too long. Signed-off-by: Kefu Chai --- .../special/ceph_objectstore_tool.py | 46 +++++++------------ 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/qa/standalone/special/ceph_objectstore_tool.py b/qa/standalone/special/ceph_objectstore_tool.py index d7d1694b358..98a2c87238f 100755 --- a/qa/standalone/special/ceph_objectstore_tool.py +++ b/qa/standalone/special/ceph_objectstore_tool.py @@ -18,18 +18,6 @@ logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.WARNING, datefmt="%FT%T") -if sys.version_info[0] >= 3: - def decode(s): - return s.decode('utf-8') - - def check_output(*args, **kwargs): # noqa - return decode(subprocess.check_output(*args, **kwargs)) -else: - def decode(s): - return s - - - def wait_for_health(): print("Wait for health_ok...", end="") tries = 0 @@ -44,7 +32,7 @@ def wait_for_health(): def get_pool_id(name, nullfd): cmd = "{path}/ceph osd pool stats {pool}".format(pool=name, path=CEPH_BIN).split() # pool {pool} id # .... grab the 4 field - return check_output(cmd, stderr=nullfd).split()[3] + return check_output(cmd, stderr=nullfd).decode().split()[3] # return a list of unique PGS given an osd subdirectory @@ -205,7 +193,7 @@ def verify(DATADIR, POOL, NAME_PREFIX, db): for key, val in db[nspace][file]["xattr"].items(): cmd = "{path}/rados -p {pool} -N '{nspace}' getxattr {name} {key}".format(pool=POOL, name=file, key=key, nspace=nspace, path=CEPH_BIN) logging.debug(cmd) - getval = check_output(cmd, shell=True, stderr=DEVNULL) + getval = check_output(cmd, shell=True, stderr=DEVNULL).decode() logging.debug("getxattr {key} {val}".format(key=key, val=getval)) if getval != val: logging.error("getxattr of key {key} returned wrong val: {get} instead of {orig}".format(key=key, get=getval, orig=val)) @@ -426,7 +414,7 @@ def set_osd_weight(CFSD_PREFIX, osd_ids, osd_path, weight): osdmap_file = tempfile.NamedTemporaryFile(delete=True) cmd = (CFSD_PREFIX + "--op get-osdmap --file {osdmap_file}").format(osd=osd_path, osdmap_file=osdmap_file.name) - output = check_output(cmd, shell=True) + output = check_output(cmd, shell=True).decode() epoch = int(re.findall('#(\d+)', output)[0]) new_crush_file = tempfile.NamedTemporaryFile(delete=True) @@ -491,7 +479,7 @@ def get_osd_weights(CFSD_PREFIX, osd_ids, osd_path): output = check_output("{path}/crushtool --tree -i {crush_file} | tail -n {num_osd}".format(crush_file=crush_file.name, num_osd=len(osd_ids), path=CEPH_BIN), stderr=DEVNULL, - shell=True) + shell=True).decode() weights = [] for line in output.strip().split('\n'): print(line) @@ -537,7 +525,7 @@ def test_get_set_inc_osdmap(CFSD_PREFIX, osd_path): file_e2 = tempfile.NamedTemporaryFile(delete=True) cmd = (CFSD_PREFIX + "--op get-inc-osdmap --file {file}").format(osd=osd_path, file=file_e2.name) - output = check_output(cmd, shell=True) + output = check_output(cmd, shell=True).decode() epoch = int(re.findall('#(\d+)', output)[0]) # backup e1 incremental before overwriting it epoch -= 1 @@ -1271,7 +1259,7 @@ def main(argv): attrkey = "_" + key cmd = (CFSD_PREFIX + " '{json}' get-attr {key}").format(osd=osd, json=JSON, key=attrkey) logging.debug(cmd) - getval = check_output(cmd, shell=True) + getval = check_output(cmd, shell=True).decode() if getval != val: logging.error("get-attr of key {key} returned wrong val: {get} instead of {orig}".format(key=attrkey, get=getval, orig=val)) ERRORS += 1 @@ -1295,7 +1283,7 @@ def main(argv): # Check the set-attr cmd = (CFSD_PREFIX + " --pgid {pg} '{json}' get-attr {key}").format(osd=osd, pg=pg, json=JSON, key=attrkey) logging.debug(cmd) - getval = check_output(cmd, shell=True) + getval = check_output(cmd, shell=True).decode() if ret != 0: logging.error("Bad exit status {ret} from get-attr".format(ret=ret)) ERRORS += 1 @@ -1338,7 +1326,7 @@ def main(argv): hdr = db[nspace][basename].get("omapheader", "") cmd = (CFSD_PREFIX + "'{json}' get-omaphdr").format(osd=osd, json=JSON) logging.debug(cmd) - gethdr = check_output(cmd, shell=True) + gethdr = check_output(cmd, shell=True).decode() if gethdr != hdr: logging.error("get-omaphdr was wrong: {get} instead of {orig}".format(get=gethdr, orig=hdr)) ERRORS += 1 @@ -1354,7 +1342,7 @@ def main(argv): # Check the set-omaphdr cmd = (CFSD_PREFIX + "'{json}' get-omaphdr").format(osd=osd, pg=pg, json=JSON) logging.debug(cmd) - gethdr = check_output(cmd, shell=True) + gethdr = check_output(cmd, shell=True).decode() if ret != 0: logging.error("Bad exit status {ret} from get-omaphdr".format(ret=ret)) ERRORS += 1 @@ -1383,7 +1371,7 @@ def main(argv): for omapkey, val in db[nspace][basename]["omap"].items(): cmd = (CFSD_PREFIX + " '{json}' get-omap {key}").format(osd=osd, json=JSON, key=omapkey) logging.debug(cmd) - getval = check_output(cmd, shell=True) + getval = check_output(cmd, shell=True).decode() if getval != val: logging.error("get-omap of key {key} returned wrong val: {get} instead of {orig}".format(key=omapkey, get=getval, orig=val)) ERRORS += 1 @@ -1407,7 +1395,7 @@ def main(argv): # Check the set-omap cmd = (CFSD_PREFIX + " --pgid {pg} '{json}' get-omap {key}").format(osd=osd, pg=pg, json=JSON, key=omapkey) logging.debug(cmd) - getval = check_output(cmd, shell=True) + getval = check_output(cmd, shell=True).decode() if ret != 0: logging.error("Bad exit status {ret} from get-omap".format(ret=ret)) ERRORS += 1 @@ -1511,7 +1499,7 @@ def main(argv): cmd = (CFSD_PREFIX + " --tty '{json}' get-attr hinfo_key").format(osd=osd, json=JSON) logging.debug("TRY: " + cmd) try: - out = check_output(cmd, shell=True, stderr=subprocess.STDOUT) + out = check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode() logging.debug("FOUND: {json} in {osd} has value '{val}'".format(osd=osd, json=JSON, val=out)) found += 1 except subprocess.CalledProcessError as e: @@ -1667,7 +1655,7 @@ def main(argv): cmd = (CFSD_PREFIX + "--op list-pgs").format(osd=osd) logging.debug(cmd) - TEST_PGS = check_output(cmd, shell=True).split("\n") + TEST_PGS = check_output(cmd, shell=True).decode().split("\n") TEST_PGS = sorted(TEST_PGS)[1:] # Skip extra blank line if TEST_PGS != CHECK_PGS: @@ -1852,7 +1840,7 @@ def main(argv): ERRORS += 1 cmd = "{path}/rados -p {pool} ls".format(pool=NEWPOOL, path=CEPH_BIN) logging.debug(cmd) - data = check_output(cmd, shell=True) + data = check_output(cmd, shell=True).decode() if data: logging.error("'{data}'".format(data=data)) logging.error("Found objects after dry-run") @@ -1891,7 +1879,7 @@ def main(argv): logging.debug(cmd) call(cmd, shell=True, stdout=nullfd, stderr=nullfd) SPLITID = get_pool_id(SPLIT_POOL, nullfd) - pool_size = int(check_output("{path}/ceph osd pool get {pool} size".format(pool=SPLIT_POOL, path=CEPH_BIN), shell=True, stderr=nullfd).split(" ")[1]) + pool_size = int(check_output("{path}/ceph osd pool get {pool} size".format(pool=SPLIT_POOL, path=CEPH_BIN), shell=True, stderr=nullfd).decode().split(" ")[1]) EXP_ERRORS = 0 RM_ERRORS = 0 IMP_ERRORS = 0 @@ -2033,11 +2021,11 @@ def remove_btrfs_subvolumes(path): return result = subprocess.Popen("stat -f -c '%%T' %s" % path, shell=True, stdout=subprocess.PIPE) for line in result.stdout: - filesystem = decode(line).rstrip('\n') + filesystem = line.decode('utf-8').rstrip('\n') if filesystem == "btrfs": result = subprocess.Popen("sudo btrfs subvolume list %s" % path, shell=True, stdout=subprocess.PIPE) for line in result.stdout: - subvolume = decode(line).split()[8] + subvolume = line.decode('utf-8').split()[8] # extracting the relative volume name m = re.search(".*(%s.*)" % path, subvolume) if m: -- 2.47.3