From: Adam King Date: Sat, 23 Sep 2023 16:15:06 +0000 (-0400) Subject: cephadm: move more funcs into data_utils.py X-Git-Tag: v19.0.0~384^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=915ac7d74fd17c1823bb4890ea21d9229358949c;p=ceph.git cephadm: move more funcs into data_utils.py Signed-off-by: Adam King --- diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py index b65d9ee21ff..2030a3b6c53 100755 --- a/src/cephadm/cephadm.py +++ b/src/cephadm/cephadm.py @@ -25,7 +25,6 @@ from typing import Dict, List, Tuple, Optional, Union, Any, Callable, IO, Sequen import re import uuid -from configparser import ConfigParser from contextlib import redirect_stdout from functools import wraps from glob import glob @@ -53,7 +52,6 @@ from cephadmlib.constants import ( DEFAULT_NVMEOF_IMAGE, DEFAULT_PROMETHEUS_IMAGE, DEFAULT_PROMTAIL_IMAGE, - DEFAULT_REGISTRY, DEFAULT_SNMP_GATEWAY_IMAGE, # other constant values CEPH_CONF, @@ -114,6 +112,11 @@ from cephadmlib.container_engines import ( from cephadmlib.data_utils import ( dict_get, dict_get_join, + get_legacy_config_fsid, + is_fsid, + normalize_image_digest, + try_convert_datetime, + read_config, with_units_to_int, ) from cephadmlib.file_utils import ( @@ -1386,55 +1389,6 @@ def is_available(ctx, what, func): time.sleep(2) -def read_config(fn): - # type: (Optional[str]) -> ConfigParser - cp = ConfigParser() - if fn: - cp.read(fn) - return cp - - -def try_convert_datetime(s): - # type: (str) -> Optional[str] - # This is super irritating because - # 1) podman and docker use different formats - # 2) python's strptime can't parse either one - # - # I've seen: - # docker 18.09.7: 2020-03-03T09:21:43.636153304Z - # podman 1.7.0: 2020-03-03T15:52:30.136257504-06:00 - # 2020-03-03 15:52:30.136257504 -0600 CST - # (In the podman case, there is a different string format for - # 'inspect' and 'inspect --format {{.Created}}'!!) - - # In *all* cases, the 9 digit second precision is too much for - # python's strptime. Shorten it to 6 digits. - p = re.compile(r'(\.[\d]{6})[\d]*') - s = p.sub(r'\1', s) - - # replace trailing Z with -0000, since (on python 3.6.8) it won't parse - if s and s[-1] == 'Z': - s = s[:-1] + '-0000' - - # cut off the redundant 'CST' part that strptime can't parse, if - # present. - v = s.split(' ') - s = ' '.join(v[0:3]) - - # try parsing with several format strings - fmts = [ - '%Y-%m-%dT%H:%M:%S.%f%z', - '%Y-%m-%d %H:%M:%S.%f %z', - ] - for f in fmts: - try: - # return timestamp normalized to UTC, rendered as DATEFMT. - return datetime.datetime.strptime(s, f).astimezone(tz=datetime.timezone.utc).strftime(DATEFMT) - except ValueError: - pass - return None - - def generate_service_id(): # type: () -> str return get_short_hostname() + '.' + ''.join(random.choice(string.ascii_lowercase) @@ -1464,15 +1418,6 @@ def make_fsid(): return str(uuid.uuid1()) -def is_fsid(s): - # type: (str) -> bool - try: - uuid.UUID(s) - except ValueError: - return False - return True - - def validate_fsid(func: FuncT) -> FuncT: @wraps(func) def _validate_fsid(ctx: CephadmContext) -> Any: @@ -1851,19 +1796,6 @@ def get_unit_name_by_daemon_name(ctx: CephadmContext, fsid: str, name: str) -> s raise Error('Failed to get unit name for {}'.format(daemon)) -def get_legacy_config_fsid(cluster, legacy_dir=None): - # type: (str, Optional[str]) -> Optional[str] - config_file = '/etc/ceph/%s.conf' % cluster - if legacy_dir is not None: - config_file = os.path.abspath(legacy_dir + config_file) - - if os.path.exists(config_file): - config = read_config(config_file) - if config.has_section('global') and config.has_option('global', 'fsid'): - return config.get('global', 'fsid') - return None - - def get_legacy_daemon_fsid(ctx, cluster, daemon_type, daemon_id, legacy_dir=None): # type: (CephadmContext, str, str, Union[int, str], Optional[str]) -> Optional[str] @@ -3954,33 +3886,6 @@ def command_inspect_image(ctx): return 0 -def normalize_image_digest(digest: str) -> str: - """ - Normal case: - >>> normalize_image_digest('ceph/ceph', 'docker.io') - 'docker.io/ceph/ceph' - - No change: - >>> normalize_image_digest('quay.ceph.io/ceph/ceph', 'docker.io') - 'quay.ceph.io/ceph/ceph' - - >>> normalize_image_digest('docker.io/ubuntu', 'docker.io') - 'docker.io/ubuntu' - - >>> normalize_image_digest('localhost/ceph', 'docker.io') - 'localhost/ceph' - """ - known_shortnames = [ - 'ceph/ceph', - 'ceph/daemon', - 'ceph/daemon-base', - ] - for image in known_shortnames: - if digest.startswith(image): - return f'{DEFAULT_REGISTRY}/{digest}' - return digest - - def get_image_info_from_inspect(out, image): # type: (str, str) -> Dict[str, Union[str,List[str]]] image_id, digests = out.split(',', 1) diff --git a/src/cephadm/cephadmlib/data_utils.py b/src/cephadm/cephadmlib/data_utils.py index eee7751e850..21fceb54ccf 100644 --- a/src/cephadm/cephadmlib/data_utils.py +++ b/src/cephadm/cephadmlib/data_utils.py @@ -1,8 +1,15 @@ # data_utils.py - assorted data management functions +import datetime +import os +import re +import uuid -from typing import Dict, Any +from configparser import ConfigParser +from typing import Dict, Any, Optional + +from .constants import DATEFMT, DEFAULT_REGISTRY from .exceptions import Error @@ -86,3 +93,101 @@ def with_units_to_int(v: str) -> int: mult = 1024 * 1024 * 1024 * 1024 v = v[:-1] return int(float(v) * mult) + + +def read_config(fn): + # type: (Optional[str]) -> ConfigParser + cp = ConfigParser() + if fn: + cp.read(fn) + return cp + + +def try_convert_datetime(s): + # type: (str) -> Optional[str] + # This is super irritating because + # 1) podman and docker use different formats + # 2) python's strptime can't parse either one + # + # I've seen: + # docker 18.09.7: 2020-03-03T09:21:43.636153304Z + # podman 1.7.0: 2020-03-03T15:52:30.136257504-06:00 + # 2020-03-03 15:52:30.136257504 -0600 CST + # (In the podman case, there is a different string format for + # 'inspect' and 'inspect --format {{.Created}}'!!) + + # In *all* cases, the 9 digit second precision is too much for + # python's strptime. Shorten it to 6 digits. + p = re.compile(r'(\.[\d]{6})[\d]*') + s = p.sub(r'\1', s) + + # replace trailing Z with -0000, since (on python 3.6.8) it won't parse + if s and s[-1] == 'Z': + s = s[:-1] + '-0000' + + # cut off the redundant 'CST' part that strptime can't parse, if + # present. + v = s.split(' ') + s = ' '.join(v[0:3]) + + # try parsing with several format strings + fmts = [ + '%Y-%m-%dT%H:%M:%S.%f%z', + '%Y-%m-%d %H:%M:%S.%f %z', + ] + for f in fmts: + try: + # return timestamp normalized to UTC, rendered as DATEFMT. + return datetime.datetime.strptime(s, f).astimezone(tz=datetime.timezone.utc).strftime(DATEFMT) + except ValueError: + pass + return None + + +def is_fsid(s): + # type: (str) -> bool + try: + uuid.UUID(s) + except ValueError: + return False + return True + + +def normalize_image_digest(digest: str) -> str: + """ + Normal case: + >>> normalize_image_digest('ceph/ceph', 'docker.io') + 'docker.io/ceph/ceph' + + No change: + >>> normalize_image_digest('quay.ceph.io/ceph/ceph', 'docker.io') + 'quay.ceph.io/ceph/ceph' + + >>> normalize_image_digest('docker.io/ubuntu', 'docker.io') + 'docker.io/ubuntu' + + >>> normalize_image_digest('localhost/ceph', 'docker.io') + 'localhost/ceph' + """ + known_shortnames = [ + 'ceph/ceph', + 'ceph/daemon', + 'ceph/daemon-base', + ] + for image in known_shortnames: + if digest.startswith(image): + return f'{DEFAULT_REGISTRY}/{digest}' + return digest + + +def get_legacy_config_fsid(cluster, legacy_dir=None): + # type: (str, Optional[str]) -> Optional[str] + config_file = '/etc/ceph/%s.conf' % cluster + if legacy_dir is not None: + config_file = os.path.abspath(legacy_dir + config_file) + + if os.path.exists(config_file): + config = read_config(config_file) + if config.has_section('global') and config.has_option('global', 'fsid'): + return config.get('global', 'fsid') + return None diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py index 565111903ea..525e22a8c91 100644 --- a/src/cephadm/tests/test_cephadm.py +++ b/src/cephadm/tests/test_cephadm.py @@ -21,6 +21,8 @@ from .fixtures import ( from pyfakefs import fake_filesystem from pyfakefs import fake_filesystem_unittest +from cephadmlib.constants import DEFAULT_REGISTRY + _cephadm = import_cephadm() @@ -785,7 +787,7 @@ class TestCephAdm(object): assert _cephadm.normalize_image_digest(s) == s s = 'ceph/ceph:latest' - assert _cephadm.normalize_image_digest(s) == f'{_cephadm.DEFAULT_REGISTRY}/{s}' + assert _cephadm.normalize_image_digest(s) == f'{DEFAULT_REGISTRY}/{s}' @pytest.mark.parametrize('fsid, ceph_conf, list_daemons, result, err, ', [