From 3fe715201c8c07cf4ea86b590f9682422eeccf33 Mon Sep 17 00:00:00 2001 From: Volker Theile Date: Thu, 10 Dec 2020 10:47:46 +0100 Subject: [PATCH] cephadm: Various properties like 'last_refresh' do not contain timezone Fixes: https://tracker.ceph.com/issues/48068 Signed-off-by: Volker Theile --- src/cephadm/cephadm | 9 ++- src/pybind/mgr/cephadm/inventory.py | 24 +++---- src/pybind/mgr/cephadm/module.py | 17 ++--- src/pybind/mgr/cephadm/serve.py | 5 +- src/pybind/mgr/cephadm/services/osd.py | 3 +- src/pybind/mgr/cephadm/tests/fixtures.py | 5 +- src/pybind/mgr/cephadm/tests/test_cephadm.py | 14 ++-- .../mgr/cephadm/tests/test_migration.py | 15 ++-- src/pybind/mgr/cephadm/tests/test_spec.py | 13 +++- src/pybind/mgr/cephadm/utils.py | 14 +--- src/pybind/mgr/orchestrator/_interface.py | 17 +++-- src/pybind/mgr/orchestrator/module.py | 6 +- .../orchestrator/tests/test_orchestrator.py | 14 ++-- src/python-common/ceph/tests/test_datetime.py | 58 ++++++++++++++++ src/python-common/ceph/utils.py | 68 +++++++++++++++++++ 15 files changed, 201 insertions(+), 81 deletions(-) create mode 100644 src/python-common/ceph/tests/test_datetime.py create mode 100644 src/python-common/ceph/utils.py diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index df516f0fe97..0be218883fb 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -48,7 +48,6 @@ import os import platform import pwd import random -import re import select import shutil import socket @@ -101,7 +100,7 @@ if sys.version_info > (3, 0): container_path = '' cached_stdin = None -DATEFMT = '%Y-%m-%dT%H:%M:%S.%f' +DATEFMT = '%Y-%m-%dT%H:%M:%S.%fZ' # Log and console output config logging_config = { @@ -1177,7 +1176,7 @@ def get_file_timestamp(fn): return datetime.datetime.fromtimestamp( mt, tz=datetime.timezone.utc ).strftime(DATEFMT) - except Exception as e: + except Exception: return None @@ -1199,11 +1198,11 @@ def try_convert_datetime(s): p = re.compile(r'(\.[\d]{6})[\d]*') s = p.sub(r'\1', s) - # replace trailling Z with -0000, since (on python 3.6.8) it won't parse + # replace trailing Z with -0000, since (on python 3.6.8) it won't parse if s and s[-1] == 'Z': s = s[:-1] + '-0000' - # cut off the redundnat 'CST' part that strptime can't parse, if + # cut off the redundant 'CST' part that strptime can't parse, if # present. v = s.split(' ') s = ' '.join(v[0:3]) diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py index f309504155f..2fa9b2f7302 100644 --- a/src/pybind/mgr/cephadm/inventory.py +++ b/src/pybind/mgr/cephadm/inventory.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Dict, List, Iterator, Optional, Any, Tuple, Se import orchestrator from ceph.deployment import inventory from ceph.deployment.service_spec import ServiceSpec -from cephadm.utils import str_to_datetime, datetime_to_str +from ceph.utils import str_to_datetime, datetime_to_str, datetime_now from orchestrator import OrchestratorError, HostSpec, OrchestratorEvent if TYPE_CHECKING: @@ -136,7 +136,7 @@ class SpecStore(): self.spec_preview[spec.service_name()] = spec return None self.specs[spec.service_name()] = spec - self.spec_created[spec.service_name()] = datetime.datetime.utcnow() + self.spec_created[spec.service_name()] = datetime_now() self.mgr.set_store( SPEC_STORE_PREFIX + spec.service_name(), json.dumps({ @@ -278,7 +278,7 @@ class HostCache(): def update_host_daemons(self, host, dm): # type: (str, Dict[str, orchestrator.DaemonDescription]) -> None self.daemons[host] = dm - self.last_daemon_update[host] = datetime.datetime.utcnow() + self.last_daemon_update[host] = datetime_now() def update_host_facts(self, host, facts): # type: (str, Dict[str, Dict[str, Any]]) -> None @@ -289,7 +289,7 @@ class HostCache(): # type: (str, List[inventory.Device], Dict[str,List[str]]) -> None self.devices[host] = dls self.networks[host] = nets - self.last_device_update[host] = datetime.datetime.utcnow() + self.last_device_update[host] = datetime_now() def update_daemon_config_deps(self, host: str, name: str, deps: List[str], stamp: datetime.datetime) -> None: self.daemon_config_deps[host][name] = { @@ -299,7 +299,7 @@ class HostCache(): def update_last_host_check(self, host): # type: (str) -> None - self.last_host_check[host] = datetime.datetime.utcnow() + self.last_host_check[host] = datetime_now() def prime_empty_host(self, host): # type: (str) -> None @@ -472,7 +472,7 @@ class HostCache(): if host in self.daemon_refresh_queue: self.daemon_refresh_queue.remove(host) return True - cutoff = datetime.datetime.utcnow() - datetime.timedelta( + cutoff = datetime_now() - datetime.timedelta( seconds=self.mgr.daemon_cache_timeout) if host not in self.last_daemon_update or self.last_daemon_update[host] < cutoff: return True @@ -507,7 +507,7 @@ class HostCache(): if host in self.device_refresh_queue: self.device_refresh_queue.remove(host) return True - cutoff = datetime.datetime.utcnow() - datetime.timedelta( + cutoff = datetime_now() - datetime.timedelta( seconds=self.mgr.device_cache_timeout) if host not in self.last_device_update or self.last_device_update[host] < cutoff: return True @@ -526,7 +526,7 @@ class HostCache(): def host_needs_check(self, host): # type: (str) -> bool - cutoff = datetime.datetime.utcnow() - datetime.timedelta( + cutoff = datetime_now() - datetime.timedelta( seconds=self.mgr.host_check_interval) return host not in self.last_host_check or self.last_host_check[host] < cutoff @@ -551,7 +551,7 @@ class HostCache(): def update_last_etc_ceph_ceph_conf(self, host: str) -> None: if not self.mgr.last_monmap: return - self.last_etc_ceph_ceph_conf[host] = datetime.datetime.utcnow() + self.last_etc_ceph_ceph_conf[host] = datetime_now() def host_needs_registry_login(self, host: str) -> bool: if host in self.mgr.offline_hosts: @@ -632,14 +632,14 @@ class EventStore(): self.events[event.kind_subject()] = self.events[event.kind_subject()][-5:] def for_service(self, spec: ServiceSpec, level: str, message: str) -> None: - e = OrchestratorEvent(datetime.datetime.utcnow(), 'service', + e = OrchestratorEvent(datetime_now(), 'service', spec.service_name(), level, message) self.add(e) def from_orch_error(self, e: OrchestratorError) -> None: if e.event_subject is not None: self.add(OrchestratorEvent( - datetime.datetime.utcnow(), + datetime_now(), e.event_subject[0], e.event_subject[1], "ERROR", @@ -647,7 +647,7 @@ class EventStore(): )) def for_daemon(self, daemon_name: str, level: str, message: str) -> None: - e = OrchestratorEvent(datetime.datetime.utcnow(), 'daemon', daemon_name, level, message) + e = OrchestratorEvent(datetime_now(), 'daemon', daemon_name, level, message) self.add(e) def for_daemon_from_exception(self, daemon_name: str, e: Exception) -> None: diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 579eb183048..de43549794a 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -26,6 +26,7 @@ from ceph.deployment.drive_group import DriveGroupSpec from ceph.deployment.service_spec import \ NFSServiceSpec, ServiceSpec, PlacementSpec, assert_valid_host, \ CustomContainerSpec, HostPlacementSpec +from ceph.utils import str_to_datetime, datetime_to_str, datetime_now from cephadm.serve import CephadmServe from cephadm.services.cephadmservice import CephadmDaemonSpec @@ -52,8 +53,7 @@ from .schedule import HostAssignment from .inventory import Inventory, SpecStore, HostCache, EventStore from .upgrade import CEPH_UPGRADE_ORDER, CephadmUpgrade from .template import TemplateMgr -from .utils import forall_hosts, CephadmNoImage, cephadmNoImage, \ - str_to_datetime, datetime_to_str +from .utils import forall_hosts, CephadmNoImage, cephadmNoImage try: import remoto @@ -89,8 +89,6 @@ Host * ConnectTimeout=30 """ -CEPH_DATEFMT = '%Y-%m-%dT%H:%M:%S.%fZ' - CEPH_TYPES = set(CEPH_UPGRADE_ORDER) @@ -498,9 +496,8 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, if notify_type == "mon_map": # get monmap mtime so we can refresh configs when mons change monmap = self.get('mon_map') - self.last_monmap = datetime.datetime.strptime( - monmap['modified'], CEPH_DATEFMT) - if self.last_monmap and self.last_monmap > datetime.datetime.utcnow(): + self.last_monmap = str_to_datetime(monmap['modified']) + if self.last_monmap and self.last_monmap > datetime_now(): self.last_monmap = None # just in case clocks are skewed if getattr(self, 'manage_etc_ceph_ceph_conf', False): # getattr, due to notify() being called before config_notify() @@ -939,7 +936,7 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, self.set_store("extra_ceph_conf", json.dumps({ 'conf': inbuf, - 'last_modified': datetime_to_str(datetime.datetime.utcnow()) + 'last_modified': datetime_to_str(datetime_now()) })) self.log.info('Set extra_ceph_conf') self._kick_serve_loop() @@ -1840,7 +1837,7 @@ To check that the host is reachable: ).service_id(), overwrite=True): image = '' - start_time = datetime.datetime.utcnow() + start_time = datetime_now() ports: List[int] = daemon_spec.ports if daemon_spec.ports else [] if daemon_spec.daemon_type == 'container': @@ -2381,7 +2378,7 @@ To check that the host is reachable: force=force, hostname=daemon.hostname, fullname=daemon.name(), - process_started_at=datetime.datetime.utcnow(), + process_started_at=datetime_now(), remove_util=self.rm_util)) except NotFoundError: return f"Unable to find OSDs: {osd_ids}" diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index e1a90ee4314..adb69ec9222 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -12,11 +12,12 @@ except ImportError: from ceph.deployment import inventory from ceph.deployment.drive_group import DriveGroupSpec from ceph.deployment.service_spec import ServiceSpec, HostPlacementSpec, RGWSpec +from ceph.utils import str_to_datetime, datetime_now import orchestrator from cephadm.schedule import HostAssignment from cephadm.upgrade import CEPH_UPGRADE_ORDER -from cephadm.utils import forall_hosts, cephadmNoImage, str_to_datetime, is_repo_digest +from cephadm.utils import forall_hosts, cephadmNoImage, is_repo_digest from orchestrator import OrchestratorError if TYPE_CHECKING: @@ -224,7 +225,7 @@ class CephadmServe: if '.' not in d['name']: continue sd = orchestrator.DaemonDescription() - sd.last_refresh = datetime.datetime.utcnow() + sd.last_refresh = datetime_now() for k in ['created', 'started', 'last_configured', 'last_deployed']: v = d.get(k, None) if v: diff --git a/src/pybind/mgr/cephadm/services/osd.py b/src/pybind/mgr/cephadm/services/osd.py index 8dd49c1e414..0929edf2f6a 100644 --- a/src/pybind/mgr/cephadm/services/osd.py +++ b/src/pybind/mgr/cephadm/services/osd.py @@ -6,10 +6,11 @@ from ceph.deployment import translate from ceph.deployment.drive_group import DriveGroupSpec from ceph.deployment.drive_selection import DriveSelection from ceph.deployment.inventory import Device +from ceph.utils import datetime_to_str, str_to_datetime from datetime import datetime import orchestrator -from cephadm.utils import forall_hosts, datetime_to_str, str_to_datetime +from cephadm.utils import forall_hosts from orchestrator import OrchestratorError from mgr_module import MonCommandFailed diff --git a/src/pybind/mgr/cephadm/tests/fixtures.py b/src/pybind/mgr/cephadm/tests/fixtures.py index 90db5309724..e03d01d5469 100644 --- a/src/pybind/mgr/cephadm/tests/fixtures.py +++ b/src/pybind/mgr/cephadm/tests/fixtures.py @@ -1,10 +1,9 @@ -import datetime import time import fnmatch from contextlib import contextmanager from ceph.deployment.service_spec import PlacementSpec, ServiceSpec -from cephadm.module import CEPH_DATEFMT +from ceph.utils import datetime_to_str, datetime_now from cephadm.serve import CephadmServe try: @@ -55,7 +54,7 @@ def with_cephadm_module(module_options=None, store=None): store = {} if '_ceph_get/mon_map' not in store: m.mock_store_set('_ceph_get', 'mon_map', { - 'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT), + 'modified': datetime_to_str(datetime_now()), 'fsid': 'foobar', }) for k, v in store.items(): diff --git a/src/pybind/mgr/cephadm/tests/test_cephadm.py b/src/pybind/mgr/cephadm/tests/test_cephadm.py index cec4939ea26..e5b99226017 100644 --- a/src/pybind/mgr/cephadm/tests/test_cephadm.py +++ b/src/pybind/mgr/cephadm/tests/test_cephadm.py @@ -1,4 +1,3 @@ -import datetime import json from contextlib import contextmanager from unittest.mock import ANY @@ -20,12 +19,13 @@ from ceph.deployment.service_spec import ServiceSpec, PlacementSpec, RGWSpec, \ NFSServiceSpec, IscsiServiceSpec, HostPlacementSpec, CustomContainerSpec from ceph.deployment.drive_selection.selector import DriveSelection from ceph.deployment.inventory import Devices, Device +from ceph.utils import datetime_to_str, datetime_now from orchestrator import ServiceDescription, DaemonDescription, InventoryHost, \ HostSpec, OrchestratorError from tests import mock from .fixtures import cephadm_module, wait, _run_cephadm, match_glob, with_host, \ with_cephadm_module, with_service, assert_rm_service, _deploy_cephadm_binary -from cephadm.module import CephadmOrchestrator, CEPH_DATEFMT +from cephadm.module import CephadmOrchestrator """ TODOs: @@ -202,7 +202,7 @@ class TestCephadm(object): # Make sure, _check_daemons does a redeploy due to monmap change: cephadm_module._store['_ceph_get/mon_map'] = { - 'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT), + 'modified': datetime_to_str(datetime_now()), 'fsid': 'foobar', } cephadm_module.notify('mon_map', None) @@ -221,7 +221,7 @@ class TestCephadm(object): # Make sure, _check_daemons does a redeploy due to monmap change: cephadm_module.mock_store_set('_ceph_get', 'mon_map', { - 'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT), + 'modified': datetime_to_str(datetime_now()), 'fsid': 'foobar', }) cephadm_module.notify('mon_map', None) @@ -301,7 +301,7 @@ class TestCephadm(object): # Make sure, _check_daemons does a redeploy due to monmap change: cephadm_module.mock_store_set('_ceph_get', 'mon_map', { - 'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT), + 'modified': datetime_to_str(datetime_now()), 'fsid': 'foobar', }) cephadm_module.notify('mon_map', None) @@ -512,7 +512,7 @@ class TestCephadm(object): force=False, hostname='test', fullname='osd.0', - process_started_at=datetime.datetime.utcnow(), + process_started_at=datetime_now(), remove_util=cephadm_module.rm_util )) cephadm_module.rm_util.process_removal_queue() @@ -877,7 +877,7 @@ class TestCephadm(object): # Make sure, _check_daemons does a redeploy due to monmap change: cephadm_module.mock_store_set('_ceph_get', 'mon_map', { - 'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT), + 'modified': datetime_to_str(datetime_now()), 'fsid': 'foobar', }) cephadm_module.notify('mon_map', mock.MagicMock()) diff --git a/src/pybind/mgr/cephadm/tests/test_migration.py b/src/pybind/mgr/cephadm/tests/test_migration.py index 8536c4e1962..f46c1024f34 100644 --- a/src/pybind/mgr/cephadm/tests/test_migration.py +++ b/src/pybind/mgr/cephadm/tests/test_migration.py @@ -1,12 +1,11 @@ import json -from datetime import datetime import pytest from ceph.deployment.service_spec import PlacementSpec, ServiceSpec, HostPlacementSpec +from ceph.utils import datetime_to_str, datetime_now from cephadm import CephadmOrchestrator from cephadm.inventory import SPEC_STORE_PREFIX -from cephadm.utils import DATEFMT from cephadm.tests.fixtures import _run_cephadm, cephadm_module, wait, with_host from orchestrator import OrchestratorError from cephadm.serve import CephadmServe @@ -48,8 +47,8 @@ def test_migrate_scheduler(cephadm_module: CephadmOrchestrator): # Sorry, for this hack, but I need to make sure, Migration thinks, # we have updated all daemons already. - cephadm_module.cache.last_daemon_update['host1'] = datetime.now() - cephadm_module.cache.last_daemon_update['host2'] = datetime.now() + cephadm_module.cache.last_daemon_update['host1'] = datetime_now() + cephadm_module.cache.last_daemon_update['host2'] = datetime_now() cephadm_module.migration_current = 0 cephadm_module.migration.migrate() @@ -72,7 +71,7 @@ def test_migrate_service_id_mon_one(cephadm_module: CephadmOrchestrator): 'hosts': ['host1'] } }, - 'created': datetime.utcnow().strftime(DATEFMT), + 'created': datetime_to_str(datetime_now()), }, sort_keys=True), ) @@ -103,7 +102,7 @@ def test_migrate_service_id_mon_two(cephadm_module: CephadmOrchestrator): 'count': 5, } }, - 'created': datetime.utcnow().strftime(DATEFMT), + 'created': datetime_to_str(datetime_now()), }, sort_keys=True), ) cephadm_module.set_store(SPEC_STORE_PREFIX + 'mon.wrong', json.dumps({ @@ -114,7 +113,7 @@ def test_migrate_service_id_mon_two(cephadm_module: CephadmOrchestrator): 'hosts': ['host1'] } }, - 'created': datetime.utcnow().strftime(DATEFMT), + 'created': datetime_to_str(datetime_now()), }, sort_keys=True), ) @@ -146,7 +145,7 @@ def test_migrate_service_id_mds_one(cephadm_module: CephadmOrchestrator): 'hosts': ['host1'] } }, - 'created': datetime.utcnow().strftime(DATEFMT), + 'created': datetime_to_str(datetime_now()), }, sort_keys=True), ) diff --git a/src/pybind/mgr/cephadm/tests/test_spec.py b/src/pybind/mgr/cephadm/tests/test_spec.py index a0d22710a3b..f4f24f352cd 100644 --- a/src/pybind/mgr/cephadm/tests/test_spec.py +++ b/src/pybind/mgr/cephadm/tests/test_spec.py @@ -273,7 +273,18 @@ def test_dd_octopus(dd_json): # https://tracker.ceph.com/issues/44934 # Those are real user data from early octopus. # Please do not modify those JSON values. - assert dd_json == DaemonDescription.from_json(dd_json).to_json() + + # Convert datetime properties to old style. + # 2020-04-03T07:29:16.926292Z -> 2020-04-03T07:29:16.926292 + def convert_to_old_style_json(j): + for k in ['last_refresh', 'created', 'started', 'last_deployed', + 'last_configured']: + if k in j: + j[k] = j[k].rstrip('Z') + return j + + assert dd_json == convert_to_old_style_json( + DaemonDescription.from_json(dd_json).to_json()) @pytest.mark.parametrize("spec,dd,valid", diff --git a/src/pybind/mgr/cephadm/utils.py b/src/pybind/mgr/cephadm/utils.py index ac788039a13..0680df0099c 100644 --- a/src/pybind/mgr/cephadm/utils.py +++ b/src/pybind/mgr/cephadm/utils.py @@ -1,11 +1,9 @@ import logging -import re import json -import datetime import socket from enum import Enum from functools import wraps -from typing import Optional, Callable, TypeVar, List, NewType, TYPE_CHECKING, Any +from typing import Callable, TypeVar, List, NewType, TYPE_CHECKING, Any from orchestrator import OrchestratorError if TYPE_CHECKING: @@ -16,8 +14,6 @@ logger = logging.getLogger(__name__) ConfEntity = NewType('ConfEntity', str) -DATEFMT = '%Y-%m-%dT%H:%M:%S.%f' - class CephadmNoImage(Enum): token = 1 @@ -94,14 +90,6 @@ def is_repo_digest(image_name: str) -> bool: return '@' in image_name -def str_to_datetime(input: str) -> datetime.datetime: - return datetime.datetime.strptime(input, DATEFMT) - - -def datetime_to_str(dt: datetime.datetime) -> str: - return dt.strftime(DATEFMT) - - def resolve_ip(hostname: str) -> str: try: return socket.getaddrinfo(hostname, None, flags=socket.AI_CANONNAME, type=socket.SOCK_STREAM)[0][4][0] diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py index 39dc20ecc2e..de09cbdaa4b 100644 --- a/src/pybind/mgr/orchestrator/_interface.py +++ b/src/pybind/mgr/orchestrator/_interface.py @@ -25,6 +25,7 @@ from ceph.deployment.service_spec import ServiceSpec, NFSServiceSpec, RGWSpec, \ ServiceSpecValidationError, IscsiServiceSpec from ceph.deployment.drive_group import DriveGroupSpec from ceph.deployment.hostspec import HostSpec +from ceph.utils import datetime_to_str, str_to_datetime from mgr_module import MgrModule, CLICommand, HandleCommandResult @@ -36,8 +37,6 @@ except ImportError: logger = logging.getLogger(__name__) -DATEFMT = '%Y-%m-%dT%H:%M:%S.%f' - T = TypeVar('T') @@ -1339,7 +1338,7 @@ class DaemonDescription(object): for k in ['last_refresh', 'created', 'started', 'last_deployed', 'last_configured']: if getattr(self, k): - out[k] = getattr(self, k).strftime(DATEFMT) + out[k] = datetime_to_str(getattr(self, k)) if self.events: out['events'] = [e.to_json() for e in self.events] @@ -1357,7 +1356,7 @@ class DaemonDescription(object): for k in ['last_refresh', 'created', 'started', 'last_deployed', 'last_configured']: if k in c: - c[k] = datetime.datetime.strptime(c[k], DATEFMT) + c[k] = str_to_datetime(c[k]) events = [OrchestratorEvent.from_json(e) for e in event_strs] return cls(events=events, **c) @@ -1445,7 +1444,7 @@ class ServiceDescription(object): } for k in ['last_refresh', 'created']: if getattr(self, k): - status[k] = getattr(self, k).strftime(DATEFMT) + status[k] = datetime_to_str(getattr(self, k)) status = {k: v for (k, v) in status.items() if v is not None} out['status'] = status if self.events: @@ -1463,7 +1462,7 @@ class ServiceDescription(object): c_status = status.copy() for k in ['last_refresh', 'created']: if k in c_status: - c_status[k] = datetime.datetime.strptime(c_status[k], DATEFMT) + c_status[k] = str_to_datetime(c_status[k]) events = [OrchestratorEvent.from_json(e) for e in event_strs] return cls(spec=spec, events=events, **c_status) @@ -1583,7 +1582,7 @@ class OrchestratorEvent: def __init__(self, created: Union[str, datetime.datetime], kind, subject, level, message): if isinstance(created, str): - created = datetime.datetime.strptime(created, DATEFMT) + created = str_to_datetime(created) self.created: datetime.datetime = created assert kind in "service daemon".split() @@ -1605,7 +1604,7 @@ class OrchestratorEvent: def to_json(self) -> str: # Make a long list of events readable. - created = self.created.strftime(DATEFMT) + created = datetime_to_str(self.created) return f'{created} {self.kind_subject()} [{self.level}] "{self.message}"' @classmethod @@ -1613,7 +1612,7 @@ class OrchestratorEvent: def from_json(cls, data) -> "OrchestratorEvent": """ >>> OrchestratorEvent.from_json('''2020-06-10T10:20:25.691255 daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on host 'ubuntu'"''').to_json() - '2020-06-10T10:20:25.691255 daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on host \\'ubuntu\\'"' + '2020-06-10T10:20:25.691255Z daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on host \\'ubuntu\\'"' :param data: :return: diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index 347f5ac8367..d33224c87be 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -1,4 +1,3 @@ -import datetime import errno import json from typing import List, Set, Optional, Iterator, cast, Dict, Any, Union @@ -11,6 +10,7 @@ from prettytable import PrettyTable from ceph.deployment.inventory import Device from ceph.deployment.drive_group import DriveGroupSpec, DeviceSelection from ceph.deployment.service_spec import PlacementSpec, ServiceSpec +from ceph.utils import datetime_now from mgr_util import format_bytes, to_pretty_timedelta, format_dimless from mgr_module import MgrModule, HandleCommandResult, Option @@ -515,7 +515,7 @@ class OrchestratorCli(OrchestratorClientMixin, MgrModule, else: return HandleCommandResult(stdout=to_format(services, format, many=True, cls=ServiceDescription)) else: - now = datetime.datetime.utcnow() + now = datetime_now() table = PrettyTable( ['NAME', 'RUNNING', 'REFRESHED', 'AGE', 'PLACEMENT', @@ -580,7 +580,7 @@ class OrchestratorCli(OrchestratorClientMixin, MgrModule, if len(daemons) == 0: return HandleCommandResult(stdout="No daemons reported") - now = datetime.datetime.utcnow() + now = datetime_now() table = PrettyTable( ['NAME', 'HOST', 'STATUS', 'REFRESHED', 'AGE', 'VERSION', 'IMAGE NAME', 'IMAGE ID', 'CONTAINER ID'], diff --git a/src/pybind/mgr/orchestrator/tests/test_orchestrator.py b/src/pybind/mgr/orchestrator/tests/test_orchestrator.py index 523c2863d34..c5858aa9531 100644 --- a/src/pybind/mgr/orchestrator/tests/test_orchestrator.py +++ b/src/pybind/mgr/orchestrator/tests/test_orchestrator.py @@ -1,6 +1,5 @@ from __future__ import absolute_import -import datetime import json import pytest @@ -8,6 +7,7 @@ import yaml from ceph.deployment.service_spec import ServiceSpec from ceph.deployment import inventory +from ceph.utils import datetime_now from mgr_module import HandleCommandResult from test_orchestrator import TestOrchestrator as _TestOrchestrator @@ -258,7 +258,7 @@ status: 1 status_desc: starting is_active: false events: -- 2020-06-10T10:08:22.933241 daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on +- 2020-06-10T10:08:22.933241Z daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on host 'ubuntu'" --- service_type: crash @@ -268,12 +268,12 @@ placement: status: container_image_id: 74803e884bea289d2d2d3ebdf6d37cd560499e955595695b1390a89800f4e37a container_image_name: docker.io/ceph/daemon-base:latest-master-devel - created: '2020-06-10T10:37:31.051288' - last_refresh: '2020-06-10T10:57:40.715637' + created: '2020-06-10T10:37:31.051288Z' + last_refresh: '2020-06-10T10:57:40.715637Z' running: 1 size: 1 events: -- 2020-06-10T10:37:31.139159 service:crash [INFO] "service was created" +- 2020-06-10T10:37:31.139159Z service:crash [INFO] "service was created" """ types = (DaemonDescription, ServiceDescription) @@ -290,10 +290,10 @@ events: def test_event_multiline(): from .._interface import OrchestratorEvent - e = OrchestratorEvent(datetime.datetime.utcnow(), 'service', 'subject', 'ERROR', 'message') + e = OrchestratorEvent(datetime_now(), 'service', 'subject', 'ERROR', 'message') assert OrchestratorEvent.from_json(e.to_json()) == e - e = OrchestratorEvent(datetime.datetime.utcnow(), 'service', + e = OrchestratorEvent(datetime_now(), 'service', 'subject', 'ERROR', 'multiline\nmessage') assert OrchestratorEvent.from_json(e.to_json()) == e diff --git a/src/python-common/ceph/tests/test_datetime.py b/src/python-common/ceph/tests/test_datetime.py new file mode 100644 index 00000000000..a5c6ec00151 --- /dev/null +++ b/src/python-common/ceph/tests/test_datetime.py @@ -0,0 +1,58 @@ +import datetime + +import pytest + +from ceph.utils import datetime_now, datetime_to_str, str_to_datetime + + +def test_datetime_to_str_1(): + dt = datetime.datetime.now() + assert type(datetime_to_str(dt)) is str + + +def test_datetime_to_str_2(): + dt = datetime.datetime.strptime('2019-04-24T17:06:53.039991', + '%Y-%m-%dT%H:%M:%S.%f') + assert datetime_to_str(dt) == '2019-04-24T17:06:53.039991Z' + + +def test_datetime_to_str_3(): + dt = datetime.datetime.strptime('2020-11-02T04:40:12.748172-0800', + '%Y-%m-%dT%H:%M:%S.%f%z') + assert datetime_to_str(dt) == '2020-11-02T12:40:12.748172Z' + + +def test_str_to_datetime_1(): + dt = str_to_datetime('2020-03-03T09:21:43.636153304Z') + assert type(dt) is datetime.datetime + assert dt.tzinfo is not None + + +def test_str_to_datetime_2(): + dt = str_to_datetime('2020-03-03T15:52:30.136257504-0600') + assert type(dt) is datetime.datetime + assert dt.tzinfo is not None + + +def test_str_to_datetime_3(): + dt = str_to_datetime('2020-03-03T15:52:30.136257504') + assert type(dt) is datetime.datetime + assert dt.tzinfo is not None + + +def test_str_to_datetime_invalid_format_1(): + with pytest.raises(ValueError): + str_to_datetime('2020-03-03 15:52:30.136257504') + + +def test_str_to_datetime_invalid_format_2(): + with pytest.raises(ValueError): + str_to_datetime('2020-03-03') + + +def test_datetime_now_1(): + dt = str_to_datetime('2020-03-03T09:21:43.636153304Z') + dt_now = datetime_now() + assert type(dt_now) is datetime.datetime + assert dt_now.tzinfo is not None + assert dt < dt_now diff --git a/src/python-common/ceph/utils.py b/src/python-common/ceph/utils.py new file mode 100644 index 00000000000..2a272fa3321 --- /dev/null +++ b/src/python-common/ceph/utils.py @@ -0,0 +1,68 @@ +import datetime +import re + + +def datetime_now() -> datetime.datetime: + """ + Return the current local date and time. + :return: Returns an aware datetime object of the current date + and time. + """ + return datetime.datetime.now(tz=datetime.timezone.utc) + + +def datetime_to_str(dt: datetime.datetime) -> str: + """ + Convert a datetime object into a ISO 8601 string, e.g. + '2019-04-24T17:06:53.039991Z'. + :param dt: The datetime object to process. + :return: Return a string representing the date in + ISO 8601 (timezone=UTC). + """ + return dt.astimezone(tz=datetime.timezone.utc).strftime( + '%Y-%m-%dT%H:%M:%S.%fZ') + + +def str_to_datetime(string: str) -> datetime.datetime: + """ + Convert an ISO 8601 string into a datetime object. + The following formats are supported: + + - 2020-03-03T09:21:43.636153304Z + - 2020-03-03T15:52:30.136257504-0600 + - 2020-03-03T15:52:30.136257504 + + :param string: The string to parse. + :return: Returns an aware datetime object of the given date + and time string. + :raises: :exc:`~exceptions.ValueError` for an unknown + datetime string. + """ + fmts = [ + '%Y-%m-%dT%H:%M:%S.%f', + '%Y-%m-%dT%H:%M:%S.%f%z' + ] + + # In *all* cases, the 9 digit second precision is too much for + # Python's strptime. Shorten it to 6 digits. + p = re.compile(r'(\.[\d]{6})[\d]*') + string = p.sub(r'\1', string) + + # Replace trailing Z with -0000, since (on Python 3.6.8) it + # won't parse. + if string and string[-1] == 'Z': + string = string[:-1] + '-0000' + + for fmt in fmts: + try: + dt = datetime.datetime.strptime(string, fmt) + # Make sure the datetime object is aware (timezone is set). + # If not, then assume the time is in UTC. + if dt.tzinfo is None: + dt = dt.replace(tzinfo=datetime.timezone.utc) + return dt + except ValueError: + pass + + raise ValueError("Time data {} does not match one of the formats {}".format( + string, str(fmts))) -- 2.39.5