]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
cephadm: Various properties like 'last_refresh' do not contain timezone 37920/head
authorVolker Theile <vtheile@suse.com>
Thu, 10 Dec 2020 09:47:46 +0000 (10:47 +0100)
committerVolker Theile <vtheile@suse.com>
Tue, 5 Jan 2021 11:22:32 +0000 (12:22 +0100)
Fixes: https://tracker.ceph.com/issues/48068
Signed-off-by: Volker Theile <vtheile@suse.com>
15 files changed:
src/cephadm/cephadm
src/pybind/mgr/cephadm/inventory.py
src/pybind/mgr/cephadm/module.py
src/pybind/mgr/cephadm/serve.py
src/pybind/mgr/cephadm/services/osd.py
src/pybind/mgr/cephadm/tests/fixtures.py
src/pybind/mgr/cephadm/tests/test_cephadm.py
src/pybind/mgr/cephadm/tests/test_migration.py
src/pybind/mgr/cephadm/tests/test_spec.py
src/pybind/mgr/cephadm/utils.py
src/pybind/mgr/orchestrator/_interface.py
src/pybind/mgr/orchestrator/module.py
src/pybind/mgr/orchestrator/tests/test_orchestrator.py
src/python-common/ceph/tests/test_datetime.py [new file with mode: 0644]
src/python-common/ceph/utils.py [new file with mode: 0644]

index df516f0fe97e7817dec8028cefdeab1866de81a0..0be218883fb1e06c8b00bb69e22f7262ad6ae9ad 100755 (executable)
@@ -48,7 +48,6 @@ import os
 import platform
 import pwd
 import random
-import re
 import select
 import shutil
 import socket
@@ -101,7 +100,7 @@ if sys.version_info > (3, 0):
 container_path = ''
 cached_stdin = None
 
-DATEFMT = '%Y-%m-%dT%H:%M:%S.%f'
+DATEFMT = '%Y-%m-%dT%H:%M:%S.%fZ'
 
 # Log and console output config
 logging_config = {
@@ -1177,7 +1176,7 @@ def get_file_timestamp(fn):
         return datetime.datetime.fromtimestamp(
             mt, tz=datetime.timezone.utc
         ).strftime(DATEFMT)
-    except Exception as e:
+    except Exception:
         return None
 
 
@@ -1199,11 +1198,11 @@ def try_convert_datetime(s):
     p = re.compile(r'(\.[\d]{6})[\d]*')
     s = p.sub(r'\1', s)
 
-    # replace trailling Z with -0000, since (on python 3.6.8) it won't parse
+    # replace trailing Z with -0000, since (on python 3.6.8) it won't parse
     if s and s[-1] == 'Z':
         s = s[:-1] + '-0000'
 
-    # cut off the redundnat 'CST' part that strptime can't parse, if
+    # cut off the redundant 'CST' part that strptime can't parse, if
     # present.
     v = s.split(' ')
     s = ' '.join(v[0:3])
index f309504155fd0e99017d816a9e65d06d6c43e44a..2fa9b2f730207fb0ab9e1ec6a908d73d029844fb 100644 (file)
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Dict, List, Iterator, Optional, Any, Tuple, Se
 import orchestrator
 from ceph.deployment import inventory
 from ceph.deployment.service_spec import ServiceSpec
-from cephadm.utils import str_to_datetime, datetime_to_str
+from ceph.utils import str_to_datetime, datetime_to_str, datetime_now
 from orchestrator import OrchestratorError, HostSpec, OrchestratorEvent
 
 if TYPE_CHECKING:
@@ -136,7 +136,7 @@ class SpecStore():
             self.spec_preview[spec.service_name()] = spec
             return None
         self.specs[spec.service_name()] = spec
-        self.spec_created[spec.service_name()] = datetime.datetime.utcnow()
+        self.spec_created[spec.service_name()] = datetime_now()
         self.mgr.set_store(
             SPEC_STORE_PREFIX + spec.service_name(),
             json.dumps({
@@ -278,7 +278,7 @@ class HostCache():
     def update_host_daemons(self, host, dm):
         # type: (str, Dict[str, orchestrator.DaemonDescription]) -> None
         self.daemons[host] = dm
-        self.last_daemon_update[host] = datetime.datetime.utcnow()
+        self.last_daemon_update[host] = datetime_now()
 
     def update_host_facts(self, host, facts):
         # type: (str, Dict[str, Dict[str, Any]]) -> None
@@ -289,7 +289,7 @@ class HostCache():
         # type: (str, List[inventory.Device], Dict[str,List[str]]) -> None
         self.devices[host] = dls
         self.networks[host] = nets
-        self.last_device_update[host] = datetime.datetime.utcnow()
+        self.last_device_update[host] = datetime_now()
 
     def update_daemon_config_deps(self, host: str, name: str, deps: List[str], stamp: datetime.datetime) -> None:
         self.daemon_config_deps[host][name] = {
@@ -299,7 +299,7 @@ class HostCache():
 
     def update_last_host_check(self, host):
         # type: (str) -> None
-        self.last_host_check[host] = datetime.datetime.utcnow()
+        self.last_host_check[host] = datetime_now()
 
     def prime_empty_host(self, host):
         # type: (str) -> None
@@ -472,7 +472,7 @@ class HostCache():
         if host in self.daemon_refresh_queue:
             self.daemon_refresh_queue.remove(host)
             return True
-        cutoff = datetime.datetime.utcnow() - datetime.timedelta(
+        cutoff = datetime_now() - datetime.timedelta(
             seconds=self.mgr.daemon_cache_timeout)
         if host not in self.last_daemon_update or self.last_daemon_update[host] < cutoff:
             return True
@@ -507,7 +507,7 @@ class HostCache():
         if host in self.device_refresh_queue:
             self.device_refresh_queue.remove(host)
             return True
-        cutoff = datetime.datetime.utcnow() - datetime.timedelta(
+        cutoff = datetime_now() - datetime.timedelta(
             seconds=self.mgr.device_cache_timeout)
         if host not in self.last_device_update or self.last_device_update[host] < cutoff:
             return True
@@ -526,7 +526,7 @@ class HostCache():
 
     def host_needs_check(self, host):
         # type: (str) -> bool
-        cutoff = datetime.datetime.utcnow() - datetime.timedelta(
+        cutoff = datetime_now() - datetime.timedelta(
             seconds=self.mgr.host_check_interval)
         return host not in self.last_host_check or self.last_host_check[host] < cutoff
 
@@ -551,7 +551,7 @@ class HostCache():
     def update_last_etc_ceph_ceph_conf(self, host: str) -> None:
         if not self.mgr.last_monmap:
             return
-        self.last_etc_ceph_ceph_conf[host] = datetime.datetime.utcnow()
+        self.last_etc_ceph_ceph_conf[host] = datetime_now()
 
     def host_needs_registry_login(self, host: str) -> bool:
         if host in self.mgr.offline_hosts:
@@ -632,14 +632,14 @@ class EventStore():
         self.events[event.kind_subject()] = self.events[event.kind_subject()][-5:]
 
     def for_service(self, spec: ServiceSpec, level: str, message: str) -> None:
-        e = OrchestratorEvent(datetime.datetime.utcnow(), 'service',
+        e = OrchestratorEvent(datetime_now(), 'service',
                               spec.service_name(), level, message)
         self.add(e)
 
     def from_orch_error(self, e: OrchestratorError) -> None:
         if e.event_subject is not None:
             self.add(OrchestratorEvent(
-                datetime.datetime.utcnow(),
+                datetime_now(),
                 e.event_subject[0],
                 e.event_subject[1],
                 "ERROR",
@@ -647,7 +647,7 @@ class EventStore():
             ))
 
     def for_daemon(self, daemon_name: str, level: str, message: str) -> None:
-        e = OrchestratorEvent(datetime.datetime.utcnow(), 'daemon', daemon_name, level, message)
+        e = OrchestratorEvent(datetime_now(), 'daemon', daemon_name, level, message)
         self.add(e)
 
     def for_daemon_from_exception(self, daemon_name: str, e: Exception) -> None:
index 579eb18304824fc1913864918a3beff789441f08..de43549794afeba2f09a5e4b5dce008b3bfa0b16 100644 (file)
@@ -26,6 +26,7 @@ from ceph.deployment.drive_group import DriveGroupSpec
 from ceph.deployment.service_spec import \
     NFSServiceSpec, ServiceSpec, PlacementSpec, assert_valid_host, \
     CustomContainerSpec, HostPlacementSpec
+from ceph.utils import str_to_datetime, datetime_to_str, datetime_now
 from cephadm.serve import CephadmServe
 from cephadm.services.cephadmservice import CephadmDaemonSpec
 
@@ -52,8 +53,7 @@ from .schedule import HostAssignment
 from .inventory import Inventory, SpecStore, HostCache, EventStore
 from .upgrade import CEPH_UPGRADE_ORDER, CephadmUpgrade
 from .template import TemplateMgr
-from .utils import forall_hosts, CephadmNoImage, cephadmNoImage, \
-    str_to_datetime, datetime_to_str
+from .utils import forall_hosts, CephadmNoImage, cephadmNoImage
 
 try:
     import remoto
@@ -89,8 +89,6 @@ Host *
   ConnectTimeout=30
 """
 
-CEPH_DATEFMT = '%Y-%m-%dT%H:%M:%S.%fZ'
-
 CEPH_TYPES = set(CEPH_UPGRADE_ORDER)
 
 
@@ -498,9 +496,8 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule,
         if notify_type == "mon_map":
             # get monmap mtime so we can refresh configs when mons change
             monmap = self.get('mon_map')
-            self.last_monmap = datetime.datetime.strptime(
-                monmap['modified'], CEPH_DATEFMT)
-            if self.last_monmap and self.last_monmap > datetime.datetime.utcnow():
+            self.last_monmap = str_to_datetime(monmap['modified'])
+            if self.last_monmap and self.last_monmap > datetime_now():
                 self.last_monmap = None  # just in case clocks are skewed
             if getattr(self, 'manage_etc_ceph_ceph_conf', False):
                 # getattr, due to notify() being called before config_notify()
@@ -939,7 +936,7 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule,
 
         self.set_store("extra_ceph_conf", json.dumps({
             'conf': inbuf,
-            'last_modified': datetime_to_str(datetime.datetime.utcnow())
+            'last_modified': datetime_to_str(datetime_now())
         }))
         self.log.info('Set extra_ceph_conf')
         self._kick_serve_loop()
@@ -1840,7 +1837,7 @@ To check that the host is reachable:
         ).service_id(), overwrite=True):
 
             image = ''
-            start_time = datetime.datetime.utcnow()
+            start_time = datetime_now()
             ports: List[int] = daemon_spec.ports if daemon_spec.ports else []
 
             if daemon_spec.daemon_type == 'container':
@@ -2381,7 +2378,7 @@ To check that the host is reachable:
                                                 force=force,
                                                 hostname=daemon.hostname,
                                                 fullname=daemon.name(),
-                                                process_started_at=datetime.datetime.utcnow(),
+                                                process_started_at=datetime_now(),
                                                 remove_util=self.rm_util))
             except NotFoundError:
                 return f"Unable to find OSDs: {osd_ids}"
index e1a90ee4314bf1b97bd390dcc80030a76f156d27..adb69ec92228a75537aa332e77924c520dd30118 100644 (file)
@@ -12,11 +12,12 @@ except ImportError:
 from ceph.deployment import inventory
 from ceph.deployment.drive_group import DriveGroupSpec
 from ceph.deployment.service_spec import ServiceSpec, HostPlacementSpec, RGWSpec
+from ceph.utils import str_to_datetime, datetime_now
 
 import orchestrator
 from cephadm.schedule import HostAssignment
 from cephadm.upgrade import CEPH_UPGRADE_ORDER
-from cephadm.utils import forall_hosts, cephadmNoImage, str_to_datetime, is_repo_digest
+from cephadm.utils import forall_hosts, cephadmNoImage, is_repo_digest
 from orchestrator import OrchestratorError
 
 if TYPE_CHECKING:
@@ -224,7 +225,7 @@ class CephadmServe:
             if '.' not in d['name']:
                 continue
             sd = orchestrator.DaemonDescription()
-            sd.last_refresh = datetime.datetime.utcnow()
+            sd.last_refresh = datetime_now()
             for k in ['created', 'started', 'last_configured', 'last_deployed']:
                 v = d.get(k, None)
                 if v:
index 8dd49c1e414cb220ae43bcc1a017939f841d11b6..0929edf2f6a8d6f3697a532f639bf4f1e3e66c19 100644 (file)
@@ -6,10 +6,11 @@ from ceph.deployment import translate
 from ceph.deployment.drive_group import DriveGroupSpec
 from ceph.deployment.drive_selection import DriveSelection
 from ceph.deployment.inventory import Device
+from ceph.utils import datetime_to_str, str_to_datetime
 
 from datetime import datetime
 import orchestrator
-from cephadm.utils import forall_hosts, datetime_to_str, str_to_datetime
+from cephadm.utils import forall_hosts
 from orchestrator import OrchestratorError
 from mgr_module import MonCommandFailed
 
index 90db5309724e0f48cb9c1240e02c08f1314ab91c..e03d01d54693dbaddfe2daf8211ceaafa0b5278e 100644 (file)
@@ -1,10 +1,9 @@
-import datetime
 import time
 import fnmatch
 from contextlib import contextmanager
 
 from ceph.deployment.service_spec import PlacementSpec, ServiceSpec
-from cephadm.module import CEPH_DATEFMT
+from ceph.utils import datetime_to_str, datetime_now
 from cephadm.serve import CephadmServe
 
 try:
@@ -55,7 +54,7 @@ def with_cephadm_module(module_options=None, store=None):
             store = {}
         if '_ceph_get/mon_map' not in store:
             m.mock_store_set('_ceph_get', 'mon_map', {
-                'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT),
+                'modified': datetime_to_str(datetime_now()),
                 'fsid': 'foobar',
             })
         for k, v in store.items():
index cec4939ea2648084fe2d9f6feba0c7d3bb53f0ee..e5b99226017dd704d23d972bd4370b455bbe5096 100644 (file)
@@ -1,4 +1,3 @@
-import datetime
 import json
 from contextlib import contextmanager
 from unittest.mock import ANY
@@ -20,12 +19,13 @@ from ceph.deployment.service_spec import ServiceSpec, PlacementSpec, RGWSpec, \
     NFSServiceSpec, IscsiServiceSpec, HostPlacementSpec, CustomContainerSpec
 from ceph.deployment.drive_selection.selector import DriveSelection
 from ceph.deployment.inventory import Devices, Device
+from ceph.utils import datetime_to_str, datetime_now
 from orchestrator import ServiceDescription, DaemonDescription, InventoryHost, \
     HostSpec, OrchestratorError
 from tests import mock
 from .fixtures import cephadm_module, wait, _run_cephadm, match_glob, with_host, \
     with_cephadm_module, with_service, assert_rm_service, _deploy_cephadm_binary
-from cephadm.module import CephadmOrchestrator, CEPH_DATEFMT
+from cephadm.module import CephadmOrchestrator
 
 """
 TODOs:
@@ -202,7 +202,7 @@ class TestCephadm(object):
 
                 # Make sure, _check_daemons does a redeploy due to monmap change:
                 cephadm_module._store['_ceph_get/mon_map'] = {
-                    'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT),
+                    'modified': datetime_to_str(datetime_now()),
                     'fsid': 'foobar',
                 }
                 cephadm_module.notify('mon_map', None)
@@ -221,7 +221,7 @@ class TestCephadm(object):
 
                     # Make sure, _check_daemons does a redeploy due to monmap change:
                     cephadm_module.mock_store_set('_ceph_get', 'mon_map', {
-                        'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT),
+                        'modified': datetime_to_str(datetime_now()),
                         'fsid': 'foobar',
                     })
                     cephadm_module.notify('mon_map', None)
@@ -301,7 +301,7 @@ class TestCephadm(object):
 
                 # Make sure, _check_daemons does a redeploy due to monmap change:
                 cephadm_module.mock_store_set('_ceph_get', 'mon_map', {
-                    'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT),
+                    'modified': datetime_to_str(datetime_now()),
                     'fsid': 'foobar',
                 })
                 cephadm_module.notify('mon_map', None)
@@ -512,7 +512,7 @@ class TestCephadm(object):
                                                       force=False,
                                                       hostname='test',
                                                       fullname='osd.0',
-                                                      process_started_at=datetime.datetime.utcnow(),
+                                                      process_started_at=datetime_now(),
                                                       remove_util=cephadm_module.rm_util
                                                       ))
             cephadm_module.rm_util.process_removal_queue()
@@ -877,7 +877,7 @@ class TestCephadm(object):
 
             # Make sure, _check_daemons does a redeploy due to monmap change:
             cephadm_module.mock_store_set('_ceph_get', 'mon_map', {
-                'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT),
+                'modified': datetime_to_str(datetime_now()),
                 'fsid': 'foobar',
             })
             cephadm_module.notify('mon_map', mock.MagicMock())
index 8536c4e19622621f46119a96a80828779adebbef..f46c1024f34e5c6319327a10787f900523579e4d 100644 (file)
@@ -1,12 +1,11 @@
 import json
-from datetime import datetime
 
 import pytest
 
 from ceph.deployment.service_spec import PlacementSpec, ServiceSpec, HostPlacementSpec
+from ceph.utils import datetime_to_str, datetime_now
 from cephadm import CephadmOrchestrator
 from cephadm.inventory import SPEC_STORE_PREFIX
-from cephadm.utils import DATEFMT
 from cephadm.tests.fixtures import _run_cephadm, cephadm_module, wait, with_host
 from orchestrator import OrchestratorError
 from cephadm.serve import CephadmServe
@@ -48,8 +47,8 @@ def test_migrate_scheduler(cephadm_module: CephadmOrchestrator):
 
             # Sorry, for this hack, but I need to make sure, Migration thinks,
             # we have updated all daemons already.
-            cephadm_module.cache.last_daemon_update['host1'] = datetime.now()
-            cephadm_module.cache.last_daemon_update['host2'] = datetime.now()
+            cephadm_module.cache.last_daemon_update['host1'] = datetime_now()
+            cephadm_module.cache.last_daemon_update['host2'] = datetime_now()
 
             cephadm_module.migration_current = 0
             cephadm_module.migration.migrate()
@@ -72,7 +71,7 @@ def test_migrate_service_id_mon_one(cephadm_module: CephadmOrchestrator):
                     'hosts': ['host1']
                 }
             },
-            'created': datetime.utcnow().strftime(DATEFMT),
+            'created': datetime_to_str(datetime_now()),
         }, sort_keys=True),
         )
 
@@ -103,7 +102,7 @@ def test_migrate_service_id_mon_two(cephadm_module: CephadmOrchestrator):
                     'count': 5,
                 }
             },
-            'created': datetime.utcnow().strftime(DATEFMT),
+            'created': datetime_to_str(datetime_now()),
         }, sort_keys=True),
         )
         cephadm_module.set_store(SPEC_STORE_PREFIX + 'mon.wrong', json.dumps({
@@ -114,7 +113,7 @@ def test_migrate_service_id_mon_two(cephadm_module: CephadmOrchestrator):
                     'hosts': ['host1']
                 }
             },
-            'created': datetime.utcnow().strftime(DATEFMT),
+            'created': datetime_to_str(datetime_now()),
         }, sort_keys=True),
         )
 
@@ -146,7 +145,7 @@ def test_migrate_service_id_mds_one(cephadm_module: CephadmOrchestrator):
                     'hosts': ['host1']
                 }
             },
-            'created': datetime.utcnow().strftime(DATEFMT),
+            'created': datetime_to_str(datetime_now()),
         }, sort_keys=True),
         )
 
index a0d22710a3b5a667456346bf569d8d00c3cf7c8e..f4f24f352cd97c460f25e4e10b72f17e9b3eaa0a 100644 (file)
@@ -273,7 +273,18 @@ def test_dd_octopus(dd_json):
     # https://tracker.ceph.com/issues/44934
     # Those are real user data from early octopus.
     # Please do not modify those JSON values.
-    assert dd_json == DaemonDescription.from_json(dd_json).to_json()
+
+    # Convert datetime properties to old style.
+    # 2020-04-03T07:29:16.926292Z -> 2020-04-03T07:29:16.926292
+    def convert_to_old_style_json(j):
+        for k in ['last_refresh', 'created', 'started', 'last_deployed',
+                  'last_configured']:
+            if k in j:
+                j[k] = j[k].rstrip('Z')
+        return j
+
+    assert dd_json == convert_to_old_style_json(
+        DaemonDescription.from_json(dd_json).to_json())
 
 
 @pytest.mark.parametrize("spec,dd,valid",
index ac788039a1340df1d91cb8446012a965f472ada5..0680df0099c41a72c6cc0dd19c304accb952cb2b 100644 (file)
@@ -1,11 +1,9 @@
 import logging
-import re
 import json
-import datetime
 import socket
 from enum import Enum
 from functools import wraps
-from typing import Optional, Callable, TypeVar, List, NewType, TYPE_CHECKING, Any
+from typing import Callable, TypeVar, List, NewType, TYPE_CHECKING, Any
 from orchestrator import OrchestratorError
 
 if TYPE_CHECKING:
@@ -16,8 +14,6 @@ logger = logging.getLogger(__name__)
 
 ConfEntity = NewType('ConfEntity', str)
 
-DATEFMT = '%Y-%m-%dT%H:%M:%S.%f'
-
 
 class CephadmNoImage(Enum):
     token = 1
@@ -94,14 +90,6 @@ def is_repo_digest(image_name: str) -> bool:
     return '@' in image_name
 
 
-def str_to_datetime(input: str) -> datetime.datetime:
-    return datetime.datetime.strptime(input, DATEFMT)
-
-
-def datetime_to_str(dt: datetime.datetime) -> str:
-    return dt.strftime(DATEFMT)
-
-
 def resolve_ip(hostname: str) -> str:
     try:
         return socket.getaddrinfo(hostname, None, flags=socket.AI_CANONNAME, type=socket.SOCK_STREAM)[0][4][0]
index 39dc20ecc2e37af337412ec61166a7f3a1455801..de09cbdaa4b07396d5a6f5cfbaa3e0252925ef7d 100644 (file)
@@ -25,6 +25,7 @@ from ceph.deployment.service_spec import ServiceSpec, NFSServiceSpec, RGWSpec, \
     ServiceSpecValidationError, IscsiServiceSpec
 from ceph.deployment.drive_group import DriveGroupSpec
 from ceph.deployment.hostspec import HostSpec
+from ceph.utils import datetime_to_str, str_to_datetime
 
 from mgr_module import MgrModule, CLICommand, HandleCommandResult
 
@@ -36,8 +37,6 @@ except ImportError:
 
 logger = logging.getLogger(__name__)
 
-DATEFMT = '%Y-%m-%dT%H:%M:%S.%f'
-
 T = TypeVar('T')
 
 
@@ -1339,7 +1338,7 @@ class DaemonDescription(object):
         for k in ['last_refresh', 'created', 'started', 'last_deployed',
                   'last_configured']:
             if getattr(self, k):
-                out[k] = getattr(self, k).strftime(DATEFMT)
+                out[k] = datetime_to_str(getattr(self, k))
 
         if self.events:
             out['events'] = [e.to_json() for e in self.events]
@@ -1357,7 +1356,7 @@ class DaemonDescription(object):
         for k in ['last_refresh', 'created', 'started', 'last_deployed',
                   'last_configured']:
             if k in c:
-                c[k] = datetime.datetime.strptime(c[k], DATEFMT)
+                c[k] = str_to_datetime(c[k])
         events = [OrchestratorEvent.from_json(e) for e in event_strs]
         return cls(events=events, **c)
 
@@ -1445,7 +1444,7 @@ class ServiceDescription(object):
         }
         for k in ['last_refresh', 'created']:
             if getattr(self, k):
-                status[k] = getattr(self, k).strftime(DATEFMT)
+                status[k] = datetime_to_str(getattr(self, k))
         status = {k: v for (k, v) in status.items() if v is not None}
         out['status'] = status
         if self.events:
@@ -1463,7 +1462,7 @@ class ServiceDescription(object):
         c_status = status.copy()
         for k in ['last_refresh', 'created']:
             if k in c_status:
-                c_status[k] = datetime.datetime.strptime(c_status[k], DATEFMT)
+                c_status[k] = str_to_datetime(c_status[k])
         events = [OrchestratorEvent.from_json(e) for e in event_strs]
         return cls(spec=spec, events=events, **c_status)
 
@@ -1583,7 +1582,7 @@ class OrchestratorEvent:
 
     def __init__(self, created: Union[str, datetime.datetime], kind, subject, level, message):
         if isinstance(created, str):
-            created = datetime.datetime.strptime(created, DATEFMT)
+            created = str_to_datetime(created)
         self.created: datetime.datetime = created
 
         assert kind in "service daemon".split()
@@ -1605,7 +1604,7 @@ class OrchestratorEvent:
 
     def to_json(self) -> str:
         # Make a long list of events readable.
-        created = self.created.strftime(DATEFMT)
+        created = datetime_to_str(self.created)
         return f'{created} {self.kind_subject()} [{self.level}] "{self.message}"'
 
     @classmethod
@@ -1613,7 +1612,7 @@ class OrchestratorEvent:
     def from_json(cls, data) -> "OrchestratorEvent":
         """
         >>> OrchestratorEvent.from_json('''2020-06-10T10:20:25.691255 daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on host 'ubuntu'"''').to_json()
-        '2020-06-10T10:20:25.691255 daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on host \\'ubuntu\\'"'
+        '2020-06-10T10:20:25.691255Z daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on host \\'ubuntu\\'"'
 
         :param data:
         :return:
index 347f5ac83673193c431e3506181d5413f8af630e..d33224c87be6bc44c006aceeffa1e228123b64ff 100644 (file)
@@ -1,4 +1,3 @@
-import datetime
 import errno
 import json
 from typing import List, Set, Optional, Iterator, cast, Dict, Any, Union
@@ -11,6 +10,7 @@ from prettytable import PrettyTable
 from ceph.deployment.inventory import Device
 from ceph.deployment.drive_group import DriveGroupSpec, DeviceSelection
 from ceph.deployment.service_spec import PlacementSpec, ServiceSpec
+from ceph.utils import datetime_now
 
 from mgr_util import format_bytes, to_pretty_timedelta, format_dimless
 from mgr_module import MgrModule, HandleCommandResult, Option
@@ -515,7 +515,7 @@ class OrchestratorCli(OrchestratorClientMixin, MgrModule,
             else:
                 return HandleCommandResult(stdout=to_format(services, format, many=True, cls=ServiceDescription))
         else:
-            now = datetime.datetime.utcnow()
+            now = datetime_now()
             table = PrettyTable(
                 ['NAME', 'RUNNING', 'REFRESHED', 'AGE',
                  'PLACEMENT',
@@ -580,7 +580,7 @@ class OrchestratorCli(OrchestratorClientMixin, MgrModule,
             if len(daemons) == 0:
                 return HandleCommandResult(stdout="No daemons reported")
 
-            now = datetime.datetime.utcnow()
+            now = datetime_now()
             table = PrettyTable(
                 ['NAME', 'HOST', 'STATUS', 'REFRESHED', 'AGE',
                  'VERSION', 'IMAGE NAME', 'IMAGE ID', 'CONTAINER ID'],
index 523c2863d34a3d403eb07fa87f3c119e3b19b531..c5858aa9531250061ab62303bebb360f20cc777b 100644 (file)
@@ -1,6 +1,5 @@
 from __future__ import absolute_import
 
-import datetime
 import json
 
 import pytest
@@ -8,6 +7,7 @@ import yaml
 
 from ceph.deployment.service_spec import ServiceSpec
 from ceph.deployment import inventory
+from ceph.utils import datetime_now
 from mgr_module import HandleCommandResult
 
 from test_orchestrator import TestOrchestrator as _TestOrchestrator
@@ -258,7 +258,7 @@ status: 1
 status_desc: starting
 is_active: false
 events:
-- 2020-06-10T10:08:22.933241 daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on
+- 2020-06-10T10:08:22.933241Z daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on
   host 'ubuntu'"
 ---
 service_type: crash
@@ -268,12 +268,12 @@ placement:
 status:
   container_image_id: 74803e884bea289d2d2d3ebdf6d37cd560499e955595695b1390a89800f4e37a
   container_image_name: docker.io/ceph/daemon-base:latest-master-devel
-  created: '2020-06-10T10:37:31.051288'
-  last_refresh: '2020-06-10T10:57:40.715637'
+  created: '2020-06-10T10:37:31.051288Z'
+  last_refresh: '2020-06-10T10:57:40.715637Z'
   running: 1
   size: 1
 events:
-- 2020-06-10T10:37:31.139159 service:crash [INFO] "service was created"
+- 2020-06-10T10:37:31.139159Z service:crash [INFO] "service was created"
 """
     types = (DaemonDescription, ServiceDescription)
 
@@ -290,10 +290,10 @@ events:
 
 def test_event_multiline():
     from .._interface import OrchestratorEvent
-    e = OrchestratorEvent(datetime.datetime.utcnow(), 'service', 'subject', 'ERROR', 'message')
+    e = OrchestratorEvent(datetime_now(), 'service', 'subject', 'ERROR', 'message')
     assert OrchestratorEvent.from_json(e.to_json()) == e
 
-    e = OrchestratorEvent(datetime.datetime.utcnow(), 'service',
+    e = OrchestratorEvent(datetime_now(), 'service',
                           'subject', 'ERROR', 'multiline\nmessage')
     assert OrchestratorEvent.from_json(e.to_json()) == e
 
diff --git a/src/python-common/ceph/tests/test_datetime.py b/src/python-common/ceph/tests/test_datetime.py
new file mode 100644 (file)
index 0000000..a5c6ec0
--- /dev/null
@@ -0,0 +1,58 @@
+import datetime
+
+import pytest
+
+from ceph.utils import datetime_now, datetime_to_str, str_to_datetime
+
+
+def test_datetime_to_str_1():
+    dt = datetime.datetime.now()
+    assert type(datetime_to_str(dt)) is str
+
+
+def test_datetime_to_str_2():
+    dt = datetime.datetime.strptime('2019-04-24T17:06:53.039991',
+                                    '%Y-%m-%dT%H:%M:%S.%f')
+    assert datetime_to_str(dt) == '2019-04-24T17:06:53.039991Z'
+
+
+def test_datetime_to_str_3():
+    dt = datetime.datetime.strptime('2020-11-02T04:40:12.748172-0800',
+                                    '%Y-%m-%dT%H:%M:%S.%f%z')
+    assert datetime_to_str(dt) == '2020-11-02T12:40:12.748172Z'
+
+
+def test_str_to_datetime_1():
+    dt = str_to_datetime('2020-03-03T09:21:43.636153304Z')
+    assert type(dt) is datetime.datetime
+    assert dt.tzinfo is not None
+
+
+def test_str_to_datetime_2():
+    dt = str_to_datetime('2020-03-03T15:52:30.136257504-0600')
+    assert type(dt) is datetime.datetime
+    assert dt.tzinfo is not None
+
+
+def test_str_to_datetime_3():
+    dt = str_to_datetime('2020-03-03T15:52:30.136257504')
+    assert type(dt) is datetime.datetime
+    assert dt.tzinfo is not None
+
+
+def test_str_to_datetime_invalid_format_1():
+    with pytest.raises(ValueError):
+        str_to_datetime('2020-03-03 15:52:30.136257504')
+
+
+def test_str_to_datetime_invalid_format_2():
+    with pytest.raises(ValueError):
+        str_to_datetime('2020-03-03')
+
+
+def test_datetime_now_1():
+    dt = str_to_datetime('2020-03-03T09:21:43.636153304Z')
+    dt_now = datetime_now()
+    assert type(dt_now) is datetime.datetime
+    assert dt_now.tzinfo is not None
+    assert dt < dt_now
diff --git a/src/python-common/ceph/utils.py b/src/python-common/ceph/utils.py
new file mode 100644 (file)
index 0000000..2a272fa
--- /dev/null
@@ -0,0 +1,68 @@
+import datetime
+import re
+
+
+def datetime_now() -> datetime.datetime:
+    """
+    Return the current local date and time.
+    :return: Returns an aware datetime object of the current date
+        and time.
+    """
+    return datetime.datetime.now(tz=datetime.timezone.utc)
+
+
+def datetime_to_str(dt: datetime.datetime) -> str:
+    """
+    Convert a datetime object into a ISO 8601 string, e.g.
+    '2019-04-24T17:06:53.039991Z'.
+    :param dt: The datetime object to process.
+    :return: Return a string representing the date in
+        ISO 8601 (timezone=UTC).
+    """
+    return dt.astimezone(tz=datetime.timezone.utc).strftime(
+        '%Y-%m-%dT%H:%M:%S.%fZ')
+
+
+def str_to_datetime(string: str) -> datetime.datetime:
+    """
+    Convert an ISO 8601 string into a datetime object.
+    The following formats are supported:
+
+    - 2020-03-03T09:21:43.636153304Z
+    - 2020-03-03T15:52:30.136257504-0600
+    - 2020-03-03T15:52:30.136257504
+
+    :param string: The string to parse.
+    :return: Returns an aware datetime object of the given date
+        and time string.
+    :raises: :exc:`~exceptions.ValueError` for an unknown
+        datetime string.
+    """
+    fmts = [
+        '%Y-%m-%dT%H:%M:%S.%f',
+        '%Y-%m-%dT%H:%M:%S.%f%z'
+    ]
+
+    # In *all* cases, the 9 digit second precision is too much for
+    # Python's strptime. Shorten it to 6 digits.
+    p = re.compile(r'(\.[\d]{6})[\d]*')
+    string = p.sub(r'\1', string)
+
+    # Replace trailing Z with -0000, since (on Python 3.6.8) it
+    # won't parse.
+    if string and string[-1] == 'Z':
+        string = string[:-1] + '-0000'
+
+    for fmt in fmts:
+        try:
+            dt = datetime.datetime.strptime(string, fmt)
+            # Make sure the datetime object is aware (timezone is set).
+            # If not, then assume the time is in UTC.
+            if dt.tzinfo is None:
+                dt = dt.replace(tzinfo=datetime.timezone.utc)
+            return dt
+        except ValueError:
+            pass
+
+    raise ValueError("Time data {} does not match one of the formats {}".format(
+        string, str(fmts)))