]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
cephadm: Various properties like 'last_refresh' do not contain timezone 39059/head
authorVolker Theile <vtheile@suse.com>
Mon, 25 Jan 2021 11:00:56 +0000 (12:00 +0100)
committerVolker Theile <vtheile@suse.com>
Fri, 29 Jan 2021 09:19:59 +0000 (10:19 +0100)
Fixes: https://tracker.ceph.com/issues/48068
Signed-off-by: Volker Theile <vtheile@suse.com>
(cherry picked from commit 3fe715201c8c07cf4ea86b590f9682422eeccf33)

15 files changed:
src/cephadm/cephadm
src/pybind/mgr/cephadm/inventory.py
src/pybind/mgr/cephadm/module.py
src/pybind/mgr/cephadm/serve.py
src/pybind/mgr/cephadm/services/osd.py
src/pybind/mgr/cephadm/tests/fixtures.py
src/pybind/mgr/cephadm/tests/test_cephadm.py
src/pybind/mgr/cephadm/tests/test_migration.py
src/pybind/mgr/cephadm/tests/test_spec.py
src/pybind/mgr/cephadm/utils.py
src/pybind/mgr/orchestrator/_interface.py
src/pybind/mgr/orchestrator/module.py
src/pybind/mgr/orchestrator/tests/test_orchestrator.py
src/python-common/ceph/tests/test_datetime.py [new file with mode: 0644]
src/python-common/ceph/utils.py [new file with mode: 0644]

index 326f236e8342998f427bb589eaed75bf23843b06..4585c4cd68f22a7c53633ba1382968394eafb5e1 100755 (executable)
@@ -48,7 +48,6 @@ import os
 import platform
 import pwd
 import random
-import re
 import select
 import shutil
 import socket
@@ -94,7 +93,7 @@ if sys.version_info > (3, 0):
 container_path = ''
 cached_stdin = None
 
-DATEFMT = '%Y-%m-%dT%H:%M:%S.%f'
+DATEFMT = '%Y-%m-%dT%H:%M:%S.%fZ'
 
 # Log and console output config
 logging_config = {
@@ -1178,7 +1177,7 @@ def get_file_timestamp(fn):
         return datetime.datetime.fromtimestamp(
             mt, tz=datetime.timezone.utc
         ).strftime(DATEFMT)
-    except Exception as e:
+    except Exception:
         return None
 
 
@@ -1200,11 +1199,11 @@ def try_convert_datetime(s):
     p = re.compile(r'(\.[\d]{6})[\d]*')
     s = p.sub(r'\1', s)
 
-    # replace trailling Z with -0000, since (on python 3.6.8) it won't parse
+    # replace trailing Z with -0000, since (on python 3.6.8) it won't parse
     if s and s[-1] == 'Z':
         s = s[:-1] + '-0000'
 
-    # cut off the redundnat 'CST' part that strptime can't parse, if
+    # cut off the redundant 'CST' part that strptime can't parse, if
     # present.
     v = s.split(' ')
     s = ' '.join(v[0:3])
index 188cd0082a68faef342fe0f565ae447cb9a5a72f..7c2bd661076fb4459f61c60310ae886d52138915 100644 (file)
@@ -9,7 +9,7 @@ import six
 import orchestrator
 from ceph.deployment import inventory
 from ceph.deployment.service_spec import ServiceSpec
-from cephadm.utils import str_to_datetime, datetime_to_str
+from ceph.utils import str_to_datetime, datetime_to_str, datetime_now
 from orchestrator import OrchestratorError, HostSpec, OrchestratorEvent
 
 if TYPE_CHECKING:
@@ -138,7 +138,7 @@ class SpecStore():
             self.spec_preview[spec.service_name()] = spec
             return None
         self.specs[spec.service_name()] = spec
-        self.spec_created[spec.service_name()] = datetime.datetime.utcnow()
+        self.spec_created[spec.service_name()] = datetime_now()
         self.mgr.set_store(
             SPEC_STORE_PREFIX + spec.service_name(),
             json.dumps({
@@ -280,7 +280,7 @@ class HostCache():
     def update_host_daemons(self, host, dm):
         # type: (str, Dict[str, orchestrator.DaemonDescription]) -> None
         self.daemons[host] = dm
-        self.last_daemon_update[host] = datetime.datetime.utcnow()
+        self.last_daemon_update[host] = datetime_now()
 
     def update_host_facts(self, host, facts):
         # type: (str, Dict[str, Dict[str, Any]]) -> None
@@ -291,7 +291,7 @@ class HostCache():
         # type: (str, List[inventory.Device], Dict[str,List[str]]) -> None
         self.devices[host] = dls
         self.networks[host] = nets
-        self.last_device_update[host] = datetime.datetime.utcnow()
+        self.last_device_update[host] = datetime_now()
 
     def update_daemon_config_deps(self, host: str, name: str, deps: List[str], stamp: datetime.datetime) -> None:
         self.daemon_config_deps[host][name] = {
@@ -301,7 +301,7 @@ class HostCache():
 
     def update_last_host_check(self, host):
         # type: (str) -> None
-        self.last_host_check[host] = datetime.datetime.utcnow()
+        self.last_host_check[host] = datetime_now()
 
     def prime_empty_host(self, host):
         # type: (str) -> None
@@ -474,7 +474,7 @@ class HostCache():
         if host in self.daemon_refresh_queue:
             self.daemon_refresh_queue.remove(host)
             return True
-        cutoff = datetime.datetime.utcnow() - datetime.timedelta(
+        cutoff = datetime_now() - datetime.timedelta(
             seconds=self.mgr.daemon_cache_timeout)
         if host not in self.last_daemon_update or self.last_daemon_update[host] < cutoff:
             return True
@@ -509,7 +509,7 @@ class HostCache():
         if host in self.device_refresh_queue:
             self.device_refresh_queue.remove(host)
             return True
-        cutoff = datetime.datetime.utcnow() - datetime.timedelta(
+        cutoff = datetime_now() - datetime.timedelta(
             seconds=self.mgr.device_cache_timeout)
         if host not in self.last_device_update or self.last_device_update[host] < cutoff:
             return True
@@ -528,7 +528,7 @@ class HostCache():
 
     def host_needs_check(self, host):
         # type: (str) -> bool
-        cutoff = datetime.datetime.utcnow() - datetime.timedelta(
+        cutoff = datetime_now() - datetime.timedelta(
             seconds=self.mgr.host_check_interval)
         return host not in self.last_host_check or self.last_host_check[host] < cutoff
 
@@ -553,7 +553,7 @@ class HostCache():
     def update_last_etc_ceph_ceph_conf(self, host: str) -> None:
         if not self.mgr.last_monmap:
             return
-        self.last_etc_ceph_ceph_conf[host] = datetime.datetime.utcnow()
+        self.last_etc_ceph_ceph_conf[host] = datetime_now()
 
     def host_needs_registry_login(self, host: str) -> bool:
         if host in self.mgr.offline_hosts:
@@ -634,14 +634,14 @@ class EventStore():
         self.events[event.kind_subject()] = self.events[event.kind_subject()][-5:]
 
     def for_service(self, spec: ServiceSpec, level: str, message: str) -> None:
-        e = OrchestratorEvent(datetime.datetime.utcnow(), 'service',
+        e = OrchestratorEvent(datetime_now(), 'service',
                               spec.service_name(), level, message)
         self.add(e)
 
     def from_orch_error(self, e: OrchestratorError) -> None:
         if e.event_subject is not None:
             self.add(OrchestratorEvent(
-                datetime.datetime.utcnow(),
+                datetime_now(),
                 e.event_subject[0],
                 e.event_subject[1],
                 "ERROR",
@@ -649,7 +649,7 @@ class EventStore():
             ))
 
     def for_daemon(self, daemon_name: str, level: str, message: str) -> None:
-        e = OrchestratorEvent(datetime.datetime.utcnow(), 'daemon', daemon_name, level, message)
+        e = OrchestratorEvent(datetime_now(), 'daemon', daemon_name, level, message)
         self.add(e)
 
     def for_daemon_from_exception(self, daemon_name: str, e: Exception) -> None:
index 371d2a4bb437e4eda92924da011e16af5de4ffc0..096d0e89336de59ec78ec71fb377a6ee4ebbb0cf 100644 (file)
@@ -27,6 +27,7 @@ from ceph.deployment.drive_group import DriveGroupSpec
 from ceph.deployment.service_spec import \
     NFSServiceSpec, ServiceSpec, PlacementSpec, assert_valid_host, \
     CustomContainerSpec, HostPlacementSpec
+from ceph.utils import str_to_datetime, datetime_to_str, datetime_now
 from cephadm.serve import CephadmServe
 from cephadm.services.cephadmservice import CephadmDaemonSpec
 
@@ -51,8 +52,7 @@ from .schedule import HostAssignment
 from .inventory import Inventory, SpecStore, HostCache, EventStore
 from .upgrade import CEPH_UPGRADE_ORDER, CephadmUpgrade
 from .template import TemplateMgr
-from .utils import forall_hosts, CephadmNoImage, cephadmNoImage, \
-    str_to_datetime, datetime_to_str
+from .utils import forall_hosts, CephadmNoImage, cephadmNoImage
 
 try:
     import remoto
@@ -88,8 +88,6 @@ Host *
   ConnectTimeout=30
 """
 
-CEPH_DATEFMT = '%Y-%m-%dT%H:%M:%S.%fZ'
-
 CEPH_TYPES = set(CEPH_UPGRADE_ORDER)
 
 
@@ -482,9 +480,8 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule):
         if notify_type == "mon_map":
             # get monmap mtime so we can refresh configs when mons change
             monmap = self.get('mon_map')
-            self.last_monmap = datetime.datetime.strptime(
-                monmap['modified'], CEPH_DATEFMT)
-            if self.last_monmap and self.last_monmap > datetime.datetime.utcnow():
+            self.last_monmap = str_to_datetime(monmap['modified'])
+            if self.last_monmap and self.last_monmap > datetime_now():
                 self.last_monmap = None  # just in case clocks are skewed
             if getattr(self, 'manage_etc_ceph_ceph_conf', False):
                 # getattr, due to notify() being called before config_notify()
@@ -923,7 +920,7 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule):
 
         self.set_store("extra_ceph_conf", json.dumps({
             'conf': inbuf,
-            'last_modified': datetime_to_str(datetime.datetime.utcnow())
+            'last_modified': datetime_to_str(datetime_now())
         }))
         self.log.info('Set extra_ceph_conf')
         self._kick_serve_loop()
@@ -1716,7 +1713,7 @@ To check that the host is reachable:
         ).service_id(), overwrite=True):
 
             image = ''
-            start_time = datetime.datetime.utcnow()
+            start_time = datetime_now()
             ports: List[int] = daemon_spec.ports if daemon_spec.ports else []
 
             if daemon_spec.daemon_type == 'container':
@@ -2225,7 +2222,7 @@ To check that the host is reachable:
                                                 force=force,
                                                 hostname=daemon.hostname,
                                                 fullname=daemon.name(),
-                                                process_started_at=datetime.datetime.utcnow(),
+                                                process_started_at=datetime_now(),
                                                 remove_util=self.to_remove_osds.rm_util))
             except NotFoundError:
                 return f"Unable to find OSDs: {osd_ids}"
index 7808dc5baded94bac0433aa411476901e7e1d059..33632099cd77dfb9517c152678142ee647addd34 100644 (file)
@@ -12,11 +12,12 @@ except ImportError:
 from ceph.deployment import inventory
 from ceph.deployment.drive_group import DriveGroupSpec
 from ceph.deployment.service_spec import ServiceSpec, HostPlacementSpec, RGWSpec
+from ceph.utils import str_to_datetime, datetime_now
 
 import orchestrator
 from cephadm.schedule import HostAssignment
 from cephadm.upgrade import CEPH_UPGRADE_ORDER
-from cephadm.utils import forall_hosts, cephadmNoImage, str_to_datetime, is_repo_digest
+from cephadm.utils import forall_hosts, cephadmNoImage, is_repo_digest
 from orchestrator import OrchestratorError
 
 if TYPE_CHECKING:
@@ -224,7 +225,7 @@ class CephadmServe:
             if '.' not in d['name']:
                 continue
             sd = orchestrator.DaemonDescription()
-            sd.last_refresh = datetime.datetime.utcnow()
+            sd.last_refresh = datetime_now()
             for k in ['created', 'started', 'last_configured', 'last_deployed']:
                 v = d.get(k, None)
                 if v:
index 15eb667478e9350f3f88258ceadb6eb18db0dc3f..8febf352b615bb4c67c1a421c21c14dd1d978ede 100644 (file)
@@ -7,10 +7,11 @@ from ceph.deployment import translate
 from ceph.deployment.drive_group import DriveGroupSpec
 from ceph.deployment.drive_selection import DriveSelection
 from ceph.deployment.inventory import Device
+from ceph.utils import datetime_to_str, str_to_datetime
 
 from datetime import datetime
 import orchestrator
-from cephadm.utils import forall_hosts, datetime_to_str, str_to_datetime
+from cephadm.utils import forall_hosts
 from orchestrator import OrchestratorError
 from mgr_module import MonCommandFailed
 
index 5548c53e2efe1400376872ddbc6a09dbf8feee52..bfc0f0b99e7a90344e603ac6ea5b790fb8664937 100644 (file)
@@ -1,10 +1,9 @@
-import datetime
 import time
 import fnmatch
 from contextlib import contextmanager
 
 from ceph.deployment.service_spec import PlacementSpec, ServiceSpec
-from cephadm.module import CEPH_DATEFMT
+from ceph.utils import datetime_to_str, datetime_now
 from cephadm.serve import CephadmServe
 
 try:
@@ -55,7 +54,7 @@ def with_cephadm_module(module_options=None, store=None):
             store = {}
         if '_ceph_get/mon_map' not in store:
             m.mock_store_set('_ceph_get', 'mon_map', {
-                'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT),
+                'modified': datetime_to_str(datetime_now()),
                 'fsid': 'foobar',
             })
         for k, v in store.items():
index d5740fa53577629bb9bfcf506605284a15106872..67fd47ee5b08a2e3b341f76530a2b87e31a4e418 100644 (file)
@@ -1,4 +1,3 @@
-import datetime
 import json
 from contextlib import contextmanager
 from unittest.mock import ANY
@@ -20,12 +19,13 @@ from ceph.deployment.service_spec import ServiceSpec, PlacementSpec, RGWSpec, \
     NFSServiceSpec, IscsiServiceSpec, HostPlacementSpec, CustomContainerSpec
 from ceph.deployment.drive_selection.selector import DriveSelection
 from ceph.deployment.inventory import Devices, Device
+from ceph.utils import datetime_to_str, datetime_now
 from orchestrator import ServiceDescription, DaemonDescription, InventoryHost, \
     HostSpec, OrchestratorError
 from tests import mock
 from .fixtures import cephadm_module, wait, _run_cephadm, match_glob, with_host, \
     with_cephadm_module, with_service, assert_rm_service
-from cephadm.module import CephadmOrchestrator, CEPH_DATEFMT
+from cephadm.module import CephadmOrchestrator
 
 """
 TODOs:
@@ -195,7 +195,7 @@ class TestCephadm(object):
 
                 # Make sure, _check_daemons does a redeploy due to monmap change:
                 cephadm_module._store['_ceph_get/mon_map'] = {
-                    'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT),
+                    'modified': datetime_to_str(datetime_now()),
                     'fsid': 'foobar',
                 }
                 cephadm_module.notify('mon_map', None)
@@ -214,7 +214,7 @@ class TestCephadm(object):
 
                     # Make sure, _check_daemons does a redeploy due to monmap change:
                     cephadm_module.mock_store_set('_ceph_get', 'mon_map', {
-                        'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT),
+                        'modified': datetime_to_str(datetime_now()),
                         'fsid': 'foobar',
                     })
                     cephadm_module.notify('mon_map', None)
@@ -294,7 +294,7 @@ class TestCephadm(object):
 
                 # Make sure, _check_daemons does a redeploy due to monmap change:
                 cephadm_module.mock_store_set('_ceph_get', 'mon_map', {
-                    'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT),
+                    'modified': datetime_to_str(datetime_now()),
                     'fsid': 'foobar',
                 })
                 cephadm_module.notify('mon_map', None)
@@ -504,7 +504,7 @@ class TestCephadm(object):
                                                       force=False,
                                                       hostname='test',
                                                       fullname='osd.0',
-                                                      process_started_at=datetime.datetime.utcnow(),
+                                                      process_started_at=datetime_now(),
                                                       remove_util=cephadm_module.to_remove_osds.rm_util
                                                       ))
             cephadm_module.to_remove_osds.process_removal_queue()
@@ -865,7 +865,7 @@ class TestCephadm(object):
 
             # Make sure, _check_daemons does a redeploy due to monmap change:
             cephadm_module.mock_store_set('_ceph_get', 'mon_map', {
-                'modified': datetime.datetime.utcnow().strftime(CEPH_DATEFMT),
+                'modified': datetime_to_str(datetime_now()),
                 'fsid': 'foobar',
             })
             cephadm_module.notify('mon_map', mock.MagicMock())
index 8536c4e19622621f46119a96a80828779adebbef..f46c1024f34e5c6319327a10787f900523579e4d 100644 (file)
@@ -1,12 +1,11 @@
 import json
-from datetime import datetime
 
 import pytest
 
 from ceph.deployment.service_spec import PlacementSpec, ServiceSpec, HostPlacementSpec
+from ceph.utils import datetime_to_str, datetime_now
 from cephadm import CephadmOrchestrator
 from cephadm.inventory import SPEC_STORE_PREFIX
-from cephadm.utils import DATEFMT
 from cephadm.tests.fixtures import _run_cephadm, cephadm_module, wait, with_host
 from orchestrator import OrchestratorError
 from cephadm.serve import CephadmServe
@@ -48,8 +47,8 @@ def test_migrate_scheduler(cephadm_module: CephadmOrchestrator):
 
             # Sorry, for this hack, but I need to make sure, Migration thinks,
             # we have updated all daemons already.
-            cephadm_module.cache.last_daemon_update['host1'] = datetime.now()
-            cephadm_module.cache.last_daemon_update['host2'] = datetime.now()
+            cephadm_module.cache.last_daemon_update['host1'] = datetime_now()
+            cephadm_module.cache.last_daemon_update['host2'] = datetime_now()
 
             cephadm_module.migration_current = 0
             cephadm_module.migration.migrate()
@@ -72,7 +71,7 @@ def test_migrate_service_id_mon_one(cephadm_module: CephadmOrchestrator):
                     'hosts': ['host1']
                 }
             },
-            'created': datetime.utcnow().strftime(DATEFMT),
+            'created': datetime_to_str(datetime_now()),
         }, sort_keys=True),
         )
 
@@ -103,7 +102,7 @@ def test_migrate_service_id_mon_two(cephadm_module: CephadmOrchestrator):
                     'count': 5,
                 }
             },
-            'created': datetime.utcnow().strftime(DATEFMT),
+            'created': datetime_to_str(datetime_now()),
         }, sort_keys=True),
         )
         cephadm_module.set_store(SPEC_STORE_PREFIX + 'mon.wrong', json.dumps({
@@ -114,7 +113,7 @@ def test_migrate_service_id_mon_two(cephadm_module: CephadmOrchestrator):
                     'hosts': ['host1']
                 }
             },
-            'created': datetime.utcnow().strftime(DATEFMT),
+            'created': datetime_to_str(datetime_now()),
         }, sort_keys=True),
         )
 
@@ -146,7 +145,7 @@ def test_migrate_service_id_mds_one(cephadm_module: CephadmOrchestrator):
                     'hosts': ['host1']
                 }
             },
-            'created': datetime.utcnow().strftime(DATEFMT),
+            'created': datetime_to_str(datetime_now()),
         }, sort_keys=True),
         )
 
index ab2d059126c828cfd9c52ef4259afb5d59b728bf..fcb18855121d2ee0864733786a75c5132c350acb 100644 (file)
@@ -273,7 +273,18 @@ def test_dd_octopus(dd_json):
     # https://tracker.ceph.com/issues/44934
     # Those are real user data from early octopus.
     # Please do not modify those JSON values.
-    assert dd_json == DaemonDescription.from_json(dd_json).to_json()
+
+    # Convert datetime properties to old style.
+    # 2020-04-03T07:29:16.926292Z -> 2020-04-03T07:29:16.926292
+    def convert_to_old_style_json(j):
+        for k in ['last_refresh', 'created', 'started', 'last_deployed',
+                  'last_configured']:
+            if k in j:
+                j[k] = j[k].rstrip('Z')
+        return j
+
+    assert dd_json == convert_to_old_style_json(
+        DaemonDescription.from_json(dd_json).to_json())
 
 
 @pytest.mark.parametrize("spec,dd,valid",
index ac788039a1340df1d91cb8446012a965f472ada5..0680df0099c41a72c6cc0dd19c304accb952cb2b 100644 (file)
@@ -1,11 +1,9 @@
 import logging
-import re
 import json
-import datetime
 import socket
 from enum import Enum
 from functools import wraps
-from typing import Optional, Callable, TypeVar, List, NewType, TYPE_CHECKING, Any
+from typing import Callable, TypeVar, List, NewType, TYPE_CHECKING, Any
 from orchestrator import OrchestratorError
 
 if TYPE_CHECKING:
@@ -16,8 +14,6 @@ logger = logging.getLogger(__name__)
 
 ConfEntity = NewType('ConfEntity', str)
 
-DATEFMT = '%Y-%m-%dT%H:%M:%S.%f'
-
 
 class CephadmNoImage(Enum):
     token = 1
@@ -94,14 +90,6 @@ def is_repo_digest(image_name: str) -> bool:
     return '@' in image_name
 
 
-def str_to_datetime(input: str) -> datetime.datetime:
-    return datetime.datetime.strptime(input, DATEFMT)
-
-
-def datetime_to_str(dt: datetime.datetime) -> str:
-    return dt.strftime(DATEFMT)
-
-
 def resolve_ip(hostname: str) -> str:
     try:
         return socket.getaddrinfo(hostname, None, flags=socket.AI_CANONNAME, type=socket.SOCK_STREAM)[0][4][0]
index 117d3d0b3d6d07ea2db164ff8486e43dc52469b0..c22ac4c2d3fefe038cfb9d03b25de13665875ca1 100644 (file)
@@ -25,6 +25,7 @@ from ceph.deployment.service_spec import ServiceSpec, NFSServiceSpec, RGWSpec, \
     ServiceSpecValidationError, IscsiServiceSpec
 from ceph.deployment.drive_group import DriveGroupSpec
 from ceph.deployment.hostspec import HostSpec
+from ceph.utils import datetime_to_str, str_to_datetime
 
 from mgr_module import MgrModule, CLICommand, HandleCommandResult
 
@@ -36,8 +37,6 @@ except ImportError:
 
 logger = logging.getLogger(__name__)
 
-DATEFMT = '%Y-%m-%dT%H:%M:%S.%f'
-
 T = TypeVar('T')
 
 
@@ -1330,7 +1329,7 @@ class DaemonDescription(object):
         for k in ['last_refresh', 'created', 'started', 'last_deployed',
                   'last_configured']:
             if getattr(self, k):
-                out[k] = getattr(self, k).strftime(DATEFMT)
+                out[k] = datetime_to_str(getattr(self, k))
 
         if self.events:
             out['events'] = [e.to_json() for e in self.events]
@@ -1348,7 +1347,7 @@ class DaemonDescription(object):
         for k in ['last_refresh', 'created', 'started', 'last_deployed',
                   'last_configured']:
             if k in c:
-                c[k] = datetime.datetime.strptime(c[k], DATEFMT)
+                c[k] = str_to_datetime(c[k])
         events = [OrchestratorEvent.from_json(e) for e in event_strs]
         return cls(events=events, **c)
 
@@ -1436,7 +1435,7 @@ class ServiceDescription(object):
         }
         for k in ['last_refresh', 'created']:
             if getattr(self, k):
-                status[k] = getattr(self, k).strftime(DATEFMT)
+                status[k] = datetime_to_str(getattr(self, k))
         status = {k: v for (k, v) in status.items() if v is not None}
         out['status'] = status
         if self.events:
@@ -1454,7 +1453,7 @@ class ServiceDescription(object):
         c_status = status.copy()
         for k in ['last_refresh', 'created']:
             if k in c_status:
-                c_status[k] = datetime.datetime.strptime(c_status[k], DATEFMT)
+                c_status[k] = str_to_datetime(c_status[k])
         events = [OrchestratorEvent.from_json(e) for e in event_strs]
         return cls(spec=spec, events=events, **c_status)
 
@@ -1574,7 +1573,7 @@ class OrchestratorEvent:
 
     def __init__(self, created: Union[str, datetime.datetime], kind, subject, level, message):
         if isinstance(created, str):
-            created = datetime.datetime.strptime(created, DATEFMT)
+            created = str_to_datetime(created)
         self.created: datetime.datetime = created
 
         assert kind in "service daemon".split()
@@ -1596,7 +1595,7 @@ class OrchestratorEvent:
 
     def to_json(self) -> str:
         # Make a long list of events readable.
-        created = self.created.strftime(DATEFMT)
+        created = datetime_to_str(self.created)
         return f'{created} {self.kind_subject()} [{self.level}] "{self.message}"'
 
     @classmethod
@@ -1604,7 +1603,7 @@ class OrchestratorEvent:
     def from_json(cls, data) -> "OrchestratorEvent":
         """
         >>> OrchestratorEvent.from_json('''2020-06-10T10:20:25.691255 daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on host 'ubuntu'"''').to_json()
-        '2020-06-10T10:20:25.691255 daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on host \\'ubuntu\\'"'
+        '2020-06-10T10:20:25.691255Z daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on host \\'ubuntu\\'"'
 
         :param data:
         :return:
index 7d9812a2e53d3b142a2b707f4edc76cfbdb9e0a6..ce69df8a1b711bb9d0ead7a7dbb8f46a00e45b02 100644 (file)
@@ -1,4 +1,3 @@
-import datetime
 import errno
 import json
 from typing import List, Set, Optional, Iterator, cast, Dict, Any, Union
@@ -12,6 +11,7 @@ from prettytable import PrettyTable
 from ceph.deployment.inventory import Device
 from ceph.deployment.drive_group import DriveGroupSpec, DeviceSelection
 from ceph.deployment.service_spec import PlacementSpec, ServiceSpec
+from ceph.utils import datetime_now
 
 from mgr_util import format_bytes, to_pretty_timedelta, format_dimless
 from mgr_module import MgrModule, HandleCommandResult
@@ -518,7 +518,7 @@ class OrchestratorCli(OrchestratorClientMixin, MgrModule):
             else:
                 return HandleCommandResult(stdout=to_format(services, format, many=True, cls=ServiceDescription))
         else:
-            now = datetime.datetime.utcnow()
+            now = datetime_now()
             table = PrettyTable(
                 ['NAME', 'RUNNING', 'REFRESHED', 'AGE',
                  'PLACEMENT',
@@ -583,7 +583,7 @@ class OrchestratorCli(OrchestratorClientMixin, MgrModule):
             if len(daemons) == 0:
                 return HandleCommandResult(stdout="No daemons reported")
 
-            now = datetime.datetime.utcnow()
+            now = datetime_now()
             table = PrettyTable(
                 ['NAME', 'HOST', 'STATUS', 'REFRESHED', 'AGE',
                  'VERSION', 'IMAGE NAME', 'IMAGE ID', 'CONTAINER ID'],
index 7767743186bccf6b5aa348e12fdd42bdbc778b47..4037252f8813fe29aca474fdc5264fb6d4a40d17 100644 (file)
@@ -1,6 +1,5 @@
 from __future__ import absolute_import
 
-import datetime
 import json
 
 import pytest
@@ -8,6 +7,7 @@ import yaml
 
 from ceph.deployment.service_spec import ServiceSpec
 from ceph.deployment import inventory
+from ceph.utils import datetime_now
 
 from test_orchestrator import TestOrchestrator as _TestOrchestrator
 from tests import mock
@@ -257,7 +257,7 @@ status: 1
 status_desc: starting
 is_active: false
 events:
-- 2020-06-10T10:08:22.933241 daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on
+- 2020-06-10T10:08:22.933241Z daemon:crash.ubuntu [INFO] "Deployed crash.ubuntu on
   host 'ubuntu'"
 ---
 service_type: crash
@@ -267,12 +267,12 @@ placement:
 status:
   container_image_id: 74803e884bea289d2d2d3ebdf6d37cd560499e955595695b1390a89800f4e37a
   container_image_name: docker.io/ceph/daemon-base:latest-master-devel
-  created: '2020-06-10T10:37:31.051288'
-  last_refresh: '2020-06-10T10:57:40.715637'
+  created: '2020-06-10T10:37:31.051288Z'
+  last_refresh: '2020-06-10T10:57:40.715637Z'
   running: 1
   size: 1
 events:
-- 2020-06-10T10:37:31.139159 service:crash [INFO] "service was created"
+- 2020-06-10T10:37:31.139159Z service:crash [INFO] "service was created"
 """
     types = (DaemonDescription, ServiceDescription)
 
@@ -289,10 +289,10 @@ events:
 
 def test_event_multiline():
     from .._interface import OrchestratorEvent
-    e = OrchestratorEvent(datetime.datetime.utcnow(), 'service', 'subject', 'ERROR', 'message')
+    e = OrchestratorEvent(datetime_now(), 'service', 'subject', 'ERROR', 'message')
     assert OrchestratorEvent.from_json(e.to_json()) == e
 
-    e = OrchestratorEvent(datetime.datetime.utcnow(), 'service',
+    e = OrchestratorEvent(datetime_now(), 'service',
                           'subject', 'ERROR', 'multiline\nmessage')
     assert OrchestratorEvent.from_json(e.to_json()) == e
 
diff --git a/src/python-common/ceph/tests/test_datetime.py b/src/python-common/ceph/tests/test_datetime.py
new file mode 100644 (file)
index 0000000..a5c6ec0
--- /dev/null
@@ -0,0 +1,58 @@
+import datetime
+
+import pytest
+
+from ceph.utils import datetime_now, datetime_to_str, str_to_datetime
+
+
+def test_datetime_to_str_1():
+    dt = datetime.datetime.now()
+    assert type(datetime_to_str(dt)) is str
+
+
+def test_datetime_to_str_2():
+    dt = datetime.datetime.strptime('2019-04-24T17:06:53.039991',
+                                    '%Y-%m-%dT%H:%M:%S.%f')
+    assert datetime_to_str(dt) == '2019-04-24T17:06:53.039991Z'
+
+
+def test_datetime_to_str_3():
+    dt = datetime.datetime.strptime('2020-11-02T04:40:12.748172-0800',
+                                    '%Y-%m-%dT%H:%M:%S.%f%z')
+    assert datetime_to_str(dt) == '2020-11-02T12:40:12.748172Z'
+
+
+def test_str_to_datetime_1():
+    dt = str_to_datetime('2020-03-03T09:21:43.636153304Z')
+    assert type(dt) is datetime.datetime
+    assert dt.tzinfo is not None
+
+
+def test_str_to_datetime_2():
+    dt = str_to_datetime('2020-03-03T15:52:30.136257504-0600')
+    assert type(dt) is datetime.datetime
+    assert dt.tzinfo is not None
+
+
+def test_str_to_datetime_3():
+    dt = str_to_datetime('2020-03-03T15:52:30.136257504')
+    assert type(dt) is datetime.datetime
+    assert dt.tzinfo is not None
+
+
+def test_str_to_datetime_invalid_format_1():
+    with pytest.raises(ValueError):
+        str_to_datetime('2020-03-03 15:52:30.136257504')
+
+
+def test_str_to_datetime_invalid_format_2():
+    with pytest.raises(ValueError):
+        str_to_datetime('2020-03-03')
+
+
+def test_datetime_now_1():
+    dt = str_to_datetime('2020-03-03T09:21:43.636153304Z')
+    dt_now = datetime_now()
+    assert type(dt_now) is datetime.datetime
+    assert dt_now.tzinfo is not None
+    assert dt < dt_now
diff --git a/src/python-common/ceph/utils.py b/src/python-common/ceph/utils.py
new file mode 100644 (file)
index 0000000..2a272fa
--- /dev/null
@@ -0,0 +1,68 @@
+import datetime
+import re
+
+
+def datetime_now() -> datetime.datetime:
+    """
+    Return the current local date and time.
+    :return: Returns an aware datetime object of the current date
+        and time.
+    """
+    return datetime.datetime.now(tz=datetime.timezone.utc)
+
+
+def datetime_to_str(dt: datetime.datetime) -> str:
+    """
+    Convert a datetime object into a ISO 8601 string, e.g.
+    '2019-04-24T17:06:53.039991Z'.
+    :param dt: The datetime object to process.
+    :return: Return a string representing the date in
+        ISO 8601 (timezone=UTC).
+    """
+    return dt.astimezone(tz=datetime.timezone.utc).strftime(
+        '%Y-%m-%dT%H:%M:%S.%fZ')
+
+
+def str_to_datetime(string: str) -> datetime.datetime:
+    """
+    Convert an ISO 8601 string into a datetime object.
+    The following formats are supported:
+
+    - 2020-03-03T09:21:43.636153304Z
+    - 2020-03-03T15:52:30.136257504-0600
+    - 2020-03-03T15:52:30.136257504
+
+    :param string: The string to parse.
+    :return: Returns an aware datetime object of the given date
+        and time string.
+    :raises: :exc:`~exceptions.ValueError` for an unknown
+        datetime string.
+    """
+    fmts = [
+        '%Y-%m-%dT%H:%M:%S.%f',
+        '%Y-%m-%dT%H:%M:%S.%f%z'
+    ]
+
+    # In *all* cases, the 9 digit second precision is too much for
+    # Python's strptime. Shorten it to 6 digits.
+    p = re.compile(r'(\.[\d]{6})[\d]*')
+    string = p.sub(r'\1', string)
+
+    # Replace trailing Z with -0000, since (on Python 3.6.8) it
+    # won't parse.
+    if string and string[-1] == 'Z':
+        string = string[:-1] + '-0000'
+
+    for fmt in fmts:
+        try:
+            dt = datetime.datetime.strptime(string, fmt)
+            # Make sure the datetime object is aware (timezone is set).
+            # If not, then assume the time is in UTC.
+            if dt.tzinfo is None:
+                dt = dt.replace(tzinfo=datetime.timezone.utc)
+            return dt
+        except ValueError:
+            pass
+
+    raise ValueError("Time data {} does not match one of the formats {}".format(
+        string, str(fmts)))