From 0e9303d0599c92e575aae7acca87328ed8ee9527 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 11 Dec 2019 10:33:17 -0600 Subject: [PATCH] pybind/mgr: index=4, sort_keys=True for json.dumps throughput - fixes balancer test on py3 by making the pool ls order deterministic - makes everybody else behave consistently I left internal json storage alone--no need to sort/normalize. I also left out some CLI commands returning non-indented json. Signed-off-by: Sage Weil --- src/pybind/mgr/balancer/module.py | 8 ++++---- src/pybind/mgr/cephadm/module.py | 2 +- src/pybind/mgr/crash/module.py | 6 +++--- src/pybind/mgr/diskprediction_cloud/module.py | 2 +- src/pybind/mgr/influx/module.py | 4 ++-- src/pybind/mgr/orchestrator_cli/module.py | 4 ++-- src/pybind/mgr/pg_autoscaler/module.py | 2 +- src/pybind/mgr/progress/module.py | 7 ++++--- src/pybind/mgr/rbd_support/module.py | 2 +- src/pybind/mgr/restful/module.py | 2 +- src/pybind/mgr/telemetry/module.py | 4 ++-- src/pybind/mgr/volumes/fs/volume.py | 4 ++-- src/pybind/mgr/zabbix/module.py | 2 +- 13 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/pybind/mgr/balancer/module.py b/src/pybind/mgr/balancer/module.py index e2ce6cf3a3e..9ea28735e7c 100644 --- a/src/pybind/mgr/balancer/module.py +++ b/src/pybind/mgr/balancer/module.py @@ -67,7 +67,7 @@ class Plan: 'plan %s final' % self.name) def dump(self): - return json.dumps(self.inc.dump(), indent=4) + return json.dumps(self.inc.dump(), indent=4, sort_keys=True) def show(self): ls = [] @@ -429,7 +429,7 @@ class Module(MgrModule): 'optimize_result': self.optimize_result, 'mode': self.get_module_option('mode'), } - return (0, json.dumps(s, indent=4), '') + return (0, json.dumps(s, indent=4, sort_keys=True), '') elif command['prefix'] == 'balancer mode': if command['mode'] == 'upmap': min_compat_client = self.get_osdmap().dump().get('require_min_compat_client', '') @@ -476,7 +476,7 @@ class Module(MgrModule): should_prune = True if should_prune: # some pools were gone, prune self.set_module_option('pool_ids', ','.join(final_ids)) - return (0, json.dumps(final_names, indent=4), '') + return (0, json.dumps(final_names, indent=4, sort_keys=True), '') elif command['prefix'] == 'balancer pool add': raw_names = command['pools'] pool_id_by_name = dict((p['pool_name'], p['pool']) for p in self.get_osdmap().dump().get('pools', [])) @@ -565,7 +565,7 @@ class Module(MgrModule): self.plans = {} return (0, '', '') elif command['prefix'] == 'balancer ls': - return (0, json.dumps([p for p in self.plans], indent=4), '') + return (0, json.dumps([p for p in self.plans], indent=4, sort_keys=True), '') elif command['prefix'] == 'balancer dump': plan = self.plans.get(command['plan']) if not plan: diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index f33a74169d5..51d0b2a56c4 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -1475,4 +1475,4 @@ class CephadmOrchestrator(MgrModule, orchestrator.Orchestrator): 'current_name': s.container_image_name, 'current_id': s.container_image_id, } - return trivial_result(json.dumps(r, indent=4)) + return trivial_result(json.dumps(r, indent=4, sort_keys=True)) diff --git a/src/pybind/mgr/crash/module.py b/src/pybind/mgr/crash/module.py index 245e88da136..5fc68e39bf8 100644 --- a/src/pybind/mgr/crash/module.py +++ b/src/pybind/mgr/crash/module.py @@ -173,7 +173,7 @@ class Module(MgrModule): crash = self.crashes.get(crashid) if not crash: return errno.EINVAL, '', 'crash info: %s not found' % crashid - val = json.dumps(crash, indent=4) + val = json.dumps(crash, indent=4, sort_keys=True) return 0, val, '' def do_post(self, cmd, inbuf): @@ -206,7 +206,7 @@ class Module(MgrModule): if 'archived' not in crash] r = sorted(t, key=lambda i: i.get('crash_id')) if cmd.get('format') == 'json' or cmd.get('format') == 'json-pretty': - return 0, json.dumps(r, indent=4), '' + return 0, json.dumps(r, indent=4, sort_keys=True), '' else: table = PrettyTable(['ID', 'ENTITY', 'NEW'], border=False) @@ -333,7 +333,7 @@ class Module(MgrModule): pname = "unknown" report[pname] += 1 - return 0, '', json.dumps(report) + return 0, '', json.dumps(report, sort_keys=True) def self_test(self): # test time conversion diff --git a/src/pybind/mgr/diskprediction_cloud/module.py b/src/pybind/mgr/diskprediction_cloud/module.py index fb918a09102..3735a766311 100644 --- a/src/pybind/mgr/diskprediction_cloud/module.py +++ b/src/pybind/mgr/diskprediction_cloud/module.py @@ -189,7 +189,7 @@ class Module(MgrModule): def _show_prediction_config(self, cmd): self.show_module_config() - return 0, json.dumps(self.config, indent=4), '' + return 0, json.dumps(self.config, indent=4, sort_keys=True), '' def _set_ssl_target_name(self, cmd): str_ssl_target = cmd.get('ssl_target_name', '') diff --git a/src/pybind/mgr/influx/module.py b/src/pybind/mgr/influx/module.py index f3b80041c2b..14c79d3fe74 100644 --- a/src/pybind/mgr/influx/module.py +++ b/src/pybind/mgr/influx/module.py @@ -439,11 +439,11 @@ class Module(MgrModule): 'df_stats': df_stats } - return json.dumps(result, indent=2) + return json.dumps(result, indent=2, sort_keys=True) def handle_command(self, inbuf, cmd): if cmd['prefix'] == 'influx config-show': - return 0, json.dumps(self.config), '' + return 0, json.dumps(self.config, sort_keys=True), '' elif cmd['prefix'] == 'influx config-set': key = cmd['key'] value = cmd['value'] diff --git a/src/pybind/mgr/orchestrator_cli/module.py b/src/pybind/mgr/orchestrator_cli/module.py index 76b8aa107d1..8e5672116ab 100644 --- a/src/pybind/mgr/orchestrator_cli/module.py +++ b/src/pybind/mgr/orchestrator_cli/module.py @@ -87,7 +87,7 @@ class OrchestratorCli(orchestrator.OrchestratorClientMixin, MgrModule): stdout=json.dumps({ 'ident': list(self.ident), 'fault': list(self.fault) - }, indent=4)) + }, indent=4, sort_keys=True)) def light_on(self, fault_ident, devid): # type: (str, str) -> HandleCommandResult @@ -184,7 +184,7 @@ class OrchestratorCli(orchestrator.OrchestratorClientMixin, MgrModule): if format == 'json': hosts = [dict(host=node.name, labels=node.labels) for node in completion.result] - output = json.dumps(hosts) + output = json.dumps(hosts, sort_keys=True) else: table = PrettyTable( ['HOST', 'LABELS'], diff --git a/src/pybind/mgr/pg_autoscaler/module.py b/src/pybind/mgr/pg_autoscaler/module.py index da1df3fb0f7..e3dc83ff2cd 100644 --- a/src/pybind/mgr/pg_autoscaler/module.py +++ b/src/pybind/mgr/pg_autoscaler/module.py @@ -115,7 +115,7 @@ class PgAutoscaler(MgrModule): ps, root_map, pool_root = self._get_pool_status(osdmap, pools) if cmd.get('format') == 'json' or cmd.get('format') == 'json-pretty': - return 0, json.dumps(ps, indent=2), '' + return 0, json.dumps(ps, indent=4, sort_keys=True), '' else: table = PrettyTable(['POOL', 'SIZE', 'TARGET SIZE', 'RATE', 'RAW CAPACITY', diff --git a/src/pybind/mgr/progress/module.py b/src/pybind/mgr/progress/module.py index 4ab545cb73d..c7b2a0f34e1 100644 --- a/src/pybind/mgr/progress/module.py +++ b/src/pybind/mgr/progress/module.py @@ -425,7 +425,7 @@ class Module(MgrModule): )) self.log.debug( - "old_up_acting: {0}".format(json.dumps(old_up_acting, indent=2))) + "old_up_acting: {0}".format(json.dumps(old_up_acting, indent=4, sort_keys=True))) # Has this OSD been assigned a new location? # (it might not be if there is no suitable place to move @@ -437,7 +437,8 @@ class Module(MgrModule): self.log.debug( "new_up_acting: {0}".format(json.dumps(new_up_acting, - indent=2))) + indent=4, + sort_keys=True))) if was_on_out_or_in_osd and is_relocated: # This PG is now in motion, track its progress @@ -702,6 +703,6 @@ class Module(MgrModule): # that never finishes) return self._handle_clear() elif cmd['prefix'] == "progress json": - return 0, json.dumps(self._json(), indent=2), "" + return 0, json.dumps(self._json(), indent=4, sort_keys=True), "" else: raise NotImplementedError(cmd['prefix']) diff --git a/src/pybind/mgr/rbd_support/module.py b/src/pybind/mgr/rbd_support/module.py index 90f00160c1c..d652050ed0b 100644 --- a/src/pybind/mgr/rbd_support/module.py +++ b/src/pybind/mgr/rbd_support/module.py @@ -1258,7 +1258,7 @@ class TaskHandler: task.refs[TASK_REF_POOL_NAMESPACE]): result.append(task.to_dict()) - return 0, json.dumps(result), "" + return 0, json.dumps(result, indent=4, sort_keys=True), "" def handle_command(self, inbuf, prefix, cmd): with self.lock: diff --git a/src/pybind/mgr/restful/module.py b/src/pybind/mgr/restful/module.py index d73973afe42..0f8257b7bed 100644 --- a/src/pybind/mgr/restful/module.py +++ b/src/pybind/mgr/restful/module.py @@ -441,7 +441,7 @@ class Module(MgrModule): self.refresh_keys() return ( 0, - json.dumps(self.keys, indent=2), + json.dumps(self.keys, indent=4, sort_keys=True), "", ) diff --git a/src/pybind/mgr/telemetry/module.py b/src/pybind/mgr/telemetry/module.py index 551da01eb6a..cb99302f7fa 100644 --- a/src/pybind/mgr/telemetry/module.py +++ b/src/pybind/mgr/telemetry/module.py @@ -730,7 +730,7 @@ class Module(MgrModule): r = {} for opt in self.MODULE_OPTIONS: r[opt['name']] = getattr(self, opt['name']) - return 0, json.dumps(r, indent=4), '' + return 0, json.dumps(r, indent=4, sort_keys=True), '' elif command['prefix'] == 'telemetry on': if command.get('license') != LICENSE: return -errno.EPERM, '', "Telemetry data is licensed under the " + LICENSE_NAME + " (" + LICENSE_URL + ").\nTo enable, add '--license " + LICENSE + "' to the 'ceph telemetry on' command." @@ -749,7 +749,7 @@ class Module(MgrModule): report = self.compile_report( channels=command.get('channels', None) ) - return 0, json.dumps(report, indent=4), '' + return 0, json.dumps(report, indent=4, sort_keys=True), '' else: return (-errno.EINVAL, '', "Command not found '{0}'".format(command['prefix'])) diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py index 842d5ced3b0..0cb46c8dcb4 100644 --- a/src/pybind/mgr/volumes/fs/volume.py +++ b/src/pybind/mgr/volumes/fs/volume.py @@ -365,7 +365,7 @@ class VolumeClient(object): fs_map = self.mgr.get("fs_map") for f in fs_map['filesystems']: result.append({'name': f['mdsmap']['fs_name']}) - return 0, json.dumps(result, indent=2), "" + return 0, json.dumps(result, indent=4, sort_keys=True), "" def group_exists(self, sv, spec): # default group need not be explicitly created (as it gets created @@ -421,7 +421,7 @@ class VolumeClient(object): namedict = [] for i in range(len(names)): namedict.append({'name': names[i].decode('utf-8')}) - return json.dumps(namedict, indent=2) + return json.dumps(namedict, indent=4, sort_keys=True) ### subvolume operations diff --git a/src/pybind/mgr/zabbix/module.py b/src/pybind/mgr/zabbix/module.py index e864735ccc0..cc84e428de9 100644 --- a/src/pybind/mgr/zabbix/module.py +++ b/src/pybind/mgr/zabbix/module.py @@ -404,7 +404,7 @@ class Module(MgrModule): def handle_command(self, inbuf, command): if command['prefix'] == 'zabbix config-show': - return 0, json.dumps(self.config), '' + return 0, json.dumps(self.config, index=4, sort_keys=True), '' elif command['prefix'] == 'zabbix config-set': key = command['key'] value = command['value'] -- 2.39.5