From: Ilya Dryomov <idryomov@gmail.com>
Date: Fri, 16 Apr 2021 11:55:56 +0000 (+0200)
Subject: common: make y2c.py choke on duplicate keys
X-Git-Tag: v17.1.0~2224^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=7bef7e9b3b19cbbc3f4f185fe0e0caa2faae544a;p=ceph.git

common: make y2c.py choke on duplicate keys

Commit 5505fc0051a3 ("common: generate legacy_config_opts.h from
.yaml.in files") inadvertently reverted a change of a default value by
adding a second "default" key with the old value.  This was corrected
in commit 75e07f8638ef ("common/options/global: correct default of
auth_mon_ticket_ttl"), but highlights that mis-merging a yaml file
is rather easy.

To prevent this happening again, fail the build if duplicate keys
exist in any of src/common/options/*.yaml.in files.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---

diff --git a/src/common/options/y2c.py b/src/common/options/y2c.py
index 8064177296f2..3ff34fa8239a 100755
--- a/src/common/options/y2c.py
+++ b/src/common/options/y2c.py
@@ -201,6 +201,7 @@ def yaml_to_h(opt):
     else:
         return ''
 
+
 TEMPLATE_CC = '''#include "common/options.h"
 {headers}
 
@@ -212,6 +213,26 @@ std::vector<Option> get_{name}_options() {{
 '''
 
 
+# PyYAML doesn't check for duplicates even though the YAML spec says
+# that mapping keys must be unique and that duplicates must be treated
+# as an error.  See https://github.com/yaml/pyyaml/issues/165.
+#
+# This workaround breaks merge keys -- in "<<: *xyz", duplicate keys
+# from xyz mapping raise an error instead of being discarded.
+class UniqueKeySafeLoader(yaml.SafeLoader):
+    def construct_mapping(self, node, deep=False):
+        mapping = super().construct_mapping(node, deep)
+        keys = set()
+        for key_node, _ in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            if key in keys:
+                raise yaml.constructor.ConstructorError(None, None,
+                                                        "found duplicate key",
+                                                        key_node.start_mark)
+            keys.add(key)
+        return mapping
+
+
 def translate(opts):
     if opts.raw:
         prelude, epilogue = '', ''
@@ -228,7 +249,7 @@ def translate(opts):
     with open(opts.input) as infile, \
          open(opts.output, 'w') as cc_file, \
          open(opts.legacy, 'w') as h_file:
-        yml = yaml.safe_load(infile)
+        yml = yaml.load(infile, Loader=UniqueKeySafeLoader)
         headers = yml.get('headers', '')
         cc_file.write(prelude.format(name=name, headers=headers))
         options = yml['options']
@@ -292,7 +313,7 @@ def readable_millisecs(value, typ):
 
 def readable(opts):
     with open(opts.input) as infile, open(opts.output, 'w') as outfile:
-        yml = yaml.safe_load(infile)
+        yml = yaml.load(infile, Loader=UniqueKeySafeLoader)
         options = yml['options']
         for option in options:
             typ = option['type']