json_spirit: use utf8 intenally when parsing \uHHHH

When the python CLI is given non-ASCII characters, it converts them to
\uHHHH escapes in JSON.  json_spirit parses these internally into 16 bit
characters, which could only work if json_spirit were built to use
std::wstring, which it isn't; it's using std::string, so the high byte
ends up being zero'd, leaving the low byte which is effectively garbage.

This hack^H^H^H^H change makes json_spirit convert to utf8 internally
instead, which can be stored just fine inside a std::string.

Note that this implementation still assumes \uHHHH escapes are four hex
digits, so it'll only cope with characters in the Basic Multilingual
Plane.  Still, that's rather a lot more characters than it could cope
with before ;)

(For characters outside the BMP, Python seems to generate escapes in the
form \uHHHHHHHH, i.e. 8 hex digits, which the current implementation
doesn't expect to see)

Fixes: #7387
Signed-off-by: Tim Serong <tserong@suse.com>
(cherry picked from commit 8add15b86e7aaef41397ab8fa9e77ee7957eb607)

Conflicts:
	src/test/mon/osd-pool-create.sh

        Changed $CEPH_MON to 127.0.0.1 -- the CEPH_MON was introduced after
        firefly to allow tests to run in parallel. Back in firefly all tests
        use the same port because 127.0.0.1 was hardcoded. We can't
        conveniently backport all that's necessary for tests to run in
        parallel, therefore we keep the 127.0.0.1 hardcoded.

diff --git a/src/json_spirit/json_spirit_reader_template.h b/src/json_spirit/json_spirit_reader_template.h
index f87b59331b73999c984b9b3d9226ad985e37200d..2eaf743efae4c15d53de4620a66d0f33907dc463 100644 (file)
--- a/src/json_spirit/json_spirit_reader_template.h
+++ b/src/json_spirit/json_spirit_reader_template.h
@@ -13,6 +13,8 @@
 #include "json_spirit_value.h"\r
 #include "json_spirit_error_position.h"\r
 \r
+#include "common/utf8.h"\r
+\r
 #define BOOST_SPIRIT_THREADSAFE  // uncomment for multithreaded use, requires linking to boost.thread\r
 \r
 #include <boost/bind.hpp>\r
@@ -71,18 +73,30 @@ namespace json_spirit
         return ( hex_to_num( c1 ) << 4 ) + hex_to_num( c2 );\r
     }       \r
 \r
-    template< class Char_type, class Iter_type >\r
-    Char_type unicode_str_to_char( Iter_type& begin )\r
+    template< class String_type, class Iter_type >\r
+    String_type unicode_str_to_utf8( Iter_type& begin );\r
+\r
+    template<>\r
+    std::string unicode_str_to_utf8( std::string::const_iterator & begin )\r
     {\r
+        typedef typename std::string::value_type Char_type;\r
+\r
         const Char_type c1( *( ++begin ) );\r
         const Char_type c2( *( ++begin ) );\r
         const Char_type c3( *( ++begin ) );\r
         const Char_type c4( *( ++begin ) );\r
 \r
-        return ( hex_to_num( c1 ) << 12 ) + \r
-               ( hex_to_num( c2 ) <<  8 ) + \r
-               ( hex_to_num( c3 ) <<  4 ) + \r
-               hex_to_num( c4 );\r
+        unsigned long uc = ( hex_to_num( c1 ) << 12 ) + \r
+                           ( hex_to_num( c2 ) <<  8 ) + \r
+                           ( hex_to_num( c3 ) <<  4 ) + \r
+                           hex_to_num( c4 );\r
+\r
+        unsigned char buf[7];  // MAX_UTF8_SZ is 6 (see src/common/utf8.c)\r
+        int r = encode_utf8(uc, buf);\r
+        if (r >= 0) {\r
+            return std::string(reinterpret_cast<char *>(buf), r);\r
+        }\r
+        return std::string("_");\r
     }\r
 \r
     template< class String_type >\r
@@ -116,7 +130,7 @@ namespace json_spirit
             {\r
                 if( end - begin >= 5 )  //  expecting "uHHHH..."\r
                 {\r
-                    s += unicode_str_to_char< Char_type >( begin );  \r
+                    s += unicode_str_to_utf8< String_type >( begin );\r
                 }\r
                 break;\r
             }\r
@@ -178,11 +192,15 @@ namespace json_spirit
         return get_str_< std::string >( begin, end );\r
     }\r
 \r
+// Need this guard else it tries to instantiate unicode_str_to_utf8 with a\r
+// std::wstring, which isn't presently implemented\r
+#if defined( JSON_SPIRIT_WMVALUE_ENABLED ) && !defined( BOOST_NO_STD_WSTRING )\r
     inline std::wstring get_str( std::wstring::const_iterator begin, std::wstring::const_iterator end )\r
     {\r
         return get_str_< std::wstring >( begin, end );\r
     }\r
-    \r
+#endif\r
+\r
     template< class String_type, class Iter_type >\r
     String_type get_str( Iter_type begin, Iter_type end )\r
     {\r

diff --git a/src/test/mon/osd-pool-create.sh b/src/test/mon/osd-pool-create.sh
index 0e4811b41d732445dceb7229e2a45fc75fc6a77d..ef18eb57bd26c7f89d23c644c64be24e20772511 100755 (executable)
--- a/src/test/mon/osd-pool-create.sh
+++ b/src/test/mon/osd-pool-create.sh
@@ -228,6 +228,21 @@ function TEST_replicated_pool() {
         grep 'cannot change to type erasure' || return 1
 }
 
+function TEST_utf8_cli() {
+    local dir=$1
+    run_mon $dir a --public-addr 127.0.0.1
+    # Hopefully it's safe to include literal UTF-8 characters to test
+    # the fix for http://tracker.ceph.com/issues/7387.  If it turns out
+    # to not be OK (when is the default encoding *not* UTF-8?), maybe
+    # the character '黄' can be replaced with the escape $'\xe9\xbb\x84'
+    ./ceph osd pool create 黄 1024 2>&1 | \
+        grep "pool '黄' created" || return 1
+    ./ceph osd lspools 2>&1 | \
+        grep "黄" || return 1
+    ./ceph -f json-pretty osd dump | \
+        python -c "import json; import sys; json.load(sys.stdin)" || return 1
+    ./ceph osd pool delete 黄 黄 --yes-i-really-really-mean-it
+}
 
 main osd-pool-create