From: Jesse F. Williamson Date: Tue, 13 Jan 2026 22:19:22 +0000 (-0800) Subject: convert spirit_json to Boost.JSON (ceph_json, Mgr) X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e060f8e00d77c7487d2d41528b8bb9e30e8be68f;p=ceph-ci.git convert spirit_json to Boost.JSON (ceph_json, Mgr) Consolidate codecs; views, buffer::list, etc.. Move JSON encoding from source file into inline header functions; reorganize. Consolidate mostly-similar implementations with templates. Allow string_view, buffer::list to ceph_json parse(); drive-by cleanups. Assorted extensions (string_view for most search functions, etc.) Signed-off-by: Jesse F. Williamson --- diff --git a/src/common/ceph_json.cc b/src/common/ceph_json.cc index ccb5f32ecfb..e68a68a182d 100644 --- a/src/common/ceph_json.cc +++ b/src/common/ceph_json.cc @@ -1,64 +1,50 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2025 International Business Machines Corp. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * +*/ + #include "common/ceph_json.h" #include "include/utime.h" -// for testing DELETE ME -#include #include - -#include #include #include +#include +#include -#include "json_spirit/json_spirit_writer_template.h" +/* Enable boost.json's header-only mode: + (see: "https://github.com/boostorg/json?tab=readme-ov-file#header-only"): */ +#include -using namespace json_spirit; +#include +#include +#include +#include using std::ifstream; -using std::pair; using std::ostream; -using std::string; -using std::vector; -using ceph::bufferlist; -using ceph::Formatter; - -#define dout_subsys ceph_subsys_rgw - -static JSONFormattable default_formattable; +using std::begin; +using std::end; -void encode_json(const char *name, const JSONObj::data_val& v, Formatter *f) -{ - if (v.quoted) { - encode_json(name, v.str, f); - } else { - f->dump_format_unquoted(name, "%s", v.str.c_str()); - } -} - -JSONObjIter::JSONObjIter() -{ -} - -JSONObjIter::~JSONObjIter() -{ -} - -void JSONObjIter::set(const JSONObjIter::map_iter_t &_cur, const JSONObjIter::map_iter_t &_last) -{ - cur = _cur; - last = _last; -} +using std::pair; +using std::vector; -void JSONObjIter::operator++() -{ - if (cur != last) - ++cur; -} +using std::string; +using std::string_view; -JSONObj *JSONObjIter::operator*() -{ - return cur->second; -} +using ceph::bufferlist; +using ceph::Formatter; // does not work, FIXME ostream& operator<<(ostream &out, const JSONObj &obj) { @@ -66,654 +52,127 @@ ostream& operator<<(ostream &out, const JSONObj &obj) { return out; } -JSONObj::~JSONObj() -{ - for (auto iter = children.begin(); iter != children.end(); ++iter) { - JSONObj *obj = iter->second; - delete obj; - } -} - - -void JSONObj::add_child(string el, JSONObj *obj) -{ - children.insert(pair(el, obj)); -} - -bool JSONObj::get_attr(string name, data_val& attr) -{ - auto iter = attr_map.find(name); - if (iter == attr_map.end()) - return false; - attr = iter->second; - return true; -} - -JSONObjIter JSONObj::find(const string& name) -{ - JSONObjIter iter; - auto first = children.find(name); - if (first != children.end()) { - auto last = children.upper_bound(name); - iter.set(first, last); - } - return iter; -} - -JSONObjIter JSONObj::find_first() -{ - JSONObjIter iter; - iter.set(children.begin(), children.end()); - return iter; -} - -JSONObjIter JSONObj::find_first(const string& name) -{ - JSONObjIter iter; - auto first = children.find(name); - iter.set(first, children.end()); - return iter; -} - -JSONObj *JSONObj::find_obj(const string& name) -{ - JSONObjIter iter = find(name); - if (iter.end()) - return NULL; - - return *iter; -} - -bool JSONObj::get_data(const string& key, data_val *dest) -{ - JSONObj *obj = find_obj(key); - if (!obj) - return false; - - *dest = obj->get_data_val(); - - return true; -} - /* accepts a JSON Array or JSON Object contained in - * a JSON Spirit Value, v, and creates a JSONObj for each + * a Boost.JSON value, v, and creates a Ceph JSONObj for each * child contained in v */ -void JSONObj::handle_value(Value v) -{ - if (v.type() == obj_type) { - Object temp_obj = v.get_obj(); - for (Object::size_type i = 0; i < temp_obj.size(); i++) { - Pair temp_pair = temp_obj[i]; - string temp_name = temp_pair.name_; - Value temp_value = temp_pair.value_; - JSONObj *child = new JSONObj; - child->init(this, temp_value, temp_name); - add_child(temp_name, child); - } - } else if (v.type() == array_type) { - Array temp_array = v.get_array(); - Value value; - - for (unsigned j = 0; j < temp_array.size(); j++) { - Value cur = temp_array[j]; - string temp_name; - - JSONObj *child = new JSONObj; - child->init(this, cur, temp_name); - add_child(child->get_name(), child); - } - } -} - -void JSONObj::init(JSONObj *p, Value v, string n) +void JSONObj::handle_value(boost::json::value v) { - name = n; - parent = p; - data = v; + if (auto op = v.if_object()) { + for (const auto& kvp : *op) { + auto child = std::make_unique(this, kvp.key(), kvp.value()); + children.emplace(std::pair { kvp.key(), std::move(child) }); + } - handle_value(v); - if (v.type() == str_type) { - val.set(v.get_str(), true); - } else { - val.set(json_spirit::write_string(v), false); + return; } - attr_map.insert(pair(name, val)); -} -JSONObj *JSONObj::get_parent() -{ - return parent; -} - -bool JSONObj::is_object() -{ - return (data.type() == obj_type); -} + if (auto ap = v.if_array()) { + for (const auto& kvp : *ap) { + auto child = std::make_unique(this, "", kvp); + children.emplace(std::pair { child->get_name(), std::move(child) }); + } + } -bool JSONObj::is_array() -{ - return (data.type() == array_type); + // unknown type is not-an-error } vector JSONObj::get_array_elements() { - vector elements; - Array temp_array; - - if (data.type() == array_type) - temp_array = data.get_array(); - - int array_size = temp_array.size(); - if (array_size > 0) - for (int i = 0; i < array_size; i++) { - Value temp_value = temp_array[i]; - string temp_string; - temp_string = write(temp_value, raw_utf8); - elements.push_back(temp_string); - } + vector elements; + if (!data.is_array()) return elements; -} - -JSONParser::JSONParser() : buf_len(0), success(true) -{ -} - -JSONParser::~JSONParser() -{ -} - - - -void JSONParser::handle_data(const char *s, int len) -{ - json_buffer.append(s, len); // check for problems with null termination FIXME - buf_len += len; -} - -// parse a supplied JSON fragment -bool JSONParser::parse(const char *buf_, int len) -{ - if (!buf_) { - set_failure(); - return false; - } - - string json_string(buf_, len); - success = read(json_string, data); - if (success) { - handle_value(data); - if (data.type() != obj_type && - data.type() != array_type) { - if (data.type() == str_type) { - val.set(data.get_str(), true); - } else { - const std::string& s = json_spirit::write_string(data); - if (s.size() == (uint64_t)len) { /* Check if entire string is read */ - val.set(s, false); - } else { - set_failure(); - } - } - } - } else { - set_failure(); - } - return success; -} + std::ranges::for_each(data.as_array(), [&elements](const auto& i) { + elements.emplace_back(boost::json::serialize(i)); + }); -// parse the internal json_buffer up to len -bool JSONParser::parse(int len) -{ - string json_string = json_buffer.substr(0, len); - success = read(json_string, data); - if (success) - handle_value(data); - else - set_failure(); - - return success; + return elements; } // parse the complete internal json_buffer bool JSONParser::parse() { - success = read(json_buffer, data); - if (success) - handle_value(data); - else - set_failure(); - - return success; -} - -// parse a supplied ifstream, for testing. DELETE ME -bool JSONParser::parse(const char *file_name) -{ - ifstream is(file_name); - success = read(is, data); - if (success) - handle_value(data); - else - set_failure(); - - return success; -} - - -void decode_json_obj(long& val, JSONObj *obj) -{ - string s = obj->get_data(); - const char *start = s.c_str(); - char *p; - - errno = 0; - val = strtol(start, &p, 10); - - /* Check for various possible errors */ - - if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) || - (errno != 0 && val == 0)) { - throw JSONDecoder::err("failed to parse number"); - } - - if (p == start) { - throw JSONDecoder::err("failed to parse number"); - } - - while (*p != '\0') { - if (!isspace(*p)) { - throw JSONDecoder::err("failed to parse number"); - } - p++; - } -} - -void decode_json_obj(unsigned long& val, JSONObj *obj) -{ - string s = obj->get_data(); - const char *start = s.c_str(); - char *p; - - errno = 0; - val = strtoul(start, &p, 10); - - /* Check for various possible errors */ - - if ((errno == ERANGE && val == ULONG_MAX) || - (errno != 0 && val == 0)) { - throw JSONDecoder::err("failed to number"); - } - - if (p == start) { - throw JSONDecoder::err("failed to parse number"); - } - - while (*p != '\0') { - if (!isspace(*p)) { - throw JSONDecoder::err("failed to parse number"); - } - p++; - } -} - -void decode_json_obj(long long& val, JSONObj *obj) -{ - string s = obj->get_data(); - const char *start = s.c_str(); - char *p; - - errno = 0; - val = strtoll(start, &p, 10); + if (!parse_json(json_buffer, data)) + return false; - /* Check for various possible errors */ + handle_value(data); - if ((errno == ERANGE && (val == LLONG_MAX || val == LLONG_MIN)) || - (errno != 0 && val == 0)) { - throw JSONDecoder::err("failed to parse number"); - } - - if (p == start) { - throw JSONDecoder::err("failed to parse number"); - } - - while (*p != '\0') { - if (!isspace(*p)) { - throw JSONDecoder::err("failed to parse number"); - } - p++; - } + return true; } -void decode_json_obj(unsigned long long& val, JSONObj *obj) +// parse the internal json_buffer up to len +bool JSONParser::parse(int len) { - string s = obj->get_data(); - const char *start = s.c_str(); - char *p; - - errno = 0; - val = strtoull(start, &p, 10); - - /* Check for various possible errors */ + if (!parse_json(std::string_view { std::begin(json_buffer), len + std::begin(json_buffer) }, data)) + return false; - if ((errno == ERANGE && val == ULLONG_MAX) || - (errno != 0 && val == 0)) { - throw JSONDecoder::err("failed to number"); - } - - if (p == start) { - throw JSONDecoder::err("failed to parse number"); - } - - while (*p != '\0') { - if (!isspace(*p)) { - throw JSONDecoder::err("failed to parse number"); - } - p++; - } -} - -void decode_json_obj(int& val, JSONObj *obj) -{ - long l; - decode_json_obj(l, obj); -#if LONG_MAX > INT_MAX - if (l > INT_MAX || l < INT_MIN) { - throw JSONDecoder::err("integer out of range"); - } -#endif + handle_value(data); - val = (int)l; + return true; } -void decode_json_obj(unsigned& val, JSONObj *obj) +// parse a supplied JSON fragment: +bool JSONParser::parse(std::string_view json_string_view) { - unsigned long l; - decode_json_obj(l, obj); -#if ULONG_MAX > UINT_MAX - if (l > UINT_MAX) { - throw JSONDecoder::err("unsigned integer out of range"); - } -#endif + // The original implementation checked this, I'm not sure we have to but we're doing it for now: + if (json_string_view.empty()) + return false; - val = (unsigned)l; -} + if (!parse_json(json_string_view, data)) + return false; -void decode_json_obj(bool& val, JSONObj *obj) -{ - string s = obj->get_data(); - if (strcasecmp(s.c_str(), "true") == 0) { - val = true; - return; - } - if (strcasecmp(s.c_str(), "false") == 0) { - val = false; - return; - } - int i; - decode_json_obj(i, obj); - val = (bool)i; -} - -void decode_json_obj(bufferlist& val, JSONObj *obj) -{ - string s = obj->get_data(); + // recursively evaluate the result: + handle_value(data); - bufferlist bl; - bl.append(s.c_str(), s.size()); - try { - val.decode_base64(bl); - } catch (ceph::buffer::error& err) { - throw JSONDecoder::err("failed to decode base64"); - } -} + if (data.is_object() or data.is_array()) + return true; -void decode_json_obj(utime_t& val, JSONObj *obj) -{ - string s = obj->get_data(); - uint64_t epoch; - uint64_t nsec; - int r = utime_t::parse_date(s, &epoch, &nsec); - if (r == 0) { - val = utime_t(epoch, nsec); - } else { - throw JSONDecoder::err("failed to decode utime_t"); - } -} + if (data.is_string()) { + val.set(data.as_string(), true); + return true; + } -void decode_json_obj(ceph::real_time& val, JSONObj *obj) -{ - const std::string& s = obj->get_data(); - uint64_t epoch; - uint64_t nsec; - int r = utime_t::parse_date(s, &epoch, &nsec); - if (r == 0) { - using namespace std::chrono; - val = real_time{seconds(epoch) + nanoseconds(nsec)}; - } else { - throw JSONDecoder::err("failed to decode real_time"); - } -} + // For any other kind of value: + std::string s = boost::json::serialize(data); -void decode_json_obj(ceph::coarse_real_time& val, JSONObj *obj) -{ - const std::string& s = obj->get_data(); - uint64_t epoch; - uint64_t nsec; - int r = utime_t::parse_date(s, &epoch, &nsec); - if (r == 0) { - using namespace std::chrono; - val = coarse_real_time{seconds(epoch) + nanoseconds(nsec)}; - } else { - throw JSONDecoder::err("failed to decode coarse_real_time"); + // Was the entire string read? + if (s.size() == static_cast(json_string_view.length())) { + val.set(s, false); + return true; } -} -void decode_json_obj(ceph_dir_layout& i, JSONObj *obj){ - - unsigned tmp; - JSONDecoder::decode_json("dir_hash", tmp, obj, true); - i.dl_dir_hash = tmp; - JSONDecoder::decode_json("unused1", tmp, obj, true); - i.dl_unused1 = tmp; - JSONDecoder::decode_json("unused2", tmp, obj, true); - i.dl_unused2 = tmp; - JSONDecoder::decode_json("unused3", tmp, obj, true); - i.dl_unused3 = tmp; + // Could not parse and convert: + return false; } -void encode_json(const char *name, std::string_view val, Formatter *f) +bool JSONParser::parse_file(const std::filesystem::path file_name) { - f->dump_string(name, val); -} + ifstream is(file_name); -void encode_json(const char *name, const string& val, Formatter *f) -{ - f->dump_string(name, val); -} - -void encode_json(const char *name, const char *val, Formatter *f) -{ - f->dump_string(name, val); -} - -void encode_json(const char *name, bool val, Formatter *f) -{ - f->dump_bool(name, val); -} - -void encode_json(const char *name, int val, Formatter *f) -{ - f->dump_int(name, val); -} - -void encode_json(const char *name, long val, Formatter *f) -{ - f->dump_int(name, val); -} - -void encode_json(const char *name, unsigned val, Formatter *f) -{ - f->dump_unsigned(name, val); -} - -void encode_json(const char *name, unsigned long val, Formatter *f) -{ - f->dump_unsigned(name, val); -} - -void encode_json(const char *name, unsigned long long val, Formatter *f) -{ - f->dump_unsigned(name, val); -} - -void encode_json(const char *name, long long val, Formatter *f) -{ - f->dump_int(name, val); -} - -void encode_json(const char *name, const utime_t& val, Formatter *f) -{ - val.gmtime(f->dump_stream(name)); -} - -void encode_json(const char *name, const ceph::real_time& val, Formatter *f) -{ - encode_json(name, utime_t{val}, f); -} - -void encode_json(const char *name, const ceph::coarse_real_time& val, Formatter *f) -{ - encode_json(name, utime_t{val}, f); -} + if (!is.is_open()) { + throw std::runtime_error(fmt::format("unable to open \"{}\"", file_name.string())); + } -void encode_json(const char *name, const bufferlist& bl, Formatter *f) -{ - /* need to copy data from bl, as it is const bufferlist */ - bufferlist src = bl; + std::error_code ec; + data = boost::json::parse(is, ec); - bufferlist b64; - src.encode_base64(b64); + if (ec) + return false; - string s(b64.c_str(), b64.length()); + handle_value(data); - encode_json(name, s, f); + return true; } - - /* JSONFormattable */ -const JSONFormattable& JSONFormattable::operator[](const string& name) const -{ - auto i = obj.find(name); - if (i == obj.end()) { - return default_formattable; - } - return i->second; -} - -const JSONFormattable& JSONFormattable::operator[](size_t index) const -{ - if (index >= arr.size()) { - return default_formattable; - } - return arr[index]; -} - -JSONFormattable& JSONFormattable::operator[](const string& name) -{ - auto i = obj.find(name); - if (i == obj.end()) { - return default_formattable; - } - return i->second; -} - -JSONFormattable& JSONFormattable::operator[](size_t index) -{ - if (index >= arr.size()) { - return default_formattable; - } - return arr[index]; -} - -bool JSONFormattable::exists(const string& name) const -{ - auto i = obj.find(name); - return (i != obj.end()); -} - -bool JSONFormattable::exists(size_t index) const -{ - return (index < arr.size()); -} - -bool JSONFormattable::find(const string& name, string *val) const -{ - auto i = obj.find(name); - if (i == obj.end()) { - return false; - } - *val = i->second.val(); - return true; -} - -int JSONFormattable::val_int() const { - return atoi(value.str.c_str()); -} - -long JSONFormattable::val_long() const { - return atol(value.str.c_str()); -} - -long long JSONFormattable::val_long_long() const { - return atoll(value.str.c_str()); -} - bool JSONFormattable::val_bool() const { return (boost::iequals(value.str, "true") || + boost::iequals(value.str, "1") || boost::iequals(value.str, "on") || - boost::iequals(value.str, "yes") || - boost::iequals(value.str, "1")); -} - -string JSONFormattable::def(const string& def_val) const { - if (type == FMT_NONE) { - return def_val; - } - return val(); -} - -int JSONFormattable::def(int def_val) const { - if (type == FMT_NONE) { - return def_val; - } - return val_int(); -} - -bool JSONFormattable::def(bool def_val) const { - if (type == FMT_NONE) { - return def_val; - } - return val_bool(); -} - -string JSONFormattable::get(const string& name, const string& def_val) const -{ - return (*this)[name].def(def_val); -} - -int JSONFormattable::get_int(const string& name, int def_val) const -{ - return (*this)[name].def(def_val); -} - -bool JSONFormattable::get_bool(const string& name, bool def_val) const -{ - return (*this)[name].def(def_val); + boost::iequals(value.str, "yes")); } struct field_entity { @@ -722,7 +181,8 @@ struct field_entity { int index{0}; /* if array */ bool append{false}; - field_entity() {} + field_entity() = default; + explicit field_entity(std::string_view sv) : is_obj(true), name(sv) {} explicit field_entity(const string& n) : is_obj(true), name(n) {} explicit field_entity(int i) : is_obj(false), index(i) {} }; @@ -737,12 +197,14 @@ static int parse_entity(const string& s, vector *result) if (ofs != 0) { return -EINVAL; } - result->push_back(field_entity(s)); + result->emplace_back(field_entity(s)); return 0; } if (next_arr > ofs) { - string field = s.substr(ofs, next_arr - ofs); - result->push_back(field_entity(field)); + + auto field = string_view(ofs + begin(s), next_arr - ofs + begin(s)); + + result->emplace_back(field_entity(field)); ofs = next_arr; } size_t end_arr = s.find(']', next_arr + 1); @@ -750,16 +212,22 @@ static int parse_entity(const string& s, vector *result) return -EINVAL; } - string index_str = s.substr(next_arr + 1, end_arr - next_arr - 1); + auto index_str = string_view(s).substr(1 + next_arr, end_arr - next_arr - 1); ofs = end_arr + 1; if (!index_str.empty()) { - result->push_back(field_entity(atoi(index_str.c_str()))); + + int x; + if (auto [_, ec] = std::from_chars(begin(index_str), end(index_str), x); std::errc() == ec) + result->emplace_back(field_entity(x)); + else + throw std::invalid_argument(fmt::format("{}", index_str)); + } else { field_entity f; f.append = true; - result->push_back(f); + result->emplace_back(f); } } return 0; @@ -784,7 +252,7 @@ int JSONFormattable::set(const string& name, const string& val) JSONParser jp; - bool is_valid_json = jp.parse(val.c_str(), val.size()); + bool is_valid_json = jp.parse(val); for (const auto& i : tok) { vector v; @@ -919,36 +387,10 @@ void JSONFormattable::derive_from(const JSONFormattable& parent) } } -void encode_json(const char *name, const JSONFormattable& v, Formatter *f) -{ - v.encode_json(name, f); -} - -void JSONFormattable::encode_json(const char *name, Formatter *f) const -{ - switch (type) { - case JSONFormattable::FMT_VALUE: - ::encode_json(name, value, f); - break; - case JSONFormattable::FMT_ARRAY: - ::encode_json(name, arr, f); - break; - case JSONFormattable::FMT_OBJ: - f->open_object_section(name); - for (auto iter : obj) { - ::encode_json(iter.first.c_str(), iter.second, f); - } - f->close_section(); - break; - case JSONFormattable::FMT_NONE: - break; - } -} - bool JSONFormattable::handle_value(std::string_view name, std::string_view s, bool quoted) { JSONFormattable *new_val; if (cur_enc->is_array()) { - cur_enc->arr.push_back(JSONFormattable()); + cur_enc->arr.emplace_back(JSONFormattable()); new_val = &cur_enc->arr.back(); } else { cur_enc->set_type(JSONFormattable::FMT_OBJ); @@ -964,7 +406,7 @@ bool JSONFormattable::handle_open_section(std::string_view name, const char *ns, bool section_is_array) { if (cur_enc->is_array()) { - cur_enc->arr.push_back(JSONFormattable()); + cur_enc->arr.emplace_back(JSONFormattable()); cur_enc = &cur_enc->arr.back(); } else if (enc_stack.size() > 1) { /* only open a new section if already nested, @@ -972,7 +414,7 @@ bool JSONFormattable::handle_open_section(std::string_view name, */ cur_enc = &cur_enc->obj[string{name}]; } - enc_stack.push_back(cur_enc); + enc_stack.emplace_back(cur_enc); if (section_is_array) { cur_enc->set_type(JSONFormattable::FMT_ARRAY); diff --git a/src/common/ceph_json.h b/src/common/ceph_json.h index a2e4fba5d7c..567ceebb316 100644 --- a/src/common/ceph_json.h +++ b/src/common/ceph_json.h @@ -1,49 +1,168 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2025 International Business Machines Corp. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * +*/ + #ifndef CEPH_JSON_H #define CEPH_JSON_H -#include +#include + #include #include -#include +#include #include + +#include +#include + +#include +#include + #include -#include "include/encoding.h" +#include + #include +#include + +#include +#include + +#include + #include #include -#include + +#include + +#include #include + +#include "common/strtol.h" #include "common/ceph_time.h" -#include "json_spirit/json_spirit.h" +#include #include "JSONFormatter.h" +#include "include/encoding.h" +class utime_t; class JSONObj; +class JSONFormattable; + +namespace ceph_json::detail { + +template +consteval bool is_any_of() +{ + return (std::is_same_v || ...); +} + +/* Note that std::is_integer<> will also pick up bool, which in our +case we need to count as a non-integer type as the JSON codecs treat +it differently ("https://en.cppreference.com/w/cpp/types/is_integral"). + +From what I can see in the extant code, it looks like the same rules +should be applied to char, etc., so I've also done so here: +*/ +template +concept json_integer = requires +{ + requires std::is_integral_v; + requires !std::is_same_v; + requires !is_any_of(); +}; + +template +concept json_signed_integer = requires +{ + requires json_integer && std::signed_integral; +}; + +template +concept json_unsigned_integer = requires +{ + requires json_integer && std::unsigned_integral; +}; + +/* Distinguish between containers with a value that's an associative kv-pair (a mapped type) and +those which are a "single" value. Note that this is not the same as the AssociativeContainer +named concept, as the rule there is that the container is key-indexed, and it does not necessarily +have to be a pair (e.g. std::set<> is an AssociativeContainer). Similarly, for sequence types +we don't want to capture standard strings and the like, even if we otherwise could consider them +a value sequence: +*/ +template +concept json_mapped_kv_seq = requires +{ + typename ContainerT::key_type; + typename ContainerT::key_compare; + + typename ContainerT::value_type; + typename ContainerT::mapped_type; +}; + +template +concept json_val_seq = requires +{ + typename ContainerT::value_type; + + requires !json_mapped_kv_seq; + requires !std::convertible_to; +}; + +} // namespace ceph_json + +class JSONObjIter final { + + using map_iter_t = boost::container::flat_map, std::less<>>::iterator; -class JSONObjIter { - typedef std::map::iterator map_iter_t; map_iter_t cur; map_iter_t last; -public: - JSONObjIter(); - ~JSONObjIter(); - void set(const JSONObjIter::map_iter_t &_cur, const JSONObjIter::map_iter_t &_end); +private: + JSONObjIter(const JSONObjIter::map_iter_t& cur_, const JSONObjIter::map_iter_t& end_) + : cur(cur_), + last(end_) + {} - void operator++(); - JSONObj *operator*(); +public: + JSONObjIter() = default; - bool end() const { - return (cur == last); +public: + void set(const JSONObjIter::map_iter_t &cur_, const JSONObjIter::map_iter_t &end_) { + cur = cur_; + last = end_; } + + void operator++() { if (cur != last) ++cur; } + + // IMPORTANT: The returned pointer is intended as NON-OWNING (i.e. JSONObjIter + // is responsible for it): + JSONObj *operator*() { return cur->second.get(); } + + bool end() const { return (cur == last); } + +private: + friend JSONObj; }; -class JSONObj +class JSONObj { - JSONObj *parent; + JSONObj *parent = nullptr; + public: struct data_val { std::string str; @@ -54,40 +173,109 @@ public: quoted = q; } }; + protected: std::string name; // corresponds to obj_type in XMLObj - json_spirit::Value data; - struct data_val val; + + boost::json::value data; + + data_val val; + bool data_quoted{false}; - std::multimap children; - std::map attr_map; - void handle_value(json_spirit::Value v); + + boost::container::flat_multimap, std::less<>> children; + boost::container::flat_map> attr_map; + + void handle_value(boost::json::value v); + +protected: + // Although the error_code contains useful information, the API constraints require + // that we throw it out: + static bool parse_json(std::string_view input, boost::json::value& data_out) + { + std::error_code ec; + + data_out = boost::json::parse(input, ec, boost::json::storage_ptr(), + { .allow_invalid_utf8 = true }); + + return ec ? false : true; + } public: + JSONObj() = default; + + JSONObj(JSONObj *parent_node, std::string_view name_in, boost::json::value data_in) + : parent { parent_node }, + name { name_in }, + data { data_in } + { + handle_value(data); + + if (auto vp = data_in.if_string()) + val.set(*vp, true); + else + val.set(boost::json::serialize(data), false); + + attr_map.insert({ name, val }); + } + + virtual ~JSONObj() = default; + +public: + std::string& get_name() noexcept { return name; } + data_val& get_data_val() noexcept { return val; } + + const std::string& get_data() const noexcept { return val.str; } + + bool get_data(std::string_view key, data_val *dest) { + + JSONObj *obj = find_obj(key); + if (!obj) + return false; + + *dest = obj->get_data_val(); + + return true; + } + + JSONObj *get_parent() const noexcept { return parent; }; + + bool get_attr(std::string_view name, data_val& attr) { + if (auto i = attr_map.find(name); end(attr_map) != i) + return (attr = i->second), true; + + return false; + } - JSONObj() : parent(NULL){} + JSONObjIter find(std::string_view name) { + auto fst = children.find(name); - virtual ~JSONObj(); + if(end(children) != fst) { + return { fst, children.upper_bound(name) }; + } + + return { fst, std::end(children) }; + } + + JSONObjIter find_first() { + return { children.begin(), children.end() }; + } - void init(JSONObj *p, json_spirit::Value v, std::string n); + JSONObjIter find_first(std::string_view name) { + return { children.find(name), children.end() }; + } - std::string& get_name() { return name; } - data_val& get_data_val() { return val; } - const std::string& get_data() { return val.str; } - bool get_data(const std::string& key, data_val *dest); - JSONObj *get_parent(); - void add_child(std::string el, JSONObj *child); - bool get_attr(std::string name, data_val& attr); - JSONObjIter find(const std::string& name); - JSONObjIter find_first(); - JSONObjIter find_first(const std::string& name); - JSONObj *find_obj(const std::string& name); + JSONObj *find_obj(std::string_view name) { + JSONObjIter i = this->find(name); + return i.end() ? nullptr : *i; + } friend std::ostream& operator<<(std::ostream &out, const JSONObj &obj); // does not work, FIXME - bool is_array(); - bool is_object(); + bool is_array() const noexcept { return data.is_array(); } + bool is_object() const noexcept { return data.is_object(); } + std::vector get_array_elements(); }; @@ -97,28 +285,40 @@ inline std::ostream& operator<<(std::ostream &out, const JSONObj::data_val& dv) return out; } -class JSONParser : public JSONObj +class JSONParser final : public JSONObj { - int buf_len; + int buf_len = 0; std::string json_buffer; - bool success; + public: - JSONParser(); - ~JSONParser() override; - void handle_data(const char *s, int len); + ~JSONParser() override = default; - bool parse(const char *buf_, int len); - bool parse(int len); +public: + // operate on the internal buffer: bool parse(); - bool parse(const char *file_name); + bool parse(int len); - const char *get_json() { return json_buffer.c_str(); } - void set_failure() { success = false; } -}; + // operate on a string/stringlike range or object: + bool parse(std::string_view sv); + + bool parse(const char *buf_, int len) { + return buf_ ? + parse(std::string_view { buf_, static_cast(len) }) + : false; + } -void encode_json(const char *name, const JSONObj::data_val& v, ceph::Formatter *f); + bool parse(ceph::buffer::list& bl) { + // JFW: return parse(bl.c_str(), bl.length()); + return parse(bl.to_str()); + } + + [[deprecated("this may not be reliable")]] bool parse_file(const std::filesystem::path file_name); + +public: + const char *get_json() const noexcept{ return json_buffer.c_str(); } +}; -class JSONDecoder { +class JSONDecoder final { public: struct err : std::runtime_error { using runtime_error::runtime_error; @@ -127,29 +327,38 @@ public: JSONParser parser; JSONDecoder(ceph::buffer::list& bl) { - if (!parser.parse(bl.c_str(), bl.length())) { - std::cout << "JSONDecoder::err()" << std::endl; + if (!parser.parse(bl.c_str(), bl.length())) throw JSONDecoder::err("failed to parse JSON input"); - } } template - static bool decode_json(const char *name, T& val, JSONObj *obj, bool mandatory = false); + static bool decode_json(std::string_view name, T& val, JSONObj *obj, bool mandatory = false); template - static bool decode_json(const char *name, C& container, void (*cb)(C&, JSONObj *obj), JSONObj *obj, bool mandatory = false); + static bool decode_json(std::string_view name, C& container, void (*cb)(C&, JSONObj *obj), JSONObj *obj, bool mandatory = false); template - static void decode_json(const char *name, T& val, const T& default_val, JSONObj *obj); + static void decode_json(std::string_view name, T& val, const T& default_val, JSONObj *obj); template - static bool decode_json(const char *name, boost::optional& val, JSONObj *obj, bool mandatory = false); + static bool decode_json(std::string_view name, boost::optional& val, JSONObj *obj, bool mandatory = false); template - static bool decode_json(const char *name, std::optional& val, JSONObj *obj, bool mandatory = false); - + static bool decode_json(std::string_view name, std::optional& val, JSONObj *obj, bool mandatory = false); }; +template +requires ceph_json::detail::json_integer +void decode_json_obj(IntegerT& val, JSONObj *obj) +{ + auto r = ceph::parse(obj->get_data()); + + if(!r) + throw JSONDecoder::err(fmt::format("failed to parse number from JSON")); + + val = *r; +} + template void decode_json_obj(T& val, JSONObj *obj) { @@ -161,183 +370,149 @@ inline void decode_json_obj(std::string& val, JSONObj *obj) val = obj->get_data(); } -static inline void decode_json_obj(JSONObj::data_val& val, JSONObj *obj) +inline void decode_json_obj(JSONObj::data_val& val, JSONObj *obj) { val = obj->get_data_val(); } -void decode_json_obj(unsigned long long& val, JSONObj *obj); -void decode_json_obj(long long& val, JSONObj *obj); -void decode_json_obj(unsigned long& val, JSONObj *obj); -void decode_json_obj(long& val, JSONObj *obj); -void decode_json_obj(unsigned& val, JSONObj *obj); -void decode_json_obj(int& val, JSONObj *obj); -void decode_json_obj(bool& val, JSONObj *obj); -void decode_json_obj(ceph::buffer::list& val, JSONObj *obj); -class utime_t; -void decode_json_obj(utime_t& val, JSONObj *obj); -void decode_json_obj(ceph_dir_layout& i, JSONObj *obj); - -void decode_json_obj(ceph::real_time& val, JSONObj *obj); -void decode_json_obj(ceph::coarse_real_time& val, JSONObj *obj); - -template -void decode_json_obj(std::list& l, JSONObj *obj) +inline void decode_json_obj(bool& val, JSONObj *obj) { - l.clear(); + std::string_view sv(obj->get_data()); - JSONObjIter iter = obj->find_first(); + if (boost::iequals(sv, "true")) + { + val = true; + return; + } - for (; !iter.end(); ++iter) { - T val; - JSONObj *o = *iter; - decode_json_obj(val, o); - l.push_back(val); + if (boost::iequals(sv, "false")) + { + val = false; + return; } + + // For 1, 0, anything else: + int i; + decode_json_obj(i, obj); + val = static_cast(i); } -template -void decode_json_obj(std::deque& l, JSONObj *obj) +inline void decode_json_obj(bufferlist& val, JSONObj *obj) { - l.clear(); + bufferlist bl; - JSONObjIter iter = obj->find_first(); + std::string_view sv = obj->get_data(); - for (; !iter.end(); ++iter) { - T val; - JSONObj *o = *iter; - decode_json_obj(val, o); - l.push_back(val); - } -} + bl.append(sv); -template -void decode_json_obj(std::set& l, JSONObj *obj) -{ - l.clear(); - - JSONObjIter iter = obj->find_first(); + try { + val.decode_base64(bl); - for (; !iter.end(); ++iter) { - T val; - JSONObj *o = *iter; - decode_json_obj(val, o); - l.insert(val); + } catch (ceph::buffer::error& err) { + throw JSONDecoder::err("failed to decode base64"); } } -template -void decode_json_obj(boost::container::flat_set& l, JSONObj *obj) +inline void decode_json_obj(utime_t& val, JSONObj *obj) { - l.clear(); - - JSONObjIter iter = obj->find_first(); - - for (; !iter.end(); ++iter) { - T val; - JSONObj *o = *iter; - decode_json_obj(val, o); - l.insert(val); + uint64_t epoch; + uint64_t nsec; + int r = utime_t::parse_date(obj->get_data(), &epoch, &nsec); + if (r == 0) { + val = utime_t(epoch, nsec); + } else { + throw JSONDecoder::err("failed to decode utime_t"); } } -template -void decode_json_obj(std::vector& l, JSONObj *obj) +inline void decode_json_obj(ceph::real_time& val, JSONObj *obj) { - l.clear(); - - JSONObjIter iter = obj->find_first(); - - for (; !iter.end(); ++iter) { - T val; - JSONObj *o = *iter; - decode_json_obj(val, o); - l.push_back(val); + uint64_t epoch; + uint64_t nsec; + int r = utime_t::parse_date(obj->get_data(), &epoch, &nsec); + if (r == 0) { + using namespace std::chrono; + val = real_time{seconds(epoch) + nanoseconds(nsec)}; + } else { + throw JSONDecoder::err("failed to decode real_time"); } } -template > -void decode_json_obj(std::map& m, JSONObj *obj) +inline void decode_json_obj(ceph::coarse_real_time& val, JSONObj *obj) { - m.clear(); - - JSONObjIter iter = obj->find_first(); - - for (; !iter.end(); ++iter) { - K key; - V val; - JSONObj *o = *iter; - JSONDecoder::decode_json("key", key, o); - JSONDecoder::decode_json("val", val, o); - m[key] = val; + uint64_t epoch; + uint64_t nsec; + int r = utime_t::parse_date(obj->get_data(), &epoch, &nsec); + if (r == 0) { + using namespace std::chrono; + val = coarse_real_time{seconds(epoch) + nanoseconds(nsec)}; + } else { + throw JSONDecoder::err("failed to decode coarse_real_time"); } } -template > -void decode_json_obj(boost::container::flat_map& m, JSONObj *obj) +inline void decode_json_obj(ceph_dir_layout& i, JSONObj *obj) { - m.clear(); - - JSONObjIter iter = obj->find_first(); - - for (; !iter.end(); ++iter) { - K key; - V val; - JSONObj *o = *iter; - JSONDecoder::decode_json("key", key, o); - JSONDecoder::decode_json("val", val, o); - m[key] = val; - } + unsigned tmp; + JSONDecoder::decode_json("dir_hash", tmp, obj, true); + i.dl_dir_hash = tmp; + JSONDecoder::decode_json("unused1", tmp, obj, true); + i.dl_unused1 = tmp; + JSONDecoder::decode_json("unused2", tmp, obj, true); + i.dl_unused2 = tmp; + JSONDecoder::decode_json("unused3", tmp, obj, true); + i.dl_unused3 = tmp; } -template -void decode_json_obj(std::multimap& m, JSONObj *obj) +template +void decode_json_obj(SeqT& seq, JSONObj *obj) { - m.clear(); - - JSONObjIter iter = obj->find_first(); + seq.clear(); - for (; !iter.end(); ++iter) { - K key; - V val; + for (auto iter = obj->find_first(); !iter.end(); ++iter) { + typename SeqT::value_type val; JSONObj *o = *iter; - JSONDecoder::decode_json("key", key, o); - JSONDecoder::decode_json("val", val, o); - m.insert(make_pair(key, val)); - } + decode_json_obj(val, o); + + if constexpr (requires { seq.emplace_back(val); }) + seq.emplace_back(val); + else + seq.emplace(val); + } } -template -void decode_json_obj(boost::container::flat_map& m, JSONObj *obj) +template +void decode_json_obj(KVSeqT& kvs, JSONObj *obj) { - m.clear(); - - JSONObjIter iter = obj->find_first(); + kvs.clear(); - for (; !iter.end(); ++iter) { - K key; - V val; + for (auto iter = obj->find_first(); !iter.end(); ++iter) { + typename KVSeqT::key_type key; + typename KVSeqT::mapped_type val; JSONObj *o = *iter; JSONDecoder::decode_json("key", key, o); JSONDecoder::decode_json("val", val, o); - m[key] = val; + + if constexpr(requires { kvs[key] = val; }) + kvs[key] = val; // i.e. insert_or_assign() + else + kvs.insert({key, val}); } } + template void decode_json_obj(C& container, void (*cb)(C&, JSONObj *obj), JSONObj *obj) { container.clear(); - JSONObjIter iter = obj->find_first(); - - for (; !iter.end(); ++iter) { + for (auto iter = obj->find_first(); !iter.end(); ++iter) { JSONObj *o = *iter; cb(container, o); } } template -bool JSONDecoder::decode_json(const char *name, T& val, JSONObj *obj, bool mandatory) +bool JSONDecoder::decode_json(std::string_view name, T& val, JSONObj *obj, bool mandatory) { JSONObjIter iter = obj->find_first(name); if (iter.end()) { @@ -363,7 +538,7 @@ bool JSONDecoder::decode_json(const char *name, T& val, JSONObj *obj, bool manda } template -bool JSONDecoder::decode_json(const char *name, C& container, void (*cb)(C&, JSONObj *), JSONObj *obj, bool mandatory) +bool JSONDecoder::decode_json(std::string_view name, C& container, void (*cb)(C&, JSONObj *), JSONObj *obj, bool mandatory) { container.clear(); @@ -388,7 +563,7 @@ bool JSONDecoder::decode_json(const char *name, C& container, void (*cb)(C&, JSO } template -void JSONDecoder::decode_json(const char *name, T& val, const T& default_val, JSONObj *obj) +void JSONDecoder::decode_json(std::string_view name, T& val, const T& default_val, JSONObj *obj) { JSONObjIter iter = obj->find_first(name); if (iter.end()) { @@ -407,7 +582,7 @@ void JSONDecoder::decode_json(const char *name, T& val, const T& default_val, JS } template -bool JSONDecoder::decode_json(const char *name, boost::optional& val, JSONObj *obj, bool mandatory) +bool JSONDecoder::decode_json(std::string_view name, boost::optional& val, JSONObj *obj, bool mandatory) { JSONObjIter iter = obj->find_first(name); if (iter.end()) { @@ -433,7 +608,7 @@ bool JSONDecoder::decode_json(const char *name, boost::optional& val, JSONObj } template -bool JSONDecoder::decode_json(const char *name, std::optional& val, JSONObj *obj, bool mandatory) +bool JSONDecoder::decode_json(std::string_view name, std::optional& val, JSONObj *obj, bool mandatory) { JSONObjIter iter = obj->find_first(name); if (iter.end()) { @@ -463,7 +638,7 @@ class JSONEncodeFilter public: class HandlerBase { public: - virtual ~HandlerBase() {} + virtual ~HandlerBase() = default; virtual std::type_index get_type() = 0; virtual void encode_json(const char *name, const void *pval, ceph::Formatter *) const = 0; @@ -472,15 +647,13 @@ public: template class Handler : public HandlerBase { public: - virtual ~Handler() {} - std::type_index get_type() override { return std::type_index(typeid(const T&)); } }; private: - std::map handlers; + boost::container::flat_map handlers; public: void register_type(HandlerBase *h) { @@ -499,8 +672,19 @@ public: } }; +void encode_json(const char *name, ceph_json::detail::json_signed_integer auto val, Formatter *f) +{ + f->dump_int(name, val); +} + +void encode_json(const char *name, ceph_json::detail::json_unsigned_integer auto val, Formatter *f) +{ + f->dump_unsigned(name, val); +} + template -static void encode_json_impl(const char *name, const T& val, ceph::Formatter *f) +requires requires(const T& val, ceph::Formatter *f) { val.dump(f); } +void encode_json_impl(const char *name, const T& val, ceph::Formatter *f) { f->open_object_section(name); val.dump(f); @@ -508,7 +692,8 @@ static void encode_json_impl(const char *name, const T& val, ceph::Formatter *f) } template -static void encode_json(const char *name, const T& val, ceph::Formatter *f) +requires requires(const T& val, ceph::Formatter *f) { encode_json_impl("", val, f); } +void encode_json(const char *name, const T& val, ceph::Formatter *f) { JSONEncodeFilter *filter = static_cast(f->get_external_feature_handler("JSONEncodeFilter")); @@ -518,150 +703,82 @@ static void encode_json(const char *name, const T& val, ceph::Formatter *f) } } -class utime_t; - -void encode_json(const char *name, std::string_view val, ceph::Formatter *f); -void encode_json(const char *name, const std::string& val, ceph::Formatter *f); -void encode_json(const char *name, const char *val, ceph::Formatter *f); -void encode_json(const char *name, bool val, ceph::Formatter *f); -void encode_json(const char *name, int val, ceph::Formatter *f); -void encode_json(const char *name, unsigned val, ceph::Formatter *f); -void encode_json(const char *name, long val, ceph::Formatter *f); -void encode_json(const char *name, unsigned long val, ceph::Formatter *f); -void encode_json(const char *name, long long val, ceph::Formatter *f); -void encode_json(const char *name, const utime_t& val, ceph::Formatter *f); -void encode_json(const char *name, const ceph::buffer::list& bl, ceph::Formatter *f); -void encode_json(const char *name, long long unsigned val, ceph::Formatter *f); - -void encode_json(const char *name, const ceph::real_time& val, ceph::Formatter *f); -void encode_json(const char *name, const ceph::coarse_real_time& val, ceph::Formatter *f); - -template -static void encode_json(const char *name, const std::list& l, ceph::Formatter *f) +inline void encode_json(const char *name, std::string_view val, Formatter *f) { - f->open_array_section(name); - for (auto iter = l.cbegin(); iter != l.cend(); ++iter) { - encode_json("obj", *iter, f); - } - f->close_section(); + f->dump_string(name, val); } -template -static void encode_json(const char *name, const std::deque& l, ceph::Formatter *f) +inline void encode_json(const char *name, const std::string& val, Formatter *f) { - f->open_array_section(name); - for (auto iter = l.cbegin(); iter != l.cend(); ++iter) { - encode_json("obj", *iter, f); - } - f->close_section(); + f->dump_string(name, val); } -template > -static void encode_json(const char *name, const std::set& l, ceph::Formatter *f) +inline void encode_json(const char *name, const char *val, Formatter *f) { - f->open_array_section(name); - for (auto iter = l.cbegin(); iter != l.cend(); ++iter) { - encode_json("obj", *iter, f); - } - f->close_section(); + f->dump_string(name, val); } -template -static void encode_json(const char *name, - const boost::container::flat_set& l, - ceph::Formatter *f) +inline void encode_json(const char *name, bool val, Formatter *f) { - f->open_array_section(name); - for (auto iter = l.cbegin(); iter != l.cend(); ++iter) { - encode_json("obj", *iter, f); - } - f->close_section(); + f->dump_bool(name, val); } -template -static void encode_json(const char *name, const std::vector& l, ceph::Formatter *f) +inline void encode_json(const char *name, const utime_t& val, Formatter *f) { - f->open_array_section(name); - for (auto iter = l.cbegin(); iter != l.cend(); ++iter) { - encode_json("obj", *iter, f); - } - f->close_section(); + val.gmtime(f->dump_stream(name)); } -template -static void encode_json(const char *name, const std::array& l, - ceph::Formatter *f) +inline void encode_json(const char *name, const ceph::real_time& val, Formatter *f) { - f->open_array_section(name); - for (auto iter = l.cbegin(); iter != l.cend(); ++iter) { - encode_json("obj", *iter, f); - } - f->close_section(); + encode_json(name, utime_t{val}, f); } -template> -static void encode_json(const char *name, const std::map& m, ceph::Formatter *f) +inline void encode_json(const char *name, const ceph::coarse_real_time& val, Formatter *f) { - f->open_array_section(name); - for (auto i = m.cbegin(); i != m.cend(); ++i) { - f->open_object_section("entry"); - encode_json("key", i->first, f); - encode_json("val", i->second, f); - f->close_section(); - } - f->close_section(); + encode_json(name, utime_t{val}, f); } -template > -static void encode_json(const char *name, const boost::container::flat_map& m, ceph::Formatter *f) +inline void encode_json(const char *name, const bufferlist& bl, Formatter *f) { - f->open_array_section(name); - for (auto i = m.cbegin(); i != m.cend(); ++i) { - f->open_object_section("entry"); - encode_json("key", i->first, f); - encode_json("val", i->second, f); - f->close_section(); - } - f->close_section(); + /* need to copy data from bl, as it is const bufferlist */ + bufferlist src = bl; + + bufferlist b64; + src.encode_base64(b64); + + std::string_view sv(b64.c_str(), b64.length()); + + encode_json(name, sv, f); } -template -static void encode_json(const char *name, const std::multimap& m, ceph::Formatter *f) +template +void encode_json(const char *name, const std::optional& o, ceph::Formatter *f) { - f->open_array_section(name); - for (auto i = m.begin(); i != m.end(); ++i) { - f->open_object_section("entry"); - encode_json("key", i->first, f); - encode_json("val", i->second, f); - f->close_section(); + if (!o) { + return; } - f->close_section(); + encode_json(name, *o, f); } -template -static void encode_json(const char *name, const boost::container::flat_map& m, ceph::Formatter *f) +inline void encode_json(const char *name, const JSONObj::data_val& v, Formatter *f) { - f->open_array_section(name); - for (auto i = m.begin(); i != m.end(); ++i) { - f->open_object_section("entry"); - encode_json("key", i->first, f); - encode_json("val", i->second, f); - f->close_section(); + if (v.quoted) { + encode_json(name, v.str, f); + } else { + f->dump_format_unquoted(name, "%s", v.str.c_str()); } - f->close_section(); } +inline void encode_json(const char *name, const JSONFormattable& v, Formatter *f); + template void encode_json_map(const char *name, const std::map& m, ceph::Formatter *f) { f->open_array_section(name); - for (auto iter = m.cbegin(); iter != m.cend(); ++iter) { - encode_json("obj", iter->second, f); - } + std::ranges::for_each(m, [&f](const auto& kv) { encode_json("obj", kv.second, f); }); f->close_section(); } - template void encode_json_map(const char *name, const char *index_name, const char *object_name, const char *value_name, @@ -700,27 +817,17 @@ void encode_json_map(const char *name, const char *index_name, const char *object_name, const char *value_name, const std::map& m, ceph::Formatter *f) { - encode_json_map(name, index_name, object_name, value_name, NULL, NULL, m, f); + encode_json_map(name, index_name, object_name, value_name, nullptr, nullptr, m, f); } template void encode_json_map(const char *name, const char *index_name, const char *value_name, const std::map& m, ceph::Formatter *f) { - encode_json_map(name, index_name, NULL, value_name, NULL, NULL, m, f); + encode_json_map(name, index_name, nullptr, value_name, nullptr, nullptr, m, f); } -template -static void encode_json(const char *name, const std::optional& o, ceph::Formatter *f) -{ - if (!o) { - return; - } - encode_json(name, *o, f); -} - - -template +template void encode_json_map(const char *name, const boost::container::flat_map& m, ceph::Formatter *f) { f->open_array_section(name); @@ -730,7 +837,6 @@ void encode_json_map(const char *name, const boost::container::flat_map& m f->close_section(); } - template void encode_json_map(const char *name, const char *index_name, const char *object_name, const char *value_name, @@ -769,21 +875,42 @@ void encode_json_map(const char *name, const char *index_name, const char *object_name, const char *value_name, const boost::container::flat_map& m, ceph::Formatter *f) { - encode_json_map(name, index_name, object_name, value_name, NULL, NULL, m, f); + encode_json_map(name, index_name, object_name, value_name, nullptr, nullptr, m, f); } template void encode_json_map(const char *name, const char *index_name, const char *value_name, const boost::container::flat_map& m, ceph::Formatter *f) { - encode_json_map(name, index_name, NULL, value_name, NULL, NULL, m, f); + encode_json_map(name, index_name, nullptr, value_name, nullptr, nullptr, m, f); } +void encode_json(const char *name, const ceph_json::detail::json_val_seq auto& val, Formatter *f) +{ + f->open_array_section(name); + std::ranges::for_each(val, [&f](const auto &obj) { + ::encode_json("obj", obj, f); + }); + f->close_section(); +} + +void encode_json(const char *name, const ceph_json::detail::json_mapped_kv_seq auto& val, Formatter *f) +{ + f->open_array_section(name); + std::ranges::for_each(val, [&f](const auto& kv) { + f->open_object_section("entry"); + ::encode_json("key", kv.first, f); + ::encode_json("val", kv.second, f); + f->close_section(); + }); + f->close_section(); +} class JSONFormattable : public ceph::JSONFormatter { + JSONObj::data_val value; std::vector arr; - std::map obj; + std::map> obj; std::vector enc_stack; JSONFormattable *cur_enc; @@ -794,7 +921,8 @@ protected: bool handle_close_section() override; public: - JSONFormattable(bool p = false) : JSONFormatter(p) { + JSONFormattable(bool p = false) + : JSONFormatter(p) { cur_enc = this; enc_stack.push_back(cur_enc); } @@ -857,6 +985,61 @@ public: break; } } + + const std::map> object() const noexcept { return obj; } + + const std::vector& array() const noexcept { return arr; } + + JSONFormattable& operator[](const std::string& name); + const JSONFormattable& operator[](const std::string& name) const; + + JSONFormattable& operator[](size_t index); + const JSONFormattable& operator[](size_t index) const; + + const std::string& val() const noexcept { return value.str; } + int val_int() const { return atoi(value.str.c_str()); } + long val_long() const { return atol(value.str.c_str()); } + long long val_long_long() const { return atoll(value.str.c_str()); } + bool val_bool() const; + + operator std::string() const noexcept { return value.str; } + explicit operator int() const { return val_int(); } + explicit operator long() const { return val_long(); } + explicit operator long long() const { return val_long_long(); } + explicit operator bool() const { return val_bool(); } + + std::string def(const std::string& def_val) const { return FMT_NONE == type ? def_val : val(); } + int def(int def_val) const { return FMT_NONE == type ? def_val : val_int(); } + bool def(bool def_val) const { return FMT_NONE == type ? def_val : val_bool(); } + + std::string operator ()(const char *def_val) const { return def(std::string(def_val)); } + int operator()(int def_val) const { return def(def_val); } + bool operator()(bool def_val) const { return def(def_val); } + + bool exists(const std::string& name) const noexcept { return obj.contains(name); } + bool exists(size_t index) const noexcept { return (index < arr.size()); } + + bool find(const std::string& name, std::string *val) const noexcept { + if (auto i = obj.find(name); end(obj) != i) + return (*val = i->second.val()), true; + + return false; + } + + std::string get(const std::string& name, const std::string& def_val) const { return (*this)[name].def(def_val); } + int get_int(const std::string& name, int def_val) const { return (*this)[name].def(def_val); } + bool get_bool(const std::string& name, bool def_val) const { return (*this)[name].def(def_val); } + + int set(const std::string& name, const std::string& val); + int erase(const std::string& name); + + void derive_from(const JSONFormattable& jf); + + void encode_json(const char *name, ceph::Formatter *f) const; + + bool is_array() const { return type == FMT_ARRAY; } + +public: static std::list generate_test_instances() { std::list o; o.emplace_back(); @@ -887,98 +1070,81 @@ public: return o; } - const std::string& val() const { - return value.str; - } - - int val_int() const; - long val_long() const; - long long val_long_long() const; - bool val_bool() const; - - const std::map object() const { - return obj; - } - - const std::vector& array() const { - return arr; - } - - const JSONFormattable& operator[](const std::string& name) const; - const JSONFormattable& operator[](size_t index) const; - - JSONFormattable& operator[](const std::string& name); - JSONFormattable& operator[](size_t index); - - operator std::string() const { - return value.str; - } - - explicit operator int() const { - return val_int(); + static void generate_test_instances(std::list& o) { + o.push_back(new JSONFormattable); + o.push_back(new JSONFormattable); + o.back()->set_type(FMT_VALUE); + o.back()->value.str = "foo"; + o.back()->value.quoted = true; + o.push_back(new JSONFormattable); + o.back()->set_type(FMT_VALUE); + o.back()->value.str = "foo"; + o.back()->value.quoted = false; + o.push_back(new JSONFormattable); + o.back()->set_type(FMT_ARRAY); + o.back()->arr.push_back(JSONFormattable()); + o.back()->arr.back().set_type(FMT_VALUE); + o.back()->arr.back().value.str = "foo"; + o.back()->arr.back().value.quoted = true; + o.back()->arr.push_back(JSONFormattable()); + o.back()->arr.back().set_type(FMT_VALUE); + o.back()->arr.back().value.str = "bar"; + o.back()->arr.back().value.quoted = true; + o.push_back(new JSONFormattable); + o.back()->set_type(FMT_OBJ); + o.back()->obj["foo"] = JSONFormattable(); + o.back()->obj["foo"].set_type(FMT_VALUE); + o.back()->obj["foo"].value.str = "bar"; + o.back()->obj["foo"].value.quoted = true; } +}; +WRITE_CLASS_ENCODER(JSONFormattable) - explicit operator long() const { - return val_long(); - } - - explicit operator long long() const { - return val_long_long(); - } - - explicit operator bool() const { - return val_bool(); - } - - template - T operator[](const std::string& name) const { - return this->operator[](name)(T()); - } - - template - T operator[](const std::string& name) { - return this->operator[](name)(T()); - } - - std::string operator ()(const char *def_val) const { - return def(std::string(def_val)); - } - - int operator()(int def_val) const { - return def(def_val); - } - - bool operator()(bool def_val) const { - return def(def_val); - } - - bool exists(const std::string& name) const; - bool exists(size_t index) const; - - std::string def(const std::string& def_val) const; - int def(int def_val) const; - bool def(bool def_val) const; - - bool find(const std::string& name, std::string *val) const; +static inline JSONFormattable default_formattable; - std::string get(const std::string& name, const std::string& def_val) const; +inline JSONFormattable& JSONFormattable::operator[](const std::string& name) { + if (const auto i = obj.find(name); end(obj) != i) + return i->second; + + return default_formattable; +} - int get_int(const std::string& name, int def_val) const; - bool get_bool(const std::string& name, bool def_val) const; +inline const JSONFormattable& JSONFormattable::operator[](const std::string& name) const { + return const_cast(this)->operator[](name); +} - int set(const std::string& name, const std::string& val); - int erase(const std::string& name); +inline JSONFormattable& JSONFormattable::operator[](size_t index) { + return index >= arr.size() ? default_formattable : arr[index]; +} - void derive_from(const JSONFormattable& jf); +inline const JSONFormattable& JSONFormattable::operator[](size_t index) const { + return const_cast(this)->operator[](index); +} - void encode_json(const char *name, ceph::Formatter *f) const; +inline void encode_json(const char *name, const JSONFormattable& v, Formatter *f) +{ + v.encode_json(name, f); +} - bool is_array() const { - return (type == FMT_ARRAY); +inline void JSONFormattable::encode_json(const char *name, Formatter *f) const +{ + switch (type) { + case JSONFormattable::FMT_VALUE: + ::encode_json(name, value, f); + break; + case JSONFormattable::FMT_ARRAY: + ::encode_json(name, arr, f); + break; + case JSONFormattable::FMT_OBJ: + f->open_object_section(name); + for (auto iter : obj) { + ::encode_json(iter.first.c_str(), iter.second, f); + } + f->close_section(); + break; + case JSONFormattable::FMT_NONE: + break; } -}; -WRITE_CLASS_ENCODER(JSONFormattable) - -void encode_json(const char *name, const JSONFormattable& v, ceph::Formatter *f); +} #endif diff --git a/src/common/cmdparse.cc b/src/common/cmdparse.cc index bd5898b54c7..498ca98c285 100644 --- a/src/common/cmdparse.cc +++ b/src/common/cmdparse.cc @@ -21,7 +21,9 @@ #include "common/strtol.h" #include "include/ceph_assert.h" // boost clobbers this #include "include/types.h" // for operator<<(std::vector) + #include "json_spirit/json_spirit.h" +#include "common/ceph_json.h" #include #include @@ -322,13 +324,15 @@ cmdmap_from_json(const vector& cmd, cmdmap_t *mapp, std::ostream& ss) json_spirit::mValue v; string fullcmd; + // First, join all cmd strings for (auto& c : cmd) fullcmd += c; try { if (!json_spirit::read(fullcmd, v)) - throw std::runtime_error("unparseable JSON " + fullcmd); + throw std::runtime_error("unparseable JSON: |" + fullcmd + '|'); + if (v.type() != json_spirit::obj_type) throw std::runtime_error("not JSON object " + fullcmd); diff --git a/src/include/encoding.h b/src/include/encoding.h index d073c1e7c62..a23465b19a3 100644 --- a/src/include/encoding.h +++ b/src/include/encoding.h @@ -246,7 +246,7 @@ inline void decode_nohead(unsigned len, std::string& s, bufferlist::const_iterat // const char* (encode only, string compatible) inline void encode(const char *s, bufferlist& bl) { - encode(std::string_view(s, strlen(s)), bl); + encode(std::string_view(s), bl); } // opaque byte vectors diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc index c23f8ff27a5..d0863e56775 100644 --- a/src/librados/RadosClient.cc +++ b/src/librados/RadosClient.cc @@ -1141,7 +1141,7 @@ int librados::RadosClient::get_inconsistent_pgs(int64_t pool_id, return 0; } JSONParser parser; - if (!parser.parse(outbl.c_str(), outbl.length())) { + if (!parser.parse(outbl)) { return -EINVAL; } vector v; @@ -1151,7 +1151,7 @@ int librados::RadosClient::get_inconsistent_pgs(int64_t pool_id, return 0; auto s = pgstat_obj->get_data(); JSONParser pg_stats; - if (!pg_stats.parse(s.c_str(), s.length())) { + if (!pg_stats.parse(s)) { return -EINVAL; } v = pg_stats.get_array_elements(); @@ -1160,7 +1160,7 @@ int librados::RadosClient::get_inconsistent_pgs(int64_t pool_id, } for (auto i : v) { JSONParser pg_json; - if (!pg_json.parse(i.c_str(), i.length())) { + if (!pg_json.parse(i)) { return -EINVAL; } string pgid; diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc index 0cf6aa3c116..e22be9a383b 100644 --- a/src/mgr/ActivePyModules.cc +++ b/src/mgr/ActivePyModules.cc @@ -42,6 +42,8 @@ #include "PyModuleRegistry.h" #include "PyUtil.h" +#include "json_spirit/json_spirit.h" + #define dout_context g_ceph_context #define dout_subsys ceph_subsys_mgr #undef dout_prefix diff --git a/src/mgr/Mgr.cc b/src/mgr/Mgr.cc index 136b453a2f9..18dd0903d0b 100644 --- a/src/mgr/Mgr.cc +++ b/src/mgr/Mgr.cc @@ -16,6 +16,7 @@ #include "osdc/Objecter.h" #include "common/errno.h" +#include "common/ceph_json.h" #include "mon/MonClient.h" #include "include/stringify.h" #include "include/str_map.h" @@ -86,21 +87,23 @@ void MetadataUpdate::finish(int r) if (r == 0) { if (key.type == "mds" || key.type == "osd" || key.type == "mgr" || key.type == "mon") { - json_spirit::mValue json_result; - bool read_ok = json_spirit::read( - outbl.to_str(), json_result); - if (!read_ok) { + + std::error_code ec; + auto json_result = boost::json::parse(outbl.to_str(), ec, boost::json::storage_ptr(), + { .allow_invalid_utf8 = true }); + + if (ec) { dout(1) << "mon returned invalid JSON for " << key << dendl; return; } - if (json_result.type() != json_spirit::obj_type) { + if (!json_result.is_object()) { dout(1) << "mon returned valid JSON " << key << " but not an object: '" << outbl.to_str() << "'" << dendl; return; } dout(4) << "mon returned valid metadata JSON for " << key << dendl; - json_spirit::mObject daemon_meta = json_result.get_obj(); + boost::json::object daemon_meta = json_result.get_object(); // Skip daemon who doesn't have hostname yet if (daemon_meta.count("hostname") == 0) { @@ -120,7 +123,7 @@ void MetadataUpdate::finish(int r) std::map m; { std::lock_guard l(state->lock); - state->hostname = daemon_meta.at("hostname").get_str(); + state->hostname = daemon_meta.at("hostname").get_string(); if (key.type == "mds" || key.type == "mgr" || key.type == "mon") { daemon_meta.erase("name"); @@ -129,14 +132,14 @@ void MetadataUpdate::finish(int r) } daemon_meta.erase("hostname"); for (const auto &[key, val] : daemon_meta) { - m.emplace(key, val.get_str()); + m.emplace(key, val.get_string()); } } daemon_state.update_metadata(state, m); } else { auto state = std::make_shared(daemon_state.types); state->key = key; - state->hostname = daemon_meta.at("hostname").get_str(); + state->hostname = daemon_meta.at("hostname").get_string(); if (key.type == "mds" || key.type == "mgr" || key.type == "mon") { daemon_meta.erase("name"); @@ -147,7 +150,7 @@ void MetadataUpdate::finish(int r) std::map m; for (const auto &[key, val] : daemon_meta) { - m.emplace(key, val.get_str()); + m.emplace(key, val.get_string()); } state->set_metadata(m); @@ -192,7 +195,7 @@ std::map Mgr::load_store() std::map loaded; for (auto &key_str : cmd.json_result.get_array()) { - std::string const key = key_str.get_str(); + boost::json::string_view key = key_str.get_string(); dout(20) << "saw key '" << key << "'" << dendl; @@ -417,45 +420,44 @@ void Mgr::load_all_metadata() ceph_assert(osd_cmd.r == 0); for (auto &metadata_val : mds_cmd.json_result.get_array()) { - json_spirit::mObject daemon_meta = metadata_val.get_obj(); + boost::json::object daemon_meta = metadata_val.get_object(); if (daemon_meta.count("hostname") == 0) { dout(1) << "Skipping incomplete metadata entry" << dendl; continue; } DaemonStatePtr dm = std::make_shared(daemon_state.types); - dm->key = DaemonKey{"mds", - daemon_meta.at("name").get_str()}; - dm->hostname = daemon_meta.at("hostname").get_str(); + dm->key = DaemonKey{"mds", boost::json::value_to(daemon_meta.at("name")) }; + dm->hostname = boost::json::value_to(daemon_meta.at("hostname")); + daemon_meta.erase("name"); daemon_meta.erase("hostname"); for (const auto &[key, val] : daemon_meta) { - dm->metadata.emplace(key, val.get_str()); + dm->metadata.emplace(key, boost::json::value_to(val)); } daemon_state.insert(dm); } for (auto &metadata_val : mon_cmd.json_result.get_array()) { - json_spirit::mObject daemon_meta = metadata_val.get_obj(); + boost::json::object daemon_meta = metadata_val.get_object(); if (daemon_meta.count("hostname") == 0) { dout(1) << "Skipping incomplete metadata entry" << dendl; continue; } DaemonStatePtr dm = std::make_shared(daemon_state.types); - dm->key = DaemonKey{"mon", - daemon_meta.at("name").get_str()}; - dm->hostname = daemon_meta.at("hostname").get_str(); + dm->key = DaemonKey{"mon", boost::json::value_to(daemon_meta.at("name")) }; + dm->hostname = daemon_meta.at("hostname").get_string(); daemon_meta.erase("name"); daemon_meta.erase("hostname"); std::map m; for (const auto &[key, val] : daemon_meta) { - m.emplace(key, val.get_str()); + m.emplace(key, val.get_string()); } dm->set_metadata(m); @@ -463,24 +465,24 @@ void Mgr::load_all_metadata() } for (auto &osd_metadata_val : osd_cmd.json_result.get_array()) { - json_spirit::mObject osd_metadata = osd_metadata_val.get_obj(); + boost::json::object osd_metadata = osd_metadata_val.get_object(); if (osd_metadata.count("hostname") == 0) { dout(1) << "Skipping incomplete metadata entry" << dendl; continue; } - dout(4) << osd_metadata.at("hostname").get_str() << dendl; + dout(4) << osd_metadata.at("hostname").get_string() << dendl; DaemonStatePtr dm = std::make_shared(daemon_state.types); dm->key = DaemonKey{"osd", - stringify(osd_metadata.at("id").get_int())}; - dm->hostname = osd_metadata.at("hostname").get_str(); + stringify(osd_metadata.at("id").get_int64())}; + dm->hostname = osd_metadata.at("hostname").get_string(); osd_metadata.erase("id"); osd_metadata.erase("hostname"); - std::map m; - for (const auto &i : osd_metadata) { - m[i.first] = i.second.get_str(); + map m; + for (const auto &[k, v] : osd_metadata) { + m[k] = v.get_string(); } dm->set_metadata(m); diff --git a/src/mgr/MgrContext.h b/src/mgr/MgrContext.h index 1647ee771e4..b3ac8e96254 100644 --- a/src/mgr/MgrContext.h +++ b/src/mgr/MgrContext.h @@ -21,6 +21,8 @@ #include "common/Cond.h" #include "mon/MonClient.h" +#include + class Command { protected: @@ -50,22 +52,22 @@ public: virtual ~Command() {} }; - class JSONCommand : public Command { public: - json_spirit::mValue json_result; + boost::json::value json_result; void wait() override { Command::wait(); - if (r == 0) { - bool read_ok = json_spirit::read( - outbl.to_str(), json_result); - if (!read_ok) { - r = -EINVAL; - } + if (0 != r) { + return; + } + + boost::system::error_code ec; + if (json_result = boost::json::parse(outbl.to_str(), ec); ec) { + r = -EINVAL; } } }; diff --git a/src/rgw/radosgw-admin/radosgw-admin.cc b/src/rgw/radosgw-admin/radosgw-admin.cc index a8d22d99e02..6db12fff041 100644 --- a/src/rgw/radosgw-admin/radosgw-admin.cc +++ b/src/rgw/radosgw-admin/radosgw-admin.cc @@ -2,8 +2,8 @@ // vim: ts=8 sw=2 sts=2 expandtab ft=cpp /* - * Copyright (C) 2025 IBM - */ + * Copyright (C) 2025-2026 IBM +*/ #include #include @@ -14,12 +14,11 @@ #include #include -extern "C" { -#include -} #include +#include + #include "auth/Crypto.h" #include "compressor/Compressor.h" @@ -1370,7 +1369,7 @@ static int read_decode_json(const string& infile, T& t) return ret; } JSONParser p; - if (!p.parse(bl.c_str(), bl.length())) { + if (!p.parse(bl)) { cout << "failed to parse JSON" << std::endl; return -EINVAL; } @@ -1394,7 +1393,7 @@ static int read_decode_json(const string& infile, T& t, K *k) return ret; } JSONParser p; - if (!p.parse(bl.c_str(), bl.length())) { + if (!p.parse(bl)) { cout << "failed to parse JSON" << std::endl; return -EINVAL; } @@ -1971,7 +1970,7 @@ static int send_to_remote_gateway(RGWRESTConn* conn, req_info& info, return ret; } - ret = parser.parse(response.c_str(), response.length()); + ret = parser.parse(response); if (ret < 0) { cerr << "failed to parse response" << std::endl; return ret; @@ -2001,14 +2000,15 @@ static int send_to_url(const string& url, if (!result) { return result.error(); } + int ret = rgw_http_error_to_errno(*result); if (ret < 0) { return ret; } - ret = parser.parse(response.c_str(), response.length()); + ret = parser.parse(response); if (ret < 0) { - cout << "failed to parse response" << std::endl; + cerr << "failed to parse remote response" << std::endl; return ret; } return 0; @@ -2053,7 +2053,7 @@ static int commit_period(rgw::sal::ConfigStore* cfgstore, realm, realm_writer, current_period, period, cerr, force, *site); if (ret < 0) { - cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl; + cerr << "commit_period(): failed to commit period: " << cpp_strerror(-ret) << std::endl; } (void) cfgstore->realm_notify_new_period(dpp(), null_yield, period); return ret; @@ -2062,7 +2062,6 @@ static int commit_period(rgw::sal::ConfigStore* cfgstore, if (remote.empty() && url.empty()) { // use the new master zone's connection remote = master_zone; - cerr << "Sending period to new master zone " << remote << std::endl; } boost::optional conn; RGWRESTConn *remote_conn = nullptr; @@ -2107,13 +2106,15 @@ static int commit_period(rgw::sal::ConfigStore* cfgstore, try { decode_json_obj(period, &p); } catch (const JSONDecoder::err& e) { - cout << "failed to decode JSON input: " << e.what() << std::endl; + cerr << "failed to decode JSON input: " << e.what() << std::endl; return -EINVAL; } + if (period.get_id().empty()) { - cerr << "Period commit got back an empty period id" << std::endl; + cerr << "commit_period(): Period commit got back an empty period id" << std::endl; return -EINVAL; } + // the master zone gave us back the period that it committed, so it's // safe to save it as our latest epoch constexpr bool exclusive = false; @@ -2170,18 +2171,21 @@ static int update_period(rgw::sal::ConfigStore* cfgstore, constexpr bool exclusive = false; ret = cfgstore->create_period(dpp(), null_yield, exclusive, period); + if (ret < 0) { cerr << "failed to driver period: " << cpp_strerror(-ret) << std::endl; return ret; } + if (commit) { ret = commit_period(cfgstore, realm, *realm_writer, period, remote, url, opt_region, access, secret, force, site); if (ret < 0) { - cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl; + cerr << "update_period(): failed to commit period: " << cpp_strerror(-ret) << std::endl; return ret; } } + encode_json("period", period, formatter); formatter->flush(cout); return 0; @@ -2231,6 +2235,7 @@ static int do_period_pull(rgw::sal::ConfigStore* cfgstore, cerr << "request failed: " << cpp_strerror(-ret) << std::endl; return ret; } + try { decode_json_obj(*period, &p); } catch (const JSONDecoder::err& e) { @@ -2242,6 +2247,7 @@ static int do_period_pull(rgw::sal::ConfigStore* cfgstore, if (ret < 0) { cerr << "Error storing period " << period->get_id() << ": " << cpp_strerror(ret) << std::endl; } + return 0; } @@ -5440,10 +5446,19 @@ int main(int argc, const char **argv) info.request_uri = "/admin/realm"; map ¶ms = info.args.get_params(); - if (!realm_id.empty()) - params["id"] = realm_id; - if (!realm_name.empty()) - params["name"] = realm_name; + + if (realm_name.empty()) { + cerr << "missing realm name" << std::endl; + return EINVAL; + } + + if (realm_id.empty()) { + cerr << "missing realm id" << std::endl; + return EINVAL; + } + + params["id"] = realm_id; + params["name"] = realm_name; bufferlist bl; JSONParser p; @@ -5457,6 +5472,7 @@ int main(int argc, const char **argv) } return -ret; } + RGWRealm realm; try { decode_json_obj(realm, &p); @@ -5464,6 +5480,7 @@ int main(int argc, const char **argv) cerr << "failed to decode JSON response: " << e.what() << std::endl; return EINVAL; } + RGWPeriod period; auto& current_period = realm.get_current_period(); if (!current_period.empty()) { @@ -7126,7 +7143,7 @@ int main(int argc, const char **argv) remote, url, opt_region, access_key, secret_key, yes_i_really_mean_it, site.get()); if (ret < 0) { - cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl; + cerr << "OPT::PERIOD_COMMIT: failed to commit period: " << cpp_strerror(-ret) << std::endl; return -ret; } diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc index 5e5c575e793..953a2f5be3a 100644 --- a/src/rgw/rgw_acl_swift.cc +++ b/src/rgw/rgw_acl_swift.cc @@ -286,7 +286,7 @@ int create_account_policy(const DoutPrefixProvider* dpp, auto& acl = policy.get_acl(); JSONParser parser; - if (!parser.parse(acl_str.c_str(), acl_str.length())) { + if (!parser.parse(acl_str)) { ldpp_dout(dpp, 0) << "ERROR: JSONParser::parse returned error=" << dendl; return -EINVAL; } diff --git a/src/rgw/rgw_auth_keystone.cc b/src/rgw/rgw_auth_keystone.cc index ee36639ff4b..0b9c0073d06 100644 --- a/src/rgw/rgw_auth_keystone.cc +++ b/src/rgw/rgw_auth_keystone.cc @@ -90,9 +90,6 @@ admin_token_retry: ret = validate.process(dpp, y); - /* NULL terminate for debug output. */ - token_body_bl.append(static_cast(0)); - /* Detect Keystone rejection earlier than during the token parsing. * Although failure at the parsing phase doesn't impose a threat, * this allows to return proper error code (EACCESS instead of EINVAL @@ -127,7 +124,7 @@ admin_token_retry: } ldpp_dout(dpp, 20) << "received response status=" << validate.get_http_status() - << ", body=" << token_body_bl.c_str() << dendl; + << ", body=" << std::string_view(token_body_bl.c_str()) << dendl; TokenEngine::token_envelope_t token_body; ret = token_body.parse(dpp, token, token_body_bl); @@ -552,7 +549,7 @@ auto EC2Engine::get_secret_from_keystone(const DoutPrefixProvider* dpp, /* now parse response */ JSONParser parser; - if (! parser.parse(token_body_bl.c_str(), token_body_bl.length())) { + if (! parser.parse(token_body_bl)) { ldpp_dout(dpp, 0) << "Keystone credential parse error: malformed json" << dendl; return make_pair(boost::none, -EINVAL); } diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index 7d102224d87..1b8b6b9e88a 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -7,7 +7,6 @@ #include #include -#include "json_spirit/json_spirit.h" #include "common/ceph_json.h" #include "common/Formatter.h" #include "common/versioned_variant.h" diff --git a/src/rgw/rgw_keystone.cc b/src/rgw/rgw_keystone.cc index 81992684df5..045efa2fa2e 100644 --- a/src/rgw/rgw_keystone.cc +++ b/src/rgw/rgw_keystone.cc @@ -263,7 +263,7 @@ int TokenEnvelope::parse(const DoutPrefixProvider *dpp, ceph::bufferlist& bl) { JSONParser parser; - if (! parser.parse(bl.c_str(), bl.length())) { + if (! parser.parse(bl)) { ldpp_dout(dpp, 0) << "Keystone token parse error: malformed json" << dendl; return -EINVAL; } diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc index 5de3d248930..ef306af4633 100644 --- a/src/rgw/rgw_metadata.cc +++ b/src/rgw/rgw_metadata.cc @@ -311,7 +311,7 @@ int RGWMetadataManager::put(string& metadata_key, bufferlist& bl, } JSONParser parser; - if (!parser.parse(bl.c_str(), bl.length())) { + if (!parser.parse(bl)) { return -EINVAL; } diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index b5fd4bb8ddf..2d19c4edd50 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -197,7 +197,7 @@ int rgw_forward_request_to_master(const DoutPrefixProvider* dpp, if (ret < 0) { return ret; } - if (jp && !jp->parse(outdata.c_str(), outdata.length())) { + if (jp && !jp->parse(outdata)) { ldpp_dout(dpp, 0) << "failed parsing response from master zonegroup" << dendl; return -EINVAL; } diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 4408bbb2e86..bc0fe181928 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -163,7 +163,7 @@ int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out, JSONParser parser; - if (!parser.parse(data.c_str(), data.length())) { + if (!parser.parse(data)) { return -EINVAL; } diff --git a/src/rgw/rgw_opa.cc b/src/rgw/rgw_opa.cc index c096fd0bd37..38c13a50071 100644 --- a/src/rgw/rgw_opa.cc +++ b/src/rgw/rgw_opa.cc @@ -79,7 +79,7 @@ int rgw_opa_authorize(RGWOp *& op, /* check OPA response */ JSONParser parser; - if (!parser.parse(bl.c_str(), bl.length())) { + if (!parser.parse(bl)) { ldpp_dout(op, 2) << "OPA parse error: malformed json" << dendl; return -EINVAL; } diff --git a/src/rgw/rgw_period.cc b/src/rgw/rgw_period.cc index ca533ce7c88..82a6a4fcbc6 100644 --- a/src/rgw/rgw_period.cc +++ b/src/rgw/rgw_period.cc @@ -154,6 +154,7 @@ void RGWPeriod::dump(Formatter *f) const encode_json("realm_epoch", realm_epoch, f); } +// Reads internal fields /from/ JSONObj: void RGWPeriod::decode_json(JSONObj *obj) { JSONDecoder::decode_json("id", id, obj); diff --git a/src/rgw/rgw_period_puller.cc b/src/rgw/rgw_period_puller.cc index fd48612f5b2..f18f19f11fe 100644 --- a/src/rgw/rgw_period_puller.cc +++ b/src/rgw/rgw_period_puller.cc @@ -53,7 +53,7 @@ int pull_period(const DoutPrefixProvider *dpp, RGWRESTConn* conn, const std::str } JSONParser parser; - r = parser.parse(data.c_str(), data.length()); + r = parser.parse(data); if (r < 0) { ldpp_dout(dpp, -1) << "request failed: " << cpp_strerror(-r) << dendl; return r; diff --git a/src/rgw/rgw_policy_s3.cc b/src/rgw/rgw_policy_s3.cc index d8fadfdc3c6..e06b54d5dba 100644 --- a/src/rgw/rgw_policy_s3.cc +++ b/src/rgw/rgw_policy_s3.cc @@ -243,14 +243,12 @@ int RGWPolicy::check(RGWPolicyEnv *env, string& err_msg) return 0; } - -int RGWPolicy::from_json(bufferlist& bl, string& err_msg) +int RGWPolicy::from_json(std::string_view json_in, string& err_msg) { JSONParser parser; - - if (!parser.parse(bl.c_str(), bl.length())) { - err_msg = "Malformed JSON"; - dout(0) << "malformed json" << dendl; + if (!parser.parse(json_in)) { + err_msg = fmt::format("malformed JSON (RGWPolicy) length = {}:\n{}\n----\n", json_in.length(), json_in); + dout(0) << err_msg << dendl; return -EINVAL; } @@ -258,7 +256,7 @@ int RGWPolicy::from_json(bufferlist& bl, string& err_msg) JSONObjIter iter = parser.find_first("expiration"); if (iter.end()) { err_msg = "Policy missing expiration"; - dout(0) << "expiration not found" << dendl; + dout(0) << err_msg << dendl; return -EINVAL; // change to a "no expiration" error following S3 } @@ -312,3 +310,9 @@ int RGWPolicy::from_json(bufferlist& bl, string& err_msg) } return 0; } + +int RGWPolicy::from_json(bufferlist& bl, string& err_msg) +{ + return from_json(std::string_view { bl.c_str(), bl.length() }, err_msg); +} + diff --git a/src/rgw/rgw_policy_s3.h b/src/rgw/rgw_policy_s3.h index 40ad3c86a41..cfe94b5ecc0 100644 --- a/src/rgw/rgw_policy_s3.h +++ b/src/rgw/rgw_policy_s3.h @@ -53,5 +53,6 @@ public: } int check(RGWPolicyEnv *env, std::string& err_msg); + int from_json(std::string_view bl, std::string& err_msg); int from_json(bufferlist& bl, std::string& err_msg); }; diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index 7b540a494e6..582ba3ef8cd 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -61,8 +61,7 @@ std::tuple rgw_rest_get_json_input_keep_data(CephContext *cct, } JSONParser parser; - - if (!parser.parse(data.c_str(), data.length())) { + if (!parser.parse(data)) { return std::make_tuple(-EINVAL, std::move(data)); } diff --git a/src/rgw/rgw_rest.sts.cc b/src/rgw/rgw_rest.sts.cc new file mode 100644 index 00000000000..15b86c3fda7 --- /dev/null +++ b/src/rgw/rgw_rest.sts.cc @@ -0,0 +1,837 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + + + +#include "ceph_ver.h" +#include "common/Formatter.h" +#include "common/utf8.h" +#include "common/ceph_json.h" + +#include "rgw_rest.h" +#include "rgw_account.h" +#include "rgw_auth.h" +#include "rgw_auth_registry.h" +#include "jwt-cpp/jwt.h" +#include "rgw_rest_sts.h" + +#include "rgw_formats.h" +#include "rgw_client_io.h" + +#include "rgw_request.h" +#include "rgw_process.h" +#include "rgw_iam_policy.h" +#include "rgw_iam_policy_keywords.h" + +#include "rgw_sts.h" +#include "rgw_rest_oidc_provider.h" + + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +namespace rgw::auth::sts { + +bool +WebTokenEngine::is_applicable(const std::string& token) const noexcept +{ + return ! token.empty(); +} + +std::string +WebTokenEngine::get_role_tenant(const string& role_arn) const +{ + string tenant; + auto r_arn = rgw::ARN::parse(role_arn); + if (r_arn) { + tenant = r_arn->account; + } + return tenant; +} + +std::string +WebTokenEngine::get_role_name(const string& role_arn) const +{ + string role_name; + auto r_arn = rgw::ARN::parse(role_arn); + if (r_arn) { + role_name = r_arn->resource; + } + if (!role_name.empty()) { + auto pos = role_name.find_last_of('/'); + if(pos != string::npos) { + role_name = role_name.substr(pos + 1); + } + } + return role_name; +} + +int WebTokenEngine::load_provider(const DoutPrefixProvider* dpp, optional_yield y, + const string& role_arn, const string& iss, + RGWOIDCProviderInfo& info) const +{ + string tenant = get_role_tenant(role_arn); + + string idp_url = iss; + auto pos = idp_url.find("http://"); + if (pos == std::string::npos) { + pos = idp_url.find("https://"); + if (pos != std::string::npos) { + idp_url.erase(pos, 8); + } else { + pos = idp_url.find("www."); + if (pos != std::string::npos) { + idp_url.erase(pos, 4); + } + } + } else { + idp_url.erase(pos, 7); + } + + return driver->load_oidc_provider(dpp, y, tenant, idp_url, info); +} + +bool +WebTokenEngine::is_client_id_valid(vector& client_ids, const string& client_id) const +{ + for (auto it : client_ids) { + if (it == client_id) { + return true; + } + } + return false; +} + +bool +WebTokenEngine::is_cert_valid(const vector& thumbprints, const string& cert) const +{ + //calculate thumbprint of cert + std::unique_ptr certbio(BIO_new_mem_buf(cert.data(), cert.size()), BIO_free_all); + std::unique_ptr keybio(BIO_new(BIO_s_mem()), BIO_free_all); + string pw=""; + std::unique_ptr x_509cert(PEM_read_bio_X509(certbio.get(), nullptr, nullptr, const_cast(pw.c_str())), X509_free); + const EVP_MD* fprint_type = EVP_sha1(); + unsigned int fprint_size; + unsigned char fprint[EVP_MAX_MD_SIZE]; + + if (!X509_digest(x_509cert.get(), fprint_type, fprint, &fprint_size)) { + return false; + } + stringstream ss; + for (unsigned int i = 0; i < fprint_size; i++) { + ss << std::setfill('0') << std::setw(2) << std::hex << (0xFF & (unsigned int)fprint[i]); + } + std::string digest = ss.str(); + + for (auto& it : thumbprints) { + if (boost::iequals(it,digest)) { + return true; + } + } + return false; +} + +template +void +WebTokenEngine::recurse_and_insert(const string& key, const jwt::claim& c, T& t) const +{ + string s_val; + jwt::claim::type c_type = c.get_type(); + switch(c_type) { + case jwt::claim::type::null: + break; + case jwt::claim::type::boolean: + case jwt::claim::type::number: + case jwt::claim::type::int64: + { + s_val = c.to_json().serialize(); + t.emplace(std::make_pair(key, s_val)); + break; + } + case jwt::claim::type::string: + { + s_val = c.to_json().to_str(); + t.emplace(std::make_pair(key, s_val)); + break; + } + case jwt::claim::type::array: + { + const picojson::array& arr = c.as_array(); + for (auto& a : arr) { + recurse_and_insert(key, jwt::claim(a), t); + } + break; + } + case jwt::claim::type::object: + { + const picojson::object& obj = c.as_object(); + for (auto& m : obj) { + recurse_and_insert(m.first, jwt::claim(m.second), t); + } + break; + } + } + return; +} + +//Extract all token claims so that they can be later used in the Condition element of Role's trust policy +WebTokenEngine::token_t +WebTokenEngine::get_token_claims(const jwt::decoded_jwt& decoded) const +{ + WebTokenEngine::token_t token; + const auto& claims = decoded.get_payload_claims(); + + for (auto& c : claims) { + if (c.first == string(princTagsNamespace)) { + continue; + } + recurse_and_insert(c.first, c.second, token); + } + return token; +} + +//Offline validation of incoming Web Token which is a signed JWT (JSON Web Token) +std::tuple, boost::optional> +WebTokenEngine::get_from_jwt(const DoutPrefixProvider* dpp, const std::string& token, const req_state* const s, + optional_yield y) const +{ + WebTokenEngine::token_t t; + WebTokenEngine::principal_tags_t principal_tags; + try { + const auto& decoded = jwt::decode(token); + + auto& payload = decoded.get_payload(); + ldpp_dout(dpp, 20) << " payload = " << payload << dendl; + + t = get_token_claims(decoded); + + string iss; + if (decoded.has_issuer()) { + iss = decoded.get_issuer(); + } + + set aud; + if (decoded.has_audience()) { + aud = decoded.get_audience(); + } + + string client_id; + if (decoded.has_payload_claim("client_id")) { + client_id = decoded.get_payload_claim("client_id").as_string(); + } + if (client_id.empty() && decoded.has_payload_claim("clientId")) { + client_id = decoded.get_payload_claim("clientId").as_string(); + } + string azp; + if (decoded.has_payload_claim("azp")) { + azp = decoded.get_payload_claim("azp").as_string(); + } + + string role_arn = s->info.args.get("RoleArn"); + RGWOIDCProviderInfo provider; + int r = load_provider(dpp, y, role_arn, iss, provider); + if (r < 0) { + ldpp_dout(dpp, 0) << "Couldn't get oidc provider info using input iss" << iss << dendl; + throw -EACCES; + } + if (decoded.has_payload_claim(string(princTagsNamespace))) { + auto& cl = decoded.get_payload_claim(string(princTagsNamespace)); + if (cl.get_type() == jwt::claim::type::object || cl.get_type() == jwt::claim::type::array) { + recurse_and_insert("dummy", cl, principal_tags); + for (auto it : principal_tags) { + ldpp_dout(dpp, 5) << "Key: " << it.first << " Value: " << it.second << dendl; + } + } else { + ldpp_dout(dpp, 0) << "Malformed principal tags" << cl.as_string() << dendl; + throw -EINVAL; + } + } + if (! provider.client_ids.empty()) { + bool found = false; + for (auto& it : aud) { + if (is_client_id_valid(provider.client_ids, it)) { + found = true; + break; + } + } + if (! found && ! is_client_id_valid(provider.client_ids, client_id) && ! is_client_id_valid(provider.client_ids, azp)) { + ldpp_dout(dpp, 0) << "Client id in token doesn't match with that registered with oidc provider" << dendl; + throw -EACCES; + } + } + //Validate signature + if (decoded.has_algorithm()) { + auto& algorithm = decoded.get_algorithm(); + try { + validate_signature(dpp, decoded, algorithm, iss, provider.thumbprints, y); + } catch (...) { + throw -EACCES; + } + } else { + return {boost::none, boost::none}; + } + } catch (int error) { + if (error == -EACCES) { + throw -EACCES; + } + ldpp_dout(dpp, 5) << "Invalid JWT token" << dendl; + return {boost::none, boost::none}; + } + catch (...) { + ldpp_dout(dpp, 5) << "Invalid JWT token" << dendl; + return {boost::none, boost::none}; + } + return {t, principal_tags}; +} + +std::string +WebTokenEngine::get_cert_url(const string& iss, const DoutPrefixProvider *dpp, optional_yield y) const +{ + string cert_url; + string openidc_wellknown_url = iss; + bufferlist openidc_resp; + + if (openidc_wellknown_url.back() == '/') { + openidc_wellknown_url.pop_back(); + } + openidc_wellknown_url.append("/.well-known/openid-configuration"); + + RGWHTTPTransceiver openidc_req(cct, "GET", openidc_wellknown_url, &openidc_resp); + + //Headers + openidc_req.append_header("Content-Type", "application/x-www-form-urlencoded"); + + int res = openidc_req.process(dpp, y); + if (res < 0) { + ldpp_dout(dpp, 10) << "HTTP request res: " << res << dendl; + throw -EINVAL; + } + + //Debug only + ldpp_dout(dpp, 20) << "HTTP status: " << openidc_req.get_http_status() << dendl; + ldpp_dout(dpp, 20) << "JSON Response is: " << openidc_resp.c_str() << dendl; + + JSONParser parser; + if (parser.parse(openidc_resp)) { + JSONObj::data_val val; + if (parser.get_data("jwks_uri", &val)) { + cert_url = val.str.c_str(); + ldpp_dout(dpp, 20) << "Cert URL is: " << cert_url.c_str() << dendl; + } else { + ldpp_dout(dpp, 0) << "Malformed json returned while fetching openidc url" << dendl; + } + } + return cert_url; +} + +void +WebTokenEngine::validate_signature(const DoutPrefixProvider* dpp, const jwt::decoded_jwt& decoded, const string& algorithm, const string& iss, const vector& thumbprints, optional_yield y) const +{ + if (algorithm != "HS256" && algorithm != "HS384" && algorithm != "HS512") { + string cert_url = get_cert_url(iss, dpp, y); + if (cert_url.empty()) { + throw -EINVAL; + } + + // Get certificate + bufferlist cert_resp; + RGWHTTPTransceiver cert_req(cct, "GET", cert_url, &cert_resp); + //Headers + cert_req.append_header("Content-Type", "application/x-www-form-urlencoded"); + + int res = cert_req.process(dpp, y); + if (res < 0) { + ldpp_dout(dpp, 10) << "HTTP request res: " << res << dendl; + throw -EINVAL; + } + //Debug only + ldpp_dout(dpp, 20) << "HTTP status: " << cert_req.get_http_status() << dendl; + ldpp_dout(dpp, 20) << "JSON Response is: " << cert_resp.c_str() << dendl; + + JSONParser parser; + if (parser.parse(cert_resp)) { + JSONObj::data_val val; + if (parser.get_data("keys", &val)) { + if (val.str[0] == '[') { + val.str.erase(0, 1); + } + if (val.str[val.str.size() - 1] == ']') { + val.str = val.str.erase(val.str.size() - 1, 1); + } + if (parser.parse(val)) { + vector x5c; + if (JSONDecoder::decode_json("x5c", x5c, &parser)) { + string cert; + bool found_valid_cert = false; + for (auto& it : x5c) { + cert = "-----BEGIN CERTIFICATE-----\n" + it + "\n-----END CERTIFICATE-----"; + ldpp_dout(dpp, 20) << "Certificate is: " << cert.c_str() << dendl; + if (is_cert_valid(thumbprints, cert)) { + found_valid_cert = true; + break; + } + } + if (! found_valid_cert) { + ldpp_dout(dpp, 0) << "Cert doesn't match that with the thumbprints registered with oidc provider: " << cert.c_str() << dendl; + throw -EINVAL; + } + try { + //verify method takes care of expired tokens also + if (algorithm == "RS256") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::rs256{cert}); + + verifier.verify(decoded); + } else if (algorithm == "RS384") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::rs384{cert}); + + verifier.verify(decoded); + } else if (algorithm == "RS512") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::rs512{cert}); + + verifier.verify(decoded); + } else if (algorithm == "ES256") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::es256{cert}); + + verifier.verify(decoded); + } else if (algorithm == "ES384") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::es384{cert}); + + verifier.verify(decoded); + } else if (algorithm == "ES512") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::es512{cert}); + + verifier.verify(decoded); + } else if (algorithm == "PS256") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::ps256{cert}); + + verifier.verify(decoded); + } else if (algorithm == "PS384") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::ps384{cert}); + + verifier.verify(decoded); + } else if (algorithm == "PS512") { + auto verifier = jwt::verify() + .allow_algorithm(jwt::algorithm::ps512{cert}); + + verifier.verify(decoded); + } else { + ldpp_dout(dpp, 0) << "Unsupported algorithm: " << algorithm << dendl; + throw -EINVAL; + } + } catch (std::runtime_error& e) { + ldpp_dout(dpp, 0) << "Signature validation failed: " << e.what() << dendl; + throw; + } + catch (...) { + ldpp_dout(dpp, 0) << "Signature validation failed" << dendl; + throw; + } + } else { + ldpp_dout(dpp, 0) << "x5c not present" << dendl; + throw -EINVAL; + } + } else { + ldpp_dout(dpp, 0) << "Malformed JSON object for keys" << dendl; + throw -EINVAL; + } + } else { + ldpp_dout(dpp, 0) << "keys not present in JSON" << dendl; + throw -EINVAL; + } //if-else get-data + } else { + ldpp_dout(dpp, 0) << "Malformed json returned while fetching cert" << dendl; + throw -EINVAL; + } //if-else parser cert_resp + } else { + ldpp_dout(dpp, 0) << "JWT signed by HMAC algos are currently not supported" << dendl; + throw -EINVAL; + } +} + +WebTokenEngine::result_t +WebTokenEngine::authenticate( const DoutPrefixProvider* dpp, + const std::string& token, + const req_state* const s, + optional_yield y) const +{ + if (! is_applicable(token)) { + return result_t::deny(); + } + + try { + auto [t, princ_tags] = get_from_jwt(dpp, token, s, y); + if (t) { + string role_session = s->info.args.get("RoleSessionName"); + if (role_session.empty()) { + ldout(s->cct, 0) << "Role Session Name is empty " << dendl; + return result_t::deny(-EACCES); + } + string role_arn = s->info.args.get("RoleArn"); + string role_tenant = get_role_tenant(role_arn); + string role_name = get_role_name(role_arn); + + rgw_account_id role_account; + if (rgw::account::validate_id(role_tenant)) { + role_account = std::move(role_tenant); + role_tenant.clear(); + } + + std::unique_ptr role = driver->get_role(role_name, role_tenant, role_account); + int ret = role->load_by_name(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "Role not found: name:" << role_name << " tenant: " << role_tenant << dendl; + return result_t::deny(-EACCES); + } + + std::optional account; + if (!role_account.empty()) { + account.emplace(); + rgw::sal::Attrs attrs; // ignored + RGWObjVersionTracker objv; // ignored + ret = driver->load_account_by_id(dpp, y, role_account, + *account, attrs, objv); + if (ret < 0) { + ldpp_dout(dpp, 0) << "Role account " << role_account << " not found" << dendl; + return result_t::deny(-EACCES); + } + } + + boost::optional> role_tags = role->get_tags(); + auto apl = apl_factory->create_apl_web_identity( + cct, s, role->get_id(), role_session, role_tenant, + *t, role_tags, princ_tags, std::move(account)); + return result_t::grant(std::move(apl)); + } + return result_t::deny(-EACCES); + } + catch (...) { + return result_t::deny(-EACCES); + } +} + +} // namespace rgw::auth::sts + +int RGWREST_STS::verify_permission(optional_yield y) +{ + STS::STSService _sts(s->cct, driver, s->user->get_id(), s->auth.identity.get()); + sts = std::move(_sts); + + string rArn = s->info.args.get("RoleArn"); + const auto& [ret, role] = sts.getRoleInfo(s, rArn, y); + if (ret < 0) { + ldpp_dout(this, 0) << "failed to get role info using role arn: " << rArn << dendl; + return ret; + } + string policy = role->get_assume_role_policy(); + + //Parse the policy + //TODO - This step should be part of Role Creation + try { + // resource policy is not restricted to the current tenant + const std::string* policy_tenant = nullptr; + + const rgw::IAM::Policy p(s->cct, policy_tenant, policy, false); + if (!s->principal_tags.empty()) { + auto res = p.eval(s->env, *s->auth.identity, rgw::IAM::stsTagSession, boost::none); + if (res != rgw::IAM::Effect::Allow) { + ldout(s->cct, 0) << "evaluating policy for stsTagSession returned deny/pass" << dendl; + return -EPERM; + } + } + uint64_t op; + if (get_type() == RGW_STS_ASSUME_ROLE_WEB_IDENTITY) { + op = rgw::IAM::stsAssumeRoleWithWebIdentity; + } else { + op = rgw::IAM::stsAssumeRole; + } + + auto res = p.eval(s->env, *s->auth.identity, op, boost::none); + if (res != rgw::IAM::Effect::Allow) { + ldout(s->cct, 0) << "evaluating policy for op: " << op << " returned deny/pass" << dendl; + return -EPERM; + } + } catch (rgw::IAM::PolicyParseException& e) { + ldpp_dout(this, 0) << "failed to parse policy: " << e.what() << dendl; + return -EPERM; + } + + return 0; +} + +void RGWREST_STS::send_response() +{ + if (op_ret) { + set_req_state_err(s, op_ret); + } + dump_errno(s); + end_header(s); +} + +int RGWSTSGetSessionToken::verify_permission(optional_yield y) +{ + rgw::Partition partition = rgw::Partition::aws; + rgw::Service service = rgw::Service::s3; + if (!verify_user_permission(this, + s, + rgw::ARN(partition, service, "", s->user->get_tenant(), ""), + rgw::IAM::stsGetSessionToken)) { + ldpp_dout(this, 0) << "User does not have permission to perform GetSessionToken" << dendl; + return -EACCES; + } + + return 0; +} + +int RGWSTSGetSessionToken::get_params() +{ + duration = s->info.args.get("DurationSeconds"); + serialNumber = s->info.args.get("SerialNumber"); + tokenCode = s->info.args.get("TokenCode"); + + if (! duration.empty()) { + string err; + uint64_t duration_in_secs = strict_strtoll(duration.c_str(), 10, &err); + if (!err.empty()) { + ldpp_dout(this, 0) << "Invalid value of input duration: " << duration << dendl; + return -EINVAL; + } + + if (duration_in_secs < STS::GetSessionTokenRequest::getMinDuration() || + duration_in_secs > s->cct->_conf->rgw_sts_max_session_duration) { + ldpp_dout(this, 0) << "Invalid duration in secs: " << duration_in_secs << dendl; + return -EINVAL; + } + } + + return 0; +} + +void RGWSTSGetSessionToken::execute(optional_yield y) +{ + if (op_ret = get_params(); op_ret < 0) { + return; + } + + STS::STSService sts(s->cct, driver, s->user->get_id(), s->auth.identity.get()); + + STS::GetSessionTokenRequest req(duration, serialNumber, tokenCode); + const auto& [ret, creds] = sts.getSessionToken(this, req); + op_ret = std::move(ret); + //Dump the output + if (op_ret == 0) { + s->formatter->open_object_section_in_ns("GetSessionTokenResponse", RGW_REST_STS_XMLNS); + s->formatter->open_object_section("GetSessionTokenResult"); + s->formatter->open_object_section("Credentials"); + creds.dump(s->formatter); + s->formatter->close_section(); + s->formatter->close_section(); + s->formatter->close_section(); + } +} + +int RGWSTSAssumeRoleWithWebIdentity::get_params() +{ + duration = s->info.args.get("DurationSeconds"); + providerId = s->info.args.get("ProviderId"); + policy = s->info.args.get("Policy"); + roleArn = s->info.args.get("RoleArn"); + roleSessionName = s->info.args.get("RoleSessionName"); + iss = s->info.args.get("provider_id"); + sub = s->info.args.get("sub"); + aud = s->info.args.get("aud"); + + if (roleArn.empty() || roleSessionName.empty() || sub.empty() || aud.empty()) { + ldpp_dout(this, 0) << "ERROR: one of role arn or role session name or token is empty" << dendl; + return -EINVAL; + } + + if (! policy.empty()) { + try { + const rgw::IAM::Policy p( + s->cct, nullptr, policy, + s->cct->_conf.get_val("rgw_policy_reject_invalid_principals")); + } + catch (rgw::IAM::PolicyParseException& e) { + ldpp_dout(this, 5) << "failed to parse policy: " << e.what() << "policy" << policy << dendl; + s->err.message = e.what(); + return -ERR_MALFORMED_DOC; + } + } + + return 0; +} + +void RGWSTSAssumeRoleWithWebIdentity::execute(optional_yield y) +{ + if (op_ret = get_params(); op_ret < 0) { + return; + } + + STS::AssumeRoleWithWebIdentityRequest req(s->cct, duration, providerId, policy, roleArn, + roleSessionName, iss, sub, aud, s->principal_tags); + STS::AssumeRoleWithWebIdentityResponse response = sts.assumeRoleWithWebIdentity(this, req); + op_ret = std::move(response.assumeRoleResp.retCode); + + //Dump the output + if (op_ret == 0) { + s->formatter->open_object_section_in_ns("AssumeRoleWithWebIdentityResponse", RGW_REST_STS_XMLNS); + s->formatter->open_object_section("AssumeRoleWithWebIdentityResult"); + encode_json("SubjectFromWebIdentityToken", response.sub , s->formatter); + encode_json("Audience", response.aud , s->formatter); + s->formatter->open_object_section("AssumedRoleUser"); + response.assumeRoleResp.user.dump(s->formatter); + s->formatter->close_section(); + s->formatter->open_object_section("Credentials"); + response.assumeRoleResp.creds.dump(s->formatter); + s->formatter->close_section(); + encode_json("Provider", response.providerId , s->formatter); + encode_json("PackedPolicySize", response.assumeRoleResp.packedPolicySize , s->formatter); + s->formatter->close_section(); + s->formatter->close_section(); + } +} + +int RGWSTSAssumeRole::get_params() +{ + duration = s->info.args.get("DurationSeconds"); + externalId = s->info.args.get("ExternalId"); + policy = s->info.args.get("Policy"); + roleArn = s->info.args.get("RoleArn"); + roleSessionName = s->info.args.get("RoleSessionName"); + serialNumber = s->info.args.get("SerialNumber"); + tokenCode = s->info.args.get("TokenCode"); + + if (roleArn.empty() || roleSessionName.empty()) { + ldpp_dout(this, 0) << "ERROR: one of role arn or role session name is empty" << dendl; + return -EINVAL; + } + + if (! policy.empty()) { + try { + const rgw::IAM::Policy p( + s->cct, nullptr, policy, + s->cct->_conf.get_val("rgw_policy_reject_invalid_principals")); + } + catch (rgw::IAM::PolicyParseException& e) { + ldpp_dout(this, 0) << "failed to parse policy: " << e.what() << "policy" << policy << dendl; + s->err.message = e.what(); + return -ERR_MALFORMED_DOC; + } + } + + return 0; +} + +void RGWSTSAssumeRole::execute(optional_yield y) +{ + if (op_ret = get_params(); op_ret < 0) { + return; + } + + STS::AssumeRoleRequest req(s->cct, duration, externalId, policy, roleArn, + roleSessionName, serialNumber, tokenCode); + STS::AssumeRoleResponse response = sts.assumeRole(s, req, y); + op_ret = std::move(response.retCode); + //Dump the output + if (op_ret == 0) { + s->formatter->open_object_section_in_ns("AssumeRoleResponse", RGW_REST_STS_XMLNS); + s->formatter->open_object_section("AssumeRoleResult"); + s->formatter->open_object_section("Credentials"); + response.creds.dump(s->formatter); + s->formatter->close_section(); + s->formatter->open_object_section("AssumedRoleUser"); + response.user.dump(s->formatter); + s->formatter->close_section(); + encode_json("PackedPolicySize", response.packedPolicySize , s->formatter); + s->formatter->close_section(); + s->formatter->close_section(); + } +} + +int RGW_Auth_STS::authorize(const DoutPrefixProvider *dpp, + rgw::sal::Driver* driver, + const rgw::auth::StrategyRegistry& auth_registry, + req_state *s, optional_yield y) +{ + return rgw::auth::Strategy::apply(dpp, auth_registry.get_sts(), s, y); +} + +using op_generator = RGWOp*(*)(); +static const std::unordered_map op_generators = { + {"AssumeRole", []() -> RGWOp* {return new RGWSTSAssumeRole;}}, + {"GetSessionToken", []() -> RGWOp* {return new RGWSTSGetSessionToken;}}, + {"AssumeRoleWithWebIdentity", []() -> RGWOp* {return new RGWSTSAssumeRoleWithWebIdentity;}} +}; + +bool RGWHandler_REST_STS::action_exists(const req_state* s) +{ + if (s->info.args.exists("Action")) { + const std::string action_name = s->info.args.get("Action"); + return op_generators.contains(action_name); + } + return false; +} + +RGWOp *RGWHandler_REST_STS::op_post() +{ + if (s->info.args.exists("Action")) { + const std::string action_name = s->info.args.get("Action"); + const auto action_it = op_generators.find(action_name); + if (action_it != op_generators.end()) { + return action_it->second(); + } + ldpp_dout(s, 10) << "unknown action '" << action_name << "' for STS handler" << dendl; + } else { + ldpp_dout(s, 10) << "missing action argument in STS handler" << dendl; + } + return nullptr; +} + +int RGWHandler_REST_STS::init(rgw::sal::Driver* driver, + req_state *s, + rgw::io::BasicClient *cio) +{ + s->dialect = "sts"; + s->prot_flags = RGW_REST_STS; + + return RGWHandler_REST::init(driver, s, cio); +} + +int RGWHandler_REST_STS::authorize(const DoutPrefixProvider* dpp, optional_yield y) +{ + if (s->info.args.exists("Action") && s->info.args.get("Action") == "AssumeRoleWithWebIdentity") { + return RGW_Auth_STS::authorize(dpp, driver, auth_registry, s, y); + } + return RGW_Auth_S3::authorize(dpp, driver, auth_registry, s, y); +} + +RGWHandler_REST* +RGWRESTMgr_STS::get_handler(rgw::sal::Driver* driver, + req_state* const s, + const rgw::auth::StrategyRegistry& auth_registry, + const std::string& frontend_prefix) +{ + return new RGWHandler_REST_STS(auth_registry); +} diff --git a/src/rgw/rgw_rest_client.cc b/src/rgw/rgw_rest_client.cc index 41bbbebd7b2..7107c62f8e5 100644 --- a/src/rgw/rgw_rest_client.cc +++ b/src/rgw/rgw_rest_client.cc @@ -466,8 +466,6 @@ auto RGWRESTSimpleRequest::forward_request(const DoutPrefixProvider *dpp, const return tl::unexpected(-ERR_SERVICE_UNAVAILABLE); } - response.append((char)0); /* NULL terminate response */ - if (outbl) { *outbl = std::move(response); } diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h index 0672e5bc138..9e66a63dc87 100644 --- a/src/rgw/rgw_rest_conn.h +++ b/src/rgw/rgw_rest_conn.h @@ -17,7 +17,7 @@ template inline int parse_decode_json(T& t, bufferlist& bl) { JSONParser p; - if (!p.parse(bl.c_str(), bl.length())) { + if (!p.parse(bl)) { return -EINVAL; } diff --git a/src/rgw/rgw_rest_role.cc b/src/rgw/rgw_rest_role.cc index 483f1b487ff..8b6e8f10813 100644 --- a/src/rgw/rgw_rest_role.cc +++ b/src/rgw/rgw_rest_role.cc @@ -449,7 +449,7 @@ int RGWModifyRoleTrustPolicy::init_processing(optional_yield y) } JSONParser p; - if (!p.parse(trust_policy.c_str(), trust_policy.length())) { + if (!p.parse(trust_policy)) { ldpp_dout(this, 20) << "ERROR: failed to parse assume role policy doc" << dendl; return -ERR_MALFORMED_DOC; } diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 7aef4284ce9..4b4b164bcc5 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -3393,16 +3393,17 @@ int RGWPostObj_ObjStore_S3::get_policy(optional_yield y) return -EINVAL; } - decoded_policy.append('\0'); // NULL terminate + if('\0' != decoded_policy.c_str()[decoded_policy.length() - 1]) + decoded_policy.append('\0'); // NULL terminate + ldpp_dout(this, 20) << "POST policy: " << decoded_policy.c_str() << dendl; - - int r = post_policy.from_json(decoded_policy, err_msg); + int r = post_policy.from_json(std::string_view { decoded_policy.c_str() }, err_msg); if (r < 0) { if (err_msg.empty()) { err_msg = "Failed to parse policy"; } - ldpp_dout(this, 0) << "failed to parse policy" << dendl; + ldpp_dout(this, 0) << err_msg << dendl; return -EINVAL; } diff --git a/src/rgw/rgw_rest_sts.cc b/src/rgw/rgw_rest_sts.cc index e2d16d56988..f2af5cc48e4 100644 --- a/src/rgw/rgw_rest_sts.cc +++ b/src/rgw/rgw_rest_sts.cc @@ -327,7 +327,7 @@ WebTokenEngine::get_cert_url(const string& iss, const DoutPrefixProvider *dpp, o ldpp_dout(dpp, 20) << "JSON Response is: " << openidc_resp.c_str() << dendl; JSONParser parser; - if (parser.parse(openidc_resp.c_str(), openidc_resp.length())) { + if (parser.parse(openidc_resp)) { JSONObj::data_val val; if (parser.get_data("jwks_uri", &val)) { cert_url = val.str.c_str(); @@ -633,7 +633,7 @@ WebTokenEngine::validate_signature(const DoutPrefixProvider* dpp, const jwt::dec ldpp_dout(dpp, 20) << "JSON Response is: " << cert_resp.c_str() << dendl; JSONParser parser; - if (parser.parse(cert_resp.c_str(), cert_resp.length())) { + if (parser.parse(cert_resp)) { JSONObj* val = parser.find_obj("keys"); if (val && val->is_array()) { vector keys = val->get_array_elements(); @@ -641,14 +641,13 @@ WebTokenEngine::validate_signature(const DoutPrefixProvider* dpp, const jwt::dec JSONParser k_parser; vector x5c; std::string use, kid; - if (k_parser.parse(key.c_str(), key.size())) { + if (k_parser.parse(key)) { if (JSONDecoder::decode_json("kid", kid, &k_parser)) { ldpp_dout(dpp, 20) << "Checking key id: " << kid << dendl; } if (JSONDecoder::decode_json("use", use, &k_parser) && use != "sig") { continue; } - if (JSONDecoder::decode_json("x5c", x5c, &k_parser)) { string cert; bool found_valid_cert = false; diff --git a/src/rgw/rgw_token.h b/src/rgw/rgw_token.h index e76221fcafd..c810089848c 100644 --- a/src/rgw/rgw_token.h +++ b/src/rgw/rgw_token.h @@ -82,13 +82,13 @@ namespace rgw { explicit RGWToken(const string& json) { JSONParser p; - p.parse(json.c_str(), json.length()); + p.parse(json); JSONDecoder::decode_json(RGWToken::type_name, *this, &p); } RGWToken& operator=(const std::string& json) { JSONParser p; - p.parse(json.c_str(), json.length()); + p.parse(json); JSONDecoder::decode_json(RGWToken::type_name, *this, &p); return *this; } diff --git a/src/test/common/test_json_formattable.cc b/src/test/common/test_json_formattable.cc index 960eef6a3c5..4997ec6370e 100644 --- a/src/test/common/test_json_formattable.cc +++ b/src/test/common/test_json_formattable.cc @@ -38,7 +38,7 @@ static void get_jf(const string& s, JSONFormattable *f) } } -TEST(formatable, str) { +TEST(formattable, str) { JSONFormattable f; get_jf("{ \"foo\": \"bar\" }", &f); ASSERT_EQ((string)f["foo"], "bar"); @@ -46,7 +46,7 @@ TEST(formatable, str) { ASSERT_EQ((string)f["fooz"]("lala"), "lala"); } -TEST(formatable, str2) { +TEST(formattable, str2) { JSONFormattable f; get_jf("{ \"foo\": \"bar\" }", &f); ASSERT_EQ((string)f["foo"], "bar"); @@ -62,13 +62,13 @@ TEST(formatable, str2) { } -TEST(formatable, str3) { +TEST(formattable, str3) { JSONFormattable f; get_jf("{ \"foo\": \"1234bar56\" }", &f); ASSERT_EQ((string)f["foo"], "1234bar56"); } -TEST(formatable, int) { +TEST(formattable, int) { JSONFormattable f; get_jf("{ \"foo\": 1 }", &f); ASSERT_EQ((int)f["foo"], 1); @@ -83,7 +83,7 @@ TEST(formatable, int) { ASSERT_EQ((int)f2["fooz"](111), 123); } -TEST(formatable, bool) { +TEST(formattable, bool) { JSONFormattable f; get_jf("{ \"foo\": \"true\" }", &f); ASSERT_EQ((bool)f["foo"], true); @@ -95,7 +95,7 @@ TEST(formatable, bool) { ASSERT_EQ((bool)f["foo"], false); } -TEST(formatable, nested) { +TEST(formattable, nested) { JSONFormattable f; get_jf("{ \"obj\": { \"foo\": 1, \"inobj\": { \"foo\": 2 } } }", &f); ASSERT_EQ((int)f["foo"], 0); @@ -103,7 +103,7 @@ TEST(formatable, nested) { ASSERT_EQ((int)f["obj"]["inobj"]["foo"], 2); } -TEST(formatable, array) { +TEST(formattable, array) { JSONFormattable f; get_jf("{ \"arr\": [ { \"foo\": 1, \"inobj\": { \"foo\": 2 } }," "{ \"foo\": 2 } ] }", &f); @@ -124,7 +124,7 @@ TEST(formatable, array) { } } -TEST(formatable, bin_encode) { +TEST(formattable, bin_encode) { JSONFormattable f, f2; get_jf("{ \"arr\": [ { \"foo\": 1, \"bar\": \"aaa\", \"inobj\": { \"foo\": 2 } }," "{ \"foo\": 2, \"inobj\": { \"foo\": 3 } } ] }", &f); @@ -156,7 +156,7 @@ TEST(formatable, bin_encode) { } -TEST(formatable, json_encode) { +TEST(formattable, json_encode) { JSONFormattable f, f2; get_jf("{ \"arr\": [ { \"foo\": 1, \"bar\": \"aaa\", \"inobj\": { \"foo\": 2 } }," "{ \"foo\": 2, \"inobj\": { \"foo\": 3 } } ] }", &f); @@ -181,7 +181,7 @@ TEST(formatable, json_encode) { } -TEST(formatable, set) { +TEST(formattable, set) { JSONFormattable f, f2; f.set("", "{ \"abc\": \"xyz\"}"); @@ -200,13 +200,13 @@ TEST(formatable, set) { ASSERT_EQ((int)f["obj"]["c"], 30); } -TEST(formatable, set2) { +TEST(formattable, set2) { JSONFormattable f; f.set("foo", "1234bar56"); ASSERT_EQ((string)f["foo"], "1234bar56"); } -TEST(formatable, erase) { +TEST(formattable, erase) { JSONFormattable f, f2; f.set("", "{ \"abc\": \"xyz\"}"); @@ -240,7 +240,7 @@ static void dumpf(const JSONFormattable& f) { dumpt(f, "f"); } -TEST(formatable, set_array) { +TEST(formattable, set_array) { JSONFormattable f, f2; f.set("asd[0]", "\"xyz\""); @@ -272,7 +272,7 @@ TEST(formatable, set_array) { ASSERT_EQ((string)f2[0]["field"], "xyz"); } -TEST(formatable, erase_array) { +TEST(formattable, erase_array) { JSONFormattable f; f.set("asd[0]", "\"xyz\""); @@ -315,7 +315,7 @@ void formatter_convert(JSONFormatter& formatter, JSONFormattable *dest) get_jf(ss.str(), dest); } -TEST(formatable, encode_simple) { +TEST(formattable, encode_simple) { JSONFormattable f; encode_json("foo", "bar", &f); @@ -422,7 +422,7 @@ struct struct2 { }; -TEST(formatable, encode_struct) { +TEST(formattable, encode_struct) { JSONFormattable f; struct2 s2; diff --git a/src/test/common/test_json_formatter.cc b/src/test/common/test_json_formatter.cc index 79a20df22da..32549912029 100644 --- a/src/test/common/test_json_formatter.cc +++ b/src/test/common/test_json_formatter.cc @@ -4,6 +4,7 @@ /* * Ceph - scalable distributed file system * + * Copyright (C) 2025 IBM * Copyright (C) 2018 Red Hat Inc. * * This library is free software; you can redistribute it and/or @@ -25,14 +26,12 @@ using namespace std; +const string outstring = +"{\"pg_ready\":true, \"pg_stats\":[ { \"pgid\":\"1.0\", \"version\":\"16'56\",\"reported_seq\":\"62\",\"reported_epoch\":\"20\",\"state\":\"active+clean+inconsistent\",\"last_fresh\":\"2018-12-18 15:21:22.173804\",\"last_change\":\"2018-12-18 15:21:22.173804\",\"last_active\":\"2018-12-18 15:21:22.173804\",\"last_peered\":\"2018-12-18 15:21:22.173804\",\"last_clean\":\"2018-12-18 15:21:22.173804\",\"last_became_active\":\"2018-12-18 15:21:21.347685\",\"last_became_peered\":\"2018-12-18 15:21:21.347685\",\"last_unstale\":\"2018-12-18 15:21:22.173804\",\"last_undegraded\":\"2018-12-18 15:21:22.173804\",\"last_fullsized\":\"2018-12-18 15:21:22.173804\",\"mapping_epoch\":19,\"log_start\":\"0'0\",\"ondisk_log_start\":\"0'0\",\"created\":7,\"last_epoch_clean\":20,\"parent\":\"0.0\",\"parent_split_bits\":0,\"last_scrub\":\"16'56\",\"last_scrub_stamp\":\"2018-12-18 15:21:22.173684\",\"last_deep_scrub\":\"0'0\",\"last_deep_scrub_stamp\":\"2018-12-18 15:21:06.514438\",\"last_clean_scrub_stamp\":\"2018-12-18 15:21:06.514438\",\"log_size\":56,\"ondisk_log_size\":56,\"stats_invalid\":false,\"dirty_stats_invalid\":false,\"omap_stats_invalid\":false,\"hitset_stats_invalid\":false,\"hitset_bytes_stats_invalid\":false,\"pin_stats_invalid\":false,\"manifest_stats_invalid\":false,\"snaptrimq_len\":0,\"stat_sum\":{\"num_bytes\":24448,\"num_objects\":36,\"num_object_clones\":20,\"num_object_copies\":36,\"num_objects_missing_on_primary\":0,\"num_objects_missing\":0,\"num_objects_degraded\":0,\"num_objects_misplaced\":0,\"num_objects_unfound\":0,\"num_objects_dirty\":36,\"num_whiteouts\":3,\"num_read\":0,\"num_read_kb\":0,\"num_write\":36,\"num_write_kb\":50,\"num_scrub_errors\":20,\"num_shallow_scrub_errors\":20,\"num_deep_scrub_errors\":0,\"num_objects_recovered\":0,\"num_bytes_recovered\":0,\"num_keys_recovered\":0,\"num_objects_omap\":0,\"num_objects_hit_set_archive\":0,\"num_bytes_hit_set_archive\":0,\"num_flush\":0,\"num_flush_kb\":0,\"num_evict\":0,\"num_evict_kb\":0,\"num_promote\":0,\"num_flush_mode_high\":0,\"num_flush_mode_low\":0,\"num_evict_mode_some\":0,\"num_evict_mode_full\":0,\"num_objects_pinned\":0,\"num_legacy_snapsets\":0,\"num_large_omap_objects\":0,\"num_objects_manifest\":0},\"up\":[0],\"acting\":[0],\"blocked_by\":[],\"up_primary\":0,\"acting_primary\":0,\"purged_snaps\":[] }]}"; TEST(formatter, bug_37706) { vector pgs; - string outstring = -"{\"pg_ready\":true, \"pg_stats\":[ { \"pgid\":\"1.0\", \"version\":\"16'56\",\"reported_seq\":\"62\",\"reported_epoch\":\"20\",\"state\":\"active+clean+inconsistent\",\"last_fresh\":\"2018-12-18 15:21:22.173804\",\"last_change\":\"2018-12-18 15:21:22.173804\",\"last_active\":\"2018-12-18 15:21:22.173804\",\"last_peered\":\"2018-12-18 15:21:22.173804\",\"last_clean\":\"2018-12-18 15:21:22.173804\",\"last_became_active\":\"2018-12-18 15:21:21.347685\",\"last_became_peered\":\"2018-12-18 15:21:21.347685\",\"last_unstale\":\"2018-12-18 15:21:22.173804\",\"last_undegraded\":\"2018-12-18 15:21:22.173804\",\"last_fullsized\":\"2018-12-18 15:21:22.173804\",\"mapping_epoch\":19,\"log_start\":\"0'0\",\"ondisk_log_start\":\"0'0\",\"created\":7,\"last_epoch_clean\":20,\"parent\":\"0.0\",\"parent_split_bits\":0,\"last_scrub\":\"16'56\",\"last_scrub_stamp\":\"2018-12-18 15:21:22.173684\",\"last_deep_scrub\":\"0'0\",\"last_deep_scrub_stamp\":\"2018-12-18 15:21:06.514438\",\"last_clean_scrub_stamp\":\"2018-12-18 15:21:06.514438\",\"log_size\":56,\"ondisk_log_size\":56,\"stats_invalid\":false,\"dirty_stats_invalid\":false,\"omap_stats_invalid\":false,\"hitset_stats_invalid\":false,\"hitset_bytes_stats_invalid\":false,\"pin_stats_invalid\":false,\"manifest_stats_invalid\":false,\"snaptrimq_len\":0,\"stat_sum\":{\"num_bytes\":24448,\"num_objects\":36,\"num_object_clones\":20,\"num_object_copies\":36,\"num_objects_missing_on_primary\":0,\"num_objects_missing\":0,\"num_objects_degraded\":0,\"num_objects_misplaced\":0,\"num_objects_unfound\":0,\"num_objects_dirty\":36,\"num_whiteouts\":3,\"num_read\":0,\"num_read_kb\":0,\"num_write\":36,\"num_write_kb\":50,\"num_scrub_errors\":20,\"num_shallow_scrub_errors\":20,\"num_deep_scrub_errors\":0,\"num_objects_recovered\":0,\"num_bytes_recovered\":0,\"num_keys_recovered\":0,\"num_objects_omap\":0,\"num_objects_hit_set_archive\":0,\"num_bytes_hit_set_archive\":0,\"num_flush\":0,\"num_flush_kb\":0,\"num_evict\":0,\"num_evict_kb\":0,\"num_promote\":0,\"num_flush_mode_high\":0,\"num_flush_mode_low\":0,\"num_evict_mode_some\":0,\"num_evict_mode_full\":0,\"num_objects_pinned\":0,\"num_legacy_snapsets\":0,\"num_large_omap_objects\":0,\"num_objects_manifest\":0},\"up\":[0],\"acting\":[0],\"blocked_by\":[],\"up_primary\":0,\"acting_primary\":0,\"purged_snaps\":[] }]}"; - - JSONParser parser; ASSERT_TRUE(parser.parse(outstring.c_str(), outstring.size())); @@ -128,3 +127,63 @@ TEST(formatter, dump_large_item) { EXPECT_TRUE(parser.parse(bl.c_str(), bl.length())); EXPECT_EQ(parser.find_obj("Location")->get_data(), full_url); } + +TEST(formatter, parse_types) { + // Check that JSONParser::parse() works with expected data types + // (otherwise at least indirectly tested elsewhere): + + const auto json_input = R"({"expiration": "2025-01-17T10:26:46Z", "conditions": [{"bucket": "user-hqfadib9zxyj2pygevewzf45t-1"}, ["starts-with", "$key", "foo"], {"acl": "private"}, ["starts-with", "$Content-Type", "text/plain"], ["content-length-range", 0, 1024]]})"; + + { + JSONParser parser; + + ASSERT_TRUE(parser.parse(outstring)); + + JSONObj *pgstat_obj = parser.find_obj("pg_stats"); + EXPECT_TRUE(pgstat_obj); + } + + { + JSONParser parser; + + ASSERT_TRUE(parser.parse(std::string_view { outstring })); + + JSONObj *pgstat_obj = parser.find_obj("pg_stats"); + EXPECT_TRUE(pgstat_obj); + } + + { + JSONParser parser; + + buffer::list bl; + bl.append(outstring); + EXPECT_TRUE(parser.parse(bl.c_str(), bl.length())); + + JSONObj *pgstat_obj = parser.find_obj("pg_stats"); + EXPECT_TRUE(pgstat_obj); + } + + { + JSONParser parser; + + buffer::list bl; + bl.append(json_input); + + ASSERT_TRUE(parser.parse(bl.c_str(), bl.length())); + + JSONObjIter oi = parser.find_first(); + + JSONObj *o0 = *oi; + ASSERT_TRUE(nullptr != o0); + + ++oi; + JSONObj *o1 = *oi; + ASSERT_TRUE(nullptr != o1); + JSONObj *conditions = parser.find_obj("conditions"); + ASSERT_TRUE(nullptr != conditions); + + JSONObj *expiration = parser.find_obj("expiration"); + ASSERT_TRUE(nullptr != expiration); + } +} + diff --git a/src/test/objectstore/allocator_replay_test.cc b/src/test/objectstore/allocator_replay_test.cc index ddab87709de..9c62d101eeb 100644 --- a/src/test/objectstore/allocator_replay_test.cc +++ b/src/test/objectstore/allocator_replay_test.cc @@ -274,7 +274,7 @@ int replay_free_dump_and_apply_raw( JSONParser p; std::cout << "parsing..." << std::endl; - bool b = p.parse(fname); + bool b = p.parse_file(fname); if (!b) { std::cerr << "Failed to parse json: " << fname << std::endl; return -1; diff --git a/src/test/rgw/rgw_multi/multisite.py b/src/test/rgw/rgw_multi/multisite.py index 77704c01ee5..92ab9472938 100644 --- a/src/test/rgw/rgw_multi/multisite.py +++ b/src/test/rgw/rgw_multi/multisite.py @@ -1,6 +1,7 @@ from abc import ABCMeta, abstractmethod from io import StringIO +import sys import json from .conn import get_gateway_connection, get_gateway_s3_resource, get_gateway_iam_connection, get_gateway_secure_connection, get_gateway_sns_client, get_gateway_sts_connection, get_gateway_temp_s3_client @@ -78,6 +79,7 @@ class SystemObject: s, r = self.command(cluster, cmd, args or [], **kwargs) if r == 0: data = json.loads(s) + # To show the entire command: json.dump(data, sys.stdout) self.load_from_json(data) self.data = data return self.data, r diff --git a/src/tools/cephfs/MetaTool.cc b/src/tools/cephfs/MetaTool.cc index 2081bf62020..77245cb8f30 100644 --- a/src/tools/cephfs/MetaTool.cc +++ b/src/tools/cephfs/MetaTool.cc @@ -354,7 +354,7 @@ void MetaTool::inode_meta_t::encode(::ceph::bufferlist& bl, uint64_t features) int MetaTool::_amend_meta(string& k, inode_meta_t& inode_meta, const string& fn, meta_op& op) { JSONParser parser; - if (!parser.parse(fn.c_str())) { + if (!parser.parse(fn)) { cout << "Error parsing create user response" << std::endl; return -1; } @@ -479,7 +479,7 @@ int MetaTool::amend_fn(meta_op &op) int MetaTool::_amend_fn(const string& fn, bool confirm) { JSONParser parser; - if (!parser.parse(fn.c_str())) { + if (!parser.parse(fn)) { cout << "Error parsing create user response : " << fn << std::endl; return -1; } diff --git a/src/tools/rbd/action/Journal.cc b/src/tools/rbd/action/Journal.cc index f1bdd6b5289..a915ef4e7d1 100644 --- a/src/tools/rbd/action/Journal.cc +++ b/src/tools/rbd/action/Journal.cc @@ -840,7 +840,7 @@ public: n++; error_count++; JSONParser p; - if (!p.parse(bl.c_str(), bl.length())) { + if (!p.parse(bl)) { std::cerr << "rbd: error parsing input (entry " << n << ")" << std::endl; r = -EINVAL;