From 1bafe7cdbd11a10446a6c1499da2be7be798f12d Mon Sep 17 00:00:00 2001 From: caleb miles Date: Mon, 1 Oct 2012 12:39:20 -0400 Subject: [PATCH] rgw: Create JSON parser Create JSON object and parser classes similar to the XMLObj and RGWXMLParser based on JSON Spirit. Signed-off-by: caleb miles Signed-off-by: Yehuda Sadeh --- src/Makefile.am | 4 +- src/rgw/rgw_json.cc | 255 ++++++++++++++++++++++++++++++++++++++++++++ src/rgw/rgw_json.h | 98 +++++++++++++++++ src/rgw/rgw_rest.cc | 54 ++-------- src/rgw/rgw_rest.h | 1 - 5 files changed, 364 insertions(+), 48 deletions(-) create mode 100644 src/rgw/rgw_json.cc create mode 100644 src/rgw/rgw_json.h diff --git a/src/Makefile.am b/src/Makefile.am index f725617cd1236..f90e07bb851e8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -151,7 +151,7 @@ base: ceph-mon ceph-osd ceph-mds \ ceph cephfs \ ceph-syn \ rados radosgw librados-config \ - ceph-conf monmaptool osdmaptool crushtool ceph-authtool \ + ceph-conf monmaptool osdmaptool crushtool jsontest ceph-authtool \ init-ceph mkcephfs @@ -316,6 +316,7 @@ librgw_a_SOURCES = \ rgw/rgw_fcgi.cc \ rgw/rgw_xml.cc \ rgw/rgw_usage.cc \ + rgw/rgw_json.cc \ rgw/rgw_user.cc \ rgw/rgw_tools.cc \ rgw/rgw_rados.cc \ @@ -1748,6 +1749,7 @@ noinst_HEADERS = \ rgw/rgw_client_io.h\ rgw/rgw_fcgi.h\ rgw/rgw_xml.h\ + rgw/rgw_json.h\ rgw/rgw_cache.h\ rgw/rgw_common.h\ rgw/rgw_formats.h\ diff --git a/src/rgw/rgw_json.cc b/src/rgw/rgw_json.cc new file mode 100644 index 0000000000000..1a6fc32dea90a --- /dev/null +++ b/src/rgw/rgw_json.cc @@ -0,0 +1,255 @@ +#include +#include + +#include "rgw_json.h" + +// for testing DELETE ME +#include + +using namespace std; +using namespace json_spirit; + +JSONObjIter::JSONObjIter() +{ +} + +JSONObjIter::~JSONObjIter() +{ +} + +void JSONObjIter::set(const JSONObjIter::map_iter_t &_cur, const JSONObjIter::map_iter_t &_last) +{ + cur = _cur; + last = _last; +} + +void JSONObjIter::operator++() +{ + if (cur != last) + ++cur; +}; + +JSONObj *JSONObjIter::operator*() +{ + return cur->second; +}; + +// does not work, FIXME +ostream& operator<<(ostream& out, JSONObj& obj) { + out << obj.name << ": " << obj.data_string; + return out; +} + +JSONObj::~JSONObj() +{ + multimap::iterator iter; + for (iter = children.begin(); iter != children.end(); ++iter) { + JSONObj *obj = iter->second; + delete obj; + } +} + + +void JSONObj::add_child(string el, JSONObj *obj) +{ + cout << "add_child: " << name << " <- " << el << endl; + children.insert(pair(el, obj)); +} + +bool JSONObj::get_attr(string name, string& attr) +{ + map::iterator iter = attr_map.find(name); + if (iter == attr_map.end()) + return false; + attr = iter->second; + return true; +} + +JSONObjIter JSONObj::find(string name) +{ + JSONObjIter iter; + map::iterator first; + map::iterator last; + first = children.find(name); + last = children.upper_bound(name); + iter.set(first, last); + return iter; +} + +JSONObjIter JSONObj::find_first() +{ + JSONObjIter iter; + iter.set(children.begin(), children.end()); + cout << "count=" << children.size() << endl; + for (map:: iterator i = children.begin(); i != children.end(); ++i) { + cout << "child: " << i->first << endl; + } + return iter; +} + +JSONObjIter JSONObj::find_first(string name) +{ + JSONObjIter iter; + map::iterator first; + first = children.find(name); + iter.set(first, children.end()); + return iter; +} + +/* accepts a JSON Array or JSON Object contained in + * a JSON Spirit Value, v, and creates a JSONObj for each + * child contained in v + */ +void JSONObj::handle_value(Value v) +{ + if (v.type() == obj_type) { + Object temp_obj = v.get_obj(); + for (Object::size_type i = 0; i < temp_obj.size(); i++) { + Pair temp_pair = temp_obj[i]; + string temp_name = temp_pair.name_; + Value temp_value = temp_pair.value_; + JSONObj *child = new JSONObj; + child->init(this, temp_value, temp_name); + add_child(temp_name, child); + } + } else if (v.type() == array_type) { + Array temp_array = v.get_array(); + Value value; + + for (unsigned j = 0; j < temp_array.size(); j++) { + Value cur = temp_array[j]; + + if (cur.type() == obj_type) { + handle_value(cur); + } else { + string temp_name; + + JSONObj *child = new JSONObj; + child->init(this, cur, temp_name); + add_child(child->get_name(), child); + } + } + } +} + +void JSONObj::init(JSONObj *p, Value v, string n) +{ + name = n; + parent = p; + data = v; + + handle_value(v); + if (v.type() == str_type) + data_string = v.get_str(); + else + data_string = write(v, raw_utf8); + attr_map.insert(pair(name, data_string)); +} + +JSONObj *JSONObj::get_parent() +{ + return parent; +} + +bool JSONObj::is_object() +{ + cout << data.type() << std::endl; + return (data.type() == obj_type); +} + +bool JSONObj::is_array() +{ + return (data.type() == array_type); +} + +vector JSONObj::get_array_elements() +{ + vector elements; + Array temp_array; + + if (data.type() == array_type) + temp_array = data.get_array(); + + int array_size = temp_array.size(); + if (array_size > 0) + for (int i = 0; i < array_size; i++) { + Value temp_value = temp_array[i]; + string temp_string; + temp_string = write(temp_value, raw_utf8); + elements.push_back(temp_string); + } + + return elements; +} + +RGWJSONParser::RGWJSONParser() : buf_len(0), success(true) +{ +} + +RGWJSONParser::~RGWJSONParser() +{ +} + + + +void RGWJSONParser::handle_data(const char *s, int len) +{ + json_buffer.append(s, len); // check for problems with null termination FIXME + buf_len += len; +} + +// parse a supplied JSON fragment +bool RGWJSONParser::parse(const char *buf_, int len) +{ + string json_string = buf_; + // make a substring to len + json_string = json_string.substr(0, len); + success = read(json_string, data); + if (success) + handle_value(data); + else + set_failure(); + + return success; +} + +// parse the internal json_buffer up to len +bool RGWJSONParser::parse(int len) +{ + string json_string = json_buffer.substr(0, len); + success = read(json_string, data); + if (success) + handle_value(data); + else + set_failure(); + + return success; +} + +// parse the complete internal json_buffer +bool RGWJSONParser::parse() +{ + success = read(json_buffer, data); + if (success) + handle_value(data); + else + set_failure(); + + return success; +} + +// parse a supplied ifstream, for testing. DELETE ME +bool RGWJSONParser::parse(const char *file_name) +{ + ifstream is(file_name); + success = read(is, data); + if (success) + handle_value(data); + else + set_failure(); + + return success; +} + + + diff --git a/src/rgw/rgw_json.h b/src/rgw/rgw_json.h new file mode 100644 index 0000000000000..a5fea2ed34cac --- /dev/null +++ b/src/rgw/rgw_json.h @@ -0,0 +1,98 @@ +#ifndef RGW_JSON_H +#define RGW_JSON_H + +#include +#include + +// for testing DELETE ME +#include + +#include "json_spirit/json_spirit.h" + + +using namespace std; +using namespace json_spirit; + + +class JSONObj; + +class JSONObjIter { + typedef map::iterator map_iter_t; + map_iter_t cur; + map_iter_t last; + +public: + JSONObjIter(); + ~JSONObjIter(); + void set(const JSONObjIter::map_iter_t &_cur, const JSONObjIter::map_iter_t &_end); + + void operator++(); + JSONObj *operator*(); + + bool end() { + return (cur == last); + } +}; + +class JSONObj +{ + JSONObj *parent; +protected: + string name; // corresponds to obj_type in XMLObj + Value data; + string data_string; + void handle_value(Value v); + +public: + + JSONObj() : parent(NULL){}; + + virtual ~JSONObj(); + + void init(JSONObj *p, Value v, string n); + + string& get_name() { return name; } + string& get_data() { return data_string; } + JSONObj *get_parent(); + void add_child(string el, JSONObj *child); + bool get_attr(string name, string& attr); + JSONObjIter find(string name); + JSONObjIter find_first(); + JSONObjIter find_first(string name); + + friend ostream& operator<<(ostream& out, JSONObj& obj); // does not work, FIXME + + bool is_array(); + bool is_object(); + vector get_array_elements(); + +private: + multimap children; + map attr_map; +}; + +class RGWJSONParser : public JSONObj +{ + int buf_len; +public: + RGWJSONParser(); + virtual ~RGWJSONParser(); + void handle_data(const char *s, int len); + + bool parse(const char *buf_, int len); + bool parse(int len); + bool parse(); + bool parse(const char *file_name); + + const char *get_json() { return json_buffer.c_str(); } + void set_failure() { success = false; } + +private: + multimap children; + map attr_map; + string json_buffer; + bool success; +}; + + +#endif diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 91a532e962595..d0bb4e405dc26 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -555,54 +555,14 @@ int RGWPostObj_ObjStore::verify_params() /* check that we have enough memory to store the object note that this test isn't exact and may fail unintentionally for large requests is */ - if ((unsigned long long)content_length > RGW_MAX_PUT_SIZE) - return -ERR_TOO_LARGE; - - return 0; -} - -int RGWPostObj_ObjStore::get_data(bufferlist& bl) -{ - size_t cl = 0; - - // try and prevent a partial read of the boundary - if (content_length - ofs > RGW_MAX_CHUNK_SIZE) - cl = RGW_MAX_CHUNK_SIZE; - else if (content_length - ofs > 0) - cl = (content_length - ofs); - else - cl = RGW_MAX_CHUNK_SIZE; - - bufferptr bp(cl); - int r = s->cio->read(bp.c_str(), cl, &len); - if (r < 0) - return r; - - // resize our buffer pointer to avoid appending garbage - bp.set_length(len); - - /* if we are at the boundary there will be two leading - newlines that we don't want */ - int start = len - boundary.size() -2; - if (start > 0) { - // read in what might be a boundary - string test_boundary; - for (int i = start; i < len -2 && i > 0 ; i++) { - test_boundary += bp.c_str()[i]; - } - - if (strcmp(test_boundary.c_str(), boundary.c_str()) == 0 ) { - // kill off bothersome newlines - bp.set_length(start-2); - data_read = true; + if (s->length) { + off_t len = atoll(s->length); + if (len > (off_t)RGW_MAX_PUT_SIZE) { + return -ERR_TOO_LARGE; } } - len = bp.length(); - bl.append(bp); - - - return len; + return 0; } int RGWPutACLs_ObjStore::get_params() @@ -1021,8 +981,10 @@ int RGWHandler_ObjStore::read_permissions(RGWOp *op_obj) break; } /* is it a 'create bucket' request? */ - if (s->object_str.size() == 0) + if ((s->op == OP_PUT) && s->object_str.size() == 0) return 0; + only_bucket = true; + break; case OP_DELETE: only_bucket = true; break; diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index 56e2b3d1da7d7..ef46c0c12a881 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -113,7 +113,6 @@ public: ~RGWPostObj_ObjStore() {} virtual int verify_params(); - int get_data(bufferlist& bl); }; class RGWPutMetadata_ObjStore : public RGWPutMetadata -- 2.39.5