From 7d75063c4168a4fc34a3a3dbc2babbeffcd882f4 Mon Sep 17 00:00:00 2001 From: Albin Antony Date: Thu, 8 Feb 2024 14:33:32 +0530 Subject: [PATCH] rgw/s3select: add json format output Signed-off-by: Albin Antony --- src/rgw/rgw_s3select.cc | 25 ++++++++++++++++++++++--- src/rgw/rgw_s3select_private.h | 5 +++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/rgw/rgw_s3select.cc b/src/rgw/rgw_s3select.cc index 800d276a6aab1..a89139d3c0e7a 100644 --- a/src/rgw/rgw_s3select.cc +++ b/src/rgw/rgw_s3select.cc @@ -427,6 +427,9 @@ int RGWSelectObj_ObjStore_S3::run_s3select_on_csv(const char* query, const char* csv.use_header_info=true; } + if(m_outputFormat == OutputFormat::JSON) { + csv.output_json_format = true; + } m_s3_csv_object.set_csv_query(&s3select_syntax, csv); m_s3_csv_object.set_external_system_functions(fp_s3select_continue, @@ -478,6 +481,10 @@ int RGWSelectObj_ObjStore_S3::run_s3select_on_parquet(const char* query) if (!m_s3_parquet_object.is_set()) { //parsing the SQL statement. s3select_syntax.parse_query(m_sql_query.c_str()); + parquet_object::csv_definitions parquet; + if(m_outputFormat == OutputFormat::JSON) { + parquet.output_json_format = true; + } m_s3_parquet_object.set_external_system_functions(fp_s3select_continue, fp_s3select_result_format, @@ -486,7 +493,7 @@ int RGWSelectObj_ObjStore_S3::run_s3select_on_parquet(const char* query) try { //at this stage the Parquet-processing requires for the meta-data that reside on Parquet object - m_s3_parquet_object.set_parquet_object(std::string("s3object"), &s3select_syntax, &m_rgw_api); + m_s3_parquet_object.set_parquet_object(std::string("s3object"), &s3select_syntax, &m_rgw_api, parquet); } catch(base_s3select_exception& e) { ldpp_dout(this, 10) << "S3select: failed upon parquet-reader construction: " << e.what() << dendl; fp_result_header_format(m_aws_response_handler.get_sql_result()); @@ -524,6 +531,11 @@ int RGWSelectObj_ObjStore_S3::run_s3select_on_json(const char* query, const char fp_s3select_result_format, fp_result_header_format, fp_debug_mesg); + const char* s3select_processTime_error = "s3select-ProcessingTime-Error"; + const char* s3select_syntax_error = "s3select-Syntax-Error"; + const char* s3select_resource_id = "resourcse-id"; + const char* s3select_json_error = "json-Format-Error"; + json_object::csv_definitions json; m_aws_response_handler.init_response(); @@ -536,6 +548,10 @@ int RGWSelectObj_ObjStore_S3::run_s3select_on_json(const char* query, const char return -EINVAL; } + if(m_outputFormat == OutputFormat::JSON) { + json.output_json_format = true; + } + //parsing the SQL statement s3select_syntax.parse_query(m_sql_query.c_str()); if (s3select_syntax.get_error_description().empty() == false) { @@ -547,8 +563,7 @@ int RGWSelectObj_ObjStore_S3::run_s3select_on_json(const char* query, const char } //initializing json processor - json_object::csv_definitions output_definition; - m_s3_json_object.set_json_query(&s3select_syntax,output_definition); + m_s3_json_object.set_json_query(&s3select_syntax, json); if (input == nullptr) { input = ""; @@ -618,6 +633,10 @@ int RGWSelectObj_ObjStore_S3::handle_aws_cli_parameters(std::string& sql_query) ldpp_dout(this, 10) << "s3select: engine is set to process Parquet objects" << dendl; } + if (m_s3select_query.find(output_tag+">", 0); diff --git a/src/rgw/rgw_s3select_private.h b/src/rgw/rgw_s3select_private.h index 7beac4f4a5d88..79abb72596d74 100644 --- a/src/rgw/rgw_s3select_private.h +++ b/src/rgw/rgw_s3select_private.h @@ -241,6 +241,11 @@ private: const char* s3select_json_error = "InvalidJsonType"; public: + enum class OutputFormat { + CSV, + JSON + }; + OutputFormat m_outputFormat = OutputFormat::CSV; unsigned int chunk_number; size_t m_requested_range; size_t m_scan_offset; -- 2.39.5