This is an extension to the [AWS SDK](https://github.com/boto/botocore/blob/develop/botocore/data/s3/2006-03-01/service-2.json).
# Users
-For the standard client to support these extensions, the ``service-2.sdk-extras.json`` file should be added. You can place it under the default folder ``~/.aws/models/s3/2006-03-01/`` or create a custom one ``/path/to/custom/folder/models/s3/2006-03-01/`` and add it to ``AWS_DATA_PATH`` environment variable.
+For the standard client to support S3 extensions, the ``service-2.sdk-extras.json`` file should be used.
+You can place it under the default folder ``~/.aws/models/s3/2006-03-01/`` or create a custom one ``/path/to/custom/folder/models/s3/2006-03-01/`` and add it to ``AWS_DATA_PATH`` environment variable.
+For S3Vectors extensions, the ``s3vectors-service-2.sdk-extras.json`` file should be used, and it should be copied to ``~/.aws/models/s3vectors/2025-07-15/service-2.sdk-extras.json`` or a custom path added to ``AWS_DATA_PATH`` as ``service-2.sdk-extras.json``.
For more information see [here](https://github.com/boto/botocore/blob/develop/botocore/loaders.py#L33).
## Python
The [boto3 client](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html) could be used with the extensions, code samples exists in this directory.
```
# Developers
-Anyone developing an extension to the S3 API supported by AWS, please modify ``service-2.sdk-extras.json`` (all extensions should go into the same file), so that boto3 could be used to test the new API.
+Anyone developing an extension to the S3 API supported by AWS, please modify ``service-2.sdk-extras.json`` (all extensions should go into the same file), so that boto3 could be used to test the new API.
+For extensions to the S3 Vectors API, modify ``s3vectors-service-2.sdk-extras.json`` instead.
In addition, python files with code samples should be added to this directory demonstrating use of the new API.
When testing you changes please:
- make sure that the modified file is in the boto3 path as explained above
--- /dev/null
+{
+"version": 1.0,
+"merge": {
+ "shapes": {
+ "MetadataConfiguration":{
+ "required":[],
+ "members":{
+ "filterableMetadataKeys":{
+ "shape":"FilterableMetadataKeys",
+ "documentation":"<p>Filterable metadata keys allow you to create typed columns that can be used as query filters during vector search.</p>"
+ }
+ }
+ },
+ "FilterableMetadataKeys":{
+ "type":"list",
+ "member":{"shape":"FilterableMetadataKeyEntry"},
+ "max":10,
+ "min":0
+ },
+ "FilterableMetadataKeyEntry":{
+ "type":"structure",
+ "required":["name"],
+ "members":{
+ "name":{
+ "shape":"MetadataKey",
+ "documentation":"<p>The name of the filterable metadata key.</p>"
+ },
+ "type":{
+ "shape":"FilterableMetadataType",
+ "documentation":"<p>The type of the filterable metadata key. Defaults to String if not specified.</p>"
+ },
+ "mustExist":{
+ "shape":"MustExist",
+ "documentation":"<p>Whether the filterable metadata key must exist in every vector. Defaults to false.</p>"
+ }
+ }
+ },
+ "MustExist":{
+ "type":"boolean"
+ },
+ "FilterableMetadataType":{
+ "type":"string",
+ "enum":[
+ "String",
+ "Number",
+ "Boolean",
+ "StringList",
+ "NumberList",
+ "BooleanList"
+ ]
+ },
+ "QueryVectorsInput":{
+ "required":[],
+ "members":{
+ "postFiltering":{
+ "shape":"PostFiltering",
+ "documentation":"<p>If true, all filtering is done via JSON post-filtering, ignoring filterable columns. This allows $or filters to mix filterable and non-filterable metadata fields. Defaults to false.</p>"
+ }
+ }
+ },
+ "PostFiltering":{
+ "type":"boolean"
+ }
+ }
+}
+}
default: 11
services:
- rgw
+- name: rgw_s3vector_topk_post_filter_factor
+ type: float
+ level: advanced
+ desc: Oversampling factor for topK when JSON metadata post-filtering is used
+ long_desc: When a vector query uses metadata filtering on non-column fields,
+ multiply topK by this factor to fetch more candidates from the index before
+ post-filtering. Higher values increase the chance of returning the full topK
+ results after filtering, at the cost of more work.
+ default: 1
+ services:
+ - rgw
+ min: 1
+ max: 10
+ with_legacy: true
- name: rgw_usage_log_key_transition
type: bool
level: advanced
-Subproject commit cdf62dec58012e7e53a7ee490049df14e1137471
+Subproject commit 02b8609baf11ce7be6dfa18923c43420acfe42eb
rgw_rest_restore.cc
rgw_rest_s3vector.cc
rgw_s3vector.cc
+ rgw_s3vector_filter.cc
rgw_s3vector_background.cc)
list(APPEND librgw_common_srcs
#include "rgw_s3vector.h"
#include "rgw_process_env.h"
#include "common/async/yield_context.h"
+#include "common/ceph_json.h"
#include "rgw_arn.h"
#include "rgw_s3vector_background.h"
class RGWS3VectorBase : public RGWDefaultResponseOp {
protected:
bufferlist in_data;
+ std::vector<rgw::s3vector::validation_error_t> validation_errors;
public:
explicit RGWS3VectorBase(bufferlist&& data) : in_data(std::move(data)) {}
protected:
return 0;
}
+
+ void send_validation_error_response() {
+ s->err.http_ret = 400;
+ s->err.err_code = "ValidationException";
+ s->err.message = "The requested action isn't valid.";
+ set_req_state_err(s, op_ret);
+ dump_errno(s);
+
+ JSONFormatter f;
+ f.open_object_section("");
+ ::encode_json("code", std::string("ValidationException"), &f);
+ ::encode_json("message", std::string("The requested action isn't valid."), &f);
+ f.open_array_section("fieldList");
+ for (const auto& err : validation_errors) {
+ f.open_object_section("");
+ ::encode_json("path", err.path, &f);
+ ::encode_json("message", err.message, &f);
+ f.close_section();
+ }
+ f.close_section(); // fieldList
+ f.close_section(); // root
+ std::stringstream ss;
+ f.flush(ss);
+ const auto body = ss.str();
+ end_header(s, this, "application/json", body.size(), false, true);
+ dump_body(s, body);
+ }
};
class RGWS3VectorCreateIndex : public RGWS3VectorBase {
ldpp_dout(this, 1) << "ERROR: failed to load s3vector bucket " << bucket_id << ". error: " << op_ret << dendl;
return;
}
- op_ret = rgw::s3vector::create_index(configuration, this, y);
+ op_ret = rgw::s3vector::create_index(configuration, this, y, validation_errors);
}
void send_response() override {
+ if (op_ret < 0 && !validation_errors.empty()) {
+ send_validation_error_response();
+ return;
+ }
if (op_ret) {
set_req_state_err(s, op_ret);
}
ldpp_dout(this, 1) << "ERROR: failed to load s3vector bucket " << bucket_id << ". error: " << op_ret << dendl;
return;
}
- op_ret = rgw::s3vector::put_vectors(configuration, this, y);
+ op_ret = rgw::s3vector::put_vectors(configuration, this, y, validation_errors);
+ }
+
+ void send_response() override {
+ if (op_ret < 0 && !validation_errors.empty()) {
+ send_validation_error_response();
+ return;
+ }
+ if (op_ret) {
+ set_req_state_err(s, op_ret);
+ }
+ dump_errno(s);
+ end_header(s, this, "application/json");
}
};
class RGWS3VectorQueryVectors : public RGWS3VectorBase {
rgw::s3vector::query_vectors_t configuration;
rgw::s3vector::query_vectors_reply_t reply;
+ std::optional<JSONParser> filter_parser;
public:
explicit RGWS3VectorQueryVectors(bufferlist&& data) : RGWS3VectorBase(std::move(data)) {}
private:
uint32_t op_mask() override { return RGW_OP_TYPE_READ; }
int init_processing(optional_yield y) override {
- return do_init_processing(configuration, y);
+ int ret = do_init_processing(configuration, y);
+ if (ret < 0) {
+ return ret;
+ }
+ if (!configuration.filter.empty()) {
+ filter_parser.emplace();
+ if (!filter_parser->parse(configuration.filter.c_str(), configuration.filter.size())) {
+ ldpp_dout(this, 1) << "ERROR: failed to parse filter JSON: " << configuration.filter << dendl;
+ return -EINVAL;
+ }
+ }
+ return 0;
}
void execute(optional_yield y) override {
ldpp_dout(this, 1) << "ERROR: failed to load s3vector bucket " << bucket_id << ". error: " << op_ret << dendl;
return;
}
- op_ret = rgw::s3vector::query_vectors(configuration, this, y, reply);
+ op_ret = rgw::s3vector::query_vectors(configuration, filter_parser, this, y, reply, validation_errors);
}
void send_response() override {
+ if (op_ret < 0 && !validation_errors.empty()) {
+ send_validation_error_response();
+ return;
+ }
if (op_ret) {
set_req_state_err(s, op_ret);
}
#include "lancedb.h"
#include <arrow/api.h>
#include <arrow/c/bridge.h>
+#include <algorithm>
#include <charconv>
+#include <cmath>
+#include <set>
#include "rgw_s3vector_background.h"
+#include "rgw_s3vector_filter.h"
#define dout_subsys ceph_subsys_rgw
case LANCEDB_INVALID_ARGUMENT:
case LANCEDB_INVALID_TABLE_NAME:
case LANCEDB_INVALID_INPUT:
+ case LANCEDB_SCHEMA:
+ case LANCEDB_ARROW:
return -EINVAL;
case LANCEDB_TABLE_NOT_FOUND:
case LANCEDB_DATABASE_NOT_FOUND:
case LANCEDB_TIMEOUT:
return -EBUSY;
case LANCEDB_CREATE_DIR:
- case LANCEDB_SCHEMA:
case LANCEDB_RUNTIME:
case LANCEDB_OBJECT_STORE:
case LANCEDB_LANCE:
case LANCEDB_HTTP:
- case LANCEDB_ARROW:
case LANCEDB_OTHER:
case LANCEDB_UNKNOWN:
return -EIO;
// utility functions for JSON encoding/decoding
- void decode_json_obj(float& val, JSONObj *obj) {
- std::string_view s = obj->get_data();
- const char* start = s.data();
- const char* end = start + s.length();
+ template <typename T>
+ void decode_from_chars(T& val, std::string_view sv) {
+ const char* start = sv.data();
+ const char* end = start + sv.length();
const auto result = std::from_chars(start, end, val);
if (result.ec == std::errc::invalid_argument) {
throw JSONDecoder::err("failed to parse number");
}
-
if (result.ec == std::errc::result_out_of_range) {
throw JSONDecoder::err("out of range number");
}
+ if (result.ptr != end) {
+ throw JSONDecoder::err("trailing characters after number");
+ }
}
void decode_json(const char* field_name, VectorData& data, JSONObj* obj) {
auto arr_it = (*it)->find("float32");
for (auto value_it = (*arr_it)->find_first(); !value_it.end(); ++value_it) {
float value;
- decode_json_obj(value, *value_it);
+ decode_from_chars(value, (*value_it)->get_data());
data.push_back(value);
}
}
// create index
- const char* distance_metric_key = "distance_metric";
+ static constexpr const char* distance_metric_key[] = {"distance_metric"};
const char* distance_metric_to_string(DistanceMetric metric) {
switch (metric) {
}
int set_table_distance_metric(const LanceDBTable* table, DistanceMetric metric, DoutPrefixProvider* dpp) {
- const char* key = distance_metric_key;
const char* value = distance_metric_to_string(metric);
char* error_message = nullptr;
- if (const auto result = lancedb_table_set_metadata(table, &key, &value, 1, &error_message); result != LANCEDB_SUCCESS) {
+ if (const auto result = lancedb_table_set_metadata(table, distance_metric_key, &value, 1, &error_message); result != LANCEDB_SUCCESS) {
ldpp_dout(dpp, 1) << "ERROR: s3vector failed to set distance_metric metadata: " << (error_message ? error_message : "unknown") << dendl;
lancedb_free_string(error_message);
return lancedb_error_to_errno(result);
return 0;
}
- DistanceMetric get_table_distance_metric(const LanceDBTable* table, DoutPrefixProvider* dpp) {
- const char* key = distance_metric_key;
+ void filterable_metadata_key_t::dump(ceph::Formatter* f) const {
+ ::encode_json("name", name, f);
+ switch (type) {
+ case FilterableMetadataType::STRING: ::encode_json("type", "String", f); break;
+ case FilterableMetadataType::NUMBER: ::encode_json("type", "Number", f); break;
+ case FilterableMetadataType::BOOLEAN: ::encode_json("type", "Boolean", f); break;
+ case FilterableMetadataType::STRING_LIST: ::encode_json("type", "StringList", f); break;
+ case FilterableMetadataType::NUMBER_LIST: ::encode_json("type", "NumberList", f); break;
+ case FilterableMetadataType::BOOLEAN_LIST: ::encode_json("type", "BooleanList", f); break;
+ }
+ ::encode_json("mustExist", must_exist, f);
+ }
+
+ void filterable_metadata_key_t::decode_json(JSONObj* obj) {
+ JSONDecoder::decode_json("name", name, obj, true);
+ std::string type_str;
+ JSONDecoder::decode_json("type", type_str, obj);
+ if (type_str.empty() || type_str == "String") {
+ type = FilterableMetadataType::STRING;
+ } else if (type_str == "Number") {
+ type = FilterableMetadataType::NUMBER;
+ } else if (type_str == "Boolean") {
+ type = FilterableMetadataType::BOOLEAN;
+ } else if (type_str == "StringList") {
+ type = FilterableMetadataType::STRING_LIST;
+ } else if (type_str == "NumberList") {
+ type = FilterableMetadataType::NUMBER_LIST;
+ } else if (type_str == "BooleanList") {
+ type = FilterableMetadataType::BOOLEAN_LIST;
+ } else {
+ throw JSONDecoder::err(fmt::format("invalid filterable metadata type: '{}'. Must be String, Number, Boolean, StringList, NumberList, or BooleanList", type_str));
+ }
+ JSONDecoder::decode_json("mustExist", must_exist, obj);
+ }
+
+ static constexpr const char* nonfilterable_metadata_key[] = {"nonfilterable_metadata"};
+
+ int set_nonfilterable_metadata(const LanceDBTable* table, const std::vector<std::string>& keys, DoutPrefixProvider* dpp) {
+ if (keys.empty()) {
+ return 0;
+ }
+ JSONFormatter f;
+ f.open_object_section("");
+ ::encode_json("keys", keys, &f);
+ f.close_section();
+ std::stringstream ss;
+ f.flush(ss);
+ const auto json_str = ss.str();
+ const char* value = json_str.c_str();
+ char* error_message = nullptr;
+ if (const auto result = lancedb_table_set_metadata(table, nonfilterable_metadata_key, &value, 1, &error_message); result != LANCEDB_SUCCESS) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector failed to set " << nonfilterable_metadata_key <<
+ " metadata: " << (error_message ? error_message : "unknown") << dendl;
+ lancedb_free_string(error_message);
+ return lancedb_error_to_errno(result);
+ }
+ return 0;
+ }
+
+ int get_nonfilterable_metadata(const LanceDBTable* table, DoutPrefixProvider* dpp, std::vector<std::string>& non_filterable_metadata_keys) {
char** keys_out = nullptr;
char** values_out = nullptr;
size_t count = 0;
char* error_message = nullptr;
- if (const auto result = lancedb_table_get_metadata(table, &key, 1, &keys_out, &values_out, &count, &error_message); result != LANCEDB_SUCCESS) {
+ if (const auto result = lancedb_table_get_metadata(table, nonfilterable_metadata_key, 1, &keys_out, &values_out, &count, &error_message); result != LANCEDB_SUCCESS) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector failed to get " << nonfilterable_metadata_key <<
+ " metadata: " << (error_message ? error_message : "unknown") << dendl;
+ lancedb_free_string(error_message);
+ return lancedb_error_to_errno(result);
+ }
+ if (count > 0) {
+ JSONParser parser;
+ if (!parser.parse(values_out[0], strlen(values_out[0]))) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector failed to parse nonfilterable metadata JSON" << dendl;
+ lancedb_free_metadata(keys_out, values_out, count);
+ return -EINVAL;
+ }
+ try {
+ JSONDecoder::decode_json("keys", non_filterable_metadata_keys, &parser);
+ } catch (const JSONDecoder::err& e) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector failed to decode nonfilterable metadata JSON: " << e.what() << dendl;
+ lancedb_free_metadata(keys_out, values_out, count);
+ return -EINVAL;
+ }
+ lancedb_free_metadata(keys_out, values_out, count);
+ }
+ return 0;
+ }
+
+
+ DistanceMetric get_distance_metric(const LanceDBTable* table, DoutPrefixProvider* dpp) {
+ char** keys_out = nullptr;
+ char** values_out = nullptr;
+ size_t count = 0;
+ char* error_message = nullptr;
+ if (const auto result = lancedb_table_get_metadata(table, distance_metric_key, 1, &keys_out, &values_out, &count, &error_message); result != LANCEDB_SUCCESS) {
ldpp_dout(dpp, 1) << "ERROR: s3vector failed to get distance_metric metadata: " << (error_message ? error_message : "unknown") << dendl;
lancedb_free_string(error_message);
return DistanceMetric::UNKNOWN;
::encode_json("indexName", index_name, f);
f->open_object_section("metadataConfiguration");
::encode_json("nonFilterableMetadataKeys", non_filterable_metadata_keys, f);
+ ::encode_json("filterableMetadataKeys", filterable_metadata_keys, f);
f->close_section();
if (vector_bucket_arn) {
::encode_json("vectorBucketArn", vector_bucket_arn->to_string(), f);
auto md_it = obj->find("metadataConfiguration");
if (!md_it.end()) {
JSONDecoder::decode_json("nonFilterableMetadataKeys", non_filterable_metadata_keys, *md_it);
+ JSONDecoder::decode_json("filterableMetadataKeys", filterable_metadata_keys, *md_it);
}
decode_vector_bucket_name(vector_bucket_name, vector_bucket_arn, obj);
}
static const std::string data_field_str{data_field};;
static constexpr const char* key_field = "key";
static const std::string key_field_str{key_field};;
+ static constexpr const char* metadata_field = "metadata";
+ static const std::string metadata_field_str{metadata_field};;
static constexpr const char* distance_field = "_distance";
static const std::string distance_field_str{distance_field};;
static constexpr const char* key_columns[] = {key_field};
static constexpr const char* data_columns[] = {data_field};
static constexpr const char* table_columns[] = {key_field, data_field};
+ static constexpr const char* table_columns_with_metadata[] = {key_field, data_field, metadata_field};
+ static constexpr const char* key_and_metadata_columns[] = {key_field, metadata_field};
static constexpr int num_key_columns = 1;
- static constexpr int num_table_columns = 2;
- int create_table_schema(unsigned int dimension, DoutPrefixProvider* dpp, ArrowSchema* c_schema) {
- const auto schema = arrow::schema(
- {
- arrow::field(key_field, arrow::utf8()),
- arrow::field(data_field, arrow::fixed_size_list(arrow::float32(), dimension))
+ std::pair<const char* const*, unsigned long> get_select_columns(bool return_data, bool return_metadata) {
+ if (return_data && return_metadata) {
+ return {table_columns_with_metadata, 3};
+ } else if (return_data) {
+ return {table_columns, 2};
+ } else if (return_metadata) {
+ return {key_and_metadata_columns, 2};
+ }
+ return {key_columns, 1};
+ }
+
+ std::shared_ptr<arrow::DataType> filterable_type_to_arrow(FilterableMetadataType type) {
+ switch (type) {
+ case FilterableMetadataType::STRING: return arrow::utf8();
+ case FilterableMetadataType::NUMBER: return arrow::float64();
+ case FilterableMetadataType::BOOLEAN: return arrow::boolean();
+ case FilterableMetadataType::STRING_LIST: return arrow::list(arrow::utf8());
+ case FilterableMetadataType::NUMBER_LIST: return arrow::list(arrow::float64());
+ case FilterableMetadataType::BOOLEAN_LIST: return arrow::list(arrow::boolean());
+ }
+ return arrow::utf8();
+ }
+
+ std::optional<FilterableMetadataType> arrow_to_filterable_type(const std::shared_ptr<arrow::DataType>& type) {
+ switch (type->id()) {
+ case arrow::Type::STRING:
+ return FilterableMetadataType::STRING;
+ case arrow::Type::DOUBLE:
+ return FilterableMetadataType::NUMBER;
+ case arrow::Type::BOOL:
+ return FilterableMetadataType::BOOLEAN;
+ case arrow::Type::LIST: {
+ const auto& value_type = std::static_pointer_cast<arrow::ListType>(type)->value_type();
+ switch (value_type->id()) {
+ case arrow::Type::STRING:
+ return FilterableMetadataType::STRING_LIST;
+ case arrow::Type::DOUBLE:
+ return FilterableMetadataType::NUMBER_LIST;
+ case arrow::Type::BOOL:
+ return FilterableMetadataType::BOOLEAN_LIST;
+ default:
+ return std::nullopt;
}
- );
+ }
+ default:
+ return std::nullopt;
+ }
+ }
+
+ std::vector<filterable_metadata_key_t> get_filterable_keys_from_schema(const std::shared_ptr<arrow::Schema>& schema) {
+ std::vector<filterable_metadata_key_t> keys;
+ for (const auto& field : schema->fields()) {
+ const auto& name = field->name();
+ if (name == key_field || name == data_field || name == metadata_field || name.starts_with('_')) {
+ continue;
+ }
+ if (const auto type = arrow_to_filterable_type(field->type()); type.has_value()) {
+ keys.push_back({name, *type, !field->nullable()});
+ }
+ }
+ return keys;
+ }
+
+ int create_table_schema(unsigned int dimension, const std::vector<filterable_metadata_key_t>& filterable_keys, DoutPrefixProvider* dpp, ArrowSchema* c_schema) {
+ arrow::FieldVector fields = {
+ arrow::field(key_field, arrow::utf8()),
+ arrow::field(data_field, arrow::fixed_size_list(arrow::float32(), dimension)),
+ arrow::field(metadata_field, arrow::utf8())
+ };
+ for (const auto& fk : filterable_keys) {
+ fields.push_back(arrow::field(fk.name, filterable_type_to_arrow(fk.type), !fk.must_exist));
+ }
+ const auto schema = arrow::schema(fields);
if (const auto status = arrow::ExportSchema(*schema, c_schema); !status.ok()) {
ldpp_dout(dpp, 1) << "ERROR: s3vector failed to export schema to C ABI: " << status.ToString() << dendl;
return -EINVAL;
return 0;
}
- int get_vector_dimension(const std::string& index_name, LanceDBTable* table, DoutPrefixProvider* dpp, unsigned int& dimension);
+ int get_vector_dimension(const std::string& index_name, const std::shared_ptr<arrow::Schema>& schema, DoutPrefixProvider* dpp, unsigned int& dimension) {
+ auto data_f = schema->GetFieldByName(data_field_str);
+ if (!data_f) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector schema missing " << data_field_str << " field for index: " << index_name << dendl;
+ return -EINVAL;
+ }
+ if (data_f->type()->id() != arrow::Type::FIXED_SIZE_LIST) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector " << data_field_str << " field is not a FixedSizeList for index: " << index_name << dendl;
+ return -EINVAL;
+ }
+ dimension = std::static_pointer_cast<arrow::FixedSizeListType>(data_f->type())->list_size();
+ return 0;
+ }
+
+ int import_table_schema(const std::string& index_name, LanceDBTable* table, DoutPrefixProvider* dpp, std::shared_ptr<arrow::Schema>& schema) {
+ struct ArrowSchema* c_schema_ptr = nullptr;
+ char* error_message = nullptr;
+ if (const LanceDBError result = lancedb_table_arrow_schema(
+ table,
+ reinterpret_cast<FFI_ArrowSchema**>(&c_schema_ptr),
+ &error_message); result != LANCEDB_SUCCESS) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector failed to get schema for index: " << index_name
+ << ". error: " << error_message << dendl;
+ lancedb_free_string(error_message);
+ return lancedb_error_to_errno(result);
+ }
+ auto imported = arrow::ImportSchema(c_schema_ptr);
+ if (!imported.ok()) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector failed to import schema for index: " << index_name
+ << ". error: " << imported.status().ToString() << dendl;
+ lancedb_free_arrow_schema(reinterpret_cast<FFI_ArrowSchema*>(c_schema_ptr));
+ return -EINVAL;
+ }
+ schema = *imported;
+ lancedb_free_arrow_schema(reinterpret_cast<FFI_ArrowSchema*>(c_schema_ptr));
+ return 0;
+ }
+
+ int get_vector_dimension(const std::string& index_name, LanceDBTable* table, DoutPrefixProvider* dpp, unsigned int& dimension) {
+ std::shared_ptr<arrow::Schema> schema;
+ if (int ret = import_table_schema(index_name, table, dpp, schema); ret < 0) {
+ return ret;
+ }
+ return get_vector_dimension(index_name, schema, dpp, dimension);
+ }
- int create_index(const create_index_t& configuration, DoutPrefixProvider* dpp, optional_yield y) {
+ int create_index(const create_index_t& configuration, DoutPrefixProvider* dpp, optional_yield y, std::vector<validation_error_t>& errors) {
log_configuration(dpp, "CreateIndex", configuration);
LanceDBConnection* conn = connect(dpp, configuration.vector_bucket_name);
if (!conn) {
return -EIO;
}
- // if the table already exists, verify the schema matches
- LanceDBTable* existing_table = lancedb_connection_open_table(conn, configuration.index_name.c_str());
- if (existing_table) {
- unsigned int existing_dimension = 0;
- if (get_vector_dimension(configuration.index_name, existing_table, dpp, existing_dimension) < 0) {
- lancedb_table_free(existing_table);
- lancedb_connection_free(conn);
- return -EEXIST;
+
+ // validate metadata key names
+ for (unsigned int i = 0; i < configuration.filterable_metadata_keys.size(); ++i) {
+ const auto& name = configuration.filterable_metadata_keys[i].name;
+ if (name.starts_with('_')) {
+ errors.push_back({fmt::format("metadataConfiguration.filterableMetadataKeys[{}].name", i),
+ fmt::format("'{}' must not start with an underscore", name)});
+ break;
}
- if (existing_dimension != configuration.dimension) {
- ldpp_dout(dpp, 1) << "ERROR: s3vector index: " << configuration.index_name
- << " already exists with dimension " << existing_dimension
- << " but requested dimension " << configuration.dimension << dendl;
- lancedb_table_free(existing_table);
- lancedb_connection_free(conn);
- return -EEXIST;
+ if (name.find('.') != std::string::npos) {
+ errors.push_back({fmt::format("metadataConfiguration.filterableMetadataKeys[{}].name", i),
+ fmt::format("'{}' must not contain '.'", name)});
+ break;
}
- const auto existing_metric = get_table_distance_metric(existing_table, dpp);
- if (existing_metric != DistanceMetric::UNKNOWN && existing_metric != configuration.distance_metric) {
- ldpp_dout(dpp, 1) << "ERROR: s3vector index: " << configuration.index_name
- << " already exists with distance metric " << distance_metric_to_string(existing_metric)
- << " but requested " << distance_metric_to_string(configuration.distance_metric) << dendl;
- lancedb_table_free(existing_table);
- lancedb_connection_free(conn);
- return -EEXIST;
+ }
+ if (!errors.empty()) {
+ lancedb_connection_free(conn);
+ return -EINVAL;
+ }
+ for (unsigned int i = 0; i < configuration.non_filterable_metadata_keys.size(); ++i) {
+ const auto& name = configuration.non_filterable_metadata_keys[i];
+ if (name.find('.') != std::string::npos) {
+ errors.push_back({fmt::format("metadataConfiguration.nonFilterableMetadataKeys[{}]", i),
+ fmt::format("'{}' must not contain '.'", name)});
+ break;
}
- ldpp_dout(dpp, 10) << "INFO: s3vector index: " << configuration.index_name
- << " already exists with matching schema, returning success" << dendl;
- lancedb_table_free(existing_table);
+ }
+ if (!errors.empty()) {
lancedb_connection_free(conn);
- return 0;
+ return -EINVAL;
+ }
+
+ // verify no overlap between filterable and non-filterable metadata keys
+ if (!configuration.non_filterable_metadata_keys.empty() && !configuration.filterable_metadata_keys.empty()) {
+ std::set<std::string> nonfilterable_names(
+ configuration.non_filterable_metadata_keys.begin(),
+ configuration.non_filterable_metadata_keys.end());
+ for (unsigned int i = 0; i < configuration.filterable_metadata_keys.size(); ++i) {
+ const auto& name = configuration.filterable_metadata_keys[i].name;
+ if (nonfilterable_names.count(name)) {
+ errors.push_back({fmt::format("metadataConfiguration.filterableMetadataKeys[{}].name", i),
+ fmt::format("'{}' appears in both filterable and non-filterable metadata keys", name)});
+ }
+ }
+ if (!errors.empty()) {
+ lancedb_connection_free(conn);
+ return -EINVAL;
+ }
}
struct ArrowSchema c_schema;
- if (int ret = create_table_schema(configuration.dimension, dpp, &c_schema); ret < 0) {
+ if (int ret = create_table_schema(configuration.dimension, configuration.filterable_metadata_keys, dpp, &c_schema); ret < 0) {
lancedb_connection_free(conn);
return ret;
}
- char* error_message;
+ char* error_message = nullptr;
LanceDBTable* table = nullptr;
if (const LanceDBError result = lancedb_table_create(conn, configuration.index_name.c_str(),
reinterpret_cast<FFI_ArrowSchema*>(&c_schema),
nullptr, &table, &error_message); result != LANCEDB_SUCCESS) {
- ldpp_dout(dpp, 1) << "ERROR: s3vector creating index: " << configuration.index_name << ", error: " << error_message << dendl;
+ ldpp_dout(dpp, 1) << "ERROR: s3vector creating index: " << configuration.index_name << ", lancedb error code: " << result << ", error: " << error_message << dendl;
+ if (result == LANCEDB_SCHEMA || result == LANCEDB_INVALID_INPUT || result == LANCEDB_ARROW || result == LANCEDB_LANCE) {
+ errors.push_back({"metadataConfiguration.filterableMetadataKeys", error_message});
+ lancedb_free_string(error_message);
+ lancedb_connection_free(conn);
+ return -EINVAL;
+ }
lancedb_free_string(error_message);
lancedb_connection_free(conn);
return lancedb_error_to_errno(result);
lancedb_connection_free(conn);
return ret;
}
+ if (int ret = set_nonfilterable_metadata(table, configuration.non_filterable_metadata_keys, dpp); ret < 0) {
+ lancedb_table_free(table);
+ lancedb_connection_free(conn);
+ return ret;
+ }
lancedb_table_free(table);
lancedb_connection_free(conn);
return 0;
::encode_json("indexName", index_name, f);
f->open_object_section("metadataConfiguration");
::encode_json("nonFilterableMetadataKeys", non_filterable_metadata_keys, f);
+ ::encode_json("filterableMetadataKeys", filterable_metadata_keys, f);
f->close_section();
::encode_json("vectorBucketName", vector_bucket_name, f);
f->close_section();
f->close_section();
}
- int get_vector_dimension(const std::string& index_name, LanceDBTable* table, DoutPrefixProvider* dpp, unsigned int& dimension) {
- // Get the Arrow schema from the table
- struct ArrowSchema* c_schema_ptr = nullptr;
- char* error_message = nullptr;
- if (const LanceDBError result = lancedb_table_arrow_schema(
- table,
- reinterpret_cast<FFI_ArrowSchema**>(&c_schema_ptr),
- &error_message); result != LANCEDB_SUCCESS) {
- ldpp_dout(dpp, 1) << "ERROR: s3vector failed to get schema for index: " << index_name
- << ". error: " << error_message << dendl;
- lancedb_free_string(error_message);
- return lancedb_error_to_errno(result);
- }
-
- // Import the schema to Arrow C++
- const auto schema = arrow::ImportSchema(c_schema_ptr);
- if (!schema.ok()) {
- ldpp_dout(dpp, 1) << "ERROR: s3vector failed to import schema for index: " << index_name
- << ". error: " << schema.status().ToString() << dendl;
- lancedb_free_arrow_schema(reinterpret_cast<FFI_ArrowSchema*>(c_schema_ptr));
- return -EINVAL;
- }
-
- // Extract dimension from the "data" field
- auto data_field = schema->get()->GetFieldByName(data_field_str);
- if (!data_field) {
- ldpp_dout(dpp, 1) << "ERROR: s3vector schema missing " << data_field_str << " field for index: " << index_name << dendl;
- lancedb_free_arrow_schema(reinterpret_cast<FFI_ArrowSchema*>(c_schema_ptr));
- return -EINVAL;
- }
-
- if (data_field->type()->id() != arrow::Type::FIXED_SIZE_LIST) {
- ldpp_dout(dpp, 1) << "ERROR: s3vector " << data_field_str << " field is not a FixedSizeList for index: " << index_name << dendl;
- lancedb_free_arrow_schema(reinterpret_cast<FFI_ArrowSchema*>(c_schema_ptr));
- return -EINVAL;
- }
-
- auto fixed_size_list_type = std::static_pointer_cast<arrow::FixedSizeListType>(data_field->type());
- dimension = fixed_size_list_type->list_size();
- lancedb_free_arrow_schema(reinterpret_cast<FFI_ArrowSchema*>(c_schema_ptr));
- return 0;
- }
int get_index(const get_index_t& configuration, const std::string& region, const std::string& account, DoutPrefixProvider* dpp, optional_yield y, get_index_reply_t& reply) {
log_configuration(dpp, "GetIndex", configuration);
return -ENOENT;
}
- if (int ret = get_vector_dimension(configuration.index_name, table, dpp, reply.dimension); ret < 0) {
+ std::shared_ptr<arrow::Schema> schema;
+ if (int ret = import_table_schema(configuration.index_name, table, dpp, schema); ret < 0) {
+ lancedb_table_free(table);
lancedb_connection_free(conn);
+ return ret;
+ }
+ reply.dimension = 0;
+ if (int ret = get_vector_dimension(configuration.index_name, schema, dpp, reply.dimension); ret < 0) {
+ lancedb_table_free(table);
+ lancedb_connection_free(conn);
+ return ret;
+ }
+ if (int ret = get_nonfilterable_metadata(table, dpp, reply.non_filterable_metadata_keys); ret < 0) {
lancedb_table_free(table);
+ lancedb_connection_free(conn);
return ret;
}
+ reply.filterable_metadata_keys = get_filterable_keys_from_schema(schema);
reply.data_type = "float32";
- reply.distance_metric = get_table_distance_metric(table, dpp);
+ reply.distance_metric = get_distance_metric(table, dpp);
reply.index_name = configuration.index_name;
reply.vector_bucket_name = configuration.vector_bucket_name;
}
reply.creation_time = get_table_creation_time(table, dpp);
- // reply.non_filterable_metadata_keys - empty for now, TODO: store and retrieve from table metadata
lancedb_table_free(table);
lancedb_connection_free(conn);
return 0;
}
}
- int put_vectors(const put_vectors_t& configuration, DoutPrefixProvider* dpp, optional_yield y) {
+ int put_vectors(const put_vectors_t& configuration, DoutPrefixProvider* dpp, optional_yield y, std::vector<validation_error_t>& errors) {
log_configuration(dpp, "PutVectors", configuration);
auto table_handle = open_table_with_session_handle(dpp, configuration.vector_bucket_name, configuration.index_name);
if (!table_handle) {
return 0;
}
- // get the schema and dimension from the table
- unsigned int dimension = 0;
- if (int ret = get_vector_dimension(configuration.index_name, table, dpp, dimension); ret < 0) {
+ // get the schema, dimension, and filterable keys from the table
+ std::shared_ptr<arrow::Schema> schema;
+ if (int ret = import_table_schema(configuration.index_name, table, dpp, schema); ret < 0) {
lancedb_table_free(table);
lancedb_connection_free(conn);
return ret;
}
-
- struct ArrowSchema* c_schema_ptr = nullptr;
- char* schema_error_message = nullptr;
- if (const LanceDBError result = lancedb_table_arrow_schema(
- table,
- reinterpret_cast<FFI_ArrowSchema**>(&c_schema_ptr),
- &schema_error_message); result != LANCEDB_SUCCESS) {
- ldpp_dout(dpp, 1) << "ERROR: s3vector failed to get schema for index: " << configuration.index_name
- << ". error: " << schema_error_message << dendl;
- lancedb_free_string(schema_error_message);
- lancedb_table_free(table);
- lancedb_connection_free(conn);
- return lancedb_error_to_errno(result);
- }
-
- const auto schema = arrow::ImportSchema(c_schema_ptr);
- if (!schema.ok()) {
- ldpp_dout(dpp, 1) << "ERROR: s3vector failed to import schema for index: " << configuration.index_name
- << ". error: " << schema.status().ToString() << dendl;
- lancedb_free_arrow_schema(reinterpret_cast<FFI_ArrowSchema*>(c_schema_ptr));
+ unsigned int dimension = 0;
+ if (int ret = get_vector_dimension(configuration.index_name, schema, dpp, dimension); ret < 0) {
lancedb_table_free(table);
lancedb_connection_free(conn);
- return -EINVAL;
+ return ret;
}
+ const auto filterable_keys = get_filterable_keys_from_schema(schema);
arrow::StringBuilder key_builder;
arrow::FloatBuilder float_builder;
arrow::FixedSizeListBuilder data_builder(arrow::default_memory_pool(),
std::make_unique<arrow::FloatBuilder>(),
dimension);
- // metadata TODO: metadata configuration should also be taken from the index configuration
+ arrow::StringBuilder metadata_builder;
+
+ // create builders for filterable columns
+ struct FilterableBuilder {
+ FilterableMetadataType type;
+ std::string name;
+ bool must_exist;
+ std::unique_ptr<arrow::ArrayBuilder> builder;
+ };
+ std::vector<FilterableBuilder> filterable_builders;
+ for (const auto& fk : filterable_keys) {
+ FilterableBuilder fb;
+ fb.type = fk.type;
+ fb.name = fk.name;
+ fb.must_exist = fk.must_exist;
+ switch (fk.type) {
+ case FilterableMetadataType::STRING:
+ fb.builder = std::make_unique<arrow::StringBuilder>();
+ break;
+ case FilterableMetadataType::NUMBER:
+ fb.builder = std::make_unique<arrow::DoubleBuilder>();
+ break;
+ case FilterableMetadataType::BOOLEAN:
+ fb.builder = std::make_unique<arrow::BooleanBuilder>();
+ break;
+ case FilterableMetadataType::STRING_LIST:
+ fb.builder = std::make_unique<arrow::ListBuilder>(
+ arrow::default_memory_pool(), std::make_unique<arrow::StringBuilder>());
+ break;
+ case FilterableMetadataType::NUMBER_LIST:
+ fb.builder = std::make_unique<arrow::ListBuilder>(
+ arrow::default_memory_pool(), std::make_unique<arrow::DoubleBuilder>());
+ break;
+ case FilterableMetadataType::BOOLEAN_LIST:
+ fb.builder = std::make_unique<arrow::ListBuilder>(
+ arrow::default_memory_pool(), std::make_unique<arrow::BooleanBuilder>());
+ break;
+ }
+ filterable_builders.push_back(std::move(fb));
+ }
+
unsigned int num_rows = 0;
- for (const auto& vector : configuration.vectors) {
+ for (size_t vi = 0; vi < configuration.vectors.size(); ++vi) {
+ const auto& vector = configuration.vectors[vi];
+ // validate key
+ if (vector.key.empty()) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector vector with empty key at index " << vi << dendl;
+ errors.push_back({fmt::format("vectors[{}].key", vi), "must not be empty"});
+ break;
+ }
+ // validate data
if (!vector.data) {
- ldpp_dout(dpp, 5) << "WARNING: s3vector skipping vector with no data" << dendl;
- continue;
+ ldpp_dout(dpp, 1) << "ERROR: s3vector vector with no data, key: " << vector.key << dendl;
+ errors.push_back({fmt::format("vectors[{}].data", vi), "missing data"});
+ break;
}
+ // validate data dimension
if (vector.data->size() != dimension) {
- ldpp_dout(dpp, 5) << "WARNING: s3vector vector dimension mismatch, expected "
- << dimension << " got " << vector.data->size() <<
- ". skip vector with key: " << vector.key << dendl;
- continue;
+ ldpp_dout(dpp, 1) << "ERROR: s3vector vector dimension mismatch, expected "
+ << dimension << " got " << vector.data->size() << " for key: " << vector.key << dendl;
+ errors.push_back({fmt::format("vectors[{}].data", vi),
+ fmt::format("expected dimension {} but got {}", dimension, vector.data->size())});
+ break;
}
- if (vector.key.empty()) {
- ldpp_dout(dpp, 5) << "WARNING: s3vector skipping vector with empty key" << dendl;
- continue;
+ // validate metadata JSON if exists
+ const bool has_metadata = !vector.metadata.empty();
+ JSONParser parser;
+ if (has_metadata && !parser.parse(vector.metadata.c_str(), vector.metadata.size())) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector invalid metadata JSON for key: " << vector.key << dendl;
+ errors.push_back({fmt::format("vectors[{}].metadata", vi), "invalid JSON"});
+ break;
}
- // metadata TODO: check if metadata is allowed based on index config
- // key column
+ if (has_metadata) {
+ bool invalid_field = false;
+ for (auto it = parser.find_first(); !it.end(); ++it) {
+ auto* field = *it;
+ const auto& name = field->get_name();
+ if (name.find('.') != std::string::npos) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector metadata field name '" << name << "' must not contain '.' in key: " << vector.key << dendl;
+ errors.push_back({fmt::format("vectors[{}].metadata.{}", vi, name), "field name must not contain '.'"});
+ invalid_field = true;
+ break;
+ }
+ const auto& dv = field->get_data_val();
+ if (!dv.quoted && dv.str == "null") {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector null metadata value for field '" << name << "' in key: " << vector.key << dendl;
+ errors.push_back({fmt::format("vectors[{}].metadata.{}", vi, name), "null values are not supported"});
+ invalid_field = true;
+ break;
+ }
+ }
+ if (invalid_field) break;
+ }
+ // add key
key_builder.Append(vector.key).ok();
- // data column
- auto
list_builder = static_cast<arrow::FloatBuilder*>(data_builder.value_builder());
- for (const auto & value : vector.data.value()) {
- list_builder->Append(value).ok();
+ // add data
+ auto
* float_list_builder = static_cast<arrow::FloatBuilder*>(data_builder.value_builder());
+ for (const auto& value : vector.data.value()) {
+ float_list_builder->Append(value).ok();
}
data_builder.Append().ok();
+ // add metadata
+ if (has_metadata) {
+ metadata_builder.Append(vector.metadata).ok();
+ } else {
+ metadata_builder.AppendNull().ok();
+ }
+ // add filterable metadata columns
+ if (!filterable_builders.empty() && !has_metadata) {
+ for (auto& fb : filterable_builders) {
+ if (fb.must_exist) {
+ errors.push_back({fmt::format("vectors[{}].metadata.{}", vi, fb.name), "field is required"});
+ break;
+ }
+ fb.builder->AppendNull().ok();
+ }
+ if (!errors.empty()) break;
+ ++num_rows;
+ continue;
+ }
+ if (has_metadata && !filterable_builders.empty()) {
+ for (auto& fb : filterable_builders) {
+ bool is_list_type = false;
+ switch (fb.type) {
+ case FilterableMetadataType::STRING:
+ case FilterableMetadataType::NUMBER:
+ case FilterableMetadataType::BOOLEAN:
+ break;
+ case FilterableMetadataType::STRING_LIST:
+ case FilterableMetadataType::NUMBER_LIST:
+ case FilterableMetadataType::BOOLEAN_LIST:
+ is_list_type = true;
+ break;
+ }
+ auto* field_obj = parser.find_obj(fb.name);
+ if (!field_obj) {
+ if (fb.must_exist) {
+ errors.push_back({fmt::format("vectors[{}].metadata.{}", vi, fb.name), "field is required"});
+ break;
+ }
+ fb.builder->AppendNull().ok();
+ continue;
+ }
+ if (is_list_type != field_obj->is_array()) {
+ // column/field type mismatch with JSON value type
+ errors.push_back({fmt::format("vectors[{}].metadata.{}", vi, fb.name), "invalid type"});
+ break;
+ }
+ std::vector<std::string> values;
+ std::string value_str;
+ try {
+ if (is_list_type) {
+ decode_json_obj(values, field_obj);
+ } else {
+ decode_json_obj(value_str, field_obj);
+ }
+ } catch (const JSONDecoder::err& e) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector failed to decode metadata field '"
+ << fb.name << "' for key: " << vector.key << ". error: " << e.what() << dendl;
+ errors.push_back({fmt::format("vectors[{}].metadata.{}", vi, fb.name), "invalid type"});
+ break;
+ }
+ switch (fb.type) {
+ case FilterableMetadataType::STRING:
+ // anything can go into a string column
+ static_cast<arrow::StringBuilder*>(fb.builder.get())->Append(value_str).ok();
+ break;
+ case FilterableMetadataType::NUMBER:
+ try {
+ double val;
+ decode_from_chars(val, value_str);
+ static_cast<arrow::DoubleBuilder*>(fb.builder.get())->Append(val).ok();
+ } catch (const JSONDecoder::err& err) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filterable metadata field '"
+ << fb.name << "' for key: " << vector.key << " expected number but got:" << value_str
+ << ". error: " << err.what() << dendl;
+ errors.push_back({fmt::format("vectors[{}].metadata.{}", vi, fb.name), err.what()});
+ }
+ break;
+ case FilterableMetadataType::BOOLEAN:
+ if (value_str == "true") {
+ static_cast<arrow::BooleanBuilder*>(fb.builder.get())->Append(true).ok();
+ } else if (value_str == "false") {
+ static_cast<arrow::BooleanBuilder*>(fb.builder.get())->Append(false).ok();
+ } else {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filterable metadata field '"
+ << fb.name << "' for key: " << vector.key << " expected boolean but got: " << value_str << dendl;
+ errors.push_back({fmt::format("vectors[{}].metadata.{}", vi, fb.name), "expected boolean"});
+ }
+ break;
+ case FilterableMetadataType::STRING_LIST: {
+ auto* list_builder = static_cast<arrow::ListBuilder*>(fb.builder.get());
+ auto* value_builder = static_cast<arrow::StringBuilder*>(list_builder->value_builder());
+ list_builder->Append().ok();
+ for (const auto& v : values) {
+ value_builder->Append(v).ok();
+ }
+ break;
+ }
+ case FilterableMetadataType::NUMBER_LIST: {
+ auto* list_builder = static_cast<arrow::ListBuilder*>(fb.builder.get());
+ auto* value_builder = static_cast<arrow::DoubleBuilder*>(list_builder->value_builder());
+ list_builder->Append().ok();
+ for (const auto& v : values) {
+ try {
+ double val;
+ decode_from_chars(val, v);
+ value_builder->Append(val).ok();
+ } catch (const JSONDecoder::err& err) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filterable metadata field '"
+ << fb.name << "' for key: " << vector.key << " expected number in list but got: " << v
+ << ". error: " << err.what() << dendl;
+ errors.push_back({fmt::format("vectors[{}].metadata.{}", vi, fb.name), err.what()});
+ break;
+ }
+ }
+ break;
+ }
+ case FilterableMetadataType::BOOLEAN_LIST: {
+ auto* list_builder = static_cast<arrow::ListBuilder*>(fb.builder.get());
+ auto* value_builder = static_cast<arrow::BooleanBuilder*>(list_builder->value_builder());
+ list_builder->Append().ok();
+ for (const auto& v : values) {
+ if (v == "true") {
+ value_builder->Append(true).ok();
+ } else if (v == "false") {
+ value_builder->Append(false).ok();
+ } else {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filterable metadata field '"
+ << fb.name << "' for key: " << vector.key << " expected boolean in list but got: " << v << dendl;
+ errors.push_back({fmt::format("vectors[{}].metadata.{}", vi, fb.name), "expected boolean"});
+ break;
+ }
+ }
+ break;
+ }
+ }
+ if (!errors.empty()) break;
+ }
+ }
+ if (!errors.empty()) break;
++num_rows;
}
- if (num_rows == 0) {
- ldpp_dout(dpp, 1) << "ERROR: s3vector no valid vectors to insert" << dendl;
+ if (!errors.empty()) {
lancedb_table_free(table);
lancedb_connection_free(conn);
return -EINVAL;
}
- std::shared_ptr<arrow::Array> key_array, data_array;
+ std::shared_ptr<arrow::Array> key_array, data_array, metadata_array;
key_builder.Finish(&key_array).ok();
data_builder.Finish(&data_array).ok();
+ metadata_builder.Finish(&metadata_array).ok();
+
+ arrow::ArrayVector arrays = {key_array, data_array, metadata_array};
+ for (auto& fb : filterable_builders) {
+ std::shared_ptr<arrow::Array> arr;
+ fb.builder->Finish(&arr).ok();
+ arrays.push_back(arr);
+ }
- auto record_batch = arrow::RecordBatch::Make(*schema, num_rows, {key_array, data_array});
+ auto record_batch = arrow::RecordBatch::Make(schema, num_rows, arrays);
ldpp_dout(dpp, 20) << "INFO: s3vector created record batch with " << num_rows << " rows" << dendl;
- struct ArrowArray c_array;
- struct ArrowSchema c_schema;
+ struct ArrowArray c_array = {};
+ struct ArrowSchema c_schema = {};
if (const auto status = arrow::ExportRecordBatch(*record_batch, &c_array, &c_schema); !status.ok()) {
ldpp_dout(dpp, 1) << "ERROR: s3vector failed to export record batch to C ABI: " << status.ToString() << dendl;
if (c_schema.release) c_schema.release(&c_schema);
const std::string& index_name,
bool use_data,
bool use_distance,
- bool vector_query) {
- unsigned long num_columns = 1;
- if (use_data) ++num_columns;
- if (vector_query) ++num_columns;
+ bool vector_query,
+ bool use_metadata,
+ const bool* matches = nullptr) {
if (auto schema = arrow::ImportSchema(c_schema_ptr); schema.ok()) {
if (auto array = arrow::ImportRecordBatch(reinterpret_cast<struct ArrowArray*>(*c_arrays_ptr), *schema); array.ok()) {
const auto& record_batch = *array;
- // return by rows instead of columns
+ const auto num_columns = static_cast<unsigned int>(record_batch->num_columns());
for (auto row = 0U; row < record_batch->num_rows(); row++) {
- const auto record_num_columns = static_cast<unsigned int>(record_batch->num_columns());
- if (record_num_columns != num_columns) {
- ldpp_dout(dpp, 1) << "ERROR: s3vector got invalid number of columns in record batch for index: " <<
- index_name << ". got: " << record_num_columns << " expected: " << num_columns << dendl;
- lancedb_free_arrow_schema(reinterpret_cast<FFI_ArrowSchema*>(c_schema_ptr));
- return -EINVAL;
- }
+ if (matches && !matches[row]) continue;
vector_item_t vector_item;
if (use_data) vector_item.data.emplace();
for (auto col = 0U; col < num_columns; col++) {
} else {
ldpp_dout(dpp, 5) << "WARNING: s3vector got no distance in record batch for index: " << index_name <<dendl;
}
+ } else if (field->name() == metadata_field_str) {
+ if (!use_metadata) continue;
+ const auto metadata_array = std::static_pointer_cast<arrow::StringArray>(column);
+ if (!metadata_array->IsNull(row)) {
+ vector_item.metadata = metadata_array->GetString(row);
+ }
} else {
ldpp_dout(dpp, 5) << "WARNING: s3vector got unknown field: " << field->name() <<
" in record batch for index: " << index_name <<dendl;
const std::string& index_name,
bool use_data,
bool use_distance,
- bool vector_query) {
+ bool vector_query,
+ bool use_metadata) {
// distance can be used only with vector queries
ceph_assert(!use_distance || vector_query);
struct ArrowArray** c_arrays_ptr = nullptr;
lancedb_free_arrow_schema(reinterpret_cast<FFI_ArrowSchema*>(c_schema_ptr));
return 0;
}
- return populate_vectors_from_arrow(dpp, c_arrays_ptr, c_schema_ptr, vectors, index_name, use_data, use_distance, vector_query);
+ return populate_vectors_from_arrow(dpp, c_arrays_ptr, c_schema_ptr, vectors, index_name, use_data, use_distance, vector_query, use_metadata);
}
int get_vectors(const get_vectors_t& configuration, DoutPrefixProvider* dpp, optional_yield y, get_vectors_reply_t& reply) {
}
char* error_message;
- const unsigned long num_columns = (configuration.return_data ? 2 : 1);
- // metadata TODO: implement fetching metadata
- if (const LanceDBError result = lancedb_query_select(query, table_columns, num_columns, &error_message) ; result != LANCEDB_SUCCESS) {
- ldpp_dout(dpp, 1) << "ERROR: s3vector failed to set select columns for query on index: " << configuration.index_name << ". error: " << error_message << dendl;
- lancedb_free_string(error_message);
- lancedb_query_free(query);
- lancedb_table_free(table);
- lancedb_connection_free(conn);
- return lancedb_error_to_errno(result);
+ {
+ const auto [columns, count] = get_select_columns(configuration.return_data, configuration.return_metadata);
+ if (const LanceDBError result = lancedb_query_select(query, columns, count, &error_message) ; result != LANCEDB_SUCCESS) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector failed to set select columns for query on index: " << configuration.index_name << ". error: " << error_message << dendl;
+ lancedb_free_string(error_message);
+ lancedb_query_free(query);
+ lancedb_table_free(table);
+ lancedb_connection_free(conn);
+ return lancedb_error_to_errno(result);
+ }
}
// build where filter for keys
return -EIO;
}
- auto ret = populate_vectors_from_query(dpp, query_result, reply.vectors, configuration.index_name, configuration.return_data, false, false);
+ auto ret = populate_vectors_from_query(dpp, query_result, reply.vectors, configuration.index_name, configuration.return_data, false, false, configuration.return_metadata);
lancedb_table_free(table);
lancedb_connection_free(conn);
return ret;
}*/
if (!next_token.empty()) {
- const char* start = next_token.data();
- const char* end = start + next_token.length();
- const auto result = std::from_chars(start, end, offset);
-
- if (result.ec == std::errc::invalid_argument) {
- throw JSONDecoder::err("failed to parse next token as offset");
- }
-
- if (result.ec == std::errc::result_out_of_range) {
- throw JSONDecoder::err("out of range offset in next token");
- }
+ decode_from_chars(offset, next_token);
}
if (segment_count > 0) {
}
char* error_message;
- const unsigned long num_columns = (configuration.return_data ? 2 : 1);
- // metadata TODO: implement metadata based queries
- if (const LanceDBError result = lancedb_query_select(query, table_columns, num_columns, &error_message) ; result != LANCEDB_SUCCESS) {
- ldpp_dout(dpp, 1) << "ERROR: s3vector failed to set select columns for query on index: " << configuration.index_name << ". error: " << error_message << dendl;
- lancedb_free_string(error_message);
- lancedb_query_free(query);
- lancedb_table_free(table);
- return lancedb_error_to_errno(result);
+ {
+ const auto [columns, count] = get_select_columns(configuration.return_data, configuration.return_metadata);
+ if (const LanceDBError result = lancedb_query_select(query, columns, count, &error_message) ; result != LANCEDB_SUCCESS) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector failed to set select columns for query on index: " << configuration.index_name << ". error: " << error_message << dendl;
+ lancedb_free_string(error_message);
+ lancedb_query_free(query);
+ lancedb_table_free(table);
+ return lancedb_error_to_errno(result);
+ }
}
if (const LanceDBError result = lancedb_query_limit(query, configuration.max_results, &error_message) ; result != LANCEDB_SUCCESS) {
}
int ret;
- if (ret = populate_vectors_from_query(dpp, query_result, reply.vectors, configuration.index_name, configuration.return_data, false, false); ret == 0) {
+ if (ret = populate_vectors_from_query(dpp, query_result, reply.vectors, configuration.index_name, configuration.return_data, false, false, configuration.return_metadata); ret == 0) {
const auto total_row_count = lancedb_table_count_rows(table);
const auto next_offset = reply.vectors.size() + configuration.offset;
if (next_offset < total_row_count) {
::encode_json("returnDistance", return_distance, f);
::encode_json("returnMetadata", return_metadata, f);
::encode_json("topK", top_k, f);
+ ::encode_json("postFiltering", post_filtering, f);
f->close_section();
}
JSONDecoder::decode_json("returnDistance", return_distance, obj);
JSONDecoder::decode_json("returnMetadata", return_metadata, obj);
JSONDecoder::decode_json("topK", top_k, obj, true);
+ JSONDecoder::decode_json("postFiltering", post_filtering, obj);
if (top_k < 1) {
throw JSONDecoder::err(fmt::format("topK must be at least 1, got {}", top_k));
throw JSONDecoder::err("queryVector cannot be empty");
}
- // metadata TODO: validate filter
}
void query_vectors_reply_t::dump(ceph::Formatter* f) const {
f->close_section();
}
- int query_vectors(const query_vectors_t& configuration, DoutPrefixProvider* dpp, optional_yield y, query_vectors_reply_t& reply) {
+ int query_vectors(const query_vectors_t& configuration, std::optional<JSONParser>& filter, DoutPrefixProvider* dpp, optional_yield y, query_vectors_reply_t& reply, std::vector<validation_error_t>& errors) {
log_configuration(dpp, "QueryVectors", configuration);
auto table_handle = open_table_with_session_handle(dpp, configuration.vector_bucket_name, configuration.index_name);
if (!table_handle) {
LanceDBTable* table = table_handle.table;
LanceDBConnection* conn = table_handle.conn_handle.conn;
+ std::shared_ptr<arrow::Schema> schema;
+ if (int ret = import_table_schema(configuration.index_name, table, dpp, schema); ret < 0) {
+ lancedb_table_free(table);
+ return ret;
+ }
+
unsigned int table_dimension;
- if (int ret = get_vector_dimension(configuration.index_name, table, dpp, table_dimension); ret < 0) {
+ if (int ret = get_vector_dimension(configuration.index_name, schema, dpp, table_dimension); ret < 0) {
lancedb_table_free(table);
lancedb_connection_free(conn);
return ret;
}
char* error_message;
- constexpr auto num_columns = 1;
- // metadata TODO: support metadata
- if (const LanceDBError result = lancedb_vector_query_select(query, key_columns, num_columns, &error_message) ; result != LANCEDB_SUCCESS) {
- ldpp_dout(dpp, 1) << "ERROR: s3vector failed to set select columns for vector query on index: " << configuration.index_name << ". error: " << error_message << dendl;
- lancedb_free_string(error_message);
- lancedb_vector_query_free(query);
- lancedb_table_free(table);
- lancedb_connection_free(conn);
- return lancedb_error_to_errno(result);
+
+ // parse filter before setting up select columns, since a JSON filter
+ // requires the metadata column to be included in the query results
+ LanceDBExpr* json_filter_expr = nullptr;
+ if (filter) {
+ const auto filterable_keys = configuration.post_filtering
+ ? std::vector<filterable_metadata_key_t>{}
+ : get_filterable_keys_from_schema(schema);
+ std::vector<std::string> nonfilterable_keys;
+ if (int ret = get_nonfilterable_metadata(table, dpp, nonfilterable_keys); ret < 0) {
+ lancedb_vector_query_free(query);
+ lancedb_table_free(table);
+ lancedb_connection_free(conn);
+ return ret;
+ }
+ auto filter_exprs = build_filter_expr(*filter, filterable_keys, nonfilterable_keys, dpp, errors);
+ if (!filter_exprs) {
+ lancedb_vector_query_free(query);
+ lancedb_table_free(table);
+ lancedb_connection_free(conn);
+ return -EINVAL;
+ }
+ if (filter_exprs->column_expr) {
+ if (const LanceDBError result = lancedb_vector_query_df_filter(query, filter_exprs->column_expr, &error_message); result != LANCEDB_SUCCESS) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector failed to apply column filter for vector query on index: " << configuration.index_name << ". error: " << error_message << dendl;
+ lancedb_free_string(error_message);
+ lancedb_expr_free(filter_exprs->json_expr);
+ lancedb_vector_query_free(query);
+ lancedb_table_free(table);
+ lancedb_connection_free(conn);
+ return lancedb_error_to_errno(result);
+ }
+ }
+ json_filter_expr = filter_exprs->json_expr;
+ }
+
+ const bool need_metadata = configuration.return_metadata || json_filter_expr;
+ {
+ const auto num_columns = need_metadata ? 2UL : 1UL;
+ const auto* columns = need_metadata ? key_and_metadata_columns : key_columns;
+ if (const LanceDBError result = lancedb_vector_query_select(query, columns, num_columns, &error_message) ; result != LANCEDB_SUCCESS) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector failed to set select columns for vector query on index: " << configuration.index_name << ". error: " << error_message << dendl;
+ lancedb_free_string(error_message);
+ lancedb_expr_free(json_filter_expr);
+ lancedb_vector_query_free(query);
+ lancedb_table_free(table);
+ lancedb_connection_free(conn);
+ return lancedb_error_to_errno(result);
+ }
}
if (const LanceDBError result = lancedb_vector_query_column(query, data_field, &error_message) ; result != LANCEDB_SUCCESS) {
ldpp_dout(dpp, 1) << "ERROR: s3vector failed to set select columns for vector query on index: " << configuration.index_name << ". error: " << error_message << dendl;
lancedb_free_string(error_message);
+ lancedb_expr_free(json_filter_expr);
lancedb_vector_query_free(query);
lancedb_table_free(table);
lancedb_connection_free(conn);
return lancedb_error_to_errno(result);
}
- if (const LanceDBError result = lancedb_vector_query_limit(query, configuration.top_k, &error_message) ; result != LANCEDB_SUCCESS) {
+ const auto effective_top_k = json_filter_expr
+ ? static_cast<unsigned int>(std::lround(configuration.top_k * dpp->get_cct()->_conf->rgw_s3vector_topk_post_filter_factor))
+ : configuration.top_k;
+ if (const LanceDBError result = lancedb_vector_query_limit(query, effective_top_k, &error_message) ; result != LANCEDB_SUCCESS) {
ldpp_dout(dpp, 1) << "ERROR: s3vector failed to set top-k for vector query on index: " << configuration.index_name << ". error: " << error_message << dendl;
lancedb_free_string(error_message);
+ lancedb_expr_free(json_filter_expr);
lancedb_vector_query_free(query);
lancedb_table_free(table);
lancedb_connection_free(conn);
return lancedb_error_to_errno(result);
}
+ // execute consumes query regardless of success/failure
LanceDBQueryResult* query_result = lancedb_vector_query_execute(query);
if (!query_result) {
ldpp_dout(dpp, 1) << "ERROR: s3vector failed to execute query on index: " << configuration.index_name << dendl;
- lancedb_vector_query_free(query);
+ lancedb_expr_free(json_filter_expr);
lancedb_table_free(table);
lancedb_connection_free(conn);
return -EIO;
}
- int ret = populate_vectors_from_query(dpp, query_result, reply.vectors, configuration.index_name, false, configuration.return_distance, true);
- reply.distance_metric = get_table_distance_metric(table, dpp);
+ int ret;
+ if (json_filter_expr) {
+ struct ArrowArray** c_arrays_ptr = nullptr;
+ struct ArrowSchema* c_schema_ptr = nullptr;
+ size_t count_out;
+ if (const LanceDBError result = lancedb_query_result_to_arrow(
+ query_result,
+ reinterpret_cast<FFI_ArrowArray***>(&c_arrays_ptr),
+ reinterpret_cast<FFI_ArrowSchema**>(&c_schema_ptr),
+ &count_out,
+ &error_message); result != LANCEDB_SUCCESS) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector failed to convert query result to arrow arrays for index: " << configuration.index_name << ". error: " << error_message << dendl;
+ lancedb_free_string(error_message);
+ lancedb_expr_free(json_filter_expr);
+ lancedb_table_free(table);
+ return lancedb_error_to_errno(result);
+ }
+
+ bool* matches = nullptr;
+ size_t match_count = 0;
+ if (count_out > 0) {
+ if (const LanceDBError result = lancedb_json_matches(
+ reinterpret_cast<FFI_ArrowArray**>(c_arrays_ptr),
+ reinterpret_cast<FFI_ArrowSchema*>(c_schema_ptr),
+ count_out,
+ json_filter_expr,
+ &matches,
+ &match_count,
+ &error_message); result != LANCEDB_SUCCESS) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector failed to apply JSON metadata filter for index: " << configuration.index_name << ". error: " << error_message << dendl;
+ lancedb_free_string(error_message);
+ lancedb_free_arrow_arrays(reinterpret_cast<FFI_ArrowArray**>(c_arrays_ptr), count_out);
+ lancedb_free_arrow_schema(reinterpret_cast<FFI_ArrowSchema*>(c_schema_ptr));
+ lancedb_table_free(table);
+ return lancedb_error_to_errno(result);
+ }
+ }
+
+ if (count_out == 0) {
+ lancedb_free_arrow_arrays(reinterpret_cast<FFI_ArrowArray**>(c_arrays_ptr), count_out);
+ lancedb_free_arrow_schema(reinterpret_cast<FFI_ArrowSchema*>(c_schema_ptr));
+ lancedb_expr_free(json_filter_expr);
+ ret = 0;
+ } else {
+ const bool need_distance = configuration.return_distance || (effective_top_k > configuration.top_k);
+ ret = populate_vectors_from_arrow(dpp, c_arrays_ptr, c_schema_ptr, reply.vectors, configuration.index_name,
+ false, need_distance, true, configuration.return_metadata, matches);
+ if (ret == 0 && reply.vectors.size() > configuration.top_k) {
+ // if we received more than k vectors (due to using the factor when post filtering)
+ // we return the top k ones based on distance
+ std::sort(reply.vectors.begin(), reply.vectors.end(),
+ [](const vector_item_t& a, const vector_item_t& b) {
+ return *a.distance < *b.distance;
+ });
+ reply.vectors.resize(configuration.top_k);
+ if (!configuration.return_distance) {
+ // if distance was not asked by the client
+ // we remove it from the reply
+ for (auto& v : reply.vectors) v.distance.reset();
+ }
+ }
+ }
+ lancedb_free_json_matches(matches);
+ } else {
+ ret = populate_vectors_from_query(dpp, query_result, reply.vectors, configuration.index_name, false, configuration.return_distance, true, configuration.return_metadata);
+ }
+
+ reply.distance_metric = get_distance_metric(table, dpp);
lancedb_table_free(table);
lancedb_connection_free(conn);
return ret;
#pragma once
+#include <optional>
#include <string>
#include <vector>
#include "include/encoding.h"
class Formatter;
}
class JSONObj;
+class JSONParser;
class DoutPrefixProvider;
namespace rgw::sal {
EUCLIDEAN,
};
+enum class FilterableMetadataType {
+ STRING,
+ NUMBER,
+ BOOLEAN,
+ STRING_LIST,
+ NUMBER_LIST,
+ BOOLEAN_LIST,
+};
+
+struct filterable_metadata_key_t {
+ std::string name;
+ FilterableMetadataType type = FilterableMetadataType::STRING;
+ bool must_exist = false;
+
+ void dump(ceph::Formatter* f) const;
+ void decode_json(JSONObj* obj);
+};
+
/*
{
"dataType": "string",
DistanceMetric distance_metric;
std::string index_name;
std::vector<std::string> non_filterable_metadata_keys;
+ std::vector<filterable_metadata_key_t> filterable_metadata_keys;
boost::optional<rgw::ARN> vector_bucket_arn;
std::string vector_bucket_name;
std::string index_arn;
std::string index_name;
std::vector<std::string> non_filterable_metadata_keys;
+ std::vector<filterable_metadata_key_t> filterable_metadata_keys;
std::string vector_bucket_name;
void dump(ceph::Formatter* f) const;
"returnDistance": boolean,
"returnMetadata": boolean,
"topK": number,
+ "postFiltering": boolean,
}
*/
struct query_vectors_t {
- std::string filter; // JSON string
+ std::string filter;
boost::optional<rgw::ARN> index_arn;
std::string index_name;
std::string vector_bucket_name;
bool return_distance = false;
bool return_metadata = false;
unsigned int top_k;
+ bool post_filtering = false;
void dump(ceph::Formatter* f) const;
void decode_json(JSONObj* obj);
);
}
-int create_index(const create_index_t& configuration, DoutPrefixProvider* dpp, optional_yield y);
+struct validation_error_t {
+ std::string path;
+ std::string message;
+};
+
+int create_index(const create_index_t& configuration, DoutPrefixProvider* dpp, optional_yield y, std::vector<validation_error_t>& errors);
int create_vector_bucket(const create_vector_bucket_t& configuration, DoutPrefixProvider* dpp, optional_yield y);
int delete_index(const delete_index_t& configuration, DoutPrefixProvider* dpp, optional_yield y);
int delete_vector_bucket(const delete_vector_bucket_t& configuration, DoutPrefixProvider* dpp, optional_yield y);
int delete_vector_bucket_policy(const delete_vector_bucket_policy_t& configuration, DoutPrefixProvider* dpp, optional_yield y);
-int put_vectors(const put_vectors_t& configuration, DoutPrefixProvider* dpp, optional_yield y);
+int put_vectors(const put_vectors_t& configuration, DoutPrefixProvider* dpp, optional_yield y, std::vector<validation_error_t>& errors);
int get_vectors(const get_vectors_t& configuration, DoutPrefixProvider* dpp, optional_yield y, get_vectors_reply_t& reply);
int list_vectors(const list_vectors_t& configuration, DoutPrefixProvider* dpp, optional_yield y, list_vectors_reply_t& reply);
int get_index(const get_index_t& configuration, const std::string& region, const std::string& account, DoutPrefixProvider* dpp, optional_yield y, get_index_reply_t& reply);
int put_vector_bucket_policy(const put_vector_bucket_policy_t& configuration, DoutPrefixProvider* dpp, optional_yield y);
int get_vector_bucket_policy(const get_vector_bucket_policy_t& configuration, DoutPrefixProvider* dpp, optional_yield y);
int delete_vectors(const delete_vectors_t& configuration, DoutPrefixProvider* dpp, optional_yield y);
-int query_vectors(const query_vectors_t& configuration, DoutPrefixProvider* dpp, optional_yield y, query_vectors_reply_t& reply);
+int query_vectors(const query_vectors_t& configuration, std::optional<JSONParser>& filter, DoutPrefixProvider* dpp, optional_yield y, query_vectors_reply_t& reply, std::vector<validation_error_t>& errors);
}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
+// vim: ts=8 sw=2 sts=2 expandtab ft=cpp
+
+#include "rgw_s3vector_filter.h"
+#include "common/ceph_json.h"
+#include "common/dout.h"
+#include "lancedb.h"
+#include <charconv>
+#include <fmt/format.h>
+
+#define dout_subsys ceph_subsys_rgw
+
+namespace rgw::s3vector {
+
+ static constexpr const char* metadata_field = "metadata";
+
+ const filterable_metadata_key_t* find_filterable_key(
+ const std::string& field_name,
+ const std::vector<filterable_metadata_key_t>& filterable_keys) {
+ for (const auto& fk : filterable_keys) {
+ if (fk.name == field_name) return &fk;
+ }
+ return nullptr;
+ }
+
+ bool is_nonfilterable_key(
+ const std::string& field_name,
+ const std::vector<std::string>& nonfilterable_keys) {
+ for (const auto& nfk : nonfilterable_keys) {
+ if (nfk == field_name) return true;
+ }
+ return false;
+ }
+
+ LanceDBExpr* build_literal_expr(JSONObj* value_obj, FilterableMetadataType type,
+ const std::string& field_name, DoutPrefixProvider* dpp,
+ std::vector<validation_error_t>& errors) {
+ const auto& dv = value_obj->get_data_val();
+ switch (type) {
+ case FilterableMetadataType::STRING:
+ return lancedb_expr_literal_string(dv.str.c_str());
+ case FilterableMetadataType::NUMBER: {
+ double val;
+ const auto [ptr, ec] = std::from_chars(dv.str.data(), dv.str.data() + dv.str.size(), val);
+ if (ec != std::errc()) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: invalid number value '" << dv.str << "'" << dendl;
+ errors.push_back({"filter", fmt::format("invalid number value '{}' for field '{}'", dv.str, field_name)});
+ return nullptr;
+ }
+ return lancedb_expr_literal_f64(val);
+ }
+ case FilterableMetadataType::BOOLEAN:
+ if (dv.str == "true") return lancedb_expr_literal_bool(true);
+ if (dv.str == "false") return lancedb_expr_literal_bool(false);
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: invalid boolean value '" << dv.str << "'" << dendl;
+ errors.push_back({"filter", fmt::format("invalid boolean value '{}' for field '{}'", dv.str, field_name)});
+ return nullptr;
+ default:
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: filtering not supported on list-type columns" << dendl;
+ errors.push_back({"filter", fmt::format("filtering not supported on list-type metadata key '{}'", field_name)});
+ return nullptr;
+ }
+ }
+
+ enum class JsonValueType { STRING, NUMBER, BOOLEAN };
+
+ std::optional<JsonValueType> infer_value_type(JSONObj* value_obj) {
+ if (value_obj->is_object() || value_obj->is_array()) return std::nullopt;
+ const auto& dv = value_obj->get_data_val();
+ if (dv.quoted) return JsonValueType::STRING;
+ if (dv.str == "true" || dv.str == "false") return JsonValueType::BOOLEAN;
+ if (dv.str == "null") return std::nullopt;
+ return JsonValueType::NUMBER;
+ }
+
+ LanceDBExpr* build_json_field_expr(const std::string& field_name, JsonValueType vtype) {
+ const char* path = field_name.c_str();
+ auto* col = lancedb_expr_column(metadata_field);
+ switch (vtype) {
+ case JsonValueType::STRING: return lancedb_expr_json_get_str(col, &path, 1);
+ case JsonValueType::NUMBER: return lancedb_expr_json_get_float(col, &path, 1);
+ case JsonValueType::BOOLEAN: return lancedb_expr_json_get_bool(col, &path, 1);
+ }
+ return nullptr;
+ }
+
+ LanceDBExpr* build_json_literal_expr(JSONObj* value_obj,
+ const std::string& field_name, DoutPrefixProvider* dpp,
+ std::vector<validation_error_t>& errors) {
+ const auto& dv = value_obj->get_data_val();
+ if (dv.quoted) return lancedb_expr_literal_string(dv.str.c_str());
+ if (dv.str == "true") return lancedb_expr_literal_bool(true);
+ if (dv.str == "false") return lancedb_expr_literal_bool(false);
+ double val;
+ auto [ptr, ec] = std::from_chars(dv.str.data(), dv.str.data() + dv.str.size(), val);
+ if (ec == std::errc()) return lancedb_expr_literal_f64(val);
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: invalid literal value '" << dv.str << "'" << dendl;
+ errors.push_back({"filter", fmt::format("invalid value '{}' for field '{}'", dv.str, field_name)});
+ return nullptr;
+ }
+
+ static const auto invalid_binary_op = static_cast<LanceDBBinaryOp>(-1);
+
+ LanceDBBinaryOp s3vector_to_lance_op(const std::string& op) {
+ if (op == "$eq") return LANCEDB_BINARY_OP_EQ;
+ if (op == "$ne") return LANCEDB_BINARY_OP_NOT_EQ;
+ if (op == "$gt") return LANCEDB_BINARY_OP_GT;
+ if (op == "$gte") return LANCEDB_BINARY_OP_GT_EQ;
+ if (op == "$lt") return LANCEDB_BINARY_OP_LT;
+ if (op == "$lte") return LANCEDB_BINARY_OP_LT_EQ;
+ return invalid_binary_op;
+ }
+
+ LanceDBExpr* build_exists_expr(
+ const std::string& field_name,
+ JSONObj* value_obj,
+ const filterable_metadata_key_t* fk,
+ DoutPrefixProvider* dpp,
+ std::vector<validation_error_t>& errors) {
+ bool exists;
+ const auto& dv = value_obj->get_data_val();
+ if (dv.quoted || value_obj->is_object() || value_obj->is_array()) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: $exists operator requires a boolean value for field '" << field_name << "'" << dendl;
+ errors.push_back({"filter", fmt::format("$exists operator requires a boolean value for field '{}'", field_name)});
+ return nullptr;
+ }
+ if (dv.str == "true") exists = true;
+ else if (dv.str == "false") exists = false;
+ else {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: $exists operator requires a boolean value for field '" << field_name << "'" << dendl;
+ errors.push_back({"filter", fmt::format("$exists operator requires a boolean value for field '{}'", field_name)});
+ return nullptr;
+ }
+ if (fk) {
+ if (fk->must_exist) {
+ // if column must exist, we can create a const boolean expression
+ return lancedb_expr_literal_bool(exists);
+ }
+ auto* col = lancedb_expr_column(fk->name.c_str());
+ // build an expression that checks if the column is null or not
+ return exists ? lancedb_expr_is_not_null(col) : lancedb_expr_is_null(col);
+ }
+ const char* path = field_name.c_str();
+ auto* col = lancedb_expr_column(metadata_field);
+ auto* contains = lancedb_expr_json_contains(col, &path, 1);
+ // build an expression that checks if the metadata JSON has this key
+ return exists ? contains : lancedb_expr_not(contains);
+ }
+
+ LanceDBExpr* build_list_expr(
+ const std::string& field_name,
+ bool negated,
+ JSONObj* value_obj,
+ const filterable_metadata_key_t* fk,
+ DoutPrefixProvider* dpp,
+ std::vector<validation_error_t>& errors) {
+ if (!value_obj->is_array()) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: $" << (negated ? "nin" : "in") << " requires an array of values for field '" << field_name << "'" << dendl;
+ errors.push_back({"filter", fmt::format("${} requires an array of values for field '{}'", negated ? "nin" : "in", field_name)});
+ return nullptr;
+ }
+ std::vector<LanceDBExpr*> list_exprs;
+ JsonValueType vtype{}; // will be initialized by the first element if field is not a column
+ for (auto it = value_obj->find_first(); !it.end(); ++it) {
+ auto* elem = *it;
+ if (!fk) {
+ auto elem_type = infer_value_type(elem);
+ if (!elem_type) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: unsupported value type for field '" << field_name << "'" << dendl;
+ errors.push_back({"filter", fmt::format("unsupported value type for field '{}'", field_name)});
+ for (auto* e : list_exprs) lancedb_expr_free(e);
+ return nullptr;
+ }
+ if (list_exprs.empty()) {
+ vtype = *elem_type;
+ } else if (*elem_type != vtype) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: mixed types in list for field '" << field_name << "'" << dendl;
+ errors.push_back({"filter", fmt::format("mixed types in list for field '{}'", field_name)});
+ for (auto* e : list_exprs) lancedb_expr_free(e);
+ return nullptr;
+ }
+ }
+ auto* elem_expr = fk ? build_literal_expr(elem, fk->type, field_name, dpp, errors) : build_json_literal_expr(elem, field_name, dpp, errors);
+ if (!elem_expr) {
+ for (auto* e : list_exprs) lancedb_expr_free(e);
+ return nullptr;
+ }
+ list_exprs.push_back(elem_expr);
+ }
+ if (list_exprs.empty()) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: empty list for field '" << field_name << "'" << dendl;
+ errors.push_back({"filter", fmt::format("empty list for field '{}'", field_name)});
+ return nullptr;
+ }
+
+ LanceDBExpr* field_expr;
+ if (!fk) {
+ field_expr = build_json_field_expr(field_name, vtype);
+ } else {
+ field_expr = lancedb_expr_column(fk->name.c_str());
+ }
+
+ char* error_message = nullptr;
+ auto* result = lancedb_expr_in_list(field_expr, list_exprs.data(), list_exprs.size(), negated, &error_message);
+ if (!result) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: failed to build list expression: " << (error_message ? error_message : "unknown") << dendl;
+ errors.push_back({"filter", fmt::format("failed to build list expression for field '{}'", field_name)});
+ lancedb_free_string(error_message);
+ }
+ return result;
+ }
+
+ LanceDBExpr* build_binary_expr(
+ const std::string& field_name,
+ const std::string& op,
+ JSONObj* value_obj,
+ const filterable_metadata_key_t* fk,
+ DoutPrefixProvider* dpp,
+ std::vector<validation_error_t>& errors) {
+ auto binary_op = s3vector_to_lance_op(op);
+ if (binary_op == invalid_binary_op) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: unknown operator '" << op << "'" << dendl;
+ errors.push_back({"filter", fmt::format("unknown filter operator '{}'", op)});
+ return nullptr;
+ }
+
+ LanceDBExpr* field_expr;
+ LanceDBExpr* literal_expr;
+ if (fk) {
+ field_expr = lancedb_expr_column(fk->name.c_str());
+ literal_expr = build_literal_expr(value_obj, fk->type, field_name, dpp, errors);
+ } else {
+ auto vtype = infer_value_type(value_obj);
+ if (!vtype) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: unsupported value type for field '" << field_name << "'" << dendl;
+ errors.push_back({"filter", fmt::format("unsupported value type for field '{}'", field_name)});
+ return nullptr;
+ }
+ field_expr = build_json_field_expr(field_name, *vtype);
+ literal_expr = build_json_literal_expr(value_obj, field_name, dpp, errors);
+ }
+ if (!literal_expr) {
+ lancedb_expr_free(field_expr);
+ return nullptr;
+ }
+ return lancedb_expr_binary(field_expr, binary_op, literal_expr);
+ }
+
+ LanceDBExpr* build_op_expr(
+ const std::string& field_name,
+ const std::string& op,
+ JSONObj* value_obj,
+ const filterable_metadata_key_t* fk,
+ DoutPrefixProvider* dpp,
+ std::vector<validation_error_t>& errors) {
+ if (op == "$exists") return build_exists_expr(field_name, value_obj, fk, dpp, errors);
+ if (op == "$in" || op == "$nin") return build_list_expr(field_name, (op == "$nin"), value_obj, fk, dpp, errors);
+ return build_binary_expr(field_name, op, value_obj, fk, dpp, errors);
+ }
+
+ void free_filter_exprs(FilterExprs& fe) {
+ lancedb_expr_free(fe.column_expr);
+ lancedb_expr_free(fe.json_expr);
+ fe.column_expr = nullptr;
+ fe.json_expr = nullptr;
+ }
+
+ void combine_filter_exprs_and(FilterExprs& combined, FilterExprs& other) {
+ if (other.column_expr) {
+ combined.column_expr = combined.column_expr
+ ? lancedb_expr_and(combined.column_expr, other.column_expr)
+ : other.column_expr;
+ }
+ if (other.json_expr) {
+ combined.json_expr = combined.json_expr
+ ? lancedb_expr_and(combined.json_expr, other.json_expr)
+ : other.json_expr;
+ }
+ }
+
+ std::optional<FilterExprs> build_field_expr(
+ const std::string& field_name,
+ JSONObj* value_obj,
+ const std::vector<filterable_metadata_key_t>& filterable_keys,
+ const std::vector<std::string>& nonfilterable_keys,
+ DoutPrefixProvider* dpp,
+ std::vector<validation_error_t>& errors) {
+ if (is_nonfilterable_key(field_name, nonfilterable_keys)) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: cannot filter on non-filterable metadata key '" << field_name << "'" << dendl;
+ errors.push_back({"filter", fmt::format("cannot filter on non-filterable metadata key '{}'", field_name)});
+ return std::nullopt;
+ }
+
+ const auto* fk = find_filterable_key(field_name, filterable_keys);
+ const bool is_column = (fk != nullptr);
+
+ LanceDBExpr* combined = nullptr;
+ if (value_obj->is_object()) {
+ for (auto it = value_obj->find_first(); !it.end(); ++it) {
+ auto* op_obj = *it;
+ auto* cmp_expr = build_op_expr(field_name, op_obj->get_name(), op_obj, fk, dpp, errors);
+ if (!cmp_expr) {
+ lancedb_expr_free(combined);
+ return std::nullopt;
+ }
+ // implicit AND between multiple operators on the same field. e.g. {"age": {"$gt": 18, "$lt": 65}} is treated as
+ // {"$and": [{"age": {"$gt": 18}}, {"age": {"$lt": 65}}]}
+ combined = combined ? lancedb_expr_and(combined, cmp_expr) : cmp_expr;
+ }
+ } else if (value_obj->is_array()) {
+ // implicit eqality with an array is not permitted
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: cannot use implicit equality with an array value for field '" << field_name << "'" << dendl;
+ errors.push_back({"filter", fmt::format("cannot use implicit equality with an array value for field '{}'", field_name)});
+ return std::nullopt;
+ } else {
+ // implicit equality if value is not an object. e.g. {"color": "red"} is treated as {"color": {"$eq": "red"}}
+ combined = build_op_expr(field_name, "$eq", value_obj, fk, dpp, errors);
+ }
+ if (!combined) return std::nullopt;
+
+ FilterExprs result;
+ if (is_column) {
+ result.column_expr = combined;
+ } else {
+ result.json_expr = combined;
+ }
+ return result;
+ }
+
+ std::optional<FilterExprs> build_filter_expr(
+ JSONObj& obj,
+ const std::vector<filterable_metadata_key_t>& filterable_keys,
+ const std::vector<std::string>& nonfilterable_keys,
+ DoutPrefixProvider* dpp,
+ std::vector<validation_error_t>& errors) {
+ FilterExprs combined;
+ for (auto it = obj.find_first(); !it.end(); ++it) {
+ auto* child = *it;
+ const auto& name = child->get_name();
+
+ if (name == "$and" || name == "$or") {
+ if (!child->is_array()) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: " << name << " requires an array of conditions" << dendl;
+ errors.push_back({"filter", fmt::format("{} requires an array of conditions", name)});
+ free_filter_exprs(combined);
+ return std::nullopt;
+ }
+ // top level logical operators
+ if (child->find_first().end()) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: " << name << " requires a non-empty array of conditions" << dendl;
+ errors.push_back({"filter", fmt::format("{} requires a non-empty array of conditions", name)});
+ free_filter_exprs(combined);
+ return std::nullopt;
+ }
+ FilterExprs logical;
+ bool first = true;
+ bool has_column = false;
+ bool has_json = false;
+ for (auto arr_it = child->find_first(); !arr_it.end(); ++arr_it) {
+ if (!(**arr_it).is_object()) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: " << name << " array elements must be filter objects" << dendl;
+ errors.push_back({"filter", fmt::format("{} array elements must be filter objects", name)});
+ free_filter_exprs(logical);
+ free_filter_exprs(combined);
+ return std::nullopt;
+ }
+ auto sub = build_filter_expr(**arr_it, filterable_keys, nonfilterable_keys, dpp, errors);
+ if (!sub) {
+ free_filter_exprs(logical);
+ free_filter_exprs(combined);
+ return std::nullopt;
+ }
+ has_column = has_column || sub->column_expr;
+ has_json = has_json || sub->json_expr;
+
+ if (name == "$or" && has_column && has_json) {
+ ldpp_dout(dpp, 1) << "ERROR: s3vector filter: $or cannot mix filterable column and JSON metadata conditions. use postFiltering=true to treat all conditions as JSON metadata conditions" << dendl;
+ errors.push_back({"filter", "$or cannot mix filterable column and JSON metadata conditions. use postFiltering=true to treat all conditions as JSON metadata conditions"});
+ free_filter_exprs(*sub);
+ free_filter_exprs(logical);
+ free_filter_exprs(combined);
+ return std::nullopt;
+ }
+
+ if (first) {
+ logical = *sub;
+ first = false;
+ } else if (name == "$and") {
+ combine_filter_exprs_and(logical, *sub);
+ } else {
+ // all children are either column expressions of JSON expressions (validated above)
+ if (sub->column_expr) {
+ logical.column_expr = lancedb_expr_or(logical.column_expr, sub->column_expr);
+ }
+ if (sub->json_expr) {
+ logical.json_expr = lancedb_expr_or(logical.json_expr, sub->json_expr);
+ }
+ }
+ }
+ combine_filter_exprs_and(combined, logical);
+ } else {
+ // top level field expression
+ auto field = build_field_expr(name, child, filterable_keys, nonfilterable_keys, dpp, errors);
+ if (!field) {
+ free_filter_exprs(combined);
+ return std::nullopt;
+ }
+ combine_filter_exprs_and(combined, *field);
+ }
+ }
+ return combined;
+ }
+
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
+// vim: ts=8 sw=2 sts=2 expandtab ft=cpp
+
+#pragma once
+
+#include <optional>
+#include <string>
+#include <vector>
+#include "rgw_s3vector.h"
+
+struct LanceDBExpr;
+class DoutPrefixProvider;
+
+namespace rgw::s3vector {
+
+struct FilterExprs {
+ LanceDBExpr* column_expr = nullptr;
+ LanceDBExpr* json_expr = nullptr;
+};
+
+std::optional<FilterExprs> build_filter_expr(
+ JSONObj& filter_obj,
+ const std::vector<filterable_metadata_key_t>& filterable_keys,
+ const std::vector<std::string>& nonfilterable_keys,
+ DoutPrefixProvider* dpp,
+ std::vector<validation_error_t>& errors);
+
+}
add_catch2_test_rgw(fdb NO_CATCH2_MAIN)
add_catch2_test_rgw(fdb_ceph NO_CATCH2_MAIN)
endif()
+
+if(WITH_RADOSGW_LANCEDB)
+add_executable(unittest_rgw_s3vector_filter test_rgw_s3vector_filter.cc)
+add_ceph_unittest(unittest_rgw_s3vector_filter)
+target_link_libraries(unittest_rgw_s3vector_filter ${rgw_libs} ${UNITTEST_LIBS})
+endif()
from datetime import datetime, timedelta, timezone
import pytest
import boto3
+import json
from botocore.config import Config
from . import(
return bash(cmd, **kwargs)
+def ceph_admin(args, **kwargs):
+ """ ceph command """
+ cmd = [test_path + 'test-rgw-call.sh', 'call_ceph', 'noname'] + args
+ return bash(cmd, **kwargs)
+
+
+def set_rgw_config_option(option, value):
+ """ change a config option """
+ client = f'client.rgw.{get_config_port()}'
+ return ceph_admin(['config', 'set', client, option, str(value)])
+
+
def gen_bucket_name():
global num_buckets
result = conn.create_index(vectorBucketName=bucket_name, indexName=index_name, dataType='float32', dimension=128, distanceMetric='euclidean')
assert result['ResponseMetadata']['HTTPStatusCode'] == 200
assert result['indexArn'] == 'arn:aws:s3vectors:::bucket/{}/index/{}'.format(bucket_name, index_name)
- # idempotent create with same definition should succeed
- result = conn.create_index(vectorBucketName=bucket_name, indexName=index_name, dataType='float32', dimension=128, distanceMetric='euclidean')
- assert result['ResponseMetadata']['HTTPStatusCode'] == 200
- # create with different dimension should fail
- pytest.raises(conn.exceptions.ClientError, conn.create_index,
- vectorBucketName=bucket_name, indexName=index_name, dataType='float32', dimension=64, distanceMetric='euclidean')
+ # create with same name should fail with ConflictException
+ with pytest.raises(conn.exceptions.ClientError) as exc_info:
+ conn.create_index(vectorBucketName=bucket_name, indexName=index_name, dataType='float32', dimension=128, distanceMetric='euclidean')
+ assert exc_info.value.response['Error']['Code'] == 'BucketAlreadyExists'
# create an index on bucket that does not exist
invalid_bucket_name = bucket_name + '-invalid'
pytest.raises(conn.exceptions.ClientError, conn.create_index, vectorBucketName=invalid_bucket_name, indexName=index_name, dataType='float32', dimension=128, distanceMetric='euclidean')
_ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+@pytest.mark.index_test
+def test_create_index_invalid_filterable_keys():
+ """Test that invalid filterable metadata key names fail with ValidationException."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ common = dict(vectorBucketName=bucket_name, dataType='float32', dimension=4, distanceMetric='euclidean')
+
+ # duplicate field names
+ assert_create_index_validation_error(conn,
+ 'metadataConfiguration.filterableMetadataKeys',
+ indexName='dup-fields',
+ metadataConfiguration={'filterableMetadataKeys': [
+ {'name': 'genre'}, {'name': 'genre'}
+ ]}, **common)
+
+ # reserved column name: key
+ assert_create_index_validation_error(conn,
+ 'metadataConfiguration.filterableMetadataKeys',
+ indexName='reserved-key',
+ metadataConfiguration={'filterableMetadataKeys': [
+ {'name': 'key'}
+ ]}, **common)
+
+ # reserved column name: data
+ assert_create_index_validation_error(conn,
+ 'metadataConfiguration.filterableMetadataKeys',
+ indexName='reserved-data',
+ metadataConfiguration={'filterableMetadataKeys': [
+ {'name': 'data'}
+ ]}, **common)
+
+ # reserved column name: metadata
+ assert_create_index_validation_error(conn,
+ 'metadataConfiguration.filterableMetadataKeys',
+ indexName='reserved-metadata',
+ metadataConfiguration={'filterableMetadataKeys': [
+ {'name': 'metadata'}
+ ]}, **common)
+
+ # filterable key name starting with underscore
+ assert_create_index_validation_error(conn,
+ 'metadataConfiguration.filterableMetadataKeys[0].name',
+ indexName='underscore-key',
+ metadataConfiguration={'filterableMetadataKeys': [
+ {'name': '_internal'}
+ ]}, **common)
+
+ # overlap between filterable and non-filterable keys
+ assert_create_index_validation_error(conn,
+ 'metadataConfiguration.filterableMetadataKeys[0].name',
+ indexName='overlap-keys',
+ metadataConfiguration={
+ 'nonFilterableMetadataKeys': ['genre', 'year'],
+ 'filterableMetadataKeys': [{'name': 'genre'}]
+ }, **common)
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
@pytest.mark.index_test
def test_get_index():
conn = connection()
_ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+@pytest.mark.index_test
+def test_non_filterable_metadata_keys():
+ """Test that nonFilterableMetadataKeys is stored on CreateIndex and returned on GetIndex."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ nonfilterable_keys = ['key1', 'key2', 'key3']
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=128, distanceMetric='euclidean',
+ metadataConfiguration={'nonFilterableMetadataKeys': nonfilterable_keys})
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ result = conn.get_index(vectorBucketName=bucket_name, indexName=index_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ returned_keys = result['index']['metadataConfiguration']['nonFilterableMetadataKeys']
+ assert set(returned_keys) == set(nonfilterable_keys), \
+ f"expected {nonfilterable_keys} but got {returned_keys}"
+
+ # create index without nonFilterableMetadataKeys
+ index_name2 = 'test-index2'
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name2,
+ dataType='float32', dimension=64, distanceMetric='cosine')
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ result = conn.get_index(vectorBucketName=bucket_name, indexName=index_name2)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ returned_keys = result['index']['metadataConfiguration']['nonFilterableMetadataKeys']
+ assert returned_keys == [], f"expected empty list but got {returned_keys}"
+
+ log.info('test_non_filterable_metadata_keys: verified nonFilterableMetadataKeys round-trip')
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.index_test
+def test_filterable_metadata_keys():
+ """Test filterableMetadataKeys: store on CreateIndex, retrieve on GetIndex,
+ and populate filterable columns via PutVectors."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ dimension = 8
+ filterable_keys = [
+ {'name': 'genre', 'type': 'String'},
+ {'name': 'year', 'type': 'Number'},
+ {'name': 'popular', 'type': 'Boolean'}
+ ]
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean',
+ metadataConfiguration={
+ 'filterableMetadataKeys': filterable_keys
+ })
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # verify filterableMetadataKeys are returned on GetIndex
+ result = conn.get_index(vectorBucketName=bucket_name, indexName=index_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ returned_filterable = result['index']['metadataConfiguration'].get('filterableMetadataKeys', [])
+ assert len(returned_filterable) == len(filterable_keys), \
+ f"expected {len(filterable_keys)} filterable keys but got {len(returned_filterable)}"
+ returned_names = {k['name'] for k in returned_filterable}
+ expected_names = {k['name'] for k in filterable_keys}
+ assert returned_names == expected_names, \
+ f"expected names {expected_names} but got {returned_names}"
+
+ # put vectors with metadata that includes filterable fields plus extra keys not in the list
+ vectors = []
+ genres = ['rock', 'jazz', 'pop', 'rock', 'jazz']
+ for i in range(5):
+ v = {
+ 'key': f'vec-{i}',
+ 'data': generate_data(dimension, i),
+ 'metadata': json.dumps({
+ 'genre': genres[i],
+ 'year': 2000 + i,
+ 'popular': i % 2 == 0,
+ 'artist': f'artist-{i}',
+ 'rating': 4.5 + i * 0.1
+ })
+ }
+ vectors.append(v)
+
+ # vectors with metadata containing only keys NOT in the filterable list
+ for i in range(5, 8):
+ v = {
+ 'key': f'vec-{i}',
+ 'data': generate_data(dimension, i),
+ 'metadata': json.dumps({
+ 'artist': f'artist-{i}',
+ 'rating': 3.0 + i * 0.1
+ })
+ }
+ vectors.append(v)
+
+ # vectors with no metadata at all
+ for i in range(8, 10):
+ v = {
+ 'key': f'vec-{i}',
+ 'data': generate_data(dimension, i),
+ }
+ vectors.append(v)
+
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # verify all vectors can be retrieved with correct metadata
+ all_keys = [f'vec-{i}' for i in range(10)]
+ result = conn.get_vectors(vectorBucketName=bucket_name, indexName=index_name,
+ keys=all_keys, returnMetadata=True)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ assert len(result['vectors']) == 10
+
+ for vector in result['vectors']:
+ key = vector['key']
+ idx = int(key.split('-')[1])
+ if idx < 5:
+ assert 'metadata' in vector, f"{key} should have metadata"
+ md = json.loads(vector['metadata'])
+ assert md['genre'] == genres[idx], f"{key} genre mismatch"
+ assert md['year'] == 2000 + idx, f"{key} year mismatch"
+ assert md['popular'] == (idx % 2 == 0), f"{key} popular mismatch"
+ assert md['artist'] == f'artist-{idx}', f"{key} artist mismatch"
+ elif idx < 8:
+ assert 'metadata' in vector, f"{key} should have metadata"
+ md = json.loads(vector['metadata'])
+ assert 'genre' not in md, f"{key} should not have genre"
+ assert md['artist'] == f'artist-{idx}', f"{key} artist mismatch"
+ else:
+ assert 'metadata' not in vector, f"{key} should not have metadata"
+
+ log.info('test_filterable_metadata_keys: verified filterable metadata round-trip')
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.index_test
+def test_filterable_metadata_list_keys():
+ """Test filterableMetadataKeys with list types: StringList, NumberList, BooleanList."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-list-index'
+ dimension = 4
+ filterable_keys = [
+ {'name': 'genre', 'type': 'String'},
+ {'name': 'tags', 'type': 'StringList'},
+ {'name': 'scores', 'type': 'NumberList'},
+ {'name': 'flags', 'type': 'BooleanList'}
+ ]
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='cosine',
+ metadataConfiguration={
+ 'filterableMetadataKeys': filterable_keys
+ })
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # verify filterableMetadataKeys with list types are returned on GetIndex
+ result = conn.get_index(vectorBucketName=bucket_name, indexName=index_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ returned_filterable = result['index']['metadataConfiguration'].get('filterableMetadataKeys', [])
+ assert len(returned_filterable) == len(filterable_keys), \
+ f"expected {len(filterable_keys)} filterable keys but got {len(returned_filterable)}"
+ for rk in returned_filterable:
+ expected = next(fk for fk in filterable_keys if fk['name'] == rk['name'])
+ assert rk['type'] == expected['type'], \
+ f"expected type {expected['type']} for {rk['name']} but got {rk['type']}"
+
+ # put vectors with list-type metadata
+ vectors = [
+ {
+ 'key': 'vec-0',
+ 'data': generate_data(dimension, 0),
+ 'metadata': json.dumps({
+ 'tags': ['rock', 'pop'],
+ 'scores': [1.5, 2.5, 3.5],
+ 'flags': [True, False, True]
+ })
+ },
+ {
+ 'key': 'vec-1',
+ 'data': generate_data(dimension, 1),
+ 'metadata': json.dumps({
+ 'genre': 'jazz',
+ 'tags': ['jazz'],
+ 'flags': [False]
+ })
+ },
+ {
+ 'key': 'vec-2',
+ 'data': generate_data(dimension, 2),
+ 'metadata': json.dumps({
+ 'description': 'no list keys here'
+ })
+ },
+ {
+ 'key': 'vec-3',
+ 'data': generate_data(dimension, 3),
+ },
+ ]
+
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # verify vectors can be retrieved with correct metadata
+ all_keys = [f'vec-{i}' for i in range(4)]
+ result = conn.get_vectors(vectorBucketName=bucket_name, indexName=index_name,
+ keys=all_keys, returnMetadata=True)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ assert len(result['vectors']) == 4
+
+ for vector in result['vectors']:
+ key = vector['key']
+ idx = int(key.split('-')[1])
+ if idx == 0:
+ md = json.loads(vector['metadata'])
+ assert md['tags'] == ['rock', 'pop'], f"{key} tags mismatch"
+ assert md['scores'] == [1.5, 2.5, 3.5], f"{key} scores mismatch"
+ assert md['flags'] == [True, False, True], f"{key} flags mismatch"
+ elif idx == 1:
+ md = json.loads(vector['metadata'])
+ assert md['genre'] == 'jazz', f"{key} genre mismatch"
+ assert md['tags'] == ['jazz'], f"{key} tags mismatch"
+ assert 'scores' not in md, f"{key} should not have scores"
+ assert md['flags'] == [False], f"{key} flags mismatch"
+ elif idx == 2:
+ md = json.loads(vector['metadata'])
+ assert 'genre' not in md, f"{key} should not have genre"
+ assert 'tags' not in md, f"{key} should not have tags"
+ assert 'scores' not in md, f"{key} should not have scores"
+ assert 'flags' not in md, f"{key} should not have flags"
+ assert md['description'] == 'no list keys here'
+ else:
+ assert 'metadata' not in vector, f"{key} should not have metadata"
+
+ log.info('test_filterable_metadata_list_keys: verified list-type filterable metadata round-trip')
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.index_test
+def test_metadata_dots_in_names_rejected():
+ """Test that dots in metadata key names are rejected at index creation."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ dimension = 4
+
+ # dots in filterable metadata key names
+ assert_create_index_validation_error(conn,
+ 'metadataConfiguration.filterableMetadataKeys[0].name',
+ vectorBucketName=bucket_name, indexName='test-dot-filterable',
+ dataType='float32', dimension=dimension, distanceMetric='euclidean',
+ metadataConfiguration={'filterableMetadataKeys': [{'name': 'user.name'}]})
+
+ # dots in non-filterable metadata key names
+ assert_create_index_validation_error(conn,
+ 'metadataConfiguration.nonFilterableMetadataKeys[0]',
+ vectorBucketName=bucket_name, indexName='test-dot-nonfilterable',
+ dataType='float32', dimension=dimension, distanceMetric='euclidean',
+ metadataConfiguration={'nonFilterableMetadataKeys': ['user.name']})
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
@pytest.mark.index_test
def test_delete_index():
conn = connection()
_ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+def assert_put_vectors_validation_error(conn, expected_paths, **kwargs):
+ """Call put_vectors expecting a ValidationException, verify the fieldList paths.
+ expected_paths can be a single string or a list of strings."""
+ if isinstance(expected_paths, str):
+ expected_paths = [expected_paths]
+ captured = {}
+ def capture(**kw):
+ captured['body'] = kw['http_response'].content
+ event = 'after-call.s3vectors.PutVectors'
+ conn.meta.events.register(event, capture)
+ try:
+ with pytest.raises(conn.exceptions.ClientError) as exc_info:
+ conn.put_vectors(**kwargs)
+ assert exc_info.value.response['Error']['Code'] == 'ValidationException'
+ body = json.loads(captured['body'])
+ assert 'fieldList' in body, f"response should contain fieldList"
+ actual_paths = [entry['path'] for entry in body['fieldList']]
+ assert actual_paths == expected_paths, \
+ f"expected fieldList paths {expected_paths} but got {actual_paths}"
+ finally:
+ conn.meta.events.unregister(event, capture)
+
+
+def assert_query_vectors_validation_error(conn, expected_paths, **kwargs):
+ """Call query_vectors expecting a ValidationException, verify the fieldList paths.
+ expected_paths can be a single string or a list of strings."""
+ if isinstance(expected_paths, str):
+ expected_paths = [expected_paths]
+ captured = {}
+ def capture(**kw):
+ captured['body'] = kw['http_response'].content
+ event = 'after-call.s3vectors.QueryVectors'
+ conn.meta.events.register(event, capture)
+ try:
+ with pytest.raises(conn.exceptions.ClientError) as exc_info:
+ conn.query_vectors(**kwargs)
+ assert exc_info.value.response['Error']['Code'] == 'ValidationException'
+ body = json.loads(captured['body'])
+ assert 'fieldList' in body, f"response should contain fieldList"
+ actual_paths = [entry['path'] for entry in body['fieldList']]
+ assert actual_paths == expected_paths, \
+ f"expected fieldList paths {expected_paths} but got {actual_paths}"
+ finally:
+ conn.meta.events.unregister(event, capture)
+
+
+def assert_create_index_validation_error(conn, expected_paths, **kwargs):
+ """Call create_index expecting a ValidationException, verify the fieldList paths.
+ expected_paths can be a single string or a list of strings."""
+ if isinstance(expected_paths, str):
+ expected_paths = [expected_paths]
+ captured = {}
+ def capture(**kw):
+ captured['body'] = kw['http_response'].content
+ event = 'after-call.s3vectors.CreateIndex'
+ conn.meta.events.register(event, capture)
+ try:
+ with pytest.raises(conn.exceptions.ClientError) as exc_info:
+ conn.create_index(**kwargs)
+ assert exc_info.value.response['Error']['Code'] == 'ValidationException'
+ body = json.loads(captured['body'])
+ assert 'fieldList' in body, f"response should contain fieldList"
+ actual_paths = [entry['path'] for entry in body['fieldList']]
+ assert actual_paths == expected_paths, \
+ f"expected fieldList paths {expected_paths} but got {actual_paths}"
+ finally:
+ conn.meta.events.unregister(event, capture)
+
+
def generate_data(dimension, index=0):
return {'float32': [random.gauss(float(index), 1.0) for _ in range(dimension)]}
-def generate_vectors(num_vectors, dimension):
+def generate_vectors(num_vectors, dimension, with_metadata=False):
vectors = []
for i in range(num_vectors):
- vectors.append({
+ v = {
'key': 'vec-' + str(i),
'data': generate_data(dimension, i)
- })
+ }
+ if with_metadata:
+ v['metadata'] = json.dumps({'genre': f'genre-{i}', 'year': 2000 + i})
+ vectors.append(v)
return vectors
# generate vectors with wrong dimension
wrong_dimension = 64
vectors = generate_vectors(10, wrong_dimension)
- # all vectors have wrong dimension, so put should fail
- pytest.raises(conn.exceptions.ClientError, conn.put_vectors,
- vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+ # all vectors have wrong dimension, bail on first error
+ assert_put_vectors_validation_error(conn,
+ 'vectors[0].data',
+ vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
# verify no vectors were inserted
result = conn.list_vectors(vectorBucketName=bucket_name, indexName=index_name, maxResults=100)
assert result['ResponseMetadata']['HTTPStatusCode'] == 200
assert len(result.get('vectors', [])) == 0
- # mix of correct and wrong dimension vectors - only correct ones should be inserted
+ # mix of correct and wrong dimension vectors - bail on first wrong one
correct_vectors = generate_vectors(5, dimension)
wrong_vectors = generate_vectors(5, wrong_dimension)
- # rename wrong vectors keys to avoid collisions
for i, v in enumerate(wrong_vectors):
v['key'] = f'wrong-{i}'
mixed_vectors = correct_vectors + wrong_vectors
- result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=mixed_vectors)
- assert result['ResponseMetadata']['HTTPStatusCode'] == 200
- # verify only the correct dimension vectors were inserted
+ assert_put_vectors_validation_error(conn,
+ 'vectors[5].data',
+ vectorBucketName=bucket_name, indexName=index_name, vectors=mixed_vectors)
+ # verify no vectors were inserted (all-or-nothing)
result = conn.list_vectors(vectorBucketName=bucket_name, indexName=index_name, maxResults=100)
assert result['ResponseMetadata']['HTTPStatusCode'] == 200
- inserted_keys = [v['key'] for v in result.get('vectors', [])]
- assert len(inserted_keys) == 5, f"expected 5 vectors but got {len(inserted_keys)}"
- for i in range(5):
- assert f'vec-{i}' in inserted_keys
- for i in range(5):
- assert f'wrong-{i}' not in inserted_keys
+ assert len(result.get('vectors', [])) == 0
# cleanup
_ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
# cleanup
_ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+@pytest.mark.vector_test
+def test_put_and_get_vectors_metadata():
+ """Test storing and retrieving a mix of vectors with and without metadata."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 8
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ result = conn.create_index(vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean')
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # create a mix: some vectors with metadata, some without
+ vectors_with_md = generate_vectors(3, dimension, with_metadata=True)
+ vectors_without_md = generate_vectors(3, dimension, with_metadata=False)
+ # rename keys to avoid collisions
+ for i, v in enumerate(vectors_without_md):
+ v['key'] = f'no-md-{i}'
+ all_vectors = vectors_with_md + vectors_without_md
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=all_vectors)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ all_keys = [v['key'] for v in all_vectors]
+
+ # get all vectors with returnMetadata=True
+ result = conn.get_vectors(vectorBucketName=bucket_name, indexName=index_name,
+ keys=all_keys, returnMetadata=True)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ assert len(result['vectors']) == len(all_vectors)
+ for vector in result['vectors']:
+ if vector['key'].startswith('vec-'):
+ assert 'metadata' in vector, f"vector {vector['key']} should have 'metadata' field"
+ md = json.loads(vector['metadata'])
+ assert 'genre' in md, f"metadata should have 'genre' key: {vector['metadata']}"
+ assert 'year' in md, f"metadata should have 'year' key: {vector['metadata']}"
+ else:
+ assert 'metadata' not in vector, \
+ f"vector {vector['key']} should not have metadata when none was stored"
+
+ # get all vectors with returnMetadata=False - no metadata should be returned
+ result = conn.get_vectors(vectorBucketName=bucket_name, indexName=index_name,
+ keys=all_keys, returnMetadata=False)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ for vector in result['vectors']:
+ assert 'metadata' not in vector, \
+ f"vector {vector['key']} should not have 'metadata' field when returnMetadata=False"
+
+ # get with both returnData and returnMetadata
+ result = conn.get_vectors(vectorBucketName=bucket_name, indexName=index_name,
+ keys=all_keys, returnData=True, returnMetadata=True)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ for vector in result['vectors']:
+ assert 'data' in vector, f"vector {vector['key']} should have 'data' field"
+ assert 'float32' in vector['data']
+ assert len(vector['data']['float32']) == dimension
+
+ log.info('test_put_and_get_vectors_metadata: verified metadata for %d vectors', len(all_vectors))
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.vector_test
+def test_list_vectors_with_metadata():
+ """Test that list_vectors returns metadata when returnMetadata=True."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 8
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ result = conn.create_index(vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean')
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ num_vectors = 5
+ vectors = generate_vectors(num_vectors, dimension, with_metadata=True)
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # list with returnMetadata=True
+ result = conn.list_vectors(vectorBucketName=bucket_name, indexName=index_name,
+ maxResults=100, returnMetadata=True)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ assert len(result['vectors']) == num_vectors
+ for vector in result['vectors']:
+ assert 'metadata' in vector, f"vector {vector['key']} should have 'metadata' field"
+ md = json.loads(vector['metadata'])
+ assert 'genre' in md, f"metadata should have 'genre' key: {vector['metadata']}"
+ assert 'year' in md, f"metadata should have 'year' key: {vector['metadata']}"
+
+ # list with returnMetadata=False
+ result = conn.list_vectors(vectorBucketName=bucket_name, indexName=index_name,
+ maxResults=100, returnMetadata=False)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ for vector in result['vectors']:
+ assert 'metadata' not in vector, \
+ f"vector {vector['key']} should not have 'metadata' field when returnMetadata=False"
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.vector_test
+def test_query_vectors_with_metadata():
+ """Test that query_vectors returns metadata when returnMetadata=True."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 8
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ result = conn.create_index(vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean')
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ num_vectors = 20
+ vectors = generate_vectors(num_vectors, dimension, with_metadata=True)
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ top_k = 5
+ query_vector = generate_data(dimension, 7)
+
+ # query with returnMetadata=True
+ result = conn.query_vectors(vectorBucketName=bucket_name, indexName=index_name,
+ queryVector=query_vector, topK=top_k, returnMetadata=True)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ assert len(result['vectors']) == top_k
+ for vector in result['vectors']:
+ assert 'metadata' in vector, f"vector {vector['key']} should have 'metadata' field"
+ md = json.loads(vector['metadata'])
+ assert 'genre' in md, f"metadata should have 'genre' key: {vector['metadata']}"
+ assert 'year' in md, f"metadata should have 'year' key: {vector['metadata']}"
+
+ # query with returnMetadata=False
+ result = conn.query_vectors(vectorBucketName=bucket_name, indexName=index_name,
+ queryVector=query_vector, topK=top_k, returnMetadata=False)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ for vector in result['vectors']:
+ assert 'metadata' not in vector, \
+ f"vector {vector['key']} should not have 'metadata' field when returnMetadata=False"
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.vector_test
+def test_put_vectors_malformed_metadata():
+ """Test that vectors with malformed JSON metadata are skipped."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 8
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ result = conn.create_index(vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean')
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # all vectors have malformed metadata - bail on first error
+ bad_vectors = [
+ {
+ 'key': f'bad-{i}',
+ 'data': generate_data(dimension, i),
+ 'metadata': '{"info": {"value": "missing end quote}}'
+ }
+ for i in range(3)
+ ]
+ assert_put_vectors_validation_error(conn,
+ 'vectors[0].metadata',
+ vectorBucketName=bucket_name, indexName=index_name, vectors=bad_vectors)
+
+ # verify no vectors were inserted
+ result = conn.list_vectors(vectorBucketName=bucket_name, indexName=index_name, maxResults=100)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ assert len(result.get('vectors', [])) == 0
+
+ # mix of good and bad metadata - bail on first bad one
+ good_vectors = [
+ {
+ 'key': f'good-{i}',
+ 'data': generate_data(dimension, i),
+ 'metadata': json.dumps({'info': f'value-{i}'})
+ }
+ for i in range(3)
+ ]
+ mixed_vectors = good_vectors + bad_vectors
+ assert_put_vectors_validation_error(conn,
+ 'vectors[3].metadata',
+ vectorBucketName=bucket_name, indexName=index_name, vectors=mixed_vectors)
+
+ # verify no vectors were inserted (all-or-nothing)
+ result = conn.list_vectors(vectorBucketName=bucket_name, indexName=index_name, maxResults=100)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ assert len(result.get('vectors', [])) == 0
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.vector_test
+def test_put_vectors_null_metadata_value():
+ """Test that vectors with null metadata values are rejected."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 4
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ result = conn.create_index(vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean')
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # null value in metadata field
+ vectors = [
+ {'key': 'v0', 'data': generate_data(dimension, 0),
+ 'metadata': '{"color": null}'},
+ ]
+ assert_put_vectors_validation_error(conn,
+ 'vectors[0].metadata.color',
+ vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+
+ # null among valid fields
+ vectors = [
+ {'key': 'v0', 'data': generate_data(dimension, 0),
+ 'metadata': '{"genre": "rock", "year": null}'},
+ ]
+ assert_put_vectors_validation_error(conn,
+ 'vectors[0].metadata.year',
+ vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+
+ # valid vector followed by null vector - all-or-nothing
+ vectors = [
+ {'key': 'v0', 'data': generate_data(dimension, 0),
+ 'metadata': json.dumps({'color': 'red'})},
+ {'key': 'v1', 'data': generate_data(dimension, 1),
+ 'metadata': '{"color": null}'},
+ ]
+ assert_put_vectors_validation_error(conn,
+ 'vectors[1].metadata.color',
+ vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+
+ # verify no vectors were inserted
+ result = conn.list_vectors(vectorBucketName=bucket_name, indexName=index_name, maxResults=100)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ assert len(result.get('vectors', [])) == 0
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.vector_test
+def test_put_vectors_dots_in_metadata_field_names():
+ """Test that vectors with dots in metadata field names are rejected."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 4
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ result = conn.create_index(vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean')
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ vectors = [
+ {'key': 'v0', 'data': generate_data(dimension, 0),
+ 'metadata': json.dumps({'user.name': 'alice'})},
+ ]
+ assert_put_vectors_validation_error(conn,
+ 'vectors[0].metadata.user.name',
+ vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.vector_test
+def test_put_vectors_missing_filterable_fields():
+ """Test that vectors with missing filterable metadata fields are inserted with nulls."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 4
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ filterable_keys = [
+ {'name': 'genre', 'type': 'String'},
+ {'name': 'year', 'type': 'Number'},
+ {'name': 'popular', 'type': 'Boolean'}
+ ]
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean',
+ metadataConfiguration={'filterableMetadataKeys': filterable_keys})
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ vectors = [
+ {
+ 'key': 'all-fields',
+ 'data': generate_data(dimension, 0),
+ 'metadata': json.dumps({'genre': 'rock', 'year': 2000, 'popular': True})
+ },
+ {
+ 'key': 'some-fields',
+ 'data': generate_data(dimension, 1),
+ 'metadata': json.dumps({'genre': 'jazz'})
+ },
+ {
+ 'key': 'no-filterable-fields',
+ 'data': generate_data(dimension, 2),
+ 'metadata': json.dumps({'artist': 'someone'})
+ },
+ {
+ 'key': 'no-metadata',
+ 'data': generate_data(dimension, 3),
+ },
+ {
+ 'key': 'nested-field',
+ 'data': generate_data(dimension, 4),
+ 'metadata': json.dumps({'info': {'genre': 'blues', 'year': 1990}})
+ },
+ ]
+
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ all_keys = [v['key'] for v in vectors]
+ result = conn.get_vectors(vectorBucketName=bucket_name, indexName=index_name,
+ keys=all_keys, returnMetadata=True)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ assert len(result['vectors']) == 5
+
+ by_key = {v['key']: v for v in result['vectors']}
+
+ md = json.loads(by_key['all-fields']['metadata'])
+ assert md['genre'] == 'rock'
+ assert md['year'] == 2000
+ assert md['popular'] is True
+
+ md = json.loads(by_key['some-fields']['metadata'])
+ assert md['genre'] == 'jazz'
+ assert 'year' not in md
+ assert 'popular' not in md
+
+ md = json.loads(by_key['no-filterable-fields']['metadata'])
+ assert 'genre' not in md
+ assert md['artist'] == 'someone'
+ assert 'year' not in md
+ assert 'popular' not in md
+
+ assert 'metadata' not in by_key['no-metadata']
+
+ # nested fields with filterable key names should not be found at top level
+ md = json.loads(by_key['nested-field']['metadata'])
+ assert 'genre' not in md
+ assert 'year' not in md
+ assert 'popular' not in md
+ assert md['info']['genre'] == 'blues'
+ assert md['info']['year'] == 1990
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.vector_test
+def test_put_vectors_invalid_filterable_types():
+ """Test that vectors with wrong types for filterable fields fail with ValidationException.
+ Each invalid type mismatch is tested individually since PutVectors is all-or-nothing."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 4
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ filterable_keys = [
+ {'name': 'genre'},
+ {'name': 'year', 'type': 'Number'},
+ {'name': 'popular', 'type': 'Boolean'},
+ {'name': 'tags', 'type': 'StringList'},
+ {'name': 'scores', 'type': 'NumberList'},
+ ]
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean',
+ metadataConfiguration={'filterableMetadataKeys': filterable_keys})
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # correct types should succeed
+ correct_vector = [{
+ 'key': 'correct-types',
+ 'data': generate_data(dimension, 0),
+ 'metadata': json.dumps({
+ 'genre': 'rock', 'year': 2000, 'popular': True,
+ 'tags': ['a', 'b'], 'scores': [1.0, 2.0]
+ })
+ }]
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=correct_vector)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # string for number field
+ assert_put_vectors_validation_error(conn, 'vectors[0].metadata.year',
+ vectorBucketName=bucket_name, indexName=index_name, vectors=[{
+ 'key': 'string-for-number',
+ 'data': generate_data(dimension, 1),
+ 'metadata': json.dumps({'year': 'not-a-number'})
+ }])
+
+ # number for string field - should succeed (number is coerced to string)
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=[{
+ 'key': 'number-for-string',
+ 'data': generate_data(dimension, 2),
+ 'metadata': json.dumps({'genre': 12345})
+ }])
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # string for boolean field
+ assert_put_vectors_validation_error(conn, 'vectors[0].metadata.popular',
+ vectorBucketName=bucket_name, indexName=index_name, vectors=[{
+ 'key': 'string-for-boolean',
+ 'data': generate_data(dimension, 3),
+ 'metadata': json.dumps({'popular': 'yes'})
+ }])
+
+ # list for scalar field
+ assert_put_vectors_validation_error(conn, 'vectors[0].metadata.genre',
+ vectorBucketName=bucket_name, indexName=index_name, vectors=[{
+ 'key': 'list-for-scalar',
+ 'data': generate_data(dimension, 4),
+ 'metadata': json.dumps({'genre': ['rock', 'pop']})
+ }])
+
+ # scalar for list field
+ assert_put_vectors_validation_error(conn, 'vectors[0].metadata.tags',
+ vectorBucketName=bucket_name, indexName=index_name, vectors=[{
+ 'key': 'scalar-for-list',
+ 'data': generate_data(dimension, 5),
+ 'metadata': json.dumps({'tags': 'single-tag'})
+ }])
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.vector_test
+def test_put_vectors_must_exist():
+ """Test the mustExist flag on filterable metadata keys."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 4
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ filterable_keys = [
+ {'name': 'genre', 'mustExist': True},
+ {'name': 'year', 'type': 'Number', 'mustExist': True},
+ {'name': 'popular', 'type': 'Boolean'},
+ ]
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean',
+ metadataConfiguration={'filterableMetadataKeys': filterable_keys})
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # all required fields present - should succeed
+ vectors = [
+ {
+ 'key': f'v{i}',
+ 'data': generate_data(dimension, i),
+ 'metadata': json.dumps({'genre': f'genre-{i}', 'year': 2000 + i})
+ }
+ for i in range(3)
+ ]
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # verify vectors were inserted
+ result = conn.list_vectors(vectorBucketName=bucket_name, indexName=index_name, maxResults=100)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ assert len(result['vectors']) == 3
+
+ # missing required field - should fail
+ vectors_with_missing = [
+ {
+ 'key': 'ok-row',
+ 'data': generate_data(dimension, 10),
+ 'metadata': json.dumps({'genre': 'rock', 'year': 2020})
+ },
+ {
+ 'key': 'ok-row-2',
+ 'data': generate_data(dimension, 11),
+ 'metadata': json.dumps({'genre': 'jazz', 'year': 2021})
+ },
+ {
+ 'key': 'bad-row-nested-ok',
+ 'data': generate_data(dimension, 12),
+ 'metadata': json.dumps({'popular': True, 'info': {'genre': 'blues', 'year': 1990}})
+ },
+ ]
+ assert_put_vectors_validation_error(conn, 'vectors[2].metadata.genre',
+ vectorBucketName=bucket_name, indexName=index_name, vectors=vectors_with_missing)
+
+ # same scenario but with mustExist=false - should succeed
+ _ = conn.delete_index(vectorBucketName=bucket_name, indexName=index_name)
+
+ filterable_keys_nullable = [
+ {'name': 'genre', 'mustExist': False},
+ {'name': 'year', 'type': 'Number', 'mustExist': False},
+ {'name': 'popular', 'type': 'Boolean'},
+ ]
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean',
+ metadataConfiguration={'filterableMetadataKeys': filterable_keys_nullable})
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=vectors_with_missing)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # verify all vectors were inserted
+ result = conn.list_vectors(vectorBucketName=bucket_name, indexName=index_name, maxResults=100)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ assert len(result['vectors']) == 3
+
+ # vector without metadata and mustExist=true - should fail
+ _ = conn.delete_index(vectorBucketName=bucket_name, indexName=index_name)
+
+ filterable_keys_not_null = [
+ {'name': 'genre', 'mustExist': True},
+ {'name': 'popular', 'type': 'Boolean'},
+ ]
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean',
+ metadataConfiguration={'filterableMetadataKeys': filterable_keys_not_null})
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ vectors_no_metadata = [
+ {
+ 'key': 'with-metadata',
+ 'data': generate_data(dimension, 20),
+ 'metadata': json.dumps({'genre': 'rock'})
+ },
+ {
+ 'key': 'no-metadata',
+ 'data': generate_data(dimension, 21),
+ },
+ ]
+ assert_put_vectors_validation_error(conn, 'vectors[1].metadata.genre',
+ vectorBucketName=bucket_name, indexName=index_name, vectors=vectors_no_metadata)
+
+ # same but with mustExist=false - should succeed
+ _ = conn.delete_index(vectorBucketName=bucket_name, indexName=index_name)
+
+ filterable_keys_all_null = [
+ {'name': 'genre', 'mustExist': False},
+ {'name': 'popular', 'type': 'Boolean'},
+ ]
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean',
+ metadataConfiguration={'filterableMetadataKeys': filterable_keys_all_null})
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=vectors_no_metadata)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ result = conn.list_vectors(vectorBucketName=bucket_name, indexName=index_name, maxResults=100)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ assert len(result['vectors']) == 2
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.vector_test
+def test_query_vectors_filter():
+ """Test metadata filtering during vector queries."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 4
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ filterable_keys = [
+ {'name': 'genre'},
+ {'name': 'year', 'type': 'Number'},
+ {'name': 'popular', 'type': 'Boolean'},
+ ]
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean',
+ metadataConfiguration={'filterableMetadataKeys': filterable_keys})
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ vectors = [
+ {'key': 'v0', 'data': generate_data(dimension, 0),
+ 'metadata': json.dumps({'genre': 'rock', 'year': 2020, 'popular': True, 'color': 'red'})},
+ {'key': 'v1', 'data': generate_data(dimension, 1),
+ 'metadata': json.dumps({'genre': 'jazz', 'year': 2019, 'popular': False, 'color': 'blue'})},
+ {'key': 'v2', 'data': generate_data(dimension, 2),
+ 'metadata': json.dumps({'genre': 'rock', 'year': 2018, 'popular': True, 'color': 'red'})},
+ {'key': 'v3', 'data': generate_data(dimension, 3),
+ 'metadata': json.dumps({'genre': 'pop', 'year': 2021, 'popular': False, 'color': 'green'})},
+ {'key': 'v4', 'data': generate_data(dimension, 4),
+ 'metadata': json.dumps({'genre': 'jazz', 'year': 2020, 'popular': True, 'color': 'red'})},
+ ]
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ query_vector = generate_data(dimension, 0)
+ top_k = 10
+
+ def query_keys(filter_expr):
+ result = conn.query_vectors(
+ vectorBucketName=bucket_name, indexName=index_name,
+ queryVector=query_vector, topK=top_k, filter=filter_expr)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ return sorted([v['key'] for v in result['vectors']])
+
+ # implicit equality
+ assert query_keys({'genre': 'rock'}) == ['v0', 'v2']
+
+ # explicit $eq
+ assert query_keys({'genre': {'$eq': 'rock'}}) == ['v0', 'v2']
+
+ # $ne
+ assert query_keys({'genre': {'$ne': 'rock'}}) == ['v1', 'v3', 'v4']
+
+ # $gt
+ assert query_keys({'year': {'$gt': 2019}}) == ['v0', 'v3', 'v4']
+
+ # range: $gte + $lte
+ assert query_keys({'year': {'$gte': 2019, '$lte': 2020}}) == ['v0', 'v1', 'v4']
+
+ # $in
+ assert query_keys({'genre': {'$in': ['rock', 'jazz']}}) == ['v0', 'v1', 'v2', 'v4']
+
+ # $nin
+ assert query_keys({'genre': {'$nin': ['rock']}}) == ['v1', 'v3', 'v4']
+
+ # $exists
+ assert query_keys({'genre': {'$exists': True}}) == ['v0', 'v1', 'v2', 'v3', 'v4']
+
+ # boolean filter
+ assert query_keys({'popular': True}) == ['v0', 'v2', 'v4']
+
+ # $and
+ assert query_keys({'$and': [{'genre': 'rock'}, {'year': {'$gt': 2019}}]}) == ['v0']
+
+ # $or
+ assert query_keys({'$or': [{'genre': 'rock'}, {'genre': 'jazz'}]}) == ['v0', 'v1', 'v2', 'v4']
+
+ # implicit AND (multiple top-level fields)
+ assert query_keys({'genre': 'jazz', 'popular': True}) == ['v4']
+
+ # mixed $and: column filter (genre) + JSON metadata filter (color)
+ assert query_keys({'$and': [{'genre': 'rock'}, {'color': 'red'}]}) == ['v0', 'v2']
+ assert query_keys({'$and': [{'genre': 'jazz'}, {'color': 'red'}]}) == ['v4']
+
+ # implicit AND with mixed column + JSON fields
+ assert query_keys({'genre': 'rock', 'color': 'red'}) == ['v0', 'v2']
+
+ # nested $and: both inner $ands mix column and JSON fields
+ assert query_keys({'$and': [
+ {'$and': [{'genre': 'rock'}, {'color': 'red'}]},
+ {'$and': [{'year': {'$gt': 2019}}, {'color': 'red'}]}
+ ]}) == ['v0']
+
+ # nested $or inside $and: each $or is homogeneous (column-only or JSON-only)
+ assert query_keys({'$and': [
+ {'$or': [{'genre': 'rock'}, {'genre': 'jazz'}]},
+ {'color': 'red'}
+ ]}) == ['v0', 'v2', 'v4']
+
+ # mixed $or: column + JSON fields should be rejected
+ assert_query_vectors_validation_error(
+ conn, 'filter',
+ vectorBucketName=bucket_name, indexName=index_name,
+ queryVector=query_vector, topK=top_k,
+ filter={'$or': [{'genre': 'rock'}, {'color': 'red'}]})
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.vector_test
+def test_query_vectors_post_filtering():
+ """Test that postFiltering forces all filtering through JSON post-filtering."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 4
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ filterable_keys = [
+ {'name': 'genre'},
+ {'name': 'year', 'type': 'Number'},
+ {'name': 'popular', 'type': 'Boolean'},
+ ]
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean',
+ metadataConfiguration={'filterableMetadataKeys': filterable_keys})
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ # v0 (index=0) and v3 (index=10) are rock; v1 and v2 are jazz.
+ # query is centered at index 0, so v0 is nearest and v3 is far away.
+ # with topK=2: pre-filtering on genre=rock searches only rock vectors
+ # and returns both (v0, v3). post-filtering fetches the 2 nearest
+ # overall (v0, v1), then filters to rock, returning only v0.
+ vectors = [
+ {'key': 'v0', 'data': generate_data(dimension, 0),
+ 'metadata': json.dumps({'genre': 'rock', 'year': 2020, 'popular': True, 'color': 'red'})},
+ {'key': 'v1', 'data': generate_data(dimension, 1),
+ 'metadata': json.dumps({'genre': 'jazz', 'year': 2019, 'popular': False, 'color': 'blue'})},
+ {'key': 'v2', 'data': generate_data(dimension, 2),
+ 'metadata': json.dumps({'genre': 'jazz', 'year': 2018, 'popular': True, 'color': 'green'})},
+ {'key': 'v3', 'data': generate_data(dimension, 10),
+ 'metadata': json.dumps({'genre': 'rock', 'year': 2021, 'popular': False, 'color': 'red'})},
+ ]
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ query_vector = generate_data(dimension, 0)
+
+ def query_keys(filter_expr, top_k, post_filtering=False):
+ kwargs = dict(vectorBucketName=bucket_name, indexName=index_name,
+ queryVector=query_vector, topK=top_k, filter=filter_expr)
+ if post_filtering:
+ kwargs['postFiltering'] = True
+ result = conn.query_vectors(**kwargs)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ return sorted([v['key'] for v in result['vectors']])
+
+ # pre-filtering on genre=rock with topK=2 returns both rock vectors
+ assert query_keys({'genre': 'rock'}, top_k=2) == ['v0', 'v3']
+ # post-filtering with topK=2 only sees the 2 nearest (v0, v1), so v3 is excluded
+ assert query_keys({'genre': 'rock'}, top_k=2, post_filtering=True) == ['v0']
+
+ # post-filtering allows mixed $or (column + JSON fields)
+ assert query_keys({'$or': [{'genre': 'rock'}, {'color': 'blue'}]}, top_k=10, post_filtering=True) == ['v0', 'v1', 'v3']
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.vector_test
+def test_query_vectors_filter_nonfilterable():
+ """Test that filtering on non-filterable keys is rejected."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 4
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean',
+ metadataConfiguration={'nonFilterableMetadataKeys': ['secret']})
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ vectors = [
+ {'key': 'v0', 'data': generate_data(dimension, 0),
+ 'metadata': json.dumps({'secret': 'hidden'})},
+ ]
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ query_vector = generate_data(dimension, 0)
+ with pytest.raises(conn.exceptions.ClientError) as exc_info:
+ conn.query_vectors(
+ vectorBucketName=bucket_name, indexName=index_name,
+ queryVector=query_vector, topK=5, filter={'secret': 'hidden'})
+ assert exc_info.value.response['ResponseMetadata']['HTTPStatusCode'] == 400
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.vector_test
+def test_query_vectors_filter_json_metadata():
+ """Test filtering on undeclared metadata fields using JSON extraction."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 4
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean')
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ vectors = [
+ {'key': 'v0', 'data': generate_data(dimension, 0),
+ 'metadata': json.dumps({'color': 'red', 'priority': 10, 'active': True})},
+ {'key': 'v1', 'data': generate_data(dimension, 1),
+ 'metadata': json.dumps({'color': 'blue', 'priority': 3, 'active': False})},
+ {'key': 'v2', 'data': generate_data(dimension, 2),
+ 'metadata': json.dumps({'color': 'red', 'priority': 7, 'active': True})},
+ ]
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ query_vector = generate_data(dimension, 0)
+ top_k = 10
+
+ def query_keys(filter_expr):
+ result = conn.query_vectors(
+ vectorBucketName=bucket_name, indexName=index_name,
+ queryVector=query_vector, topK=top_k, filter=filter_expr)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ return sorted([v['key'] for v in result['vectors']])
+
+ # string field
+ assert query_keys({'color': 'red'}) == ['v0', 'v2']
+
+ # number field comparison
+ assert query_keys({'priority': {'$gt': 5}}) == ['v0', 'v2']
+
+ # boolean field
+ assert query_keys({'active': True}) == ['v0', 'v2']
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.vector_test
+def test_query_vectors_filter_errors():
+ """Test that invalid filter expressions are rejected with 400."""
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 4
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ filterable_keys = [
+ {'name': 'genre'},
+ {'name': 'year', 'type': 'Number'},
+ {'name': 'popular', 'type': 'Boolean'},
+ ]
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean',
+ metadataConfiguration={'filterableMetadataKeys': filterable_keys})
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ vectors = [
+ {'key': 'v0', 'data': generate_data(dimension, 0),
+ 'metadata': json.dumps({'genre': 'rock', 'year': 2020, 'popular': True})},
+ ]
+ result = conn.put_vectors(vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ query_vector = generate_data(dimension, 0)
+ query_args = dict(vectorBucketName=bucket_name, indexName=index_name,
+ queryVector=query_vector, topK=5)
+
+ def expect_error(filter_expr):
+ assert_query_vectors_validation_error(
+ conn, 'filter', filter=filter_expr, **query_args)
+
+ # unknown operator
+ expect_error({'genre': {'$regex': 'r.*'}})
+
+ # invalid boolean value for boolean column
+ expect_error({'popular': {'$eq': 'yes'}})
+
+ # invalid number value for number column
+ expect_error({'year': {'$eq': 'not_a_number'}})
+
+ # empty $in list
+ expect_error({'genre': {'$in': []}})
+
+ # mixed types in $in list (JSON field)
+ expect_error({'color': {'$in': ['red', 42]}})
+
+ # mixed $or: column + JSON fields
+ expect_error({'$or': [{'genre': 'rock'}, {'color': 'red'}]})
+
+ # nested mixed $or via $and
+ expect_error({'$or': [{'genre': 'rock'}, {'$and': [{'genre': 'jazz'}, {'color': 'blue'}]}]})
+
+ # mixed $or nested inside $and
+ expect_error({'$and': [{'$or': [{'genre': 'rock'}, {'color': 'red'}]}, {'popular': True}]})
+
+ # object value in $eq (JSON field)
+ expect_error({'color': {'$eq': {'nested': 'value'}}})
+
+ # implicit $eq with an array value (JSON field)
+ expect_error({'color': ['red', 'blue']})
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+
+
+@pytest.mark.vector_test
+def test_query_vectors_post_filter_topk():
+ """ Test topK oversampling with JSON post-filtering """
+
+ set_rgw_config_option('rgw_s3vector_topk_post_filter_factor', 1.5)
+ conn = connection()
+ bucket_name = gen_bucket_name()
+ dimension = 4
+ result = conn.create_vector_bucket(vectorBucketName=bucket_name)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ index_name = 'test-index'
+ result = conn.create_index(
+ vectorBucketName=bucket_name, indexName=index_name,
+ dataType='float32', dimension=dimension, distanceMetric='euclidean')
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ vectors = []
+ for i in range(7):
+ vectors.append({
+ 'key': f'red-{i}',
+ 'data': generate_data(dimension, i),
+ 'metadata': json.dumps({'color': 'red'}),
+ })
+ for i in range(13):
+ vectors.append({
+ 'key': f'blue-{i}',
+ 'data': generate_data(dimension, 100 + i),
+ 'metadata': json.dumps({'color': 'blue'}),
+ })
+ result = conn.put_vectors(
+ vectorBucketName=bucket_name, indexName=index_name, vectors=vectors)
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+
+ top_k = 9
+
+ # test 1: fewer than k matches after filtering — return all matches
+ # query near red vectors (index 0), so all 7 red are in the top results
+ result = conn.query_vectors(
+ vectorBucketName=bucket_name, indexName=index_name,
+ queryVector=generate_data(dimension, 0), topK=top_k, filter={'color': 'red'})
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ assert len(result['vectors']) == 7
+
+ # test 2: more than k matches after filtering — return exactly k
+ # query near blue vectors (index 106), so all 13 blue are in the top results
+ # do not request returnDistance
+ result = conn.query_vectors(
+ vectorBucketName=bucket_name, indexName=index_name,
+ queryVector=generate_data(dimension, 106), topK=top_k, filter={'color': 'blue'})
+ assert result['ResponseMetadata']['HTTPStatusCode'] == 200
+ assert len(result['vectors']) == top_k
+ # verify distance is not in the response
+ for v in result['vectors']:
+ log.info(v)
+ assert 'distance' not in v, "distance should not be in response when not requested"
+
+ # cleanup
+ _ = conn.delete_vector_bucket(vectorBucketName=bucket_name)
+ set_rgw_config_option('rgw_s3vector_topk_post_filter_factor', 1)
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
+// vim: ts=8 sw=2 sts=2 expandtab ft=cpp
+
+#include "gtest/gtest.h"
+#include "rgw_s3vector_filter.h"
+#include "common/ceph_json.h"
+#include "common/dout.h"
+#include "lancedb.h"
+#include "global/global_init.h"
+#include "common/ceph_argparse.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace rgw::s3vector;
+
+class S3VectorFilterTest : public ::testing::Test {
+protected:
+ NoDoutPrefix no_dpp{g_ceph_context, dout_subsys};
+ DoutPrefixProvider* dpp = &no_dpp;
+ std::vector<rgw::s3vector::validation_error_t> errors;
+
+ // parse a JSON string into a JSONParser and call build_filter_expr
+ std::optional<FilterExprs> build(
+ const std::string& json,
+ const std::vector<filterable_metadata_key_t>& filterable_keys = {},
+ const std::vector<std::string>& nonfilterable_keys = {}) {
+ errors.clear();
+ JSONParser parser;
+ EXPECT_TRUE(parser.parse(json.c_str(), json.size()));
+ return build_filter_expr(parser, filterable_keys, nonfilterable_keys, dpp, errors);
+ }
+
+ void free_exprs(FilterExprs& fe) {
+ lancedb_expr_free(fe.column_expr);
+ lancedb_expr_free(fe.json_expr);
+ }
+};
+
+// ---- implicit $eq ----
+
+TEST_F(S3VectorFilterTest, ImplicitEqOnColumn) {
+ std::vector<filterable_metadata_key_t> keys = {{"genre", FilterableMetadataType::STRING, false}};
+ auto result = build(R"({"genre": "rock"})", keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_EQ(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, ImplicitEqOnJson) {
+ auto result = build(R"({"color": "red"})");
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+// ---- explicit operators on columns ----
+
+TEST_F(S3VectorFilterTest, ExplicitEqOnColumn) {
+ std::vector<filterable_metadata_key_t> keys = {{"genre", FilterableMetadataType::STRING, false}};
+ auto result = build(R"({"genre": {"$eq": "rock"}})", keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_EQ(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, NumericRangeOnColumn) {
+ std::vector<filterable_metadata_key_t> keys = {{"year", FilterableMetadataType::NUMBER, false}};
+ auto result = build(R"({"year": {"$gt": 2019, "$lt": 2026}})", keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_EQ(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, BooleanEqOnColumn) {
+ std::vector<filterable_metadata_key_t> keys = {{"active", FilterableMetadataType::BOOLEAN, false}};
+ auto result = build(R"({"active": {"$eq": true}})", keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_EQ(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+// ---- explicit operators on JSON metadata ----
+
+TEST_F(S3VectorFilterTest, ExplicitEqOnJson) {
+ auto result = build(R"({"color": {"$eq": "red"}})");
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, NumericRangeOnJson) {
+ auto result = build(R"({"score": {"$gte": 0.5, "$lte": 1.0}})");
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, NotEqOnJson) {
+ auto result = build(R"({"color": {"$ne": "blue"}})");
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+// ---- $exists ----
+
+TEST_F(S3VectorFilterTest, ExistsOnNullableColumn) {
+ std::vector<filterable_metadata_key_t> keys = {{"genre", FilterableMetadataType::STRING, false}};
+ auto result = build(R"({"genre": {"$exists": true}})", keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_EQ(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, ExistsTrueOnNonNullableColumn) {
+ std::vector<filterable_metadata_key_t> keys = {{"year", FilterableMetadataType::NUMBER, false}};
+ auto result = build(R"({"year": {"$exists": true}})", keys);
+ ASSERT_TRUE(result.has_value());
+ // for non-nullable column, $exists returns a constant boolean expression
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_EQ(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, ExistsFalseOnNonNullableColumn) {
+ std::vector<filterable_metadata_key_t> keys = {{"year", FilterableMetadataType::NUMBER, false}};
+ auto result = build(R"({"year": {"$exists": false}})", keys);
+ ASSERT_TRUE(result.has_value());
+ // for non-nullable column, $exists: false returns a constant false expression
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_EQ(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, ExistsOnJson) {
+ auto result = build(R"({"color": {"$exists": true}})");
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, NotExistsOnJson) {
+ auto result = build(R"({"color": {"$exists": false}})");
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, ExistsWithQuotedTrueRejected) {
+ auto result = build(R"({"color": {"$exists": "true"}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, ExistsWithNumberRejected) {
+ auto result = build(R"({"color": {"$exists": 1}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, ExistsWithEmptyValueRejected) {
+ auto result = build(R"({"color": {"$exists": ""}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+// ---- $in / $nin ----
+
+TEST_F(S3VectorFilterTest, InOnColumn) {
+ std::vector<filterable_metadata_key_t> keys = {{"genre", FilterableMetadataType::STRING, false}};
+ auto result = build(R"({"genre": {"$in": ["rock", "jazz"]}})", keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_EQ(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, NinOnColumn) {
+ std::vector<filterable_metadata_key_t> keys = {{"genre", FilterableMetadataType::STRING, false}};
+ auto result = build(R"({"genre": {"$nin": ["pop"]}})", keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_EQ(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, InOnJson) {
+ auto result = build(R"({"color": {"$in": ["red", "blue"]}})");
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, InNumericOnJson) {
+ auto result = build(R"({"score": {"$in": [1, 2, 3]}})");
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+// ---- $and ----
+
+TEST_F(S3VectorFilterTest, AndColumnsOnly) {
+ std::vector<filterable_metadata_key_t> keys = {
+ {"genre", FilterableMetadataType::STRING, false},
+ {"year", FilterableMetadataType::NUMBER, false},
+ };
+ auto result = build(R"({"$and": [{"genre": "rock"}, {"year": {"$gt": 2020}}]})", keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_EQ(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, AndJsonOnly) {
+ auto result = build(R"({"$and": [{"color": "red"}, {"size": {"$gt": 10}}]})");
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, AndMixedColumnAndJson) {
+ std::vector<filterable_metadata_key_t> keys = {{"genre", FilterableMetadataType::STRING, false}};
+ auto result = build(R"({"$and": [{"genre": "rock"}, {"color": "red"}]})", keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+// ---- implicit AND (top-level fields) ----
+
+TEST_F(S3VectorFilterTest, ImplicitAndMixedColumnAndJson) {
+ std::vector<filterable_metadata_key_t> keys = {{"genre", FilterableMetadataType::STRING, false}};
+ auto result = build(R"({"genre": "rock", "color": "red"})", keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+// ---- $or ----
+
+TEST_F(S3VectorFilterTest, OrColumnsOnly) {
+ std::vector<filterable_metadata_key_t> keys = {
+ {"genre", FilterableMetadataType::STRING, false},
+ {"year", FilterableMetadataType::NUMBER, false},
+ };
+ auto result = build(R"({"$or": [{"genre": "rock"}, {"year": {"$gt": 2020}}]})", keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_EQ(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, OrJsonOnly) {
+ auto result = build(R"({"$or": [{"color": "red"}, {"size": {"$gt": 10}}]})");
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, OrColumnsNestedInAndWithJson) {
+ // $or is column-only, combined via top-level $and with a JSON field — valid
+ std::vector<filterable_metadata_key_t> keys = {
+ {"genre", FilterableMetadataType::STRING, false},
+ {"year", FilterableMetadataType::NUMBER, false},
+ };
+ auto result = build(
+ R"({"$and": [{"$or": [{"genre": "rock"}, {"year": {"$gt": 2020}}]}, {"color": "red"}]})",
+ keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, OrJsonNestedInAndWithColumn) {
+ // $or is JSON-only, combined via top-level $and with a column field — valid
+ std::vector<filterable_metadata_key_t> keys = {{"genre", FilterableMetadataType::STRING, false}};
+ auto result = build(
+ R"({"$and": [{"$or": [{"color": "red"}, {"size": {"$gt": 10}}]}, {"genre": "rock"}]})",
+ keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, OrInsideOrAllColumns) {
+ // nested $or within $or, all column fields — valid
+ std::vector<filterable_metadata_key_t> keys = {
+ {"genre", FilterableMetadataType::STRING, false},
+ {"year", FilterableMetadataType::NUMBER, false},
+ {"active", FilterableMetadataType::BOOLEAN, false},
+ };
+ auto result = build(
+ R"({"$or": [{"genre": "rock"}, {"$or": [{"year": {"$gt": 2020}}, {"active": true}]}]})",
+ keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_EQ(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, OrInsideOrAllJson) {
+ // nested $or within $or, all JSON fields — valid
+ auto result = build(
+ R"({"$or": [{"color": "red"}, {"$or": [{"size": {"$gt": 10}}, {"weight": 5}]}]})");
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+// ---- complex expressions ----
+
+TEST_F(S3VectorFilterTest, MultipleOperatorsOnSameField) {
+ std::vector<filterable_metadata_key_t> keys = {{"year", FilterableMetadataType::NUMBER, false}};
+ auto result = build(R"({"year": {"$gte": 2000, "$lte": 2025}})", keys);
+ ASSERT_TRUE(result.has_value());
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_EQ(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, DeeplyNested) {
+ // $and [ $or [ $and [ field, field ], field ], field ]
+ // mixes column and JSON at different levels, all connected by $and at the top
+ std::vector<filterable_metadata_key_t> keys = {
+ {"genre", FilterableMetadataType::STRING, false},
+ {"year", FilterableMetadataType::NUMBER, false},
+ };
+ auto result = build(
+ R"({
+ "$and": [
+ {"$or": [
+ {"$and": [
+ {"genre": {"$in": ["rock", "jazz"]}},
+ {"genre": {"$ne": "blues"}}
+ ]},
+ {"genre": {"$exists": true}}
+ ]},
+ {"year": {"$gte": 2000, "$lte": 2025}},
+ {"color": {"$eq": "red"}}
+ ]
+ })",
+ keys);
+ ASSERT_TRUE(result.has_value());
+ // genre and year go to column_expr, color goes to json_expr
+ EXPECT_NE(result->column_expr, nullptr);
+ EXPECT_NE(result->json_expr, nullptr);
+ free_exprs(*result);
+}
+
+TEST_F(S3VectorFilterTest, AllSixComparisonOps) {
+ // verify all comparison operators produce valid expressions on JSON fields
+ for (const auto& op : {"$eq", "$ne", "$gt", "$gte", "$lt", "$lte"}) {
+ std::string json = R"({"score": {")" + std::string(op) + R"(": 42}})";
+ auto result = build(json);
+ ASSERT_TRUE(result.has_value()) << "failed for operator " << op;
+ EXPECT_NE(result->json_expr, nullptr) << "null expr for operator " << op;
+ free_exprs(*result);
+ }
+}
+
+// ---- error cases ----
+
+TEST_F(S3VectorFilterTest, NonfilterableKeyRejected) {
+ std::vector<std::string> nonfilterable = {"secret"};
+ auto result = build(R"({"secret": "value"})", {}, nonfilterable);
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, MixedOrRejected) {
+ std::vector<filterable_metadata_key_t> keys = {{"genre", FilterableMetadataType::STRING, false}};
+ auto result = build(R"({"$or": [{"genre": "rock"}, {"color": "red"}]})", keys);
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, MixedOrNestedViaAndRejected) {
+ // $or child is a $and that returns both column_expr and json_expr — makes $or mixed
+ std::vector<filterable_metadata_key_t> keys = {{"genre", FilterableMetadataType::STRING, false}};
+ auto result = build(
+ R"({"$or": [{"genre": "rock"}, {"$and": [{"genre": "jazz"}, {"color": "blue"}]}]})",
+ keys);
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, MixedOrNestedViaOrRejected) {
+ // outer $or has one column child and one nested $or with JSON — mix detected across children
+ std::vector<filterable_metadata_key_t> keys = {{"genre", FilterableMetadataType::STRING, false}};
+ auto result = build(
+ R"({"$or": [{"genre": "rock"}, {"$or": [{"color": "red"}, {"color": "blue"}]}]})",
+ keys);
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, MixedOrDeeplyNestedRejected) {
+ // mix appears 3 levels deep: $and -> $or -> $and produces mixed FilterExprs for the $or
+ std::vector<filterable_metadata_key_t> keys = {{"genre", FilterableMetadataType::STRING, false}};
+ auto result = build(
+ R"({"$and": [{"$or": [{"genre": "rock"}, {"$and": [{"color": "red"}, {"genre": "jazz"}]}]}]})",
+ keys);
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, UnknownOperatorRejected) {
+ auto result = build(R"({"color": {"$regex": "r.*"}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, EmptyOrArrayRejected) {
+ auto result = build(R"({"$or": []})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, EmptyAndArrayRejected) {
+ auto result = build(R"({"$and": []})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, InvalidBooleanValueRejected) {
+ std::vector<filterable_metadata_key_t> keys = {{"active", FilterableMetadataType::BOOLEAN, false}};
+ auto result = build(R"({"active": {"$eq": "yes"}})", keys);
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, InvalidNumberValueRejected) {
+ std::vector<filterable_metadata_key_t> keys = {{"year", FilterableMetadataType::NUMBER, false}};
+ auto result = build(R"({"year": {"$eq": "not_a_number"}})", keys);
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, OrWithObjectValueRejected) {
+ auto result = build(R"({"$or": {"$eq": {"field": "value"}}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, OrWithScalarValueRejected) {
+ auto result = build(R"({"$or": "value"})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, OrWithScalarArrayRejected) {
+ auto result = build(R"({"$or": [1, 2, 3]})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, AndWithObjectValueRejected) {
+ auto result = build(R"({"$and": {"genre": "rock"}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, AndWithScalarValueRejected) {
+ auto result = build(R"({"$and": 42})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, InWithObjectValueRejected) {
+ auto result = build(R"({"color": {"$in": {"field": "value"}}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, InWithScalarValueRejected) {
+ auto result = build(R"({"color": {"$in": "red"}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, NinWithObjectValueRejected) {
+ auto result = build(R"({"color": {"$nin": {"field": "value"}}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, EmptyInListRejected) {
+ auto result = build(R"({"color": {"$in": []}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, MixedTypesInListRejected) {
+ auto result = build(R"({"color": {"$in": ["red", 42]}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, NullImplicitEqRejected) {
+ auto result = build(R"({"color": null})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, NullExplicitEqRejected) {
+ auto result = build(R"({"color": {"$eq": null}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, NullNeRejected) {
+ auto result = build(R"({"color": {"$ne": null}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, NullInListElementRejected) {
+ auto result = build(R"({"color": {"$in": [null, "red"]}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, NullGtRejected) {
+ auto result = build(R"({"score": {"$gt": null}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, ListTypeFilteringRejected) {
+ std::vector<filterable_metadata_key_t> keys = {{"tags", FilterableMetadataType::STRING_LIST, false}};
+ auto result = build(R"({"tags": {"$eq": "foo"}})", keys);
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, ArrayValueInJsonEqRejected) {
+ // array value in $eq on a JSON field — not supported yet
+ auto result = build(R"({"tags": {"$eq": ["a", "b"]}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, ObjectValueInJsonEqRejected) {
+ // object value rejected by infer_value_type
+ auto result = build(R"({"color": {"$eq": {"nested": "value"}}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, ObjectValueInJsonNeRejected) {
+ auto result = build(R"({"color": {"$ne": {"nested": "value"}}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+TEST_F(S3VectorFilterTest, ObjectValueInJsonInListRejected) {
+ auto result = build(R"({"color": {"$in": [{"nested": "value"}]}})");
+ EXPECT_FALSE(result.has_value());
+ EXPECT_FALSE(errors.empty());
+}
+
+int main(int argc, char** argv) {
+ auto args = argv_to_vec(argc, argv);
+ auto cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT,
+ CODE_ENVIRONMENT_UTILITY,
+ CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+ common_init_finish(g_ceph_context);
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
projects / ceph.git / commitdiff