fix: nullable scalar field filter leaks null documents without inverted index (#410)
When a nullable scalar field has no inverted index, the forward filter path fails to handle null values from Arrow's filter evaluation: 1. get_forward_bit(): BooleanArray::operator[] returns nullopt for null entries, which is_filtered() treats as "no filter" (not filtered), letting null docs through. Fix: use value_or(false) to treat null as "not matched". 2. is_matched_by_forward_filter(): reads BooleanScalar.value without checking is_valid, which is UB for null scalars. Fix: check is_valid first. Closes #409 Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
3e02031eb8
commit
a9008b0fd2
@@ -98,7 +98,7 @@ std::optional<bool> DocFilter::get_forward_bit(uint64_t id) const {
|
||||
const auto &arr = forward_bitmap_->chunk(c);
|
||||
if (id < rows_seen + arr->length()) {
|
||||
auto *bool_array = static_cast<arrow::BooleanArray *>(arr.get());
|
||||
return (*bool_array)[id - rows_seen];
|
||||
return (*bool_array)[id - rows_seen].value_or(false);
|
||||
}
|
||||
rows_seen += arr->length();
|
||||
}
|
||||
@@ -155,7 +155,8 @@ std::optional<bool> DocFilter::is_matched_by_forward_filter(uint64_t id) const {
|
||||
}
|
||||
arrow::Datum datum = maybe_result.MoveValueUnsafe();
|
||||
if (datum.is_scalar()) {
|
||||
return datum.scalar_as<arrow::BooleanScalar>().value;
|
||||
const auto &scalar = datum.scalar_as<arrow::BooleanScalar>();
|
||||
return scalar.is_valid && scalar.value;
|
||||
}
|
||||
LOG_ERROR("Datum is not scalar, id[%zu] type[%s]", (size_t)id,
|
||||
datum.type()->ToString().c_str());
|
||||
|
||||
Reference in New Issue
Block a user