feat(fts): add stemmer token filter based on Snowball 3.1.1 (#513)

This commit is contained in:
egolearner
2026-07-07 14:14:13 +08:00
committed by GitHub
parent 659951b984
commit c35d24e215
19 changed files with 703 additions and 21 deletions
+52 -6
View File
@@ -261,9 +261,30 @@ Attributes:
"whitespace").
Default is "standard".
filters (list[str]): List of token filter names applied after tokenization.
Supported filters are "lowercase" and "ascii_folding". Default is
["lowercase"].
extra_params (str): Additional parameters passed to the tokenizer.
Supported values include "lowercase", "ascii_folding", and "stemmer".
Default is ["lowercase"].
extra_params (str): Additional tokenizer/filter parameters as an empty
string or JSON object string. Supported keys are grouped by component:
Tokenizers:
standard:
- "max_token_length" (positive integer).
jieba:
- "jieba_dict_dir" (directory containing jieba.dict.utf8 and
hmm_model.utf8).
- "user_dict_path" (user dictionary path).
- "cut_mode" ("search", "mix", "full", or "hmm"; default
"search").
whitespace:
- no extra_params.
Filters:
lowercase:
- no extra_params.
ascii_folding:
- no extra_params.
stemmer:
- "stemmer_lang" (Snowball language/algorithm; default
"english"), for example {"stemmer_lang":"porter"} for ES
behaviour.
Default is "".
Examples:
@@ -272,6 +293,11 @@ Examples:
... )
>>> print(params.tokenizer_name)
jieba
>>> params = FtsIndexParam(
... tokenizer_name="standard",
... filters=["lowercase", "stemmer"],
... extra_params='{"stemmer_lang":"porter"}',
... )
)pbdoc");
fts_index_params
.def(py::init<std::string, std::vector<std::string>, std::string>(),
@@ -283,9 +309,29 @@ Constructs an FtsIndexParam instance.
Args:
tokenizer_name (str, optional): Tokenizer name. Defaults to "standard".
filters (list[str], optional): Token filter names. Supports "lowercase" and
"ascii_folding". Defaults to ["lowercase"].
extra_params (str, optional): Extra tokenizer parameters. Defaults to "".
filters (list[str], optional): Token filter names. Supports "lowercase",
"ascii_folding", and "stemmer". Defaults to ["lowercase"].
extra_params (str, optional): Extra tokenizer/filter parameters as an empty
string or JSON object string. Supported keys:
Tokenizers:
standard:
- "max_token_length" (positive integer).
jieba:
- "jieba_dict_dir".
- "user_dict_path".
- "cut_mode" ("search", "mix", "full", or "hmm"; default
"search").
whitespace:
- no extra_params.
Filters:
lowercase:
- no extra_params.
ascii_folding:
- no extra_params.
stemmer:
- "stemmer_lang" (Snowball language/algorithm; default
"english").
Defaults to "".
)pbdoc")
.def_property_readonly("tokenizer_name", &FtsIndexParams::tokenizer_name,
"str: Name of the tokenizer.")
+1
View File
@@ -45,6 +45,7 @@ cc_library(
libprotobuf
FastPFOR
cppjieba
snowball
Arrow::arrow_static
Arrow::parquet_static
Arrow::arrow_compute
+1
View File
@@ -29,6 +29,7 @@ cc_library(
Arrow::arrow_compute
Arrow::arrow_dataset
cppjieba
snowball
FastPFOR
utf8proc
INCS . ${PROJECT_ROOT_DIR}/src
@@ -0,0 +1,80 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "stemmer_token_filter.h"
#include <unordered_map>
#include <zvec/ailego/logger/logger.h>
extern "C" {
#include <libstemmer.h>
}
namespace zvec::fts {
struct ThreadLocalStemmerCache {
std::unordered_map<std::string, struct sb_stemmer *> stemmers;
~ThreadLocalStemmerCache() {
for (auto &[_, s] : stemmers) {
sb_stemmer_delete(s);
}
}
struct sb_stemmer *get(const std::string &lang) {
auto it = stemmers.find(lang);
if (it != stemmers.end()) {
return it->second;
}
auto *s = sb_stemmer_new(lang.c_str(), nullptr);
if (s) {
stemmers[lang] = s;
}
return s;
}
};
bool StemmerTokenFilter::init(const ailego::JsonObject &config) {
std::string lang;
if (config.get("stemmer_lang", &lang) && !lang.empty()) {
language_ = lang;
}
auto *test_stemmer = sb_stemmer_new(language_.c_str(), nullptr);
if (!test_stemmer) {
LOG_ERROR("[StemmerTokenFilter] failed to create stemmer for language: %s",
language_.c_str());
return false;
}
sb_stemmer_delete(test_stemmer);
return true;
}
std::vector<Token> StemmerTokenFilter::filter(std::vector<Token> tokens) const {
static thread_local ThreadLocalStemmerCache tls_cache;
auto *stemmer = tls_cache.get(language_);
if (!stemmer) {
return tokens;
}
for (auto &token : tokens) {
const auto *result = sb_stemmer_stem(
stemmer, reinterpret_cast<const unsigned char *>(token.text.data()),
static_cast<int>(token.text.size()));
if (result) {
int len = sb_stemmer_length(stemmer);
token.text.assign(reinterpret_cast<const char *>(result), len);
}
}
return tokens;
}
} // namespace zvec::fts
@@ -0,0 +1,46 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <string>
#include <vector>
#include "token_filter.h"
namespace zvec::fts {
/*! Snowball stemmer token filter.
* Configure the language with extra_params key "stemmer_lang". When omitted,
* the language is "english".
*/
class StemmerTokenFilter : public TokenFilter {
public:
StemmerTokenFilter() = default;
~StemmerTokenFilter() override = default;
StemmerTokenFilter(const StemmerTokenFilter &) = delete;
StemmerTokenFilter &operator=(const StemmerTokenFilter &) = delete;
bool init(const ailego::JsonObject &config) override;
std::vector<Token> filter(std::vector<Token> tokens) const override;
const char *name() const override {
return "stemmer";
}
private:
std::string language_{"english"};
};
} // namespace zvec::fts
@@ -17,6 +17,7 @@
#include <memory>
#include <string>
#include <vector>
#include <zvec/ailego/encoding/json/mod_json_plus.h>
#include "tokenizer.h"
namespace zvec::fts {
@@ -29,6 +30,15 @@ class TokenFilter {
public:
virtual ~TokenFilter() = default;
/*! Initialise the filter from a JSON configuration object.
* Must be called once before filter().
* \param config JSON object containing filter-specific parameters.
* \return true on success, false on failure.
*/
virtual bool init(const ailego::JsonObject & /*config*/) {
return true;
}
/*! Filter/transform a list of tokens.
* \param tokens input token list (may be modified in place)
* \return processed token list
@@ -18,6 +18,7 @@
#include "ascii_folding_token_filter.h"
#include "jieba_tokenizer.h"
#include "standard_tokenizer.h"
#include "stemmer_token_filter.h"
#include "whitespace_tokenizer.h"
namespace zvec::fts {
@@ -56,6 +57,11 @@ TokenizerPipelinePtr TokenizerFactory::create(const FtsIndexParams &params) {
filter_name.c_str());
return nullptr;
}
if (!filter->init(extra_json)) {
LOG_ERROR("[TokenizerFactory] failed to init filter: %s",
filter_name.c_str());
return nullptr;
}
filters.push_back(std::move(filter));
}
@@ -99,6 +105,10 @@ TokenFilterPtr TokenizerFactory::create_filter(const std::string &filter_name) {
return std::make_shared<LowercaseTokenFilter>();
} else if (filter_name == "ascii_folding") {
return std::make_shared<AsciiFoldingTokenFilter>();
} else if (filter_name == "stemmer") {
// The stemmer filter uses Snowball and defaults to "english" unless
// extra_params overrides stemmer_lang.
return std::make_shared<StemmerTokenFilter>();
}
LOG_ERROR("[TokenizerFactory] unknown filter name: %s", filter_name.c_str());
return nullptr;
@@ -44,15 +44,14 @@ using TokenizerPipelinePtr = std::shared_ptr<TokenizerPipeline>;
/*! Tokenizer factory
* Create TokenizerPipeline based on FtsIndexParams configuration.
* Supported tokenizers: standard, jieba, whitespace.
* Supported filters: lowercase, ascii_folding.
*/
class TokenizerFactory {
public:
/*! Create tokenizer pipeline from FtsIndexParams.
* \param params FTS index parameters containing tokenizer_name, filters,
* and extra_params (JSON string for tokenizer-specific
* configuration).
* configuration). The stemmer filter reads stemmer_lang from
* extra_params and uses Snowball English by default.
* \return Tokenizer pipeline, returns nullptr on failure
*/
static TokenizerPipelinePtr create(const FtsIndexParams &params);
+28
View File
@@ -25,6 +25,8 @@
#include "db/common/constants.h"
#include "db/common/typedef.h"
#include "db/common/utils.h"
#include "db/index/column/fts_column/fts_types.h"
#include "db/index/column/fts_column/tokenizer/tokenizer_factory.h"
#include "db/index/common/type_helper.h"
namespace zvec {
@@ -61,6 +63,28 @@ std::unordered_set<IndexType> support_dense_vector_index = {
std::unordered_set<IndexType> support_sparse_vector_index = {IndexType::FLAT,
IndexType::HNSW};
static Status validate_fts_index_params(const FieldSchema &field) {
auto params = std::dynamic_pointer_cast<FtsIndexParams>(field.index_params());
if (!params) {
return Status::InvalidArgument(
"schema validate failed: FTS index requires FtsIndexParams, but field[",
field.name(), "] has incompatible index params");
}
fts::FtsIndexParams internal_params;
internal_params.tokenizer_name = params->tokenizer_name();
internal_params.filters = params->filters();
internal_params.extra_params = params->extra_params();
auto pipeline = fts::TokenizerFactory::create(internal_params);
if (!pipeline) {
return Status::InvalidArgument(
"schema validate failed: invalid FTS index params for field[",
field.name(), "]");
}
return Status::OK();
}
Status FieldSchema::validate() const {
if (data_type_ == DataType::UNDEFINED) {
return Status::InvalidArgument("schema validate failed: field[", name_,
@@ -261,6 +285,10 @@ Status FieldSchema::validate() const {
"but field[",
name_, "]'s data_type is ", DataTypeCodeBook::AsString(data_type_));
}
if (index_params_->type() == IndexType::FTS) {
auto s = validate_fts_index_params(*this);
CHECK_RETURN_STATUS(s);
}
}
}
return Status::OK();
+25 -6
View File
@@ -1089,12 +1089,31 @@ ZVEC_EXPORT zvec_error_code_t ZVEC_CALL zvec_index_params_set_invert_params(
/**
* @brief Set FTS index specific parameters
* @param params Index parameters (must be FTS type)
* @param tokenizer_name Tokenizer name: "standard", "jieba", or "whitespace"
* (NULL keeps current value)
* @param filters Token filter names: "lowercase" and/or "ascii_folding"
* (NULL keeps current value)
* @param extra_params Additional tokenizer parameters (NULL keeps current
* value)
* @param tokenizer_name Tokenizer pipeline name (NULL keeps current value).
* Supported values are "standard", "jieba", and "whitespace".
* @param filters Token filter names (NULL keeps current value). Supported
* values are "lowercase", "ascii_folding", and "stemmer".
* @param extra_params Additional tokenizer/filter parameters (NULL keeps
* current value). Must be empty or a JSON object string. Supported keys by
* tokenizer/filter:
* Tokenizers:
* standard:
* - "max_token_length" (positive integer).
* jieba:
* - "jieba_dict_dir" (directory containing jieba.dict.utf8 and
* hmm_model.utf8).
* - "user_dict_path" (user dictionary path).
* - "cut_mode" ("search", "mix", "full", or "hmm"; default "search").
* whitespace:
* - no extra_params.
* Filters:
* lowercase:
* - no extra_params.
* ascii_folding:
* - no extra_params.
* stemmer:
* - "stemmer_lang" (Snowball language/algorithm; default "english"),
* for example {"stemmer_lang":"porter"} for ES behaviour.
* @return ZVEC_OK on success, error code on failure
*/
ZVEC_EXPORT zvec_error_code_t ZVEC_CALL zvec_index_params_set_fts_params(
+22 -1
View File
@@ -718,7 +718,28 @@ class VamanaIndexParams : public VectorIndexParams {
/*
* FTS (Full-Text Search) index params
* Supported tokenizers: "standard", "jieba", "whitespace".
* Supported filters: "lowercase", "ascii_folding".
* Supported filters: "lowercase", "ascii_folding", "stemmer".
*
* extra_params must be either empty or a JSON object string. Supported keys are
* grouped by tokenizer/filter:
* Tokenizers:
* standard:
* - "max_token_length" (positive integer).
* jieba:
* - "jieba_dict_dir" (directory containing jieba.dict.utf8 and
* hmm_model.utf8).
* - "user_dict_path" (user dictionary path).
* - "cut_mode" ("search", "mix", "full", or "hmm"; default "search").
* whitespace:
* - no extra_params.
* Filters:
* lowercase:
* - no extra_params.
* ascii_folding:
* - no extra_params.
* stemmer:
* - "stemmer_lang" (Snowball language/algorithm; default "english"),
* for example {"stemmer_lang":"porter"} for ES behaviour.
*
* Not copyable. Use shared_ptr<FtsIndexParams> for shared ownership.
*/