feat(fts): add stemmer token filter based on Snowball 3.1.1 (#513)
This commit is contained in:
@@ -261,9 +261,30 @@ Attributes:
|
||||
"whitespace").
|
||||
Default is "standard".
|
||||
filters (list[str]): List of token filter names applied after tokenization.
|
||||
Supported filters are "lowercase" and "ascii_folding". Default is
|
||||
["lowercase"].
|
||||
extra_params (str): Additional parameters passed to the tokenizer.
|
||||
Supported values include "lowercase", "ascii_folding", and "stemmer".
|
||||
Default is ["lowercase"].
|
||||
extra_params (str): Additional tokenizer/filter parameters as an empty
|
||||
string or JSON object string. Supported keys are grouped by component:
|
||||
Tokenizers:
|
||||
standard:
|
||||
- "max_token_length" (positive integer).
|
||||
jieba:
|
||||
- "jieba_dict_dir" (directory containing jieba.dict.utf8 and
|
||||
hmm_model.utf8).
|
||||
- "user_dict_path" (user dictionary path).
|
||||
- "cut_mode" ("search", "mix", "full", or "hmm"; default
|
||||
"search").
|
||||
whitespace:
|
||||
- no extra_params.
|
||||
Filters:
|
||||
lowercase:
|
||||
- no extra_params.
|
||||
ascii_folding:
|
||||
- no extra_params.
|
||||
stemmer:
|
||||
- "stemmer_lang" (Snowball language/algorithm; default
|
||||
"english"), for example {"stemmer_lang":"porter"} for ES
|
||||
behaviour.
|
||||
Default is "".
|
||||
|
||||
Examples:
|
||||
@@ -272,6 +293,11 @@ Examples:
|
||||
... )
|
||||
>>> print(params.tokenizer_name)
|
||||
jieba
|
||||
>>> params = FtsIndexParam(
|
||||
... tokenizer_name="standard",
|
||||
... filters=["lowercase", "stemmer"],
|
||||
... extra_params='{"stemmer_lang":"porter"}',
|
||||
... )
|
||||
)pbdoc");
|
||||
fts_index_params
|
||||
.def(py::init<std::string, std::vector<std::string>, std::string>(),
|
||||
@@ -283,9 +309,29 @@ Constructs an FtsIndexParam instance.
|
||||
|
||||
Args:
|
||||
tokenizer_name (str, optional): Tokenizer name. Defaults to "standard".
|
||||
filters (list[str], optional): Token filter names. Supports "lowercase" and
|
||||
"ascii_folding". Defaults to ["lowercase"].
|
||||
extra_params (str, optional): Extra tokenizer parameters. Defaults to "".
|
||||
filters (list[str], optional): Token filter names. Supports "lowercase",
|
||||
"ascii_folding", and "stemmer". Defaults to ["lowercase"].
|
||||
extra_params (str, optional): Extra tokenizer/filter parameters as an empty
|
||||
string or JSON object string. Supported keys:
|
||||
Tokenizers:
|
||||
standard:
|
||||
- "max_token_length" (positive integer).
|
||||
jieba:
|
||||
- "jieba_dict_dir".
|
||||
- "user_dict_path".
|
||||
- "cut_mode" ("search", "mix", "full", or "hmm"; default
|
||||
"search").
|
||||
whitespace:
|
||||
- no extra_params.
|
||||
Filters:
|
||||
lowercase:
|
||||
- no extra_params.
|
||||
ascii_folding:
|
||||
- no extra_params.
|
||||
stemmer:
|
||||
- "stemmer_lang" (Snowball language/algorithm; default
|
||||
"english").
|
||||
Defaults to "".
|
||||
)pbdoc")
|
||||
.def_property_readonly("tokenizer_name", &FtsIndexParams::tokenizer_name,
|
||||
"str: Name of the tokenizer.")
|
||||
|
||||
@@ -45,6 +45,7 @@ cc_library(
|
||||
libprotobuf
|
||||
FastPFOR
|
||||
cppjieba
|
||||
snowball
|
||||
Arrow::arrow_static
|
||||
Arrow::parquet_static
|
||||
Arrow::arrow_compute
|
||||
|
||||
@@ -29,6 +29,7 @@ cc_library(
|
||||
Arrow::arrow_compute
|
||||
Arrow::arrow_dataset
|
||||
cppjieba
|
||||
snowball
|
||||
FastPFOR
|
||||
utf8proc
|
||||
INCS . ${PROJECT_ROOT_DIR}/src
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "stemmer_token_filter.h"
|
||||
#include <unordered_map>
|
||||
#include <zvec/ailego/logger/logger.h>
|
||||
|
||||
extern "C" {
|
||||
#include <libstemmer.h>
|
||||
}
|
||||
|
||||
namespace zvec::fts {
|
||||
|
||||
struct ThreadLocalStemmerCache {
|
||||
std::unordered_map<std::string, struct sb_stemmer *> stemmers;
|
||||
|
||||
~ThreadLocalStemmerCache() {
|
||||
for (auto &[_, s] : stemmers) {
|
||||
sb_stemmer_delete(s);
|
||||
}
|
||||
}
|
||||
|
||||
struct sb_stemmer *get(const std::string &lang) {
|
||||
auto it = stemmers.find(lang);
|
||||
if (it != stemmers.end()) {
|
||||
return it->second;
|
||||
}
|
||||
auto *s = sb_stemmer_new(lang.c_str(), nullptr);
|
||||
if (s) {
|
||||
stemmers[lang] = s;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
};
|
||||
|
||||
bool StemmerTokenFilter::init(const ailego::JsonObject &config) {
|
||||
std::string lang;
|
||||
if (config.get("stemmer_lang", &lang) && !lang.empty()) {
|
||||
language_ = lang;
|
||||
}
|
||||
auto *test_stemmer = sb_stemmer_new(language_.c_str(), nullptr);
|
||||
if (!test_stemmer) {
|
||||
LOG_ERROR("[StemmerTokenFilter] failed to create stemmer for language: %s",
|
||||
language_.c_str());
|
||||
return false;
|
||||
}
|
||||
sb_stemmer_delete(test_stemmer);
|
||||
return true;
|
||||
}
|
||||
|
||||
std::vector<Token> StemmerTokenFilter::filter(std::vector<Token> tokens) const {
|
||||
static thread_local ThreadLocalStemmerCache tls_cache;
|
||||
auto *stemmer = tls_cache.get(language_);
|
||||
if (!stemmer) {
|
||||
return tokens;
|
||||
}
|
||||
for (auto &token : tokens) {
|
||||
const auto *result = sb_stemmer_stem(
|
||||
stemmer, reinterpret_cast<const unsigned char *>(token.text.data()),
|
||||
static_cast<int>(token.text.size()));
|
||||
if (result) {
|
||||
int len = sb_stemmer_length(stemmer);
|
||||
token.text.assign(reinterpret_cast<const char *>(result), len);
|
||||
}
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
} // namespace zvec::fts
|
||||
@@ -0,0 +1,46 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "token_filter.h"
|
||||
|
||||
namespace zvec::fts {
|
||||
|
||||
/*! Snowball stemmer token filter.
|
||||
* Configure the language with extra_params key "stemmer_lang". When omitted,
|
||||
* the language is "english".
|
||||
*/
|
||||
class StemmerTokenFilter : public TokenFilter {
|
||||
public:
|
||||
StemmerTokenFilter() = default;
|
||||
~StemmerTokenFilter() override = default;
|
||||
|
||||
StemmerTokenFilter(const StemmerTokenFilter &) = delete;
|
||||
StemmerTokenFilter &operator=(const StemmerTokenFilter &) = delete;
|
||||
|
||||
bool init(const ailego::JsonObject &config) override;
|
||||
std::vector<Token> filter(std::vector<Token> tokens) const override;
|
||||
|
||||
const char *name() const override {
|
||||
return "stemmer";
|
||||
}
|
||||
|
||||
private:
|
||||
std::string language_{"english"};
|
||||
};
|
||||
|
||||
} // namespace zvec::fts
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <zvec/ailego/encoding/json/mod_json_plus.h>
|
||||
#include "tokenizer.h"
|
||||
|
||||
namespace zvec::fts {
|
||||
@@ -29,6 +30,15 @@ class TokenFilter {
|
||||
public:
|
||||
virtual ~TokenFilter() = default;
|
||||
|
||||
/*! Initialise the filter from a JSON configuration object.
|
||||
* Must be called once before filter().
|
||||
* \param config JSON object containing filter-specific parameters.
|
||||
* \return true on success, false on failure.
|
||||
*/
|
||||
virtual bool init(const ailego::JsonObject & /*config*/) {
|
||||
return true;
|
||||
}
|
||||
|
||||
/*! Filter/transform a list of tokens.
|
||||
* \param tokens input token list (may be modified in place)
|
||||
* \return processed token list
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
#include "ascii_folding_token_filter.h"
|
||||
#include "jieba_tokenizer.h"
|
||||
#include "standard_tokenizer.h"
|
||||
#include "stemmer_token_filter.h"
|
||||
#include "whitespace_tokenizer.h"
|
||||
|
||||
namespace zvec::fts {
|
||||
@@ -56,6 +57,11 @@ TokenizerPipelinePtr TokenizerFactory::create(const FtsIndexParams ¶ms) {
|
||||
filter_name.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
if (!filter->init(extra_json)) {
|
||||
LOG_ERROR("[TokenizerFactory] failed to init filter: %s",
|
||||
filter_name.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
filters.push_back(std::move(filter));
|
||||
}
|
||||
|
||||
@@ -99,6 +105,10 @@ TokenFilterPtr TokenizerFactory::create_filter(const std::string &filter_name) {
|
||||
return std::make_shared<LowercaseTokenFilter>();
|
||||
} else if (filter_name == "ascii_folding") {
|
||||
return std::make_shared<AsciiFoldingTokenFilter>();
|
||||
} else if (filter_name == "stemmer") {
|
||||
// The stemmer filter uses Snowball and defaults to "english" unless
|
||||
// extra_params overrides stemmer_lang.
|
||||
return std::make_shared<StemmerTokenFilter>();
|
||||
}
|
||||
LOG_ERROR("[TokenizerFactory] unknown filter name: %s", filter_name.c_str());
|
||||
return nullptr;
|
||||
|
||||
@@ -44,15 +44,14 @@ using TokenizerPipelinePtr = std::shared_ptr<TokenizerPipeline>;
|
||||
|
||||
/*! Tokenizer factory
|
||||
* Create TokenizerPipeline based on FtsIndexParams configuration.
|
||||
* Supported tokenizers: standard, jieba, whitespace.
|
||||
* Supported filters: lowercase, ascii_folding.
|
||||
*/
|
||||
class TokenizerFactory {
|
||||
public:
|
||||
/*! Create tokenizer pipeline from FtsIndexParams.
|
||||
* \param params FTS index parameters containing tokenizer_name, filters,
|
||||
* and extra_params (JSON string for tokenizer-specific
|
||||
* configuration).
|
||||
* configuration). The stemmer filter reads stemmer_lang from
|
||||
* extra_params and uses Snowball English by default.
|
||||
* \return Tokenizer pipeline, returns nullptr on failure
|
||||
*/
|
||||
static TokenizerPipelinePtr create(const FtsIndexParams ¶ms);
|
||||
|
||||
@@ -25,6 +25,8 @@
|
||||
#include "db/common/constants.h"
|
||||
#include "db/common/typedef.h"
|
||||
#include "db/common/utils.h"
|
||||
#include "db/index/column/fts_column/fts_types.h"
|
||||
#include "db/index/column/fts_column/tokenizer/tokenizer_factory.h"
|
||||
#include "db/index/common/type_helper.h"
|
||||
|
||||
namespace zvec {
|
||||
@@ -61,6 +63,28 @@ std::unordered_set<IndexType> support_dense_vector_index = {
|
||||
std::unordered_set<IndexType> support_sparse_vector_index = {IndexType::FLAT,
|
||||
IndexType::HNSW};
|
||||
|
||||
static Status validate_fts_index_params(const FieldSchema &field) {
|
||||
auto params = std::dynamic_pointer_cast<FtsIndexParams>(field.index_params());
|
||||
if (!params) {
|
||||
return Status::InvalidArgument(
|
||||
"schema validate failed: FTS index requires FtsIndexParams, but field[",
|
||||
field.name(), "] has incompatible index params");
|
||||
}
|
||||
|
||||
fts::FtsIndexParams internal_params;
|
||||
internal_params.tokenizer_name = params->tokenizer_name();
|
||||
internal_params.filters = params->filters();
|
||||
internal_params.extra_params = params->extra_params();
|
||||
|
||||
auto pipeline = fts::TokenizerFactory::create(internal_params);
|
||||
if (!pipeline) {
|
||||
return Status::InvalidArgument(
|
||||
"schema validate failed: invalid FTS index params for field[",
|
||||
field.name(), "]");
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status FieldSchema::validate() const {
|
||||
if (data_type_ == DataType::UNDEFINED) {
|
||||
return Status::InvalidArgument("schema validate failed: field[", name_,
|
||||
@@ -261,6 +285,10 @@ Status FieldSchema::validate() const {
|
||||
"but field[",
|
||||
name_, "]'s data_type is ", DataTypeCodeBook::AsString(data_type_));
|
||||
}
|
||||
if (index_params_->type() == IndexType::FTS) {
|
||||
auto s = validate_fts_index_params(*this);
|
||||
CHECK_RETURN_STATUS(s);
|
||||
}
|
||||
}
|
||||
}
|
||||
return Status::OK();
|
||||
|
||||
@@ -1089,12 +1089,31 @@ ZVEC_EXPORT zvec_error_code_t ZVEC_CALL zvec_index_params_set_invert_params(
|
||||
/**
|
||||
* @brief Set FTS index specific parameters
|
||||
* @param params Index parameters (must be FTS type)
|
||||
* @param tokenizer_name Tokenizer name: "standard", "jieba", or "whitespace"
|
||||
* (NULL keeps current value)
|
||||
* @param filters Token filter names: "lowercase" and/or "ascii_folding"
|
||||
* (NULL keeps current value)
|
||||
* @param extra_params Additional tokenizer parameters (NULL keeps current
|
||||
* value)
|
||||
* @param tokenizer_name Tokenizer pipeline name (NULL keeps current value).
|
||||
* Supported values are "standard", "jieba", and "whitespace".
|
||||
* @param filters Token filter names (NULL keeps current value). Supported
|
||||
* values are "lowercase", "ascii_folding", and "stemmer".
|
||||
* @param extra_params Additional tokenizer/filter parameters (NULL keeps
|
||||
* current value). Must be empty or a JSON object string. Supported keys by
|
||||
* tokenizer/filter:
|
||||
* Tokenizers:
|
||||
* standard:
|
||||
* - "max_token_length" (positive integer).
|
||||
* jieba:
|
||||
* - "jieba_dict_dir" (directory containing jieba.dict.utf8 and
|
||||
* hmm_model.utf8).
|
||||
* - "user_dict_path" (user dictionary path).
|
||||
* - "cut_mode" ("search", "mix", "full", or "hmm"; default "search").
|
||||
* whitespace:
|
||||
* - no extra_params.
|
||||
* Filters:
|
||||
* lowercase:
|
||||
* - no extra_params.
|
||||
* ascii_folding:
|
||||
* - no extra_params.
|
||||
* stemmer:
|
||||
* - "stemmer_lang" (Snowball language/algorithm; default "english"),
|
||||
* for example {"stemmer_lang":"porter"} for ES behaviour.
|
||||
* @return ZVEC_OK on success, error code on failure
|
||||
*/
|
||||
ZVEC_EXPORT zvec_error_code_t ZVEC_CALL zvec_index_params_set_fts_params(
|
||||
|
||||
@@ -718,7 +718,28 @@ class VamanaIndexParams : public VectorIndexParams {
|
||||
/*
|
||||
* FTS (Full-Text Search) index params
|
||||
* Supported tokenizers: "standard", "jieba", "whitespace".
|
||||
* Supported filters: "lowercase", "ascii_folding".
|
||||
* Supported filters: "lowercase", "ascii_folding", "stemmer".
|
||||
*
|
||||
* extra_params must be either empty or a JSON object string. Supported keys are
|
||||
* grouped by tokenizer/filter:
|
||||
* Tokenizers:
|
||||
* standard:
|
||||
* - "max_token_length" (positive integer).
|
||||
* jieba:
|
||||
* - "jieba_dict_dir" (directory containing jieba.dict.utf8 and
|
||||
* hmm_model.utf8).
|
||||
* - "user_dict_path" (user dictionary path).
|
||||
* - "cut_mode" ("search", "mix", "full", or "hmm"; default "search").
|
||||
* whitespace:
|
||||
* - no extra_params.
|
||||
* Filters:
|
||||
* lowercase:
|
||||
* - no extra_params.
|
||||
* ascii_folding:
|
||||
* - no extra_params.
|
||||
* stemmer:
|
||||
* - "stemmer_lang" (Snowball language/algorithm; default "english"),
|
||||
* for example {"stemmer_lang":"porter"} for ES behaviour.
|
||||
*
|
||||
* Not copyable. Use shared_ptr<FtsIndexParams> for shared ownership.
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user