feat(fts): add stemmer token filter based on Snowball 3.1.1 (#513)

This commit is contained in:
egolearner
2026-07-07 14:14:13 +08:00
committed by GitHub
parent 659951b984
commit c35d24e215
19 changed files with 703 additions and 21 deletions
+52 -6
View File
@@ -261,9 +261,30 @@ Attributes:
"whitespace").
Default is "standard".
filters (list[str]): List of token filter names applied after tokenization.
Supported filters are "lowercase" and "ascii_folding". Default is
["lowercase"].
extra_params (str): Additional parameters passed to the tokenizer.
Supported values include "lowercase", "ascii_folding", and "stemmer".
Default is ["lowercase"].
extra_params (str): Additional tokenizer/filter parameters as an empty
string or JSON object string. Supported keys are grouped by component:
Tokenizers:
standard:
- "max_token_length" (positive integer).
jieba:
- "jieba_dict_dir" (directory containing jieba.dict.utf8 and
hmm_model.utf8).
- "user_dict_path" (user dictionary path).
- "cut_mode" ("search", "mix", "full", or "hmm"; default
"search").
whitespace:
- no extra_params.
Filters:
lowercase:
- no extra_params.
ascii_folding:
- no extra_params.
stemmer:
- "stemmer_lang" (Snowball language/algorithm; default
"english"), for example {"stemmer_lang":"porter"} for ES
behaviour.
Default is "".
Examples:
@@ -272,6 +293,11 @@ Examples:
... )
>>> print(params.tokenizer_name)
jieba
>>> params = FtsIndexParam(
... tokenizer_name="standard",
... filters=["lowercase", "stemmer"],
... extra_params='{"stemmer_lang":"porter"}',
... )
)pbdoc");
fts_index_params
.def(py::init<std::string, std::vector<std::string>, std::string>(),
@@ -283,9 +309,29 @@ Constructs an FtsIndexParam instance.
Args:
tokenizer_name (str, optional): Tokenizer name. Defaults to "standard".
filters (list[str], optional): Token filter names. Supports "lowercase" and
"ascii_folding". Defaults to ["lowercase"].
extra_params (str, optional): Extra tokenizer parameters. Defaults to "".
filters (list[str], optional): Token filter names. Supports "lowercase",
"ascii_folding", and "stemmer". Defaults to ["lowercase"].
extra_params (str, optional): Extra tokenizer/filter parameters as an empty
string or JSON object string. Supported keys:
Tokenizers:
standard:
- "max_token_length" (positive integer).
jieba:
- "jieba_dict_dir".
- "user_dict_path".
- "cut_mode" ("search", "mix", "full", or "hmm"; default
"search").
whitespace:
- no extra_params.
Filters:
lowercase:
- no extra_params.
ascii_folding:
- no extra_params.
stemmer:
- "stemmer_lang" (Snowball language/algorithm; default
"english").
Defaults to "".
)pbdoc")
.def_property_readonly("tokenizer_name", &FtsIndexParams::tokenizer_name,
"str: Name of the tokenizer.")