feat(fts): add stemmer token filter based on Snowball 3.1.1 (#513)
This commit is contained in:
@@ -261,9 +261,30 @@ Attributes:
|
||||
"whitespace").
|
||||
Default is "standard".
|
||||
filters (list[str]): List of token filter names applied after tokenization.
|
||||
Supported filters are "lowercase" and "ascii_folding". Default is
|
||||
["lowercase"].
|
||||
extra_params (str): Additional parameters passed to the tokenizer.
|
||||
Supported values include "lowercase", "ascii_folding", and "stemmer".
|
||||
Default is ["lowercase"].
|
||||
extra_params (str): Additional tokenizer/filter parameters as an empty
|
||||
string or JSON object string. Supported keys are grouped by component:
|
||||
Tokenizers:
|
||||
standard:
|
||||
- "max_token_length" (positive integer).
|
||||
jieba:
|
||||
- "jieba_dict_dir" (directory containing jieba.dict.utf8 and
|
||||
hmm_model.utf8).
|
||||
- "user_dict_path" (user dictionary path).
|
||||
- "cut_mode" ("search", "mix", "full", or "hmm"; default
|
||||
"search").
|
||||
whitespace:
|
||||
- no extra_params.
|
||||
Filters:
|
||||
lowercase:
|
||||
- no extra_params.
|
||||
ascii_folding:
|
||||
- no extra_params.
|
||||
stemmer:
|
||||
- "stemmer_lang" (Snowball language/algorithm; default
|
||||
"english"), for example {"stemmer_lang":"porter"} for ES
|
||||
behaviour.
|
||||
Default is "".
|
||||
|
||||
Examples:
|
||||
@@ -272,6 +293,11 @@ Examples:
|
||||
... )
|
||||
>>> print(params.tokenizer_name)
|
||||
jieba
|
||||
>>> params = FtsIndexParam(
|
||||
... tokenizer_name="standard",
|
||||
... filters=["lowercase", "stemmer"],
|
||||
... extra_params='{"stemmer_lang":"porter"}',
|
||||
... )
|
||||
)pbdoc");
|
||||
fts_index_params
|
||||
.def(py::init<std::string, std::vector<std::string>, std::string>(),
|
||||
@@ -283,9 +309,29 @@ Constructs an FtsIndexParam instance.
|
||||
|
||||
Args:
|
||||
tokenizer_name (str, optional): Tokenizer name. Defaults to "standard".
|
||||
filters (list[str], optional): Token filter names. Supports "lowercase" and
|
||||
"ascii_folding". Defaults to ["lowercase"].
|
||||
extra_params (str, optional): Extra tokenizer parameters. Defaults to "".
|
||||
filters (list[str], optional): Token filter names. Supports "lowercase",
|
||||
"ascii_folding", and "stemmer". Defaults to ["lowercase"].
|
||||
extra_params (str, optional): Extra tokenizer/filter parameters as an empty
|
||||
string or JSON object string. Supported keys:
|
||||
Tokenizers:
|
||||
standard:
|
||||
- "max_token_length" (positive integer).
|
||||
jieba:
|
||||
- "jieba_dict_dir".
|
||||
- "user_dict_path".
|
||||
- "cut_mode" ("search", "mix", "full", or "hmm"; default
|
||||
"search").
|
||||
whitespace:
|
||||
- no extra_params.
|
||||
Filters:
|
||||
lowercase:
|
||||
- no extra_params.
|
||||
ascii_folding:
|
||||
- no extra_params.
|
||||
stemmer:
|
||||
- "stemmer_lang" (Snowball language/algorithm; default
|
||||
"english").
|
||||
Defaults to "".
|
||||
)pbdoc")
|
||||
.def_property_readonly("tokenizer_name", &FtsIndexParams::tokenizer_name,
|
||||
"str: Name of the tokenizer.")
|
||||
|
||||
Reference in New Issue
Block a user