feat(fts): add UTF-8 support for tokenizer and token filters via utf8proc (#515)

This commit is contained in:
egolearner
2026-07-02 11:31:44 +08:00
committed by GitHub
parent 6ca2fb09e7
commit b9ce030593
22 changed files with 1194 additions and 57 deletions
+9 -4
View File
@@ -714,15 +714,19 @@ class FtsIndexParam(IndexParam):
Attributes:
type (IndexType): Always ``IndexType.FTS``.
tokenizer_name (str): Name of the tokenizer (e.g., "standard", "jieba").
tokenizer_name (str): Name of the tokenizer (one of "standard", "jieba",
"whitespace").
Default is "standard".
filters (list[str]): List of token filter names applied after tokenization.
Default is ["lowercase"].
Supported filters are "lowercase" and "ascii_folding". Default is
["lowercase"].
extra_params (str): Additional parameters passed to the tokenizer.
Default is "".
Examples:
>>> params = FtsIndexParam(tokenizer_name="jieba", filters=["lowercase"])
>>> params = FtsIndexParam(
... tokenizer_name="jieba", filters=["lowercase", "ascii_folding"]
... )
>>> print(params.tokenizer_name)
jieba
"""
@@ -739,7 +743,8 @@ class FtsIndexParam(IndexParam):
Args:
tokenizer_name (str, optional): Tokenizer name. Defaults to "standard".
filters (list[str], optional): Token filter names. Defaults to ["lowercase"].
filters (list[str], optional): Token filter names. Supports
"lowercase" and "ascii_folding". Defaults to ["lowercase"].
extra_params (str, optional): Extra tokenizer parameters. Defaults to "".
"""