feat(fts): add ngram tokenizer (#593)

This commit is contained in:
egolearner
2026-08-03 14:06:15 +08:00
committed by GitHub
parent be6d7b7cfb
commit 58375ff7b8
26 changed files with 895 additions and 198 deletions
+16 -2
View File
@@ -834,8 +834,8 @@ class FtsIndexParam(IndexParam):
Attributes:
type (IndexType): Always ``IndexType.FTS``.
tokenizer_name (str): Name of the tokenizer (one of "standard", "jieba",
"whitespace").
tokenizer_name (str): Name of the tokenizer (one of "standard", "ngram",
"jieba", "whitespace").
Default is "standard".
filters (list[str]): List of token filter names applied after tokenization.
Supported filters are "lowercase", "ascii_folding", and "stemmer".
@@ -845,6 +845,13 @@ class FtsIndexParam(IndexParam):
Tokenizers:
standard:
- "max_token_length" (positive integer).
ngram:
- "ngram_min" (positive integer, default 2).
- "ngram_max" (positive integer, default 2).
- "token_chars" (array of "letter", "digit",
"whitespace", "punctuation", "symbol"; default [] keeps
all valid UTF-8 characters). custom_token_chars is not
supported.
jieba:
- "jieba_dict_dir" (directory containing jieba.dict.utf8 and
hmm_model.utf8).
@@ -892,6 +899,13 @@ class FtsIndexParam(IndexParam):
Tokenizers:
standard:
- "max_token_length" (positive integer).
ngram:
- "ngram_min" (positive integer, default 2).
- "ngram_max" (positive integer, default 2).
- "token_chars" (array of "letter", "digit",
"whitespace", "punctuation", "symbol"; default []
keeps all valid UTF-8 characters). custom_token_chars
is not supported.
jieba:
- "jieba_dict_dir".
- "user_dict_path".