feat(fts): add UTF-8 support for tokenizer and token filters via utf8proc (#515)
This commit is contained in:
egolearner
GitHub
parent
6ca2fb09e7
commit
b9ce030593
|
|
@ -53,3 +53,6 @@
|
|||
[submodule "thirdparty/limonp/limonp-v1.0.2"]
|
||||
path = thirdparty/limonp/limonp-v1.0.2
|
||||
url = https://github.com/yanyiwu/limonp.git
|
||||
[submodule "thirdparty/utf8proc/utf8proc-2.11.3"]
|
||||
path = thirdparty/utf8proc/utf8proc-2.11.3
|
||||
url = https://github.com/JuliaStrings/utf8proc.git
|
||||
|
|
|
|||
|
|
@ -714,15 +714,19 @@ class FtsIndexParam(IndexParam):
|
|||
|
||||
Attributes:
|
||||
type (IndexType): Always ``IndexType.FTS``.
|
||||
tokenizer_name (str): Name of the tokenizer (e.g., "standard", "jieba").
|
||||
tokenizer_name (str): Name of the tokenizer (one of "standard", "jieba",
|
||||
"whitespace").
|
||||
Default is "standard".
|
||||
filters (list[str]): List of token filter names applied after tokenization.
|
||||
Default is ["lowercase"].
|
||||
Supported filters are "lowercase" and "ascii_folding". Default is
|
||||
["lowercase"].
|
||||
extra_params (str): Additional parameters passed to the tokenizer.
|
||||
Default is "".
|
||||
|
||||
Examples:
|
||||
>>> params = FtsIndexParam(tokenizer_name="jieba", filters=["lowercase"])
|
||||
>>> params = FtsIndexParam(
|
||||
... tokenizer_name="jieba", filters=["lowercase", "ascii_folding"]
|
||||
... )
|
||||
>>> print(params.tokenizer_name)
|
||||
jieba
|
||||
"""
|
||||
|
|
@ -739,7 +743,8 @@ class FtsIndexParam(IndexParam):
|
|||
|
||||
Args:
|
||||
tokenizer_name (str, optional): Tokenizer name. Defaults to "standard".
|
||||
filters (list[str], optional): Token filter names. Defaults to ["lowercase"].
|
||||
filters (list[str], optional): Token filter names. Supports
|
||||
"lowercase" and "ascii_folding". Defaults to ["lowercase"].
|
||||
extra_params (str, optional): Extra tokenizer parameters. Defaults to "".
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -265,15 +265,19 @@ Controls the tokenizer pipeline used during indexing and querying.
|
|||
|
||||
Attributes:
|
||||
type (IndexType): Always ``IndexType.FTS``.
|
||||
tokenizer_name (str): Name of the tokenizer (e.g., "standard", "jieba").
|
||||
tokenizer_name (str): Name of the tokenizer (one of "standard", "jieba",
|
||||
"whitespace").
|
||||
Default is "standard".
|
||||
filters (list[str]): List of token filter names applied after tokenization.
|
||||
Default is ["lowercase"].
|
||||
Supported filters are "lowercase" and "ascii_folding". Default is
|
||||
["lowercase"].
|
||||
extra_params (str): Additional parameters passed to the tokenizer.
|
||||
Default is "".
|
||||
|
||||
Examples:
|
||||
>>> params = FtsIndexParam(tokenizer_name="jieba", filters=["lowercase"])
|
||||
>>> params = FtsIndexParam(
|
||||
... tokenizer_name="jieba", filters=["lowercase", "ascii_folding"]
|
||||
... )
|
||||
>>> print(params.tokenizer_name)
|
||||
jieba
|
||||
)pbdoc");
|
||||
|
|
@ -287,7 +291,8 @@ Constructs an FtsIndexParam instance.
|
|||
|
||||
Args:
|
||||
tokenizer_name (str, optional): Tokenizer name. Defaults to "standard".
|
||||
filters (list[str], optional): Token filter names. Defaults to ["lowercase"].
|
||||
filters (list[str], optional): Token filter names. Supports "lowercase" and
|
||||
"ascii_folding". Defaults to ["lowercase"].
|
||||
extra_params (str, optional): Extra tokenizer parameters. Defaults to "".
|
||||
)pbdoc")
|
||||
.def_property_readonly("tokenizer_name", &FtsIndexParams::tokenizer_name,
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ cc_library(
|
|||
Arrow::arrow_compute
|
||||
Arrow::arrow_dataset
|
||||
Arrow::arrow_acero
|
||||
utf8proc
|
||||
DEPS zvec_proto
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ cc_library(
|
|||
Arrow::arrow_dataset
|
||||
cppjieba
|
||||
FastPFOR
|
||||
utf8proc
|
||||
INCS . ${PROJECT_ROOT_DIR}/src
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -50,7 +50,9 @@ struct FtsSegmentStats {
|
|||
};
|
||||
|
||||
struct FtsIndexParams {
|
||||
// Supported tokenizers: standard, jieba, whitespace.
|
||||
std::string tokenizer_name{"standard"};
|
||||
// Supported filters: lowercase, ascii_folding.
|
||||
std::vector<std::string> filters{"lowercase"};
|
||||
std::string extra_params;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,291 @@
|
|||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "ascii_folding_token_filter.h"
|
||||
#include <utf8proc.h>
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
#include <cstdlib>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
namespace zvec::fts {
|
||||
|
||||
namespace {
|
||||
|
||||
// Supplementary folding table for codepoints that NFKD+STRIPMARK does not
|
||||
// reduce to ASCII. Inspired by Lucene's ASCIIFoldingFilter.
|
||||
// Each entry maps a single codepoint to a short ASCII string.
|
||||
struct FoldEntry {
|
||||
utf8proc_int32_t codepoint;
|
||||
const char *ascii;
|
||||
};
|
||||
|
||||
// clang-format off
|
||||
// Sorted by codepoint — binary search via std::lower_bound.
|
||||
static const FoldEntry kExtraFolds[] = {
|
||||
{0x00C6, "AE"}, // Æ LATIN CAPITAL LETTER AE
|
||||
{0x00D0, "D"}, // Ð LATIN CAPITAL LETTER ETH
|
||||
{0x00D8, "O"}, // Ø LATIN CAPITAL LETTER O WITH STROKE
|
||||
{0x00DE, "TH"}, // Þ LATIN CAPITAL LETTER THORN
|
||||
{0x00DF, "ss"}, // ß LATIN SMALL LETTER SHARP S
|
||||
{0x00E6, "ae"}, // æ LATIN SMALL LETTER AE
|
||||
{0x00F0, "d"}, // ð LATIN SMALL LETTER ETH
|
||||
{0x00F8, "o"}, // ø LATIN SMALL LETTER O WITH STROKE
|
||||
{0x00FE, "th"}, // þ LATIN SMALL LETTER THORN
|
||||
{0x0110, "D"}, // Đ LATIN CAPITAL LETTER D WITH STROKE
|
||||
{0x0111, "d"}, // đ LATIN SMALL LETTER D WITH STROKE
|
||||
{0x0126, "H"}, // Ħ LATIN CAPITAL LETTER H WITH STROKE
|
||||
{0x0127, "h"}, // ħ LATIN SMALL LETTER H WITH STROKE
|
||||
{0x0131, "i"}, // ı LATIN SMALL LETTER DOTLESS I
|
||||
{0x0132, "IJ"}, // IJ LATIN CAPITAL LIGATURE IJ
|
||||
{0x0133, "ij"}, // ij LATIN SMALL LIGATURE IJ
|
||||
{0x0138, "k"}, // ĸ LATIN SMALL LETTER KRA
|
||||
{0x0141, "L"}, // Ł LATIN CAPITAL LETTER L WITH STROKE
|
||||
{0x0142, "l"}, // ł LATIN SMALL LETTER L WITH STROKE
|
||||
{0x014A, "N"}, // Ŋ LATIN CAPITAL LETTER ENG
|
||||
{0x014B, "n"}, // ŋ LATIN SMALL LETTER ENG
|
||||
{0x0152, "OE"}, // ΠLATIN CAPITAL LIGATURE OE
|
||||
{0x0153, "oe"}, // œ LATIN SMALL LIGATURE OE
|
||||
{0x0166, "T"}, // Ŧ LATIN CAPITAL LETTER T WITH STROKE
|
||||
{0x0167, "t"}, // ŧ LATIN SMALL LETTER T WITH STROKE
|
||||
{0x0180, "b"}, // ƀ LATIN SMALL LETTER B WITH STROKE
|
||||
{0x0181, "B"}, // Ɓ LATIN CAPITAL LETTER B WITH HOOK
|
||||
{0x0182, "B"}, // Ƃ LATIN CAPITAL LETTER B WITH TOPBAR
|
||||
{0x0183, "b"}, // ƃ LATIN SMALL LETTER B WITH TOPBAR
|
||||
{0x0187, "C"}, // Ƈ LATIN CAPITAL LETTER C WITH HOOK
|
||||
{0x0188, "c"}, // ƈ LATIN SMALL LETTER C WITH HOOK
|
||||
{0x0189, "D"}, // Ɖ LATIN CAPITAL LETTER AFRICAN D
|
||||
{0x018A, "D"}, // Ɗ LATIN CAPITAL LETTER D WITH HOOK
|
||||
{0x018B, "D"}, // Ƌ LATIN CAPITAL LETTER D WITH TOPBAR
|
||||
{0x018C, "d"}, // ƌ LATIN SMALL LETTER D WITH TOPBAR
|
||||
{0x018E, "E"}, // Ǝ LATIN CAPITAL LETTER REVERSED E
|
||||
{0x018F, "A"}, // Ə LATIN CAPITAL LETTER SCHWA
|
||||
{0x0190, "E"}, // Ɛ LATIN CAPITAL LETTER OPEN E
|
||||
{0x0191, "F"}, // Ƒ LATIN CAPITAL LETTER F WITH HOOK
|
||||
{0x0192, "f"}, // ƒ LATIN SMALL LETTER F WITH HOOK
|
||||
{0x0193, "G"}, // Ɠ LATIN CAPITAL LETTER G WITH HOOK
|
||||
{0x0195, "hv"}, // ƕ LATIN SMALL LETTER HV
|
||||
{0x0196, "I"}, // Ɩ LATIN CAPITAL LETTER IOTA
|
||||
{0x0197, "I"}, // Ɨ LATIN CAPITAL LETTER I WITH STROKE
|
||||
{0x0198, "K"}, // Ƙ LATIN CAPITAL LETTER K WITH HOOK
|
||||
{0x0199, "k"}, // ƙ LATIN SMALL LETTER K WITH HOOK
|
||||
{0x019A, "l"}, // ƚ LATIN SMALL LETTER L WITH BAR
|
||||
{0x019D, "N"}, // Ɲ LATIN CAPITAL LETTER N WITH LEFT HOOK
|
||||
{0x019E, "n"}, // ƞ LATIN SMALL LETTER N WITH LONG RIGHT LEG
|
||||
{0x01A0, "O"}, // Ơ LATIN CAPITAL LETTER O WITH HORN
|
||||
{0x01A1, "o"}, // ơ LATIN SMALL LETTER O WITH HORN
|
||||
{0x01A6, "R"}, // Ʀ LATIN LETTER YR
|
||||
{0x01A9, "S"}, // Ʃ LATIN CAPITAL LETTER ESH
|
||||
{0x01AB, "t"}, // ƫ LATIN SMALL LETTER T WITH PALATAL HOOK
|
||||
{0x01AC, "T"}, // Ƭ LATIN CAPITAL LETTER T WITH HOOK
|
||||
{0x01AD, "t"}, // ƭ LATIN SMALL LETTER T WITH HOOK
|
||||
{0x01AE, "T"}, // Ʈ LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
|
||||
{0x01AF, "U"}, // Ư LATIN CAPITAL LETTER U WITH HORN
|
||||
{0x01B0, "u"}, // ư LATIN SMALL LETTER U WITH HORN
|
||||
{0x01B2, "V"}, // Ʋ LATIN CAPITAL LETTER V WITH HOOK
|
||||
{0x01B3, "Y"}, // Ƴ LATIN CAPITAL LETTER Y WITH HOOK
|
||||
{0x01B4, "y"}, // ƴ LATIN SMALL LETTER Y WITH HOOK
|
||||
{0x01B5, "Z"}, // Ƶ LATIN CAPITAL LETTER Z WITH STROKE
|
||||
{0x01B6, "z"}, // ƶ LATIN SMALL LETTER Z WITH STROKE
|
||||
{0x01C4, "DZ"}, // DŽ LATIN CAPITAL LETTER DZ WITH CARON
|
||||
{0x01C5, "Dz"}, // Dž LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON
|
||||
{0x01C6, "dz"}, // dž LATIN SMALL LETTER DZ WITH CARON
|
||||
{0x01E4, "G"}, // Ǥ LATIN CAPITAL LETTER G WITH STROKE
|
||||
{0x01E5, "g"}, // ǥ LATIN SMALL LETTER G WITH STROKE
|
||||
{0x0221, "d"}, // ȡ LATIN SMALL LETTER D WITH CURL
|
||||
{0x0222, "OU"}, // Ȣ LATIN CAPITAL LETTER OU
|
||||
{0x0223, "ou"}, // ȣ LATIN SMALL LETTER OU
|
||||
{0x0234, "l"}, // ȴ LATIN SMALL LETTER L WITH CURL
|
||||
{0x0235, "n"}, // ȵ LATIN SMALL LETTER N WITH CURL
|
||||
{0x0236, "t"}, // ȶ LATIN SMALL LETTER T WITH CURL
|
||||
{0x023A, "A"}, // Ⱥ LATIN CAPITAL LETTER A WITH STROKE
|
||||
{0x023B, "C"}, // Ȼ LATIN CAPITAL LETTER C WITH STROKE
|
||||
{0x023C, "c"}, // ȼ LATIN SMALL LETTER C WITH STROKE
|
||||
{0x023E, "T"}, // Ⱦ LATIN CAPITAL LETTER T WITH DIAGONAL STROKE
|
||||
{0x0243, "B"}, // Ƀ LATIN CAPITAL LETTER B WITH STROKE
|
||||
{0x0246, "E"}, // Ɇ LATIN CAPITAL LETTER E WITH STROKE
|
||||
{0x0247, "e"}, // ɇ LATIN SMALL LETTER E WITH STROKE
|
||||
{0x0248, "J"}, // Ɉ LATIN CAPITAL LETTER J WITH STROKE
|
||||
{0x0249, "j"}, // ɉ LATIN SMALL LETTER J WITH STROKE
|
||||
{0x024C, "R"}, // Ɍ LATIN CAPITAL LETTER R WITH STROKE
|
||||
{0x024D, "r"}, // ɍ LATIN SMALL LETTER R WITH STROKE
|
||||
{0x024E, "Y"}, // Ɏ LATIN CAPITAL LETTER Y WITH STROKE
|
||||
{0x024F, "y"}, // ɏ LATIN SMALL LETTER Y WITH STROKE
|
||||
{0x0250, "a"}, // ɐ LATIN SMALL LETTER TURNED A
|
||||
{0x0251, "a"}, // ɑ LATIN SMALL LETTER ALPHA
|
||||
{0x0253, "b"}, // ɓ LATIN SMALL LETTER B WITH HOOK
|
||||
{0x0255, "c"}, // ɕ LATIN SMALL LETTER C WITH CURL
|
||||
{0x0256, "d"}, // ɖ LATIN SMALL LETTER D WITH TAIL
|
||||
{0x0257, "d"}, // ɗ LATIN SMALL LETTER D WITH HOOK
|
||||
{0x0258, "e"}, // ɘ LATIN SMALL LETTER REVERSED E
|
||||
{0x0259, "e"}, // ə LATIN SMALL LETTER SCHWA
|
||||
{0x025B, "e"}, // ɛ LATIN SMALL LETTER OPEN E
|
||||
{0x025C, "e"}, // ɜ LATIN SMALL LETTER REVERSED OPEN E
|
||||
{0x0260, "g"}, // ɠ LATIN SMALL LETTER G WITH HOOK
|
||||
{0x0261, "g"}, // ɡ LATIN SMALL LETTER SCRIPT G
|
||||
{0x0262, "G"}, // ɢ LATIN LETTER SMALL CAPITAL G
|
||||
{0x0265, "h"}, // ɥ LATIN SMALL LETTER TURNED H
|
||||
{0x0266, "h"}, // ɦ LATIN SMALL LETTER H WITH HOOK
|
||||
{0x0268, "i"}, // ɨ LATIN SMALL LETTER I WITH STROKE
|
||||
{0x026A, "I"}, // ɪ LATIN LETTER SMALL CAPITAL I
|
||||
{0x026B, "l"}, // ɫ LATIN SMALL LETTER L WITH MIDDLE TILDE
|
||||
{0x026C, "l"}, // ɬ LATIN SMALL LETTER L WITH BELT
|
||||
{0x026D, "l"}, // ɭ LATIN SMALL LETTER L WITH RETROFLEX HOOK
|
||||
{0x0271, "m"}, // ɱ LATIN SMALL LETTER M WITH HOOK
|
||||
{0x0272, "n"}, // ɲ LATIN SMALL LETTER N WITH LEFT HOOK
|
||||
{0x0273, "n"}, // ɳ LATIN SMALL LETTER N WITH RETROFLEX HOOK
|
||||
{0x0274, "N"}, // ɴ LATIN LETTER SMALL CAPITAL N
|
||||
{0x0275, "o"}, // ɵ LATIN SMALL LETTER BARRED O
|
||||
{0x027D, "r"}, // ɽ LATIN SMALL LETTER R WITH TAIL
|
||||
{0x0282, "s"}, // ʂ LATIN SMALL LETTER S WITH HOOK
|
||||
{0x0283, "s"}, // ʃ LATIN SMALL LETTER ESH
|
||||
{0x0288, "t"}, // ʈ LATIN SMALL LETTER T WITH RETROFLEX HOOK
|
||||
{0x028B, "v"}, // ʋ LATIN SMALL LETTER V WITH HOOK
|
||||
{0x0290, "z"}, // ʐ LATIN SMALL LETTER Z WITH RETROFLEX HOOK
|
||||
{0x0291, "z"}, // ʑ LATIN SMALL LETTER Z WITH CURL
|
||||
{0x0292, "z"}, // ʒ LATIN SMALL LETTER EZH
|
||||
{0x029D, "j"}, // ʝ LATIN SMALL LETTER J WITH CROSSED-TAIL
|
||||
{0x029E, "k"}, // ʞ LATIN SMALL LETTER TURNED K
|
||||
{0x1D6D, "d"}, // ᵭ LATIN SMALL LETTER D WITH MIDDLE TILDE
|
||||
{0x1D6E, "f"}, // ᵮ LATIN SMALL LETTER F WITH MIDDLE TILDE
|
||||
{0x1D6F, "g"}, // ᵯ LATIN SMALL LETTER G WITH MIDDLE TILDE
|
||||
{0x1D70, "r"}, // ᵰ LATIN SMALL LETTER R WITH MIDDLE TILDE
|
||||
{0x1D71, "s"}, // ᵱ LATIN SMALL LETTER S WITH MIDDLE TILDE
|
||||
{0x1D72, "t"}, // ᵲ LATIN SMALL LETTER T WITH MIDDLE TILDE
|
||||
{0x1D7D, "p"}, // ᵽ LATIN SMALL LETTER P WITH STROKE
|
||||
{0x1D85, "l"}, // ᶅ LATIN SMALL LETTER L WITH PALATAL HOOK
|
||||
{0x1D86, "m"}, // ᶆ LATIN SMALL LETTER M WITH PALATAL HOOK
|
||||
{0x1D87, "n"}, // ᶇ LATIN SMALL LETTER N WITH PALATAL HOOK
|
||||
{0x1D88, "p"}, // ᶈ LATIN SMALL LETTER P WITH PALATAL HOOK
|
||||
{0x1D89, "r"}, // ᶉ LATIN SMALL LETTER R WITH PALATAL HOOK
|
||||
{0x1D8A, "s"}, // ᶊ LATIN SMALL LETTER S WITH PALATAL HOOK
|
||||
{0x1D8C, "v"}, // ᶌ LATIN SMALL LETTER V WITH PALATAL HOOK
|
||||
{0x1D8E, "z"}, // ᶎ LATIN SMALL LETTER Z WITH PALATAL HOOK
|
||||
{0x1E9E, "SS"}, // ẞ LATIN CAPITAL LETTER SHARP S
|
||||
{0x2039, "<"}, // ‹ SINGLE LEFT-POINTING ANGLE QUOTATION MARK
|
||||
{0x203A, ">"}, // › SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
|
||||
{0x2190, "<-"}, // ← LEFTWARDS ARROW
|
||||
{0x2192, "->"}, // → RIGHTWARDS ARROW
|
||||
};
|
||||
// clang-format on
|
||||
|
||||
#ifndef NDEBUG
|
||||
struct FoldTableSortChecker {
|
||||
FoldTableSortChecker() {
|
||||
for (size_t i = 1; i < std::size(kExtraFolds); ++i) {
|
||||
assert(kExtraFolds[i - 1].codepoint < kExtraFolds[i].codepoint);
|
||||
}
|
||||
}
|
||||
};
|
||||
static const FoldTableSortChecker kSortChecker;
|
||||
#endif
|
||||
|
||||
const char *lookup_extra_fold(utf8proc_int32_t cp) {
|
||||
auto it = std::lower_bound(std::begin(kExtraFolds), std::end(kExtraFolds), cp,
|
||||
[](const FoldEntry &entry, utf8proc_int32_t val) {
|
||||
return entry.codepoint < val;
|
||||
});
|
||||
if (it != std::end(kExtraFolds) && it->codepoint == cp) {
|
||||
return it->ascii;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
bool fold_codepoint_to_ascii(const utf8proc_uint8_t *data, utf8proc_ssize_t len,
|
||||
std::string *out) {
|
||||
utf8proc_uint8_t *mapped_raw = nullptr;
|
||||
utf8proc_ssize_t mapped_len = utf8proc_map(
|
||||
data, len, &mapped_raw,
|
||||
static_cast<utf8proc_option_t>(UTF8PROC_STABLE | UTF8PROC_COMPAT |
|
||||
UTF8PROC_DECOMPOSE | UTF8PROC_STRIPMARK));
|
||||
// RAII guard: utf8proc_map allocates with malloc, free with free().
|
||||
std::unique_ptr<utf8proc_uint8_t, decltype(&free)> mapped(mapped_raw, &free);
|
||||
if (mapped_len <= 0) {
|
||||
return false;
|
||||
}
|
||||
for (utf8proc_ssize_t i = 0; i < mapped_len; ++i) {
|
||||
if (mapped_raw[i] >= 0x80) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
out->assign(reinterpret_cast<const char *>(mapped_raw),
|
||||
static_cast<size_t>(mapped_len));
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::vector<Token> AsciiFoldingTokenFilter::filter(
|
||||
std::vector<Token> tokens) const {
|
||||
for (auto &token : tokens) {
|
||||
bool all_ascii = true;
|
||||
for (unsigned char c : token.text) {
|
||||
if (c >= 0x80) {
|
||||
all_ascii = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (all_ascii) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::string result;
|
||||
result.reserve(token.text.size());
|
||||
const auto *str =
|
||||
reinterpret_cast<const utf8proc_uint8_t *>(token.text.data());
|
||||
const auto len = static_cast<utf8proc_ssize_t>(token.text.size());
|
||||
utf8proc_ssize_t pos = 0;
|
||||
while (pos < len) {
|
||||
if (str[pos] < 0x80) {
|
||||
result.push_back(static_cast<char>(str[pos]));
|
||||
++pos;
|
||||
continue;
|
||||
}
|
||||
|
||||
utf8proc_int32_t cp;
|
||||
utf8proc_ssize_t bytes = utf8proc_iterate(str + pos, len - pos, &cp);
|
||||
if (bytes < 1) {
|
||||
result.push_back(static_cast<char>(str[pos]));
|
||||
++pos;
|
||||
continue;
|
||||
}
|
||||
|
||||
const char *fold = lookup_extra_fold(cp);
|
||||
if (fold) {
|
||||
result.append(fold);
|
||||
pos += bytes;
|
||||
continue;
|
||||
}
|
||||
|
||||
std::string ascii;
|
||||
if (fold_codepoint_to_ascii(str + pos, bytes, &ascii)) {
|
||||
result.append(ascii);
|
||||
} else {
|
||||
// Keep the original codepoint when it has no ASCII equivalent.
|
||||
result.append(token.text, static_cast<size_t>(pos),
|
||||
static_cast<size_t>(bytes));
|
||||
}
|
||||
pos += bytes;
|
||||
}
|
||||
token.text = std::move(result);
|
||||
}
|
||||
// Folding may leave empty tokens from empty input. Remove them.
|
||||
tokens.erase(std::remove_if(tokens.begin(), tokens.end(),
|
||||
[](const Token &t) { return t.text.empty(); }),
|
||||
tokens.end());
|
||||
return tokens;
|
||||
}
|
||||
|
||||
} // namespace zvec::fts
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "token_filter.h"
|
||||
|
||||
namespace zvec::fts {
|
||||
|
||||
/*! ASCII Folding Token Filter
|
||||
* Convert Unicode characters to their ASCII equivalents per codepoint via
|
||||
* NFKD decomposition (utf8proc) with a supplementary folding table for
|
||||
* characters that lack decomposition mappings (e.g. ø→o, đ→d, ß→ss).
|
||||
* Characters without a reasonable ASCII equivalent are kept as-is.
|
||||
*/
|
||||
class AsciiFoldingTokenFilter : public TokenFilter {
|
||||
public:
|
||||
std::vector<Token> filter(std::vector<Token> tokens) const override;
|
||||
|
||||
const char *name() const override {
|
||||
return "ascii_folding";
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace zvec::fts
|
||||
|
|
@ -13,13 +13,58 @@
|
|||
// limitations under the License.
|
||||
|
||||
#include "standard_tokenizer.h"
|
||||
#include <cctype>
|
||||
#include <utf8proc.h>
|
||||
|
||||
namespace zvec::fts {
|
||||
|
||||
namespace {
|
||||
|
||||
bool is_word_start_char(utf8proc_category_t cat) {
|
||||
switch (cat) {
|
||||
case UTF8PROC_CATEGORY_LU: // Letter, uppercase
|
||||
case UTF8PROC_CATEGORY_LL: // Letter, lowercase
|
||||
case UTF8PROC_CATEGORY_LT: // Letter, titlecase
|
||||
case UTF8PROC_CATEGORY_LM: // Letter, modifier
|
||||
case UTF8PROC_CATEGORY_LO: // Letter, other
|
||||
case UTF8PROC_CATEGORY_ND: // Number, decimal digit
|
||||
case UTF8PROC_CATEGORY_NL: // Number, letter
|
||||
case UTF8PROC_CATEGORY_NO: // Number, other
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool is_word_continue_char(utf8proc_category_t cat) {
|
||||
switch (cat) {
|
||||
case UTF8PROC_CATEGORY_MN: // Mark, nonspacing
|
||||
case UTF8PROC_CATEGORY_MC: // Mark, spacing combining
|
||||
case UTF8PROC_CATEGORY_ME: // Mark, enclosing
|
||||
return true;
|
||||
default:
|
||||
return is_word_start_char(cat);
|
||||
}
|
||||
}
|
||||
|
||||
bool is_cjk_ideograph(utf8proc_int32_t cp) {
|
||||
return (cp >= 0x4E00 && cp <= 0x9FFF) || // CJK Unified Ideographs
|
||||
(cp >= 0x3400 && cp <= 0x4DBF) || // CJK Extension A
|
||||
(cp >= 0xF900 && cp <= 0xFAFF) || // CJK Compatibility Ideographs
|
||||
(cp >= 0x20000 && cp <= 0x2A6DF) || // CJK Extension B
|
||||
(cp >= 0x2A700 && cp <= 0x2B73F) || // CJK Extension C
|
||||
(cp >= 0x2B740 && cp <= 0x2B81F) || // CJK Extension D
|
||||
(cp >= 0x2B820 && cp <= 0x2CEAF) || // CJK Extension E
|
||||
(cp >= 0x2CEB0 && cp <= 0x2EBEF) || // CJK Extension F
|
||||
(cp >= 0x2EBF0 && cp <= 0x2EE5F) || // CJK Extension I
|
||||
(cp >= 0x2F800 && cp <= 0x2FA1F) || // CJK Compatibility Supplement
|
||||
(cp >= 0x30000 && cp <= 0x3134F) || // CJK Extension G
|
||||
(cp >= 0x31350 && cp <= 0x323AF) || // CJK Extension H
|
||||
(cp >= 0x323B0 && cp <= 0x3347F); // CJK Extension J
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool StandardTokenizer::init(const ailego::JsonObject &config) {
|
||||
// Read optional max_token_length; keep default (255) if not present or
|
||||
// if the provided value is zero.
|
||||
auto length_val = config["max_token_length"];
|
||||
if (length_val.is_integer()) {
|
||||
uint32_t configured_length = static_cast<uint32_t>(length_val.as_integer());
|
||||
|
|
@ -33,41 +78,84 @@ bool StandardTokenizer::init(const ailego::JsonObject &config) {
|
|||
std::vector<Token> StandardTokenizer::tokenize(const std::string &text) const {
|
||||
std::vector<Token> tokens;
|
||||
uint32_t position = 0;
|
||||
size_t index = 0;
|
||||
const size_t text_length = text.size();
|
||||
const auto *str = reinterpret_cast<const utf8proc_uint8_t *>(text.data());
|
||||
auto len = static_cast<utf8proc_ssize_t>(text.size());
|
||||
utf8proc_ssize_t index = 0;
|
||||
|
||||
while (index < text_length) {
|
||||
// Skip non-alphanumeric characters (delimiters / punctuation).
|
||||
while (index < text_length &&
|
||||
!std::isalnum(static_cast<unsigned char>(text[index]))) {
|
||||
while (index < len) {
|
||||
// Decode current codepoint.
|
||||
utf8proc_int32_t cp;
|
||||
utf8proc_ssize_t bytes = utf8proc_iterate(str + index, len - index, &cp);
|
||||
if (bytes < 1) {
|
||||
++index;
|
||||
}
|
||||
if (index >= text_length) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Mark the start of an alphanumeric run.
|
||||
const uint32_t token_start = static_cast<uint32_t>(index);
|
||||
|
||||
// Advance to the end of the alphanumeric run.
|
||||
while (index < text_length &&
|
||||
std::isalnum(static_cast<unsigned char>(text[index]))) {
|
||||
++index;
|
||||
}
|
||||
|
||||
const uint32_t token_length = static_cast<uint32_t>(index) - token_start;
|
||||
|
||||
// Discard tokens that exceed the configured length limit.
|
||||
if (token_length > max_token_length_) {
|
||||
++position;
|
||||
continue;
|
||||
}
|
||||
|
||||
Token token;
|
||||
token.text = text.substr(token_start, token_length);
|
||||
token.offset = token_start;
|
||||
token.position = position++;
|
||||
tokens.push_back(std::move(token));
|
||||
// CJK ideograph → emit as a single-character token (always 1 codepoint,
|
||||
// which cannot exceed max_token_length_ since its minimum value is 1).
|
||||
if (is_cjk_ideograph(cp)) {
|
||||
Token token;
|
||||
token.text =
|
||||
text.substr(static_cast<size_t>(index), static_cast<size_t>(bytes));
|
||||
token.offset = static_cast<uint32_t>(index);
|
||||
token.position = position++;
|
||||
tokens.push_back(std::move(token));
|
||||
index += bytes;
|
||||
continue;
|
||||
}
|
||||
|
||||
auto cat = utf8proc_category(cp);
|
||||
// Skip delimiters and continuation-only characters that cannot start words.
|
||||
if (!is_word_start_char(cat)) {
|
||||
index += bytes;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Accumulate a word token. Marks can continue a token, but cannot start
|
||||
// one. Split at max_token_length_ codepoints (aligned with ES behavior).
|
||||
utf8proc_ssize_t token_start = index;
|
||||
uint32_t codepoint_count = 1;
|
||||
index += bytes;
|
||||
while (index < len) {
|
||||
utf8proc_int32_t next_cp;
|
||||
utf8proc_ssize_t next_bytes =
|
||||
utf8proc_iterate(str + index, len - index, &next_cp);
|
||||
if (next_bytes < 1) {
|
||||
break;
|
||||
}
|
||||
if (is_cjk_ideograph(next_cp)) {
|
||||
break;
|
||||
}
|
||||
auto next_cat = utf8proc_category(next_cp);
|
||||
if (!is_word_continue_char(next_cat)) {
|
||||
break;
|
||||
}
|
||||
// Emit a segment when codepoint count reaches the limit, but do not
|
||||
// create a segment that starts with a continuation-only mark.
|
||||
if (codepoint_count >= max_token_length_ &&
|
||||
is_word_start_char(next_cat)) {
|
||||
Token token;
|
||||
token.text = text.substr(static_cast<size_t>(token_start),
|
||||
static_cast<size_t>(index - token_start));
|
||||
token.offset = static_cast<uint32_t>(token_start);
|
||||
token.position = position++;
|
||||
tokens.push_back(std::move(token));
|
||||
token_start = index;
|
||||
codepoint_count = 0;
|
||||
}
|
||||
++codepoint_count;
|
||||
index += next_bytes;
|
||||
}
|
||||
|
||||
// Emit the remaining segment (if any).
|
||||
if (index > token_start) {
|
||||
Token token;
|
||||
token.text = text.substr(static_cast<size_t>(token_start),
|
||||
static_cast<size_t>(index - token_start));
|
||||
token.offset = static_cast<uint32_t>(token_start);
|
||||
token.position = position++;
|
||||
tokens.push_back(std::move(token));
|
||||
}
|
||||
}
|
||||
|
||||
return tokens;
|
||||
|
|
|
|||
|
|
@ -20,16 +20,17 @@
|
|||
namespace zvec::fts {
|
||||
|
||||
/*! Standard tokenizer
|
||||
* Splits text on non-alphanumeric characters (punctuation, whitespace, etc.)
|
||||
* and discards the delimiters. Produces lowercase-ready tokens composed of
|
||||
* letters and digits only.
|
||||
* Unicode-aware tokenizer aligned with Elasticsearch's standard tokenizer.
|
||||
* Splits text on non-alphanumeric characters (punctuation, whitespace,
|
||||
* symbols) using Unicode categories via utf8proc. CJK ideographs are emitted
|
||||
* as individual single-character tokens.
|
||||
*/
|
||||
class StandardTokenizer : public Tokenizer {
|
||||
public:
|
||||
/*! Initialise from JSON config.
|
||||
* Supported keys:
|
||||
* "max_token_length" (uint32, default 255): tokens longer than this limit
|
||||
* are silently discarded.
|
||||
* "max_token_length" (uint32, default 255): tokens with more codepoints
|
||||
* than this limit are split at the boundary into multiple tokens.
|
||||
* Always returns true.
|
||||
*/
|
||||
bool init(const ailego::JsonObject &config) override;
|
||||
|
|
@ -41,7 +42,7 @@ class StandardTokenizer : public Tokenizer {
|
|||
}
|
||||
|
||||
private:
|
||||
// Tokens whose byte length exceeds this value are discarded.
|
||||
// Word tokens with more codepoints than this value are split.
|
||||
uint32_t max_token_length_{255};
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -13,18 +13,37 @@
|
|||
// limitations under the License.
|
||||
|
||||
#include "token_filter.h"
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <utf8proc.h>
|
||||
#include <string>
|
||||
|
||||
namespace zvec::fts {
|
||||
|
||||
std::vector<Token> LowercaseTokenFilter::filter(
|
||||
std::vector<Token> tokens) const {
|
||||
for (auto &token : tokens) {
|
||||
std::transform(token.text.begin(), token.text.end(), token.text.begin(),
|
||||
[](unsigned char character) {
|
||||
return static_cast<char>(std::tolower(character));
|
||||
});
|
||||
std::string result;
|
||||
result.reserve(token.text.size());
|
||||
const auto *str =
|
||||
reinterpret_cast<const utf8proc_uint8_t *>(token.text.data());
|
||||
auto len = static_cast<utf8proc_ssize_t>(token.text.size());
|
||||
utf8proc_ssize_t pos = 0;
|
||||
while (pos < len) {
|
||||
utf8proc_int32_t codepoint;
|
||||
utf8proc_ssize_t bytes =
|
||||
utf8proc_iterate(str + pos, len - pos, &codepoint);
|
||||
if (bytes < 1) {
|
||||
result.push_back(token.text[pos]);
|
||||
++pos;
|
||||
continue;
|
||||
}
|
||||
utf8proc_int32_t lower = utf8proc_tolower(codepoint);
|
||||
utf8proc_uint8_t buf[4];
|
||||
utf8proc_ssize_t written = utf8proc_encode_char(lower, buf);
|
||||
result.append(reinterpret_cast<const char *>(buf),
|
||||
static_cast<size_t>(written));
|
||||
pos += bytes;
|
||||
}
|
||||
token.text = std::move(result);
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ class TokenFilter {
|
|||
using TokenFilterPtr = std::shared_ptr<TokenFilter>;
|
||||
|
||||
/*! Lowercase Token Filter
|
||||
* Convert all token text to lowercase (only handles ASCII characters)
|
||||
* Convert all token text to lowercase (supports full Unicode via utf8proc)
|
||||
*/
|
||||
class LowercaseTokenFilter : public TokenFilter {
|
||||
public:
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
#include "tokenizer_factory.h"
|
||||
#include <zvec/ailego/encoding/json/mod_json_plus.h>
|
||||
#include <zvec/ailego/logger/logger.h>
|
||||
#include "ascii_folding_token_filter.h"
|
||||
#include "jieba_tokenizer.h"
|
||||
#include "standard_tokenizer.h"
|
||||
#include "whitespace_tokenizer.h"
|
||||
|
|
@ -96,6 +97,8 @@ TokenizerPtr TokenizerFactory::create_tokenizer(
|
|||
TokenFilterPtr TokenizerFactory::create_filter(const std::string &filter_name) {
|
||||
if (filter_name == "lowercase") {
|
||||
return std::make_shared<LowercaseTokenFilter>();
|
||||
} else if (filter_name == "ascii_folding") {
|
||||
return std::make_shared<AsciiFoldingTokenFilter>();
|
||||
}
|
||||
LOG_ERROR("[TokenizerFactory] unknown filter name: %s", filter_name.c_str());
|
||||
return nullptr;
|
||||
|
|
|
|||
|
|
@ -44,6 +44,8 @@ using TokenizerPipelinePtr = std::shared_ptr<TokenizerPipeline>;
|
|||
|
||||
/*! Tokenizer factory
|
||||
* Create TokenizerPipeline based on FtsIndexParams configuration.
|
||||
* Supported tokenizers: standard, jieba, whitespace.
|
||||
* Supported filters: lowercase, ascii_folding.
|
||||
*/
|
||||
class TokenizerFactory {
|
||||
public:
|
||||
|
|
|
|||
|
|
@ -1089,8 +1089,10 @@ ZVEC_EXPORT zvec_error_code_t ZVEC_CALL zvec_index_params_set_invert_params(
|
|||
/**
|
||||
* @brief Set FTS index specific parameters
|
||||
* @param params Index parameters (must be FTS type)
|
||||
* @param tokenizer_name Tokenizer pipeline name (NULL keeps current value)
|
||||
* @param filters Token filter names (NULL keeps current value)
|
||||
* @param tokenizer_name Tokenizer name: "standard", "jieba", or "whitespace"
|
||||
* (NULL keeps current value)
|
||||
* @param filters Token filter names: "lowercase" and/or "ascii_folding"
|
||||
* (NULL keeps current value)
|
||||
* @param extra_params Additional tokenizer parameters (NULL keeps current
|
||||
* value)
|
||||
* @return ZVEC_OK on success, error code on failure
|
||||
|
|
|
|||
|
|
@ -717,6 +717,8 @@ class VamanaIndexParams : public VectorIndexParams {
|
|||
|
||||
/*
|
||||
* FTS (Full-Text Search) index params
|
||||
* Supported tokenizers: "standard", "jieba", "whitespace".
|
||||
* Supported filters: "lowercase", "ascii_folding".
|
||||
*
|
||||
* Not copyable. Use shared_ptr<FtsIndexParams> for shared ownership.
|
||||
*/
|
||||
|
|
@ -795,4 +797,4 @@ class FtsIndexParams : public IndexParams {
|
|||
friend struct detail::FtsPipelineHelper;
|
||||
};
|
||||
|
||||
} // namespace zvec
|
||||
} // namespace zvec
|
||||
|
|
|
|||
|
|
@ -0,0 +1,265 @@
|
|||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "db/index/column/fts_column/tokenizer/ascii_folding_token_filter.h"
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
using namespace zvec::fts;
|
||||
|
||||
static std::vector<Token> make_tokens(const std::vector<std::string> &texts) {
|
||||
std::vector<Token> tokens;
|
||||
for (size_t i = 0; i < texts.size(); ++i) {
|
||||
tokens.push_back({texts[i], 0, static_cast<uint32_t>(i)});
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
class AsciiFoldingTokenFilterTest : public ::testing::Test {
|
||||
protected:
|
||||
AsciiFoldingTokenFilter filter_;
|
||||
};
|
||||
|
||||
// --- Pure ASCII passthrough ---
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, AsciiPassthrough) {
|
||||
auto result = filter_.filter(make_tokens({"hello", "world", "123"}));
|
||||
ASSERT_EQ(result.size(), 3u);
|
||||
EXPECT_EQ(result[0].text, "hello");
|
||||
EXPECT_EQ(result[1].text, "world");
|
||||
EXPECT_EQ(result[2].text, "123");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, EmptyToken) {
|
||||
auto result = filter_.filter(make_tokens({""}));
|
||||
EXPECT_TRUE(result.empty());
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, EmptyList) {
|
||||
std::vector<Token> tokens;
|
||||
auto result = filter_.filter(std::move(tokens));
|
||||
EXPECT_TRUE(result.empty());
|
||||
}
|
||||
|
||||
// --- Latin diacritics (NFKD + STRIPMARK handles these) ---
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, LatinAccentedVowels) {
|
||||
// àáâãäå → aaaaaa
|
||||
auto result = filter_.filter(
|
||||
make_tokens({"\xC3\xA0\xC3\xA1\xC3\xA2\xC3\xA3\xC3\xA4\xC3\xA5"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "aaaaaa");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, LatinAccentedConsonants) {
|
||||
// ñ → n, ç → c
|
||||
auto result = filter_.filter(make_tokens({"\xC3\xB1", "\xC3\xA7"}));
|
||||
ASSERT_EQ(result.size(), 2u);
|
||||
EXPECT_EQ(result[0].text, "n");
|
||||
EXPECT_EQ(result[1].text, "c");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, UppercaseAccented) {
|
||||
// ÜBER → UBER
|
||||
auto result =
|
||||
filter_.filter(make_tokens({"\xC3\x9C"
|
||||
"BER"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "UBER");
|
||||
}
|
||||
|
||||
// --- Supplementary table entries ---
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, SharpS) {
|
||||
// ß → ss
|
||||
auto result = filter_.filter(make_tokens({"\xC3\x9F"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "ss");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, OWithStroke) {
|
||||
// ø → o
|
||||
auto result = filter_.filter(make_tokens({"\xC3\xB8"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "o");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, DWithStroke) {
|
||||
// đ → d
|
||||
auto result = filter_.filter(make_tokens({"\xC4\x91"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "d");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, LWithStroke) {
|
||||
// ł → l
|
||||
auto result = filter_.filter(make_tokens({"\xC5\x82"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "l");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, Eth) {
|
||||
// ð → d
|
||||
auto result = filter_.filter(make_tokens({"\xC3\xB0"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "d");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, Thorn) {
|
||||
// þ → th
|
||||
auto result = filter_.filter(make_tokens({"\xC3\xBE"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "th");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, OeLigature) {
|
||||
// œ → oe
|
||||
auto result = filter_.filter(make_tokens({"\xC5\x93"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "oe");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, AeLigature) {
|
||||
// Æ → AE, æ → ae
|
||||
auto result = filter_.filter(make_tokens({"\xC3\x86", "\xC3\xA6"}));
|
||||
ASSERT_EQ(result.size(), 2u);
|
||||
EXPECT_EQ(result[0].text, "AE");
|
||||
EXPECT_EQ(result[1].text, "ae");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, CapitalThorn) {
|
||||
// Þ → TH
|
||||
auto result = filter_.filter(make_tokens({"\xC3\x9E"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "TH");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, IjLigature) {
|
||||
// IJ → IJ, ij → ij
|
||||
auto result = filter_.filter(make_tokens({"\xC4\xB2", "\xC4\xB3"}));
|
||||
ASSERT_EQ(result.size(), 2u);
|
||||
EXPECT_EQ(result[0].text, "IJ");
|
||||
EXPECT_EQ(result[1].text, "ij");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, CapitalSharpS) {
|
||||
// ẞ (U+1E9E) → SS
|
||||
auto result = filter_.filter(make_tokens({"\xE1\xBA\x9E"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "SS");
|
||||
}
|
||||
|
||||
// --- Ligatures handled by NFKD ---
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, FiLigature) {
|
||||
// fi (U+FB01) → fi
|
||||
auto result = filter_.filter(make_tokens({"\xEF\xAC\x81"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "fi");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, FlLigature) {
|
||||
// fl (U+FB02) → fl
|
||||
auto result = filter_.filter(make_tokens({"\xEF\xAC\x82"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "fl");
|
||||
}
|
||||
|
||||
// --- Fullwidth forms ---
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, FullwidthLatinCapital) {
|
||||
// Ａ (U+FF21) → A
|
||||
auto result = filter_.filter(make_tokens({"\xEF\xBC\xA1"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "A");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, FullwidthDigit) {
|
||||
// ０ (U+FF10) → 0
|
||||
auto result = filter_.filter(make_tokens({"\xEF\xBC\x90"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "0");
|
||||
}
|
||||
|
||||
// --- CJK passthrough (no ASCII equivalent) ---
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, CJKPassthrough) {
|
||||
// 中文 should remain unchanged
|
||||
auto result = filter_.filter(make_tokens({"\xE4\xB8\xAD\xE6\x96\x87"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "\xE4\xB8\xAD\xE6\x96\x87");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, GreekTonosPassthrough) {
|
||||
// Greek ά has no ASCII equivalent, so it should remain unchanged.
|
||||
auto result = filter_.filter(make_tokens({"\xCE\xAC"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "\xCE\xAC");
|
||||
}
|
||||
|
||||
// --- Mixed content ---
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, MixedAsciiAndAccented) {
|
||||
// café → cafe
|
||||
auto result = filter_.filter(make_tokens({"caf\xC3\xA9"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "cafe");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, MixedAsciiAndCJK) {
|
||||
// hello中文 → hello中文 (ASCII kept, CJK kept)
|
||||
auto result = filter_.filter(make_tokens({"hello\xE4\xB8\xAD\xE6\x96\x87"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "hello\xE4\xB8\xAD\xE6\x96\x87");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, MixedLatinAndGreekTonos) {
|
||||
// caféά → cafeά (Latin folds, Greek stays original)
|
||||
auto result = filter_.filter(make_tokens({"caf\xC3\xA9\xCE\xAC"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "cafe\xCE\xAC");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, DecomposedLatinCombiningMarkPassthrough) {
|
||||
// Align with ES/Lucene asciifolding: combining marks are not folded by
|
||||
// themselves, so decomposed "cafe + U+0301" remains unchanged.
|
||||
auto result = filter_.filter(make_tokens({"cafe\xCC\x81"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "cafe\xCC\x81");
|
||||
}
|
||||
|
||||
// --- Preserves offset and position ---
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, PreservesOffsetAndPosition) {
|
||||
std::vector<Token> tokens = {{"\xC3\xA9", 10, 5}};
|
||||
auto result = filter_.filter(std::move(tokens));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "e");
|
||||
EXPECT_EQ(result[0].offset, 10u);
|
||||
EXPECT_EQ(result[0].position, 5u);
|
||||
}
|
||||
|
||||
// --- Name ---
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, FilterName) {
|
||||
EXPECT_STREQ(filter_.name(), "ascii_folding");
|
||||
}
|
||||
|
||||
TEST_F(AsciiFoldingTokenFilterTest, StandaloneCombiningMarkPassthrough) {
|
||||
// U+0301 COMBINING ACUTE ACCENT has no ASCII equivalent on its own.
|
||||
auto result = filter_.filter(make_tokens({"\xCC\x81"}));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "\xCC\x81");
|
||||
}
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <gtest/gtest.h>
|
||||
#include "db/index/column/fts_column/tokenizer/token_filter.h"
|
||||
|
||||
using namespace zvec::fts;
|
||||
|
||||
static std::vector<Token> make_tokens(const std::vector<std::string> &texts) {
|
||||
std::vector<Token> tokens;
|
||||
for (size_t i = 0; i < texts.size(); ++i) {
|
||||
tokens.push_back({texts[i], 0, static_cast<uint32_t>(i)});
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
class LowercaseTokenFilterTest : public ::testing::Test {
|
||||
protected:
|
||||
LowercaseTokenFilter filter_;
|
||||
};
|
||||
|
||||
TEST_F(LowercaseTokenFilterTest, AsciiBasic) {
|
||||
auto tokens = make_tokens({"Hello", "WORLD", "FoO"});
|
||||
auto result = filter_.filter(std::move(tokens));
|
||||
ASSERT_EQ(result.size(), 3u);
|
||||
EXPECT_EQ(result[0].text, "hello");
|
||||
EXPECT_EQ(result[1].text, "world");
|
||||
EXPECT_EQ(result[2].text, "foo");
|
||||
}
|
||||
|
||||
TEST_F(LowercaseTokenFilterTest, AlreadyLowercase) {
|
||||
auto tokens = make_tokens({"already", "lower"});
|
||||
auto result = filter_.filter(std::move(tokens));
|
||||
ASSERT_EQ(result.size(), 2u);
|
||||
EXPECT_EQ(result[0].text, "already");
|
||||
EXPECT_EQ(result[1].text, "lower");
|
||||
}
|
||||
|
||||
TEST_F(LowercaseTokenFilterTest, EmptyToken) {
|
||||
auto tokens = make_tokens({""});
|
||||
auto result = filter_.filter(std::move(tokens));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "");
|
||||
}
|
||||
|
||||
TEST_F(LowercaseTokenFilterTest, EmptyList) {
|
||||
std::vector<Token> tokens;
|
||||
auto result = filter_.filter(std::move(tokens));
|
||||
EXPECT_TRUE(result.empty());
|
||||
}
|
||||
|
||||
TEST_F(LowercaseTokenFilterTest, LatinExtended) {
|
||||
// German uppercase with umlauts
|
||||
auto tokens =
|
||||
make_tokens({"\xC3\x9C"
|
||||
"BER"}); // "ÜBER"
|
||||
auto result = filter_.filter(std::move(tokens));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text,
|
||||
"\xC3\xBC"
|
||||
"ber"); // "über"
|
||||
}
|
||||
|
||||
TEST_F(LowercaseTokenFilterTest, Cyrillic) {
|
||||
// "МОСКВА" -> "москва"
|
||||
auto tokens =
|
||||
make_tokens({"\xD0\x9C\xD0\x9E\xD0\xA1\xD0\x9A\xD0\x92\xD0\x90"});
|
||||
auto result = filter_.filter(std::move(tokens));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "\xD0\xBC\xD0\xBE\xD1\x81\xD0\xBA\xD0\xB2\xD0\xB0");
|
||||
}
|
||||
|
||||
TEST_F(LowercaseTokenFilterTest, Greek) {
|
||||
// "ΔΕΛΤΑ" -> "δελτα"
|
||||
auto tokens = make_tokens({"\xCE\x94\xCE\x95\xCE\x9B\xCE\xA4\xCE\x91"});
|
||||
auto result = filter_.filter(std::move(tokens));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "\xCE\xB4\xCE\xB5\xCE\xBB\xCF\x84\xCE\xB1");
|
||||
}
|
||||
|
||||
TEST_F(LowercaseTokenFilterTest, MixedScripts) {
|
||||
// "Hello Мир" -> "hello мир"
|
||||
auto tokens = make_tokens({"Hello \xD0\x9C\xD0\xB8\xD1\x80"});
|
||||
auto result = filter_.filter(std::move(tokens));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "hello \xD0\xBC\xD0\xB8\xD1\x80");
|
||||
}
|
||||
|
||||
TEST_F(LowercaseTokenFilterTest, NumbersAndPunctuation) {
|
||||
auto tokens = make_tokens({"ABC123!@#"});
|
||||
auto result = filter_.filter(std::move(tokens));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "abc123!@#");
|
||||
}
|
||||
|
||||
TEST_F(LowercaseTokenFilterTest, CJKPassthrough) {
|
||||
// CJK characters have no case — should pass through unchanged
|
||||
auto tokens = make_tokens({"\xE4\xB8\xAD\xE6\x96\x87"}); // "中文"
|
||||
auto result = filter_.filter(std::move(tokens));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "\xE4\xB8\xAD\xE6\x96\x87");
|
||||
}
|
||||
|
||||
TEST_F(LowercaseTokenFilterTest, PreservesOffsetAndPosition) {
|
||||
std::vector<Token> tokens = {{"ABC", 5, 3}};
|
||||
auto result = filter_.filter(std::move(tokens));
|
||||
ASSERT_EQ(result.size(), 1u);
|
||||
EXPECT_EQ(result[0].text, "abc");
|
||||
EXPECT_EQ(result[0].offset, 5);
|
||||
EXPECT_EQ(result[0].position, 3);
|
||||
}
|
||||
|
|
@ -0,0 +1,281 @@
|
|||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <gtest/gtest.h>
|
||||
#include "db/index/column/fts_column/fts_types.h"
|
||||
#include "db/index/column/fts_column/tokenizer/tokenizer_factory.h"
|
||||
|
||||
using namespace zvec::fts;
|
||||
|
||||
class StandardTokenizerTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
FtsIndexParams params;
|
||||
params.tokenizer_name = "standard";
|
||||
params.filters.clear();
|
||||
pipeline_ = TokenizerFactory::create(params);
|
||||
ASSERT_NE(pipeline_, nullptr);
|
||||
}
|
||||
|
||||
std::vector<Token> tokenize(const std::string &text) {
|
||||
return pipeline_->process(text);
|
||||
}
|
||||
|
||||
TokenizerPipelinePtr pipeline_;
|
||||
};
|
||||
|
||||
// --- ASCII basics (existing behavior preserved) ---
|
||||
|
||||
TEST_F(StandardTokenizerTest, SimpleAsciiWords) {
|
||||
auto tokens = tokenize("hello world");
|
||||
ASSERT_EQ(tokens.size(), 2u);
|
||||
EXPECT_EQ(tokens[0].text, "hello");
|
||||
EXPECT_EQ(tokens[1].text, "world");
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, PunctuationAsDelimiter) {
|
||||
auto tokens = tokenize("hello,world.test");
|
||||
ASSERT_EQ(tokens.size(), 3u);
|
||||
EXPECT_EQ(tokens[0].text, "hello");
|
||||
EXPECT_EQ(tokens[1].text, "world");
|
||||
EXPECT_EQ(tokens[2].text, "test");
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, LettersAndDigitsTogether) {
|
||||
auto tokens = tokenize("abc123 xyz");
|
||||
ASSERT_EQ(tokens.size(), 2u);
|
||||
EXPECT_EQ(tokens[0].text, "abc123");
|
||||
EXPECT_EQ(tokens[1].text, "xyz");
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, EmptyInput) {
|
||||
auto tokens = tokenize("");
|
||||
EXPECT_TRUE(tokens.empty());
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, OnlyDelimiters) {
|
||||
auto tokens = tokenize(" .,;! ");
|
||||
EXPECT_TRUE(tokens.empty());
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, OffsetAndPosition) {
|
||||
auto tokens = tokenize(" hello world");
|
||||
ASSERT_EQ(tokens.size(), 2u);
|
||||
EXPECT_EQ(tokens[0].offset, 2u);
|
||||
EXPECT_EQ(tokens[0].position, 0u);
|
||||
EXPECT_EQ(tokens[1].offset, 8u);
|
||||
EXPECT_EQ(tokens[1].position, 1u);
|
||||
}
|
||||
|
||||
// --- Accented Latin ---
|
||||
|
||||
TEST_F(StandardTokenizerTest, AccentedLatin) {
|
||||
// café résumé → ["café", "résumé"]
|
||||
auto tokens = tokenize("caf\xC3\xA9 r\xC3\xA9sum\xC3\xA9");
|
||||
ASSERT_EQ(tokens.size(), 2u);
|
||||
EXPECT_EQ(tokens[0].text, "caf\xC3\xA9");
|
||||
EXPECT_EQ(tokens[1].text, "r\xC3\xA9sum\xC3\xA9");
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, MarksContinueButDoNotStartTokens) {
|
||||
// e + U+0301 keeps the combining mark with the base letter.
|
||||
// Standalone U+0301 and the heart variation selector are not indexed.
|
||||
auto tokens = tokenize(
|
||||
"e\xCC\x81 "
|
||||
"\xCC\x81 "
|
||||
"\xE2\x9D\xA4\xEF\xB8\x8F");
|
||||
ASSERT_EQ(tokens.size(), 1u);
|
||||
EXPECT_EQ(tokens[0].text, "e\xCC\x81");
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, MaxTokenLengthDoesNotCreateMarkOnlyToken) {
|
||||
FtsIndexParams params;
|
||||
params.tokenizer_name = "standard";
|
||||
params.filters.clear();
|
||||
params.extra_params = R"({"max_token_length":2})";
|
||||
auto pipeline = TokenizerFactory::create(params);
|
||||
ASSERT_NE(pipeline, nullptr);
|
||||
|
||||
// ab + U+0301 + c should not split into a standalone combining mark token.
|
||||
auto tokens = pipeline->process(
|
||||
"ab\xCC\x81"
|
||||
"c");
|
||||
ASSERT_EQ(tokens.size(), 2u);
|
||||
EXPECT_EQ(tokens[0].text, "ab\xCC\x81");
|
||||
EXPECT_EQ(tokens[1].text, "c");
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, GermanUmlaut) {
|
||||
// Über Straße → ["Über", "Straße"]
|
||||
auto tokens = tokenize(
|
||||
"\xC3\x9C"
|
||||
"ber Stra\xC3\x9F"
|
||||
"e");
|
||||
ASSERT_EQ(tokens.size(), 2u);
|
||||
EXPECT_EQ(tokens[0].text,
|
||||
"\xC3\x9C"
|
||||
"ber");
|
||||
EXPECT_EQ(tokens[1].text,
|
||||
"Stra\xC3\x9F"
|
||||
"e");
|
||||
}
|
||||
|
||||
// --- Cyrillic ---
|
||||
|
||||
TEST_F(StandardTokenizerTest, Cyrillic) {
|
||||
// Москва Россия → ["Москва", "Россия"]
|
||||
auto tokens = tokenize(
|
||||
"\xD0\x9C\xD0\xBE\xD1\x81\xD0\xBA\xD0\xB2\xD0\xB0 "
|
||||
"\xD0\xA0\xD0\xBE\xD1\x81\xD1\x81\xD0\xB8\xD1\x8F");
|
||||
ASSERT_EQ(tokens.size(), 2u);
|
||||
EXPECT_EQ(tokens[0].text, "\xD0\x9C\xD0\xBE\xD1\x81\xD0\xBA\xD0\xB2\xD0\xB0");
|
||||
EXPECT_EQ(tokens[1].text, "\xD0\xA0\xD0\xBE\xD1\x81\xD1\x81\xD0\xB8\xD1\x8F");
|
||||
}
|
||||
|
||||
// --- CJK single-character tokenization ---
|
||||
|
||||
TEST_F(StandardTokenizerTest, CJKSingleChar) {
|
||||
// 全文检索 → ["全", "文", "检", "索"]
|
||||
auto tokens = tokenize("\xE5\x85\xA8\xE6\x96\x87\xE6\xA3\x80\xE7\xB4\xA2");
|
||||
ASSERT_EQ(tokens.size(), 4u);
|
||||
EXPECT_EQ(tokens[0].text, "\xE5\x85\xA8"); // 全
|
||||
EXPECT_EQ(tokens[1].text, "\xE6\x96\x87"); // 文
|
||||
EXPECT_EQ(tokens[2].text, "\xE6\xA3\x80"); // 检
|
||||
EXPECT_EQ(tokens[3].text, "\xE7\xB4\xA2"); // 索
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, CJKWithSpaces) {
|
||||
// 你 好 → ["你", "好"]
|
||||
auto tokens = tokenize("\xE4\xBD\xA0 \xE5\xA5\xBD");
|
||||
ASSERT_EQ(tokens.size(), 2u);
|
||||
EXPECT_EQ(tokens[0].text, "\xE4\xBD\xA0");
|
||||
EXPECT_EQ(tokens[1].text, "\xE5\xA5\xBD");
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, CJKUnicode17ExtensionBlocks) {
|
||||
// U+2EBF0 (Extension I), U+31350 (Extension H), U+323B0 (Extension J)
|
||||
// should each be emitted as an individual CJK token.
|
||||
auto tokens = tokenize("\xF0\xAE\xAF\xB0\xF0\xB1\x8D\x90\xF0\xB2\x8E\xB0");
|
||||
ASSERT_EQ(tokens.size(), 3u);
|
||||
EXPECT_EQ(tokens[0].text, "\xF0\xAE\xAF\xB0");
|
||||
EXPECT_EQ(tokens[1].text, "\xF0\xB1\x8D\x90");
|
||||
EXPECT_EQ(tokens[2].text, "\xF0\xB2\x8E\xB0");
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, CJKCompatibilitySupplement) {
|
||||
// U+2F800 CJK Compatibility Ideographs Supplement.
|
||||
auto tokens = tokenize("\xF0\xAF\xA0\x80");
|
||||
ASSERT_EQ(tokens.size(), 1u);
|
||||
EXPECT_EQ(tokens[0].text, "\xF0\xAF\xA0\x80");
|
||||
}
|
||||
|
||||
// --- Mixed scripts ---
|
||||
|
||||
TEST_F(StandardTokenizerTest, MixedLatinAndCJK) {
|
||||
// hello世界test → ["hello", "世", "界", "test"]
|
||||
auto tokens = tokenize("hello\xE4\xB8\x96\xE7\x95\x8Ctest");
|
||||
ASSERT_EQ(tokens.size(), 4u);
|
||||
EXPECT_EQ(tokens[0].text, "hello");
|
||||
EXPECT_EQ(tokens[1].text, "\xE4\xB8\x96"); // 世
|
||||
EXPECT_EQ(tokens[2].text, "\xE7\x95\x8C"); // 界
|
||||
EXPECT_EQ(tokens[3].text, "test");
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, CJKWithLatinAndDigits) {
|
||||
// ES标准分词器v2 → ["ES", "标", "准", "分", "词", "器", "v2"]
|
||||
auto tokens = tokenize(
|
||||
"ES\xE6\xA0\x87\xE5\x87\x86\xE5\x88\x86"
|
||||
"\xE8\xAF\x8D\xE5\x99\xA8v2");
|
||||
ASSERT_EQ(tokens.size(), 7u);
|
||||
EXPECT_EQ(tokens[0].text, "ES");
|
||||
EXPECT_EQ(tokens[1].text, "\xE6\xA0\x87"); // 标
|
||||
EXPECT_EQ(tokens[2].text, "\xE5\x87\x86"); // 准
|
||||
EXPECT_EQ(tokens[3].text, "\xE5\x88\x86"); // 分
|
||||
EXPECT_EQ(tokens[4].text, "\xE8\xAF\x8D"); // 词
|
||||
EXPECT_EQ(tokens[5].text, "\xE5\x99\xA8"); // 器
|
||||
EXPECT_EQ(tokens[6].text, "v2");
|
||||
}
|
||||
|
||||
// --- Consecutive positions ---
|
||||
|
||||
TEST_F(StandardTokenizerTest, CJKPositionsAreConsecutive) {
|
||||
auto tokens = tokenize("\xE4\xB8\xAD\xE6\x96\x87"); // 中文
|
||||
ASSERT_EQ(tokens.size(), 2u);
|
||||
EXPECT_EQ(tokens[0].position, 0u);
|
||||
EXPECT_EQ(tokens[1].position, 1u);
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, CJKRespectsMaxTokenLength) {
|
||||
// With max_token_length=1, multi-codepoint words are split.
|
||||
// CJK chars are always 1 codepoint each — unaffected.
|
||||
FtsIndexParams params;
|
||||
params.tokenizer_name = "standard";
|
||||
params.filters.clear();
|
||||
params.extra_params = R"({"max_token_length":1})";
|
||||
auto pipeline = TokenizerFactory::create(params);
|
||||
ASSERT_NE(pipeline, nullptr);
|
||||
|
||||
// "a中bc" → "a", "中", "b", "c" (bc split into b and c)
|
||||
auto tokens = pipeline->process(
|
||||
"a\xE4\xB8\xAD"
|
||||
"bc");
|
||||
ASSERT_EQ(tokens.size(), 4u);
|
||||
EXPECT_EQ(tokens[0].text, "a");
|
||||
EXPECT_EQ(tokens[1].text, "\xE4\xB8\xAD");
|
||||
EXPECT_EQ(tokens[2].text, "b");
|
||||
EXPECT_EQ(tokens[3].text, "c");
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, MaxTokenLengthSplitsLongWords) {
|
||||
// "abcdefgh" with max_token_length=5 → ["abcde", "fgh"]
|
||||
FtsIndexParams params;
|
||||
params.tokenizer_name = "standard";
|
||||
params.filters.clear();
|
||||
params.extra_params = R"({"max_token_length":5})";
|
||||
auto pipeline = TokenizerFactory::create(params);
|
||||
ASSERT_NE(pipeline, nullptr);
|
||||
|
||||
auto tokens = pipeline->process("abcdefgh");
|
||||
ASSERT_EQ(tokens.size(), 2u);
|
||||
EXPECT_EQ(tokens[0].text, "abcde");
|
||||
EXPECT_EQ(tokens[1].text, "fgh");
|
||||
}
|
||||
|
||||
TEST_F(StandardTokenizerTest, MaxTokenLengthCountsCodepointsNotBytes) {
|
||||
// "café" is 4 codepoints but 5 bytes.
|
||||
// With max_token_length=4 it fits in one token.
|
||||
FtsIndexParams params4;
|
||||
params4.tokenizer_name = "standard";
|
||||
params4.filters.clear();
|
||||
params4.extra_params = R"({"max_token_length":4})";
|
||||
auto pipeline4 = TokenizerFactory::create(params4);
|
||||
ASSERT_NE(pipeline4, nullptr);
|
||||
auto tokens4 = pipeline4->process("caf\xC3\xA9");
|
||||
ASSERT_EQ(tokens4.size(), 1u);
|
||||
EXPECT_EQ(tokens4[0].text, "caf\xC3\xA9");
|
||||
|
||||
// With max_token_length=3 it splits into ["caf", "é"].
|
||||
FtsIndexParams params3;
|
||||
params3.tokenizer_name = "standard";
|
||||
params3.filters.clear();
|
||||
params3.extra_params = R"({"max_token_length":3})";
|
||||
auto pipeline3 = TokenizerFactory::create(params3);
|
||||
ASSERT_NE(pipeline3, nullptr);
|
||||
auto tokens3 = pipeline3->process("caf\xC3\xA9");
|
||||
ASSERT_EQ(tokens3.size(), 2u);
|
||||
EXPECT_EQ(tokens3[0].text, "caf");
|
||||
EXPECT_EQ(tokens3[1].text, "\xC3\xA9");
|
||||
}
|
||||
|
|
@ -30,3 +30,4 @@ add_subdirectory(CRoaring CRoaring EXCLUDE_FROM_ALL)
|
|||
add_subdirectory(FastPFOR FastPFOR EXCLUDE_FROM_ALL)
|
||||
add_subdirectory(limonp limonp EXCLUDE_FROM_ALL)
|
||||
add_subdirectory(cppjieba cppjieba EXCLUDE_FROM_ALL)
|
||||
add_subdirectory(utf8proc utf8proc EXCLUDE_FROM_ALL)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
set(UTF8PROC_INSTALL OFF CACHE BOOL "" FORCE)
|
||||
add_subdirectory(utf8proc-2.11.3 utf8proc EXCLUDE_FROM_ALL)
|
||||
|
||||
set(utf8proc_FOUND TRUE PARENT_SCOPE)
|
||||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit e5e799221b45bbb90f5fdc5c69b6b8dfbf017e78
|
||||
Loading…
Reference in New Issue