xiaoxue_admin
/
zvec
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Implement standard tokenizer based on UAX29 (#547)

This commit is contained in:
egolearner 2026-07-09 10:01:56 +08:00 committed by GitHub
parent 05524045f9
commit df6af2f05d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 2910 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
NOTICE
View File

@ -9,6 +9,54 @@ their own licenses, as listed below.
Third-Party Components
================================================================================
--------------------------------------------------------------------------------
Unicode Character Database
--------------------------------------------------------------------------------
Project: Unicode Character Database
Homepage: https://www.unicode.org/
License: Unicode License V3
Used in: src/db/index/column/fts_column/tokenizer/standard_tokenizer_unicode.inc
The generated standard tokenizer lookup tables are derived from Unicode 17.0.0
data files: auxiliary/WordBreakProperty.txt, emoji/emoji-data.txt,
LineBreak.txt, and Scripts.txt.
Unicode License V3 copyright and permission notice:
UNICODE LICENSE V3 COPYRIGHT AND PERMISSION NOTICE
Copyright © 1991-2026 Unicode, Inc.
NOTICE TO USER: Carefully read the following legal agreement. BY DOWNLOADING,
INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR SOFTWARE, YOU
UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE TERMS AND
CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL,
COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE.
Permission is hereby granted, free of charge, to any person obtaining a copy
of data files and any associated documentation (the "Data Files") or software
and any associated documentation (the "Software") to deal in the Data Files
or Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, and/or sell copies of the Data
Files or Software, and to permit persons to whom the Data Files or Software
are furnished to do so, provided that either (a) this copyright and
permission notice appear with all copies of the Data Files or Software, or
(b) this copyright and permission notice appear in associated Documentation.
THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
THIRD PARTY RIGHTS.
IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE
LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR
ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA FILES OR SOFTWARE.
Except as contained in this notice, the name of a copyright holder shall not
be used in advertising or otherwise to promote the sale, use or other
dealings in these Data Files or Software without prior written authorization
of the copyright holder.
--------------------------------------------------------------------------------
pyglass
--------------------------------------------------------------------------------

177
scripts/generate_standard_tokenizer_unicode.py Executable file
View File

@ -0,0 +1,177 @@
#!/usr/bin/env python3
#
# Generate Unicode lookup tables for the standard tokenizer.
#
# Usage:
# scripts/generate_standard_tokenizer_unicode.py \
# --ucd-dir /path/to/Public/17.0.0/ucd \
# --out src/db/index/column/fts_column/tokenizer/standard_tokenizer_unicode.inc
from __future__ import annotations
import argparse
import pathlib
WORD_BREAK_CLASSES = {
"ALetter": "ALetter",
"CR": "CR",
"Double_Quote": "DoubleQuote",
"Extend": "Extend",
"ExtendNumLet": "ExtendNumLet",
"Format": "Format",
"Hebrew_Letter": "HebrewLetter",
"Katakana": "Katakana",
"LF": "LF",
"MidLetter": "MidLetter",
"MidNum": "MidNum",
"MidNumLet": "MidNumLet",
"Newline": "Newline",
"Numeric": "Numeric",
"Regional_Indicator": "RegionalIndicator",
"Single_Quote": "SingleQuote",
"WSegSpace": "WSegSpace",
"ZWJ": "ZWJ",
}
SCRIPT_CLASSES = {
"Han": "Ideographic",
"Hangul": "Hangul",
"Hiragana": "Hiragana",
}
LINE_BREAK_COMPLEX_CONTEXT = {
"Complex_Context",
"SA",
}
def parse_codepoint_range(field):
if ".." in field:
start, end = field.split("..", 1)
return int(start, 16), int(end, 16)
cp = int(field, 16)
return cp, cp
def parse_property_file(path, accepted_properties):
ranges = []
with path.open("r", encoding="utf-8") as fin:
for raw_line in fin:
line = raw_line.split("#", 1)[0].strip()
if not line:
continue
fields = [field.strip() for field in line.split(";")]
if len(fields) < 2:
continue
prop = fields[1]
if prop not in accepted_properties:
continue
start, end = parse_codepoint_range(fields[0])
ranges.append((start, end, accepted_properties[prop]))
return merge_class_ranges(ranges)
def parse_range_properties(path, accepted_properties):
ranges = []
with path.open("r", encoding="utf-8") as fin:
for raw_line in fin:
line = raw_line.split("#", 1)[0].strip()
if not line:
continue
fields = [field.strip() for field in line.split(";")]
if len(fields) < 2:
continue
if fields[1] not in accepted_properties:
continue
ranges.append(parse_codepoint_range(fields[0]))
return merge_ranges(ranges)
def merge_class_ranges(ranges):
merged = []
for start, end, cls in sorted(ranges):
if merged and merged[-1][2] == cls and merged[-1][1] + 1 == start:
merged[-1] = (merged[-1][0], end, cls)
else:
merged.append((start, end, cls))
return merged
def merge_ranges(ranges):
merged = []
for start, end in sorted(ranges):
if merged and merged[-1][1] + 1 >= start:
merged[-1] = (merged[-1][0], max(merged[-1][1], end))
else:
merged.append((start, end))
return merged
def parse_binary_property(path, property_name):
return parse_range_properties(path, {property_name})
def write_class_table(out, name, ranges):
out.write(f"constexpr UnicodeClassRange {name}[] = {{\n")
for start, end, cls in ranges:
out.write(f" {{0x{start:04X}, 0x{end:04X}, WordBreakClass::{cls}}},\n")
out.write("};\n\n")
def write_range_table(out, name, ranges):
out.write(f"constexpr UnicodeRange {name}[] = {{\n")
for start, end in ranges:
out.write(f" {{0x{start:04X}, 0x{end:04X}}},\n")
out.write("};\n\n")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--ucd-dir", required=True, type=pathlib.Path)
parser.add_argument("--out", required=True, type=pathlib.Path)
args = parser.parse_args()
word_break_path = args.ucd_dir / "auxiliary" / "WordBreakProperty.txt"
emoji_data_path = args.ucd_dir / "emoji" / "emoji-data.txt"
line_break_path = args.ucd_dir / "LineBreak.txt"
scripts_path = args.ucd_dir / "Scripts.txt"
word_break_ranges = parse_property_file(word_break_path, WORD_BREAK_CLASSES)
script_ranges = parse_property_file(scripts_path, SCRIPT_CLASSES)
extended_pictographic_ranges = parse_binary_property(
emoji_data_path, "Extended_Pictographic"
)
emoji_modifier_base_ranges = parse_binary_property(
emoji_data_path, "Emoji_Modifier_Base"
)
emoji_modifier_ranges = parse_binary_property(emoji_data_path, "Emoji_Modifier")
line_break_complex_context_ranges = parse_range_properties(
line_break_path, LINE_BREAK_COMPLEX_CONTEXT
)
args.out.parent.mkdir(parents=True, exist_ok=True)
with args.out.open("w", encoding="utf-8", newline="\n") as fout:
fout.write("// Generated by scripts/generate_standard_tokenizer_unicode.py\n")
fout.write(
"// Source: Unicode 17.0.0 WordBreakProperty, emoji-data, LineBreak, Scripts.\n"
)
fout.write(
"// Derived from Unicode data files licensed under Unicode License V3; see NOTICE.\n"
)
fout.write("// Do not edit by hand.\n\n")
fout.write("// clang-format off\n\n")
write_class_table(fout, "kWordBreakRanges", word_break_ranges)
write_class_table(fout, "kScriptClassRanges", script_ranges)
write_range_table(
fout, "kExtendedPictographicRanges", extended_pictographic_ranges
)
write_range_table(fout, "kEmojiModifierBaseRanges", emoji_modifier_base_ranges)
write_range_table(fout, "kEmojiModifierRanges", emoji_modifier_ranges)
write_range_table(
fout, "kLineBreakComplexContextRanges", line_break_complex_context_ranges
)
fout.write("// clang-format on\n")
if __name__ == "__main__":
main()

1166
src/db/index/column/fts_column/tokenizer/standard_tokenizer.cc
View File

File diff suppressed because it is too large Load Diff

14
src/db/index/column/fts_column/tokenizer/standard_tokenizer.h
View File

@ -21,17 +21,19 @@ namespace zvec::fts {
/*! Standard tokenizer
* Unicode-aware tokenizer aligned with Elasticsearch's standard tokenizer.
* Splits text on non-alphanumeric characters (punctuation, whitespace,
* symbols) using Unicode categories via utf8proc. CJK ideographs are emitted
* as individual single-character tokens.
* Uses a UAX #29 word-boundary profile with Lucene/Elasticsearch compatible
* token selection. CJK ideographs are emitted as individual single-character
* tokens.
*/
class StandardTokenizer : public Tokenizer {
public:
/*! Initialise from JSON config.
* Supported keys:
* "max_token_length" (uint32, default 255): tokens with more codepoints
* than this limit are split at the boundary into multiple tokens.
* Always returns true.
* "max_token_length" (uint32, default 255, range [1, 1048576]): long
* tokens are split into smaller segments. Combining marks and other
* ignored word-break characters may stay attached to the previous
* segment to avoid creating mark-only tokens.
* Returns false when the configuration is invalid.
*/
bool init(const ailego::JsonObject &config) override;

1398
src/db/index/column/fts_column/tokenizer/standard_tokenizer_unicode.inc Normal file
View File

File diff suppressed because it is too large Load Diff

210
tests/db/index/column/fts_column/standard_tokenizer_test.cc
View File

@ -20,6 +20,15 @@
using namespace zvec::fts;
static std::vector<std::string> token_texts(const std::vector<Token> &tokens) {
std::vector<std::string> texts;
texts.reserve(tokens.size());
for (const auto &token : tokens) {
texts.push_back(token.text);
}
return texts;
}
class StandardTokenizerTest : public ::testing::Test {
protected:
void SetUp() override {
@ -47,7 +56,7 @@ TEST_F(StandardTokenizerTest, SimpleAsciiWords) {
}
TEST_F(StandardTokenizerTest, PunctuationAsDelimiter) {
auto tokens = tokenize("hello,world.test");
auto tokens = tokenize("hello,world!test");
ASSERT_EQ(tokens.size(), 3u);
EXPECT_EQ(tokens[0].text, "hello");
EXPECT_EQ(tokens[1].text, "world");
@ -71,6 +80,19 @@ TEST_F(StandardTokenizerTest, OnlyDelimiters) {
EXPECT_TRUE(tokens.empty());
}
TEST_F(StandardTokenizerTest, MalformedUtf8BreaksTokens) {
std::string text = "ab";
text.push_back(static_cast<char>(0xFF));
text += "cd";
auto tokens = tokenize(text);
ASSERT_EQ(tokens.size(), 2u);
EXPECT_EQ(tokens[0].text, "ab");
EXPECT_EQ(tokens[0].offset, 0u);
EXPECT_EQ(tokens[1].text, "cd");
EXPECT_EQ(tokens[1].offset, 3u);
}
TEST_F(StandardTokenizerTest, OffsetAndPosition) {
auto tokens = tokenize(" hello world");
ASSERT_EQ(tokens.size(), 2u);
@ -92,11 +114,11 @@ TEST_F(StandardTokenizerTest, AccentedLatin) {
TEST_F(StandardTokenizerTest, MarksContinueButDoNotStartTokens) {
// e + U+0301 keeps the combining mark with the base letter.
// Standalone U+0301 and the heart variation selector are not indexed.
// Standalone U+0301 and U+FE0F are not indexed.
auto tokens = tokenize(
"e\xCC\x81 "
"\xCC\x81 "
"\xE2\x9D\xA4\xEF\xB8\x8F");
"\xEF\xB8\x8F");
ASSERT_EQ(tokens.size(), 1u);
EXPECT_EQ(tokens[0].text, "e\xCC\x81");
}
@ -182,6 +204,13 @@ TEST_F(StandardTokenizerTest, CJKCompatibilitySupplement) {
EXPECT_EQ(tokens[0].text, "\xF0\xAF\xA0\x80");
}
TEST_F(StandardTokenizerTest, CJKSingleCharKeepsTrailingMarks) {
auto tokens = tokenize("\xE4\xB8\xAD\xEF\xB8\x80\xE6\x96\x87");
ASSERT_EQ(tokens.size(), 2u);
EXPECT_EQ(tokens[0].text, "\xE4\xB8\xAD\xEF\xB8\x80");
EXPECT_EQ(tokens[1].text, "\xE6\x96\x87");
}
// --- Mixed scripts ---
TEST_F(StandardTokenizerTest, MixedLatinAndCJK) {
@ -279,3 +308,178 @@ TEST_F(StandardTokenizerTest, MaxTokenLengthCountsCodepointsNotBytes) {
EXPECT_EQ(tokens3[0].text, "caf");
EXPECT_EQ(tokens3[1].text, "\xC3\xA9");
}
TEST_F(StandardTokenizerTest, MaxTokenLengthDropsConnectorOnlySplitSegments) {
FtsIndexParams params3;
params3.tokenizer_name = "standard";
params3.filters.clear();
params3.extra_params = R"({"max_token_length":3})";
auto pipeline3 = TokenizerFactory::create(params3);
ASSERT_NE(pipeline3, nullptr);
auto tokens3 = pipeline3->process("dog's");
ASSERT_EQ(tokens3.size(), 2u);
EXPECT_EQ(tokens3[0].text, "dog");
EXPECT_EQ(tokens3[1].text, "s");
FtsIndexParams params1;
params1.tokenizer_name = "standard";
params1.filters.clear();
params1.extra_params = R"({"max_token_length":1})";
auto pipeline1 = TokenizerFactory::create(params1);
ASSERT_NE(pipeline1, nullptr);
auto leading = pipeline1->process("_lead");
std::vector<std::string> expected_leading = {"l", "e", "a", "d"};
EXPECT_EQ(token_texts(leading), expected_leading);
auto internal = pipeline1->process("abc__def");
std::vector<std::string> expected_internal = {"a", "b", "c", "d", "e", "f"};
EXPECT_EQ(token_texts(internal), expected_internal);
}
TEST_F(StandardTokenizerTest, IntraWordPunctuation) {
auto tokens = tokenize(
"dog's 3.14 1,000 example.com hello,world host:port a:b "
"host:9200");
std::vector<std::string> expected = {
"dog's", "3.14", "1,000", "example.com", "hello",
"world", "host:port", "a:b", "host", "9200"};
EXPECT_EQ(token_texts(tokens), expected);
}
TEST_F(StandardTokenizerTest, ExtendNumLetConnectsWordsAndNumbers) {
auto tokens = tokenize("foo_bar v1_2 _lead __123 _");
std::vector<std::string> expected = {"foo_bar", "v1_2", "_lead", "__123"};
EXPECT_EQ(token_texts(tokens), expected);
}
TEST_F(StandardTokenizerTest, EmojiZwjSequence) {
auto tokens = tokenize(
"\xF0\x9F\x91\xA9\xE2\x80\x8D\xF0\x9F\x92\xBB "
"\xE2\x9D\xA4\xEF\xB8\x8F");
ASSERT_EQ(tokens.size(), 2u);
EXPECT_EQ(tokens[0].text, "\xF0\x9F\x91\xA9\xE2\x80\x8D\xF0\x9F\x92\xBB");
EXPECT_EQ(tokens[1].text, "\xE2\x9D\xA4\xEF\xB8\x8F");
}
TEST_F(StandardTokenizerTest, EmojiKeycapSequences) {
auto tokens = tokenize(
"1\xEF\xB8\x8F\xE2\x83\xA3 "
"#\xE2\x83\xA3 "
"*\xEF\xB8\x8F\xE2\x83\xA3");
ASSERT_EQ(tokens.size(), 3u);
EXPECT_EQ(tokens[0].text, "1\xEF\xB8\x8F\xE2\x83\xA3");
EXPECT_EQ(tokens[1].text, "#\xE2\x83\xA3");
EXPECT_EQ(tokens[2].text, "*\xEF\xB8\x8F\xE2\x83\xA3");
}
TEST_F(StandardTokenizerTest, EmojiModifierSequences) {
auto tokens = tokenize(
"\xF0\x9F\x91\x8D\xF0\x9F\x8F\xBD "
"\xE2\x98\x9D\xEF\xB8\x8F\xF0\x9F\x8F\xBB "
"\xF0\x9F\x8F\xBD");
ASSERT_EQ(tokens.size(), 3u);
EXPECT_EQ(tokens[0].text, "\xF0\x9F\x91\x8D\xF0\x9F\x8F\xBD");
EXPECT_EQ(tokens[1].text, "\xE2\x98\x9D\xEF\xB8\x8F\xF0\x9F\x8F\xBB");
EXPECT_EQ(tokens[2].text, "\xF0\x9F\x8F\xBD");
}
TEST_F(StandardTokenizerTest, EmojiModifierInsideZwjSequence) {
auto tokens =
tokenize("\xF0\x9F\x91\xA9\xF0\x9F\x8F\xBD\xE2\x80\x8D\xF0\x9F\x92\xBB");
ASSERT_EQ(tokens.size(), 1u);
EXPECT_EQ(tokens[0].text,
"\xF0\x9F\x91\xA9\xF0\x9F\x8F\xBD\xE2\x80\x8D\xF0\x9F\x92\xBB");
}
TEST_F(StandardTokenizerTest, RegionalIndicatorPairs) {
auto tokens = tokenize(
"\xF0\x9F\x87\xBA\xF0\x9F\x87\xB8"
"\xF0\x9F\x87\xA8\xF0\x9F\x87\xA6"
"\xF0\x9F\x87\xAF");
ASSERT_EQ(tokens.size(), 3u);
EXPECT_EQ(tokens[0].text, "\xF0\x9F\x87\xBA\xF0\x9F\x87\xB8");
EXPECT_EQ(tokens[1].text, "\xF0\x9F\x87\xA8\xF0\x9F\x87\xA6");
EXPECT_EQ(tokens[2].text, "\xF0\x9F\x87\xAF");
}
TEST_F(StandardTokenizerTest, RegionalIndicatorPairsIgnoreExtendAndZwj) {
auto tokens = tokenize(
"\xF0\x9F\x87\xA6\xCC\x88\xF0\x9F\x87\xA7 "
"\xF0\x9F\x87\xA6\xE2\x80\x8D\xF0\x9F\x87\xA7\xF0\x9F\x87\xA8");
ASSERT_EQ(tokens.size(), 3u);
EXPECT_EQ(tokens[0].text, "\xF0\x9F\x87\xA6\xCC\x88\xF0\x9F\x87\xA7");
EXPECT_EQ(tokens[1].text, "\xF0\x9F\x87\xA6\xE2\x80\x8D\xF0\x9F\x87\xA7");
EXPECT_EQ(tokens[2].text, "\xF0\x9F\x87\xA8");
}
TEST_F(StandardTokenizerTest, MinimalWb3cZwjExtendedPictographic) {
auto tokens = tokenize(
"\xE2\x80\x8D\xF0\x9F\x9B\x91 "
"a\xE2\x80\x8D\xF0\x9F\x9B\x91 "
"\xE2\x80\x8D\xE2\x93\x82");
ASSERT_EQ(tokens.size(), 3u);
EXPECT_EQ(tokens[0].text, "\xE2\x80\x8D\xF0\x9F\x9B\x91");
EXPECT_EQ(tokens[1].text, "a\xE2\x80\x8D\xF0\x9F\x9B\x91");
EXPECT_EQ(tokens[2].text, "\xE2\x80\x8D\xE2\x93\x82");
}
TEST_F(StandardTokenizerTest, HiraganaTokensAreSingleCharacters) {
auto tokens = tokenize("\xE3\x81\x8B\xE3\x82\x99\xE3\x81\xAA");
ASSERT_EQ(tokens.size(), 2u);
EXPECT_EQ(tokens[0].text, "\xE3\x81\x8B\xE3\x82\x99");
EXPECT_EQ(tokens[1].text, "\xE3\x81\xAA");
}
TEST_F(StandardTokenizerTest, JapaneseKoreanAndSoutheastAsianScripts) {
auto tokens = tokenize(
"\xE3\x81\xB2\xE3\x82\x89\xE3\x81\x8C\xE3\x81\xAA "
"\xE3\x82\xAB\xE3\x82\xBF\xE3\x82\xAB\xE3\x83\x8A "
"\xED\x95\x9C\xEA\xB5\xAD "
"\xE0\xB9\x84\xE0\xB8\x97\xE0\xB8\xA2 "
"\xE1\x80\x99\xE1\x80\x94");
ASSERT_EQ(tokens.size(), 8u);
EXPECT_EQ(tokens[0].text, "\xE3\x81\xB2");
EXPECT_EQ(tokens[1].text, "\xE3\x82\x89");
EXPECT_EQ(tokens[2].text, "\xE3\x81\x8C");
EXPECT_EQ(tokens[3].text, "\xE3\x81\xAA");
EXPECT_EQ(tokens[4].text, "\xE3\x82\xAB\xE3\x82\xBF\xE3\x82\xAB\xE3\x83\x8A");
EXPECT_EQ(tokens[5].text, "\xED\x95\x9C\xEA\xB5\xAD");
EXPECT_EQ(tokens[6].text, "\xE0\xB9\x84\xE0\xB8\x97\xE0\xB8\xA2");
EXPECT_EQ(tokens[7].text, "\xE1\x80\x99\xE1\x80\x94");
}
TEST_F(StandardTokenizerTest, SoutheastAsianMarksContinueButDoNotStartTokens) {
auto tokens = tokenize(
"\xE0\xB8\x81\xE0\xB8\xB1 "
"\xE0\xB8\xB1");
ASSERT_EQ(tokens.size(), 1u);
EXPECT_EQ(tokens[0].text, "\xE0\xB8\x81\xE0\xB8\xB1");
}
TEST_F(StandardTokenizerTest, HangulSymbolsOutsideWordClassAreIgnored) {
auto tokens = tokenize("\xE3\x89\xA0 \xED\x95\x9C\xEA\xB5\xAD");
ASSERT_EQ(tokens.size(), 1u);
EXPECT_EQ(tokens[0].text, "\xED\x95\x9C\xEA\xB5\xAD");
}
TEST_F(StandardTokenizerTest, HebrewSingleQuoteStaysWithLetter) {
auto tokens = tokenize("\xD7\x90' \xD7\x90\"\xD7\x91");
ASSERT_EQ(tokens.size(), 2u);
EXPECT_EQ(tokens[0].text, "\xD7\x90'");
EXPECT_EQ(tokens[1].text, "\xD7\x90\"\xD7\x91");
}
TEST(StandardTokenizerConfigTest, MaxTokenLengthValidation) {
FtsIndexParams params;
params.tokenizer_name = "standard";
params.filters.clear();
params.extra_params = R"({"max_token_length":0})";
EXPECT_EQ(TokenizerFactory::create(params), nullptr);
params.extra_params = R"({"max_token_length":1048577})";
EXPECT_EQ(TokenizerFactory::create(params), nullptr);
params.extra_params = R"({"max_token_length":1})";
EXPECT_NE(TokenizerFactory::create(params), nullptr);
}

28
tests/db/sqlengine/fts_parser_test.cc
View File

@ -27,10 +27,9 @@ namespace zvec::fts {
class FtsParserTest : public ::testing::Test {
protected:
void SetUp() override {
// Standard tokenizer + lowercase filter: ASCII tests behave the same as
// the previous whitespace split (alnum runs become tokens, delimiters
// get dropped) while CJK tests can exercise the per-character tokens
// standard produces from non-alnum bytes.
// Standard tokenizer + lowercase filter. These parser tests cover
// punctuation that standard still treats as delimiters, while CJK tests
// exercise the per-character tokens standard produces for ideographs.
FtsIndexParams params;
params.tokenizer_name = "standard";
params.filters = {"lowercase"};
@ -100,8 +99,8 @@ TEST_F(FtsParserTest, SingleTermNumeric) {
TEST_F(FtsParserTest, SingleTermWithHyphen) {
// The lexer's REGULAR_ID rule keeps hyphenated text as one token, but the
// standard tokenizer on the parser side splits non-alphanumerics. With the
// default OR operator the term decomposes into Or[full, text] so query
// standard tokenizer on the parser side splits this hyphen delimiter. With
// the default OR operator the term decomposes into Or[full, text] so query
// segmentation matches the index segmentation.
auto ast = parse("full-text");
ASSERT_NE(ast, nullptr);
@ -112,6 +111,12 @@ TEST_F(FtsParserTest, SingleTermWithHyphen) {
EXPECT_EQ(as_term(*or_node.children[1]).term, "text");
}
TEST_F(FtsParserTest, BareColonQueryIsFieldPrefixSyntax) {
auto ast = parse("host:port");
EXPECT_EQ(ast, nullptr);
EXPECT_EQ(err_msg(), "field-prefixed queries are not supported");
}
// ============================================================
// Must (+) and must_not (-/NOT) modifiers
// ============================================================
@ -754,7 +759,7 @@ TEST_F(FtsParserTest, MultiTokenBareTermPreservesMustModifier) {
TEST_F(FtsParserTest, PhraseTokensRunThroughPipeline) {
// The phrase body is tokenized exactly like document text. With the
// standard tokenizer, mixed delimiters between alnum runs collapse so
// standard tokenizer, comma and exclamation delimiters collapse so
// "machine, learning!" becomes ["machine", "learning"].
auto ast = parse("\"machine, learning!\"");
ASSERT_NE(ast, nullptr);
@ -765,6 +770,15 @@ TEST_F(FtsParserTest, PhraseTokensRunThroughPipeline) {
EXPECT_EQ(phrase.terms[1], "learning");
}
TEST_F(FtsParserTest, PhraseCanSearchLiteralColonToken) {
auto ast = parse("\"host:port\"");
ASSERT_NE(ast, nullptr);
ASSERT_EQ(ast->type(), FtsNodeType::PHRASE);
const auto &phrase = as_phrase(*ast);
ASSERT_EQ(phrase.terms.size(), 1u);
EXPECT_EQ(phrase.terms[0], "host:port");
}
TEST_F(FtsParserTest, PhraseLowercaseFilterApplies) {
// The lowercase filter is part of the pipeline so phrase tokens come back
// lowercased even when the input mixed case.