diff --git a/NOTICE b/NOTICE index 1ea6ef8..5e26964 100644 --- a/NOTICE +++ b/NOTICE @@ -9,6 +9,54 @@ their own licenses, as listed below. Third-Party Components ================================================================================ +-------------------------------------------------------------------------------- +Unicode Character Database +-------------------------------------------------------------------------------- +Project: Unicode Character Database +Homepage: https://www.unicode.org/ +License: Unicode License V3 +Used in: src/db/index/column/fts_column/tokenizer/standard_tokenizer_unicode.inc + +The generated standard tokenizer lookup tables are derived from Unicode 17.0.0 +data files: auxiliary/WordBreakProperty.txt, emoji/emoji-data.txt, +LineBreak.txt, and Scripts.txt. + +Unicode License V3 copyright and permission notice: + + UNICODE LICENSE V3 COPYRIGHT AND PERMISSION NOTICE + Copyright © 1991-2026 Unicode, Inc. + NOTICE TO USER: Carefully read the following legal agreement. BY DOWNLOADING, + INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR SOFTWARE, YOU + UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE TERMS AND + CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, + COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of data files and any associated documentation (the "Data Files") or software + and any associated documentation (the "Software") to deal in the Data Files + or Software without restriction, including without limitation the rights to + use, copy, modify, merge, publish, distribute, and/or sell copies of the Data + Files or Software, and to permit persons to whom the Data Files or Software + are furnished to do so, provided that either (a) this copyright and + permission notice appear with all copies of the Data Files or Software, or + (b) this copyright and permission notice appear in associated Documentation. + + THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + THIRD PARTY RIGHTS. + + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE + LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR + ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER + IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA FILES OR SOFTWARE. + + Except as contained in this notice, the name of a copyright holder shall not + be used in advertising or otherwise to promote the sale, use or other + dealings in these Data Files or Software without prior written authorization + of the copyright holder. + -------------------------------------------------------------------------------- pyglass -------------------------------------------------------------------------------- diff --git a/scripts/generate_standard_tokenizer_unicode.py b/scripts/generate_standard_tokenizer_unicode.py new file mode 100755 index 0000000..1944af3 --- /dev/null +++ b/scripts/generate_standard_tokenizer_unicode.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# +# Generate Unicode lookup tables for the standard tokenizer. +# +# Usage: +# scripts/generate_standard_tokenizer_unicode.py \ +# --ucd-dir /path/to/Public/17.0.0/ucd \ +# --out src/db/index/column/fts_column/tokenizer/standard_tokenizer_unicode.inc +from __future__ import annotations + +import argparse +import pathlib + +WORD_BREAK_CLASSES = { + "ALetter": "ALetter", + "CR": "CR", + "Double_Quote": "DoubleQuote", + "Extend": "Extend", + "ExtendNumLet": "ExtendNumLet", + "Format": "Format", + "Hebrew_Letter": "HebrewLetter", + "Katakana": "Katakana", + "LF": "LF", + "MidLetter": "MidLetter", + "MidNum": "MidNum", + "MidNumLet": "MidNumLet", + "Newline": "Newline", + "Numeric": "Numeric", + "Regional_Indicator": "RegionalIndicator", + "Single_Quote": "SingleQuote", + "WSegSpace": "WSegSpace", + "ZWJ": "ZWJ", +} + +SCRIPT_CLASSES = { + "Han": "Ideographic", + "Hangul": "Hangul", + "Hiragana": "Hiragana", +} + + +LINE_BREAK_COMPLEX_CONTEXT = { + "Complex_Context", + "SA", +} + + +def parse_codepoint_range(field): + if ".." in field: + start, end = field.split("..", 1) + return int(start, 16), int(end, 16) + cp = int(field, 16) + return cp, cp + + +def parse_property_file(path, accepted_properties): + ranges = [] + with path.open("r", encoding="utf-8") as fin: + for raw_line in fin: + line = raw_line.split("#", 1)[0].strip() + if not line: + continue + fields = [field.strip() for field in line.split(";")] + if len(fields) < 2: + continue + prop = fields[1] + if prop not in accepted_properties: + continue + start, end = parse_codepoint_range(fields[0]) + ranges.append((start, end, accepted_properties[prop])) + return merge_class_ranges(ranges) + + +def parse_range_properties(path, accepted_properties): + ranges = [] + with path.open("r", encoding="utf-8") as fin: + for raw_line in fin: + line = raw_line.split("#", 1)[0].strip() + if not line: + continue + fields = [field.strip() for field in line.split(";")] + if len(fields) < 2: + continue + if fields[1] not in accepted_properties: + continue + ranges.append(parse_codepoint_range(fields[0])) + return merge_ranges(ranges) + + +def merge_class_ranges(ranges): + merged = [] + for start, end, cls in sorted(ranges): + if merged and merged[-1][2] == cls and merged[-1][1] + 1 == start: + merged[-1] = (merged[-1][0], end, cls) + else: + merged.append((start, end, cls)) + return merged + + +def merge_ranges(ranges): + merged = [] + for start, end in sorted(ranges): + if merged and merged[-1][1] + 1 >= start: + merged[-1] = (merged[-1][0], max(merged[-1][1], end)) + else: + merged.append((start, end)) + return merged + + +def parse_binary_property(path, property_name): + return parse_range_properties(path, {property_name}) + + +def write_class_table(out, name, ranges): + out.write(f"constexpr UnicodeClassRange {name}[] = {{\n") + for start, end, cls in ranges: + out.write(f" {{0x{start:04X}, 0x{end:04X}, WordBreakClass::{cls}}},\n") + out.write("};\n\n") + + +def write_range_table(out, name, ranges): + out.write(f"constexpr UnicodeRange {name}[] = {{\n") + for start, end in ranges: + out.write(f" {{0x{start:04X}, 0x{end:04X}}},\n") + out.write("};\n\n") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--ucd-dir", required=True, type=pathlib.Path) + parser.add_argument("--out", required=True, type=pathlib.Path) + args = parser.parse_args() + + word_break_path = args.ucd_dir / "auxiliary" / "WordBreakProperty.txt" + emoji_data_path = args.ucd_dir / "emoji" / "emoji-data.txt" + line_break_path = args.ucd_dir / "LineBreak.txt" + scripts_path = args.ucd_dir / "Scripts.txt" + + word_break_ranges = parse_property_file(word_break_path, WORD_BREAK_CLASSES) + script_ranges = parse_property_file(scripts_path, SCRIPT_CLASSES) + extended_pictographic_ranges = parse_binary_property( + emoji_data_path, "Extended_Pictographic" + ) + emoji_modifier_base_ranges = parse_binary_property( + emoji_data_path, "Emoji_Modifier_Base" + ) + emoji_modifier_ranges = parse_binary_property(emoji_data_path, "Emoji_Modifier") + line_break_complex_context_ranges = parse_range_properties( + line_break_path, LINE_BREAK_COMPLEX_CONTEXT + ) + + args.out.parent.mkdir(parents=True, exist_ok=True) + with args.out.open("w", encoding="utf-8", newline="\n") as fout: + fout.write("// Generated by scripts/generate_standard_tokenizer_unicode.py\n") + fout.write( + "// Source: Unicode 17.0.0 WordBreakProperty, emoji-data, LineBreak, Scripts.\n" + ) + fout.write( + "// Derived from Unicode data files licensed under Unicode License V3; see NOTICE.\n" + ) + fout.write("// Do not edit by hand.\n\n") + fout.write("// clang-format off\n\n") + write_class_table(fout, "kWordBreakRanges", word_break_ranges) + write_class_table(fout, "kScriptClassRanges", script_ranges) + write_range_table( + fout, "kExtendedPictographicRanges", extended_pictographic_ranges + ) + write_range_table(fout, "kEmojiModifierBaseRanges", emoji_modifier_base_ranges) + write_range_table(fout, "kEmojiModifierRanges", emoji_modifier_ranges) + write_range_table( + fout, "kLineBreakComplexContextRanges", line_break_complex_context_ranges + ) + fout.write("// clang-format on\n") + + +if __name__ == "__main__": + main() diff --git a/src/db/index/column/fts_column/tokenizer/standard_tokenizer.cc b/src/db/index/column/fts_column/tokenizer/standard_tokenizer.cc index e2971ab..fc612e8 100644 --- a/src/db/index/column/fts_column/tokenizer/standard_tokenizer.cc +++ b/src/db/index/column/fts_column/tokenizer/standard_tokenizer.cc @@ -14,148 +14,1084 @@ #include "standard_tokenizer.h" #include +#include +#include namespace zvec::fts { namespace { -bool is_word_start_char(utf8proc_category_t cat) { - switch (cat) { - case UTF8PROC_CATEGORY_LU: // Letter, uppercase - case UTF8PROC_CATEGORY_LL: // Letter, lowercase - case UTF8PROC_CATEGORY_LT: // Letter, titlecase - case UTF8PROC_CATEGORY_LM: // Letter, modifier - case UTF8PROC_CATEGORY_LO: // Letter, other - case UTF8PROC_CATEGORY_ND: // Number, decimal digit - case UTF8PROC_CATEGORY_NL: // Number, letter - case UTF8PROC_CATEGORY_NO: // Number, other +constexpr uint32_t kDefaultMaxTokenLength = 255; +constexpr uint32_t kMinMaxTokenLength = 1; +constexpr uint32_t kMaxMaxTokenLength = 1048576; +constexpr uint32_t kVariationSelector16 = 0xFE0F; +constexpr uint32_t kKeycap = 0x20E3; +constexpr uint32_t kUnicodeMaxCodepoint = 0x10FFFF; +constexpr uint32_t kUnicodePageShift = 8; +constexpr size_t kUnicodePageCount = + (kUnicodeMaxCodepoint >> kUnicodePageShift) + 1; +constexpr size_t kCodepointCacheSize = 1024; +constexpr size_t kMaxInitialTokenCapacity = 4096; + +enum class WordBreakClass : uint8_t { + Other, + CR, + LF, + Newline, + Extend, + Format, + ZWJ, + ALetter, + HebrewLetter, + Numeric, + Katakana, + ExtendNumLet, + RegionalIndicator, + WSegSpace, + MidLetter, + MidNum, + MidNumLet, + SingleQuote, + DoubleQuote, + Hiragana, + Hangul, + SoutheastAsian, + Ideographic, + ExtendedPictographic, +}; + +struct Codepoint { + utf8proc_int32_t cp{0}; + uint32_t start{0}; + uint32_t end{0}; + WordBreakClass cls{WordBreakClass::Other}; + bool extended_pictographic{false}; + bool emoji_modifier_base{false}; + bool emoji_modifier{false}; +}; + +struct CodepointProperties { + WordBreakClass cls{WordBreakClass::Other}; + bool extended_pictographic{false}; + bool emoji_modifier_base{false}; + bool emoji_modifier{false}; +}; + +struct CodepointCacheEntry { + uint32_t cp{0}; + CodepointProperties props; + bool valid{false}; +}; + +struct UnicodeClassRange { + uint32_t first{0}; + uint32_t last{0}; + WordBreakClass cls{WordBreakClass::Other}; +}; + +struct UnicodeRange { + uint32_t first{0}; + uint32_t last{0}; +}; + +struct UnicodeRangeIndex { + uint32_t first{0}; + uint32_t last{0}; +}; + +template +struct UnicodeRangeIndexTable { + std::array pages; +}; + +#include "standard_tokenizer_unicode.inc" + +std::array build_ascii_word_break_class_table() { + std::array table{}; + table.fill(WordBreakClass::Other); + table['\n'] = WordBreakClass::LF; + table['\r'] = WordBreakClass::CR; + table[0x0B] = WordBreakClass::Newline; + table[0x0C] = WordBreakClass::Newline; + table[' '] = WordBreakClass::WSegSpace; + table['"'] = WordBreakClass::DoubleQuote; + table['\''] = WordBreakClass::SingleQuote; + table[','] = WordBreakClass::MidNum; + table['.'] = WordBreakClass::MidNumLet; + table[':'] = WordBreakClass::MidLetter; + table[';'] = WordBreakClass::MidNum; + table['_'] = WordBreakClass::ExtendNumLet; + for (uint32_t cp = '0'; cp <= '9'; ++cp) { + table[cp] = WordBreakClass::Numeric; + } + for (uint32_t cp = 'A'; cp <= 'Z'; ++cp) { + table[cp] = WordBreakClass::ALetter; + } + for (uint32_t cp = 'a'; cp <= 'z'; ++cp) { + table[cp] = WordBreakClass::ALetter; + } + return table; +} + +const std::array kAsciiWordBreakClasses = + build_ascii_word_break_class_table(); + +size_t estimate_token_capacity(size_t byte_size) { + size_t capacity = byte_size / 4 + 1; + if (capacity > kMaxInitialTokenCapacity) { + return kMaxInitialTokenCapacity; + } + return capacity; +} + +size_t estimate_codepoint_capacity(size_t byte_size) { + return byte_size / 2 + 1; +} + +template +constexpr UnicodeRangeIndexTable build_range_index( + const Range (&ranges)[N]) { + UnicodeRangeIndexTable index{}; + size_t first = 0; + for (size_t page = 0; page < kUnicodePageCount; ++page) { + uint32_t page_first = static_cast(page << kUnicodePageShift); + uint32_t page_last = page_first + ((1u << kUnicodePageShift) - 1); + while (first < N && ranges[first].last < page_first) { + ++first; + } + size_t last = first; + while (last < N && ranges[last].first <= page_last) { + ++last; + } + index.pages[page] = {static_cast(first), + static_cast(last)}; + } + return index; +} + +constexpr auto kWordBreakRangeIndex = build_range_index(kWordBreakRanges); +constexpr auto kScriptClassRangeIndex = build_range_index(kScriptClassRanges); +constexpr auto kExtendedPictographicRangeIndex = + build_range_index(kExtendedPictographicRanges); +constexpr auto kEmojiModifierBaseRangeIndex = + build_range_index(kEmojiModifierBaseRanges); +constexpr auto kEmojiModifierRangeIndex = + build_range_index(kEmojiModifierRanges); +constexpr auto kLineBreakComplexContextRangeIndex = + build_range_index(kLineBreakComplexContextRanges); + +template +constexpr size_t range_count(const Range (&)[N]) { + return N; +} + +WordBreakClass lookup_class_range(const UnicodeClassRange *ranges, + size_t range_count, + const UnicodeRangeIndex *index, uint32_t cp) { + size_t page = cp >> kUnicodePageShift; + if (page >= kUnicodePageCount) { + return WordBreakClass::Other; + } + size_t left = index[page].first; + size_t right = index[page].last; + if (right > range_count) { + right = range_count; + } + if (left > right) { + left = right; + } + while (left < right) { + size_t mid = left + (right - left) / 2; + if (cp < ranges[mid].first) { + right = mid; + } else if (cp > ranges[mid].last) { + left = mid + 1; + } else { + return ranges[mid].cls; + } + } + return WordBreakClass::Other; +} + +bool contains_range(const UnicodeRange *ranges, size_t range_count, + const UnicodeRangeIndex *index, uint32_t cp) { + size_t page = cp >> kUnicodePageShift; + if (page >= kUnicodePageCount) { + return false; + } + size_t left = index[page].first; + size_t right = index[page].last; + if (right > range_count) { + right = range_count; + } + if (left > right) { + left = right; + } + while (left < right) { + size_t mid = left + (right - left) / 2; + if (cp < ranges[mid].first) { + right = mid; + } else if (cp > ranges[mid].last) { + left = mid + 1; + } else { return true; - default: + } + } + return false; +} + +WordBreakClass lookup_word_break_class(uint32_t cp) { + return lookup_class_range(kWordBreakRanges, range_count(kWordBreakRanges), + kWordBreakRangeIndex.pages.data(), cp); +} + +WordBreakClass lookup_script_class(uint32_t cp) { + return lookup_class_range(kScriptClassRanges, range_count(kScriptClassRanges), + kScriptClassRangeIndex.pages.data(), cp); +} + +bool is_extended_pictographic(uint32_t cp) { + return contains_range(kExtendedPictographicRanges, + range_count(kExtendedPictographicRanges), + kExtendedPictographicRangeIndex.pages.data(), cp); +} + +bool is_emoji_modifier_base(uint32_t cp) { + return contains_range(kEmojiModifierBaseRanges, + range_count(kEmojiModifierBaseRanges), + kEmojiModifierBaseRangeIndex.pages.data(), cp); +} + +bool is_emoji_modifier(uint32_t cp) { + return contains_range(kEmojiModifierRanges, range_count(kEmojiModifierRanges), + kEmojiModifierRangeIndex.pages.data(), cp); +} + +bool is_line_break_complex_context(uint32_t cp) { + return contains_range(kLineBreakComplexContextRanges, + range_count(kLineBreakComplexContextRanges), + kLineBreakComplexContextRangeIndex.pages.data(), cp); +} + +bool is_ahletter(WordBreakClass cls) { + return cls == WordBreakClass::ALetter || cls == WordBreakClass::HebrewLetter; +} + +bool is_ignored(WordBreakClass cls) { + return cls == WordBreakClass::Extend || cls == WordBreakClass::Format || + cls == WordBreakClass::ZWJ; +} + +bool is_extend_or_format(WordBreakClass cls) { + return cls == WordBreakClass::Extend || cls == WordBreakClass::Format; +} + +bool previous_is_zwj(const std::vector &codepoints, size_t index) { + return index > 0 && codepoints[index - 1].cls == WordBreakClass::ZWJ; +} + +bool is_extended_pictographic_codepoint(const Codepoint &codepoint) { + return codepoint.cls == WordBreakClass::ExtendedPictographic || + codepoint.extended_pictographic; +} + +bool is_token_start(WordBreakClass cls) { + return is_ahletter(cls) || cls == WordBreakClass::Numeric || + cls == WordBreakClass::Katakana || + cls == WordBreakClass::RegionalIndicator || + cls == WordBreakClass::Hiragana || cls == WordBreakClass::Hangul || + cls == WordBreakClass::SoutheastAsian || + cls == WordBreakClass::Ideographic || + cls == WordBreakClass::ExtendedPictographic; +} + +bool is_connector(WordBreakClass cls) { + return is_ahletter(cls) || cls == WordBreakClass::Numeric || + cls == WordBreakClass::Katakana || cls == WordBreakClass::ExtendNumLet; +} + +WordBreakClass lookup_ascii_word_break_class(uint32_t cp) { + if (cp <= 0x7F) { + return kAsciiWordBreakClasses[cp]; + } + return WordBreakClass::Other; +} + +bool is_ascii_letter_or_digit(unsigned char ch) { + return (ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || + (ch >= 'a' && ch <= 'z'); +} + +bool is_ascii_word_body_char(unsigned char ch) { + return is_ascii_letter_or_digit(ch) || ch == '_'; +} + +bool is_hangul_syllable(uint32_t codepoint) { + return codepoint >= 0xAC00 && codepoint <= 0xD7A3; +} + +bool is_utf8_continuation(unsigned char ch) { + return (ch & 0xC0) == 0x80; +} + +bool decode_utf8_codepoint(const utf8proc_uint8_t *str, size_t len, + size_t index, utf8proc_int32_t *cp, size_t *bytes) { + unsigned char lead = str[index]; + if (lead < 0x80) { + *cp = lead; + *bytes = 1; + return true; + } + + if ((lead & 0xE0) == 0xC0) { + if (index + 1 >= len || !is_utf8_continuation(str[index + 1])) { return false; + } + uint32_t value = ((lead & 0x1F) << 6) | (str[index + 1] & 0x3F); + if (value < 0x80) { + return false; + } + *cp = static_cast(value); + *bytes = 2; + return true; } + + if ((lead & 0xF0) == 0xE0) { + if (index + 2 >= len || !is_utf8_continuation(str[index + 1]) || + !is_utf8_continuation(str[index + 2])) { + return false; + } + uint32_t value = ((lead & 0x0F) << 12) | ((str[index + 1] & 0x3F) << 6) | + (str[index + 2] & 0x3F); + if (value < 0x800 || (value >= 0xD800 && value <= 0xDFFF)) { + return false; + } + *cp = static_cast(value); + *bytes = 3; + return true; + } + + if ((lead & 0xF8) == 0xF0) { + if (index + 3 >= len || !is_utf8_continuation(str[index + 1]) || + !is_utf8_continuation(str[index + 2]) || + !is_utf8_continuation(str[index + 3])) { + return false; + } + uint32_t value = ((lead & 0x07) << 18) | ((str[index + 1] & 0x3F) << 12) | + ((str[index + 2] & 0x3F) << 6) | (str[index + 3] & 0x3F); + if (value < 0x10000 || value > kUnicodeMaxCodepoint) { + return false; + } + *cp = static_cast(value); + *bytes = 4; + return true; + } + + return false; } -bool is_word_continue_char(utf8proc_category_t cat) { - switch (cat) { - case UTF8PROC_CATEGORY_MN: // Mark, nonspacing - case UTF8PROC_CATEGORY_MC: // Mark, spacing combining - case UTF8PROC_CATEGORY_ME: // Mark, enclosing +WordBreakClass classify_codepoint(utf8proc_int32_t cp) { + uint32_t codepoint = static_cast(cp); + if (codepoint <= 0x7F) { + return lookup_ascii_word_break_class(codepoint); + } + + WordBreakClass script_cls = lookup_script_class(codepoint); + if (script_cls == WordBreakClass::Ideographic || + script_cls == WordBreakClass::Hiragana) { + return script_cls; + } + if (is_hangul_syllable(codepoint)) { + return WordBreakClass::Hangul; + } + + WordBreakClass word_break_cls = lookup_word_break_class(codepoint); + if (word_break_cls == WordBreakClass::ALetter || + word_break_cls == WordBreakClass::Katakana || + word_break_cls == WordBreakClass::Other) { + if (is_line_break_complex_context(codepoint)) { + return WordBreakClass::SoutheastAsian; + } + if (script_cls != WordBreakClass::Other) { + if (script_cls == WordBreakClass::Hangul && + !is_ahletter(word_break_cls)) { + return word_break_cls; + } + return script_cls; + } + } + if (word_break_cls != WordBreakClass::Other) { + return word_break_cls; + } + if (is_extended_pictographic(codepoint)) { + return WordBreakClass::ExtendedPictographic; + } + return WordBreakClass::Other; +} + +CodepointProperties lookup_codepoint_properties(uint32_t codepoint) { + CodepointProperties props; + props.cls = classify_codepoint(static_cast(codepoint)); + props.extended_pictographic = + props.cls == WordBreakClass::ExtendedPictographic; + if (!props.extended_pictographic && codepoint >= 0x00A9) { + props.extended_pictographic = is_extended_pictographic(codepoint); + } + if (codepoint >= 0x261D) { + props.emoji_modifier_base = is_emoji_modifier_base(codepoint); + } + if (codepoint >= 0x1F3FB) { + props.emoji_modifier = is_emoji_modifier(codepoint); + } + return props; +} + +CodepointProperties get_codepoint_properties( + uint32_t codepoint, + std::array *cache) { + size_t slot = (codepoint * 2654435761u) & (kCodepointCacheSize - 1); + CodepointCacheEntry &entry = (*cache)[slot]; + if (entry.valid && entry.cp == codepoint) { + return entry.props; + } + + CodepointProperties props = lookup_codepoint_properties(codepoint); + entry.cp = codepoint; + entry.props = props; + entry.valid = true; + return props; +} + +std::array *codepoint_cache() { + thread_local std::array cache{}; + return &cache; +} + +std::vector decode_utf8(const std::string &text) { + std::vector codepoints; + codepoints.reserve(estimate_codepoint_capacity(text.size())); + auto *cache = codepoint_cache(); + const auto *str = reinterpret_cast(text.data()); + size_t len = text.size(); + size_t index = 0; + + while (index < len) { + utf8proc_int32_t cp; + size_t bytes = 0; + if (!decode_utf8_codepoint(str, len, index, &cp, &bytes)) { + Codepoint item; + item.start = static_cast(index); + item.end = static_cast(index + 1); + item.cls = WordBreakClass::Other; + codepoints.push_back(item); + ++index; + continue; + } + + Codepoint item; + item.cp = cp; + item.start = static_cast(index); + item.end = static_cast(index + bytes); + uint32_t codepoint = static_cast(cp); + CodepointProperties props = get_codepoint_properties(codepoint, cache); + item.cls = props.cls; + item.extended_pictographic = props.extended_pictographic; + item.emoji_modifier_base = props.emoji_modifier_base; + item.emoji_modifier = props.emoji_modifier; + codepoints.push_back(item); + index += bytes; + } + return codepoints; +} + +size_t next_significant(const std::vector &codepoints, + size_t index) { + while (index < codepoints.size() && is_ignored(codepoints[index].cls)) { + ++index; + } + return index; +} + +bool punctuation_connects(WordBreakClass left, WordBreakClass punct, + WordBreakClass right) { + if (is_ahletter(left) && is_ahletter(right) && + (punct == WordBreakClass::MidLetter || + punct == WordBreakClass::MidNumLet || + punct == WordBreakClass::SingleQuote)) { + return true; + } + if (left == WordBreakClass::HebrewLetter && + right == WordBreakClass::HebrewLetter && + punct == WordBreakClass::DoubleQuote) { + return true; + } + if (left == WordBreakClass::Numeric && right == WordBreakClass::Numeric && + (punct == WordBreakClass::MidNum || punct == WordBreakClass::MidNumLet || + punct == WordBreakClass::SingleQuote)) { + return true; + } + return false; +} + +bool significant_connects(WordBreakClass left, WordBreakClass right) { + if (is_ahletter(left) && is_ahletter(right)) { + return true; + } + if ((is_ahletter(left) && right == WordBreakClass::Numeric) || + (left == WordBreakClass::Numeric && is_ahletter(right))) { + return true; + } + if (left == WordBreakClass::Numeric && right == WordBreakClass::Numeric) { + return true; + } + if (left == WordBreakClass::Katakana && right == WordBreakClass::Katakana) { + return true; + } + if (is_connector(left) && right == WordBreakClass::ExtendNumLet) { + return true; + } + if (left == WordBreakClass::ExtendNumLet && is_connector(right)) { + return true; + } + if (left == WordBreakClass::ExtendNumLet && + right == WordBreakClass::ExtendNumLet) { + return true; + } + if (left == WordBreakClass::Hangul && right == WordBreakClass::Hangul) { + return true; + } + if (left == WordBreakClass::SoutheastAsian && + right == WordBreakClass::SoutheastAsian) { + return true; + } + return false; +} + +bool is_keycap_base(const Codepoint &codepoint) { + return (codepoint.cp >= '0' && codepoint.cp <= '9') || codepoint.cp == '#' || + codepoint.cp == '*'; +} + +size_t consume_extend_or_format(const std::vector &codepoints, + size_t index) { + while (index < codepoints.size() && + is_extend_or_format(codepoints[index].cls)) { + ++index; + } + return index; +} + +size_t consume_extend_format_and_modifier( + const std::vector &codepoints, size_t index) { + while (index < codepoints.size() && + is_extend_or_format(codepoints[index].cls)) { + ++index; + } + if (index < codepoints.size() && codepoints[index].emoji_modifier) { + index = consume_extend_or_format(codepoints, index + 1); + } + return index; +} + +size_t scan_keycap_token(const std::vector &codepoints, + size_t start) { + if (!is_keycap_base(codepoints[start])) { + return start; + } + + size_t index = start + 1; + if (index < codepoints.size() && + static_cast(codepoints[index].cp) == kVariationSelector16) { + ++index; + } + if (index >= codepoints.size() || + static_cast(codepoints[index].cp) != kKeycap) { + return start; + } + + return consume_extend_or_format(codepoints, index + 1); +} + +size_t scan_emoji_modifier_token(const std::vector &codepoints, + size_t start) { + if (codepoints[start].emoji_modifier) { + return consume_extend_or_format(codepoints, start + 1); + } + if (!codepoints[start].emoji_modifier_base) { + return start; + } + + size_t index = start + 1; + while (index < codepoints.size() && + is_extend_or_format(codepoints[index].cls) && + !codepoints[index].emoji_modifier) { + ++index; + } + if (index >= codepoints.size() || !codepoints[index].emoji_modifier) { + return start; + } + + return consume_extend_or_format(codepoints, index + 1); +} + +size_t scan_emoji_token(const std::vector &codepoints, + size_t start) { + size_t index = consume_extend_format_and_modifier(codepoints, start + 1); + + while (index < codepoints.size()) { + if (codepoints[index].cls != WordBreakClass::ZWJ) { + break; + } + ++index; + while (index < codepoints.size() && + is_extend_or_format(codepoints[index].cls)) { + ++index; + } + if (index >= codepoints.size() || + !is_extended_pictographic_codepoint(codepoints[index])) { + break; + } + ++index; + index = consume_extend_format_and_modifier(codepoints, index); + } + return index; +} + +size_t scan_zwj_ext_pict_token(const std::vector &codepoints, + size_t start) { + size_t index = start + 1; + if (index >= codepoints.size() || + !is_extended_pictographic_codepoint(codepoints[index])) { + return start; + } + return scan_emoji_token(codepoints, index); +} + +size_t scan_regional_indicator_token(const std::vector &codepoints, + size_t start) { + size_t index = start + 1; + while (index < codepoints.size() && is_ignored(codepoints[index].cls)) { + ++index; + } + if (index < codepoints.size() && + codepoints[index].cls == WordBreakClass::RegionalIndicator) { + ++index; + while (index < codepoints.size() && is_ignored(codepoints[index].cls)) { + ++index; + } + } + return index; +} + +size_t scan_single_token(const std::vector &codepoints, + size_t start) { + size_t index = start + 1; + while (index < codepoints.size() && is_ignored(codepoints[index].cls)) { + ++index; + } + return index; +} + +size_t scan_word_token(const std::vector &codepoints, size_t start) { + size_t end = start + 1; + size_t last_sig = start; + + while (end < codepoints.size()) { + WordBreakClass cls = codepoints[end].cls; + if (is_ignored(cls)) { + ++end; + continue; + } + if (cls == WordBreakClass::ExtendNumLet) { + if (!significant_connects(codepoints[last_sig].cls, cls)) { + break; + } + last_sig = end; + ++end; + continue; + } + if (cls == WordBreakClass::SingleQuote && + codepoints[last_sig].cls == WordBreakClass::HebrewLetter) { + last_sig = end; + ++end; + continue; + } + if (is_extended_pictographic_codepoint(codepoints[end]) && + previous_is_zwj(codepoints, end)) { + end = scan_emoji_token(codepoints, end); + last_sig = end - 1; + continue; + } + if (cls == WordBreakClass::Ideographic || + cls == WordBreakClass::ExtendedPictographic || + cls == WordBreakClass::RegionalIndicator || !is_token_start(cls)) { + size_t right = next_significant(codepoints, end + 1); + if (right < codepoints.size() && + punctuation_connects(codepoints[last_sig].cls, cls, + codepoints[right].cls)) { + end = right + 1; + last_sig = right; + continue; + } + break; + } + + if (!significant_connects(codepoints[last_sig].cls, cls)) { + break; + } + last_sig = end; + ++end; + } + return end; +} + +bool span_has_core_token(const std::vector &codepoints, size_t start, + size_t end) { + for (size_t index = start; index < end; ++index) { + WordBreakClass cls = codepoints[index].cls; + if (is_token_start(cls)) { return true; - default: - return is_word_start_char(cat); + } } + return false; } -bool is_cjk_ideograph(utf8proc_int32_t cp) { - return (cp >= 0x4E00 && cp <= 0x9FFF) || // CJK Unified Ideographs - (cp >= 0x3400 && cp <= 0x4DBF) || // CJK Extension A - (cp >= 0xF900 && cp <= 0xFAFF) || // CJK Compatibility Ideographs - (cp >= 0x20000 && cp <= 0x2A6DF) || // CJK Extension B - (cp >= 0x2A700 && cp <= 0x2B73F) || // CJK Extension C - (cp >= 0x2B740 && cp <= 0x2B81F) || // CJK Extension D - (cp >= 0x2B820 && cp <= 0x2CEAF) || // CJK Extension E - (cp >= 0x2CEB0 && cp <= 0x2EBEF) || // CJK Extension F - (cp >= 0x2EBF0 && cp <= 0x2EE5F) || // CJK Extension I - (cp >= 0x2F800 && cp <= 0x2FA1F) || // CJK Compatibility Supplement - (cp >= 0x30000 && cp <= 0x3134F) || // CJK Extension G - (cp >= 0x31350 && cp <= 0x323AF) || // CJK Extension H - (cp >= 0x323B0 && cp <= 0x3347F); // CJK Extension J +size_t trim_non_core_suffix(const std::vector &codepoints, + size_t start, size_t end) { + size_t trimmed = end; + while (trimmed > start) { + WordBreakClass cls = codepoints[trimmed - 1].cls; + if (is_token_start(cls) || is_ignored(cls)) { + break; + } + --trimmed; + } + return trimmed; } -} // namespace +void emit_non_empty_core_span(const std::string &text, + const std::vector &codepoints, + size_t start, size_t end, uint32_t *position, + std::vector *tokens) { + if (start >= end || !span_has_core_token(codepoints, start, end)) { + return; + } + Token token; + token.text = text.substr(codepoints[start].start, + codepoints[end - 1].end - codepoints[start].start); + token.offset = codepoints[start].start; + token.position = (*position)++; + tokens->push_back(std::move(token)); +} -bool StandardTokenizer::init(const ailego::JsonObject &config) { - auto length_val = config["max_token_length"]; - if (length_val.is_integer()) { - uint32_t configured_length = static_cast(length_val.as_integer()); - if (configured_length > 0) { - max_token_length_ = configured_length; +void emit_token_span(const std::string &text, + const std::vector &codepoints, size_t start, + size_t end, uint32_t max_token_length, uint32_t *position, + std::vector *tokens) { + if (end - start <= max_token_length) { + Token token; + token.text = text.substr(codepoints[start].start, + codepoints[end - 1].end - codepoints[start].start); + token.offset = codepoints[start].start; + token.position = (*position)++; + tokens->push_back(std::move(token)); + return; + } + + size_t token_start = start; + uint32_t codepoint_count = 0; + size_t index = start; + while (index < end) { + if (codepoint_count >= max_token_length && + is_token_start(codepoints[index].cls)) { + size_t emit_end = trim_non_core_suffix(codepoints, token_start, index); + emit_non_empty_core_span(text, codepoints, token_start, emit_end, + position, tokens); + token_start = index; + codepoint_count = 0; + continue; + } + ++codepoint_count; + ++index; + } + + size_t token_end = end; + if (end - token_start > max_token_length) { + token_end = trim_non_core_suffix(codepoints, token_start, end); + } + emit_non_empty_core_span(text, codepoints, token_start, token_end, position, + tokens); +} + +bool is_ascii_text(const std::string &text) { + for (unsigned char ch : text) { + if ((ch & 0x80) != 0) { + return false; } } return true; } -std::vector StandardTokenizer::tokenize(const std::string &text) const { - std::vector tokens; - uint32_t position = 0; - const auto *str = reinterpret_cast(text.data()); - auto len = static_cast(text.size()); - utf8proc_ssize_t index = 0; +size_t scan_ascii_word_token(const std::string &text, size_t start) { + size_t end = start + 1; + size_t last_sig = start; - while (index < len) { - // Decode current codepoint. - utf8proc_int32_t cp; - utf8proc_ssize_t bytes = utf8proc_iterate(str + index, len - index, &cp); - if (bytes < 1) { + while (end < text.size()) { + auto ch = static_cast(text[end]); + if (is_ascii_word_body_char(ch)) { + last_sig = end; + ++end; + continue; + } + + WordBreakClass cls = lookup_ascii_word_break_class(ch); + if (cls == WordBreakClass::ExtendNumLet) { + WordBreakClass left = lookup_ascii_word_break_class( + static_cast(text[last_sig])); + if (!significant_connects(left, cls)) { + break; + } + last_sig = end; + ++end; + continue; + } + if (!is_token_start(cls)) { + size_t right = end + 1; + if (right < text.size()) { + WordBreakClass left = lookup_ascii_word_break_class( + static_cast(text[last_sig])); + WordBreakClass right_cls = lookup_ascii_word_break_class( + static_cast(text[right])); + if (punctuation_connects(left, cls, right_cls)) { + end = right + 1; + last_sig = right; + continue; + } + } + break; + } + + WordBreakClass left = lookup_ascii_word_break_class( + static_cast(text[last_sig])); + if (!significant_connects(left, cls)) { + break; + } + last_sig = end; + ++end; + } + return end; +} + +bool ascii_span_has_core_token(const std::string &text, size_t start, + size_t end) { + for (size_t index = start; index < end; ++index) { + auto ch = static_cast(text[index]); + if (is_ascii_letter_or_digit(ch)) { + return true; + } + } + return false; +} + +size_t trim_ascii_non_core_suffix(const std::string &text, size_t start, + size_t end) { + size_t trimmed = end; + while (trimmed > start) { + auto ch = static_cast(text[trimmed - 1]); + if (is_ascii_letter_or_digit(ch)) { + break; + } + --trimmed; + } + return trimmed; +} + +void emit_non_empty_ascii_core_span(const std::string &text, size_t start, + size_t end, uint32_t *position, + std::vector *tokens) { + if (start >= end || !ascii_span_has_core_token(text, start, end)) { + return; + } + Token token; + token.text = text.substr(start, end - start); + token.offset = static_cast(start); + token.position = (*position)++; + tokens->push_back(std::move(token)); +} + +void emit_ascii_token_span(const std::string &text, size_t start, size_t end, + uint32_t max_token_length, uint32_t *position, + std::vector *tokens) { + if (end - start <= max_token_length) { + Token token; + token.text = text.substr(start, end - start); + token.offset = static_cast(start); + token.position = (*position)++; + tokens->push_back(std::move(token)); + return; + } + + size_t token_start = start; + uint32_t codepoint_count = 0; + size_t index = start; + while (index < end) { + auto ch = static_cast(text[index]); + if (codepoint_count >= max_token_length && is_ascii_letter_or_digit(ch)) { + size_t emit_end = trim_ascii_non_core_suffix(text, token_start, index); + emit_non_empty_ascii_core_span(text, token_start, emit_end, position, + tokens); + token_start = index; + codepoint_count = 0; + continue; + } + ++codepoint_count; + ++index; + } + + size_t token_end = end; + if (end - token_start > max_token_length) { + token_end = trim_ascii_non_core_suffix(text, token_start, end); + } + emit_non_empty_ascii_core_span(text, token_start, token_end, position, + tokens); +} + +std::vector tokenize_ascii(const std::string &text, + uint32_t max_token_length) { + std::vector tokens; + tokens.reserve(estimate_token_capacity(text.size())); + uint32_t position = 0; + size_t index = 0; + while (index < text.size()) { + auto ch = static_cast(text[index]); + if (is_ascii_letter_or_digit(ch)) { + size_t end = scan_ascii_word_token(text, index); + emit_ascii_token_span(text, index, end, max_token_length, &position, + &tokens); + index = end; + continue; + } + + WordBreakClass cls = lookup_ascii_word_break_class(ch); + if (cls == WordBreakClass::ExtendNumLet) { + size_t end = scan_ascii_word_token(text, index); + if (ascii_span_has_core_token(text, index, end)) { + emit_ascii_token_span(text, index, end, max_token_length, &position, + &tokens); + } + index = end; + continue; + } + ++index; + } + return tokens; +} + +} // namespace + +bool StandardTokenizer::init(const ailego::JsonObject &config) { + max_token_length_ = kDefaultMaxTokenLength; + auto length_val = config["max_token_length"]; + if (!length_val.is_null()) { + if (!length_val.is_integer()) { + LOG_ERROR("StandardTokenizer: max_token_length must be integer"); + return false; + } + auto configured_length = length_val.as_integer(); + if (configured_length < kMinMaxTokenLength || + configured_length > kMaxMaxTokenLength) { + LOG_ERROR("StandardTokenizer: max_token_length out of range: %zu", + (size_t)configured_length); + return false; + } + max_token_length_ = static_cast(configured_length); + } + return true; +} + +std::vector StandardTokenizer::tokenize(const std::string &text) const { + if (is_ascii_text(text)) { + return tokenize_ascii(text, max_token_length_); + } + + std::vector tokens; + tokens.reserve(estimate_token_capacity(text.size())); + uint32_t position = 0; + std::vector codepoints = decode_utf8(text); + + size_t index = 0; + while (index < codepoints.size()) { + WordBreakClass cls = codepoints[index].cls; + if (cls == WordBreakClass::Ideographic || cls == WordBreakClass::Hiragana) { + size_t end = scan_single_token(codepoints, index); + emit_token_span(text, codepoints, index, end, max_token_length_, + &position, &tokens); + index = end; + continue; + } + if (cls == WordBreakClass::Hangul || + cls == WordBreakClass::SoutheastAsian) { + size_t end = scan_word_token(codepoints, index); + emit_token_span(text, codepoints, index, end, max_token_length_, + &position, &tokens); + index = end; + continue; + } + if (cls == WordBreakClass::RegionalIndicator) { + size_t end = scan_regional_indicator_token(codepoints, index); + emit_token_span(text, codepoints, index, end, max_token_length_, + &position, &tokens); + index = end; + continue; + } + size_t end = scan_keycap_token(codepoints, index); + if (end > index) { + emit_token_span(text, codepoints, index, end, max_token_length_, + &position, &tokens); + index = end; + continue; + } + if (cls == WordBreakClass::ExtendedPictographic) { + end = scan_emoji_token(codepoints, index); + emit_token_span(text, codepoints, index, end, max_token_length_, + &position, &tokens); + index = end; + continue; + } + if (cls == WordBreakClass::ZWJ) { + end = scan_zwj_ext_pict_token(codepoints, index); + if (end > index) { + emit_token_span(text, codepoints, index, end, max_token_length_, + &position, &tokens); + index = end; + continue; + } + } + end = scan_emoji_modifier_token(codepoints, index); + if (end > index) { + emit_token_span(text, codepoints, index, end, max_token_length_, + &position, &tokens); + index = end; + continue; + } + if (cls == WordBreakClass::ExtendNumLet) { + end = scan_word_token(codepoints, index); + if (span_has_core_token(codepoints, index, end)) { + emit_token_span(text, codepoints, index, end, max_token_length_, + &position, &tokens); + } + index = end; + continue; + } + if (!is_token_start(cls)) { ++index; continue; } - // CJK ideograph → emit as a single-character token (always 1 codepoint, - // which cannot exceed max_token_length_ since its minimum value is 1). - if (is_cjk_ideograph(cp)) { - Token token; - token.text = - text.substr(static_cast(index), static_cast(bytes)); - token.offset = static_cast(index); - token.position = position++; - tokens.push_back(std::move(token)); - index += bytes; - continue; - } - - auto cat = utf8proc_category(cp); - // Skip delimiters and continuation-only characters that cannot start words. - if (!is_word_start_char(cat)) { - index += bytes; - continue; - } - - // Accumulate a word token. Marks can continue a token, but cannot start - // one. Split at max_token_length_ codepoints (aligned with ES behavior). - utf8proc_ssize_t token_start = index; - uint32_t codepoint_count = 1; - index += bytes; - while (index < len) { - utf8proc_int32_t next_cp; - utf8proc_ssize_t next_bytes = - utf8proc_iterate(str + index, len - index, &next_cp); - if (next_bytes < 1) { - break; - } - if (is_cjk_ideograph(next_cp)) { - break; - } - auto next_cat = utf8proc_category(next_cp); - if (!is_word_continue_char(next_cat)) { - break; - } - // Emit a segment when codepoint count reaches the limit, but do not - // create a segment that starts with a continuation-only mark. - if (codepoint_count >= max_token_length_ && - is_word_start_char(next_cat)) { - Token token; - token.text = text.substr(static_cast(token_start), - static_cast(index - token_start)); - token.offset = static_cast(token_start); - token.position = position++; - tokens.push_back(std::move(token)); - token_start = index; - codepoint_count = 0; - } - ++codepoint_count; - index += next_bytes; - } - - // Emit the remaining segment (if any). - if (index > token_start) { - Token token; - token.text = text.substr(static_cast(token_start), - static_cast(index - token_start)); - token.offset = static_cast(token_start); - token.position = position++; - tokens.push_back(std::move(token)); - } + end = scan_word_token(codepoints, index); + emit_token_span(text, codepoints, index, end, max_token_length_, &position, + &tokens); + index = end; } return tokens; diff --git a/src/db/index/column/fts_column/tokenizer/standard_tokenizer.h b/src/db/index/column/fts_column/tokenizer/standard_tokenizer.h index 8fbc9ef..9c46b3b 100644 --- a/src/db/index/column/fts_column/tokenizer/standard_tokenizer.h +++ b/src/db/index/column/fts_column/tokenizer/standard_tokenizer.h @@ -21,17 +21,19 @@ namespace zvec::fts { /*! Standard tokenizer * Unicode-aware tokenizer aligned with Elasticsearch's standard tokenizer. - * Splits text on non-alphanumeric characters (punctuation, whitespace, - * symbols) using Unicode categories via utf8proc. CJK ideographs are emitted - * as individual single-character tokens. + * Uses a UAX #29 word-boundary profile with Lucene/Elasticsearch compatible + * token selection. CJK ideographs are emitted as individual single-character + * tokens. */ class StandardTokenizer : public Tokenizer { public: /*! Initialise from JSON config. * Supported keys: - * "max_token_length" (uint32, default 255): tokens with more codepoints - * than this limit are split at the boundary into multiple tokens. - * Always returns true. + * "max_token_length" (uint32, default 255, range [1, 1048576]): long + * tokens are split into smaller segments. Combining marks and other + * ignored word-break characters may stay attached to the previous + * segment to avoid creating mark-only tokens. + * Returns false when the configuration is invalid. */ bool init(const ailego::JsonObject &config) override; diff --git a/src/db/index/column/fts_column/tokenizer/standard_tokenizer_unicode.inc b/src/db/index/column/fts_column/tokenizer/standard_tokenizer_unicode.inc new file mode 100644 index 0000000..36b9493 --- /dev/null +++ b/src/db/index/column/fts_column/tokenizer/standard_tokenizer_unicode.inc @@ -0,0 +1,1398 @@ +// Generated by scripts/generate_standard_tokenizer_unicode.py +// Source: Unicode 17.0.0 WordBreakProperty, emoji-data, LineBreak, Scripts. +// Derived from Unicode data files licensed under Unicode License V3; see NOTICE. +// Do not edit by hand. + +// clang-format off + +constexpr UnicodeClassRange kWordBreakRanges[] = { + {0x000A, 0x000A, WordBreakClass::LF}, + {0x000B, 0x000C, WordBreakClass::Newline}, + {0x000D, 0x000D, WordBreakClass::CR}, + {0x0020, 0x0020, WordBreakClass::WSegSpace}, + {0x0022, 0x0022, WordBreakClass::DoubleQuote}, + {0x0027, 0x0027, WordBreakClass::SingleQuote}, + {0x002C, 0x002C, WordBreakClass::MidNum}, + {0x002E, 0x002E, WordBreakClass::MidNumLet}, + {0x0030, 0x0039, WordBreakClass::Numeric}, + {0x003A, 0x003A, WordBreakClass::MidLetter}, + {0x003B, 0x003B, WordBreakClass::MidNum}, + {0x0041, 0x005A, WordBreakClass::ALetter}, + {0x005F, 0x005F, WordBreakClass::ExtendNumLet}, + {0x0061, 0x007A, WordBreakClass::ALetter}, + {0x0085, 0x0085, WordBreakClass::Newline}, + {0x00AA, 0x00AA, WordBreakClass::ALetter}, + {0x00AD, 0x00AD, WordBreakClass::Format}, + {0x00B5, 0x00B5, WordBreakClass::ALetter}, + {0x00B7, 0x00B7, WordBreakClass::MidLetter}, + {0x00B8, 0x00B8, WordBreakClass::ALetter}, + {0x00BA, 0x00BA, WordBreakClass::ALetter}, + {0x00C0, 0x00D6, WordBreakClass::ALetter}, + {0x00D8, 0x00F6, WordBreakClass::ALetter}, + {0x00F8, 0x02D7, WordBreakClass::ALetter}, + {0x02DE, 0x02FF, WordBreakClass::ALetter}, + {0x0300, 0x036F, WordBreakClass::Extend}, + {0x0370, 0x0374, WordBreakClass::ALetter}, + {0x0376, 0x0377, WordBreakClass::ALetter}, + {0x037A, 0x037D, WordBreakClass::ALetter}, + {0x037E, 0x037E, WordBreakClass::MidNum}, + {0x037F, 0x037F, WordBreakClass::ALetter}, + {0x0386, 0x0386, WordBreakClass::ALetter}, + {0x0387, 0x0387, WordBreakClass::MidLetter}, + {0x0388, 0x038A, WordBreakClass::ALetter}, + {0x038C, 0x038C, WordBreakClass::ALetter}, + {0x038E, 0x03A1, WordBreakClass::ALetter}, + {0x03A3, 0x03F5, WordBreakClass::ALetter}, + {0x03F7, 0x0481, WordBreakClass::ALetter}, + {0x0483, 0x0489, WordBreakClass::Extend}, + {0x048A, 0x052F, WordBreakClass::ALetter}, + {0x0531, 0x0556, WordBreakClass::ALetter}, + {0x0559, 0x055C, WordBreakClass::ALetter}, + {0x055E, 0x055E, WordBreakClass::ALetter}, + {0x055F, 0x055F, WordBreakClass::MidLetter}, + {0x0560, 0x0588, WordBreakClass::ALetter}, + {0x0589, 0x0589, WordBreakClass::MidNum}, + {0x058A, 0x058A, WordBreakClass::ALetter}, + {0x0591, 0x05BD, WordBreakClass::Extend}, + {0x05BF, 0x05BF, WordBreakClass::Extend}, + {0x05C1, 0x05C2, WordBreakClass::Extend}, + {0x05C4, 0x05C5, WordBreakClass::Extend}, + {0x05C7, 0x05C7, WordBreakClass::Extend}, + {0x05D0, 0x05EA, WordBreakClass::HebrewLetter}, + {0x05EF, 0x05F2, WordBreakClass::HebrewLetter}, + {0x05F3, 0x05F3, WordBreakClass::ALetter}, + {0x05F4, 0x05F4, WordBreakClass::MidLetter}, + {0x0600, 0x0605, WordBreakClass::Numeric}, + {0x060C, 0x060D, WordBreakClass::MidNum}, + {0x0610, 0x061A, WordBreakClass::Extend}, + {0x061C, 0x061C, WordBreakClass::Format}, + {0x0620, 0x064A, WordBreakClass::ALetter}, + {0x064B, 0x065F, WordBreakClass::Extend}, + {0x0660, 0x0669, WordBreakClass::Numeric}, + {0x066B, 0x066B, WordBreakClass::Numeric}, + {0x066C, 0x066C, WordBreakClass::MidNum}, + {0x066E, 0x066F, WordBreakClass::ALetter}, + {0x0670, 0x0670, WordBreakClass::Extend}, + {0x0671, 0x06D3, WordBreakClass::ALetter}, + {0x06D5, 0x06D5, WordBreakClass::ALetter}, + {0x06D6, 0x06DC, WordBreakClass::Extend}, + {0x06DD, 0x06DD, WordBreakClass::Numeric}, + {0x06DF, 0x06E4, WordBreakClass::Extend}, + {0x06E5, 0x06E6, WordBreakClass::ALetter}, + {0x06E7, 0x06E8, WordBreakClass::Extend}, + {0x06EA, 0x06ED, WordBreakClass::Extend}, + {0x06EE, 0x06EF, WordBreakClass::ALetter}, + {0x06F0, 0x06F9, WordBreakClass::Numeric}, + {0x06FA, 0x06FC, WordBreakClass::ALetter}, + {0x06FF, 0x06FF, WordBreakClass::ALetter}, + {0x070F, 0x0710, WordBreakClass::ALetter}, + {0x0711, 0x0711, WordBreakClass::Extend}, + {0x0712, 0x072F, WordBreakClass::ALetter}, + {0x0730, 0x074A, WordBreakClass::Extend}, + {0x074D, 0x07A5, WordBreakClass::ALetter}, + {0x07A6, 0x07B0, WordBreakClass::Extend}, + {0x07B1, 0x07B1, WordBreakClass::ALetter}, + {0x07C0, 0x07C9, WordBreakClass::Numeric}, + {0x07CA, 0x07EA, WordBreakClass::ALetter}, + {0x07EB, 0x07F3, WordBreakClass::Extend}, + {0x07F4, 0x07F5, WordBreakClass::ALetter}, + {0x07F8, 0x07F8, WordBreakClass::MidNum}, + {0x07FA, 0x07FA, WordBreakClass::ALetter}, + {0x07FD, 0x07FD, WordBreakClass::Extend}, + {0x0800, 0x0815, WordBreakClass::ALetter}, + {0x0816, 0x0819, WordBreakClass::Extend}, + {0x081A, 0x081A, WordBreakClass::ALetter}, + {0x081B, 0x0823, WordBreakClass::Extend}, + {0x0824, 0x0824, WordBreakClass::ALetter}, + {0x0825, 0x0827, WordBreakClass::Extend}, + {0x0828, 0x0828, WordBreakClass::ALetter}, + {0x0829, 0x082D, WordBreakClass::Extend}, + {0x0840, 0x0858, WordBreakClass::ALetter}, + {0x0859, 0x085B, WordBreakClass::Extend}, + {0x0860, 0x086A, WordBreakClass::ALetter}, + {0x0870, 0x0887, WordBreakClass::ALetter}, + {0x0889, 0x088F, WordBreakClass::ALetter}, + {0x0890, 0x0891, WordBreakClass::Numeric}, + {0x0897, 0x089F, WordBreakClass::Extend}, + {0x08A0, 0x08C9, WordBreakClass::ALetter}, + {0x08CA, 0x08E1, WordBreakClass::Extend}, + {0x08E2, 0x08E2, WordBreakClass::Numeric}, + {0x08E3, 0x0903, WordBreakClass::Extend}, + {0x0904, 0x0939, WordBreakClass::ALetter}, + {0x093A, 0x093C, WordBreakClass::Extend}, + {0x093D, 0x093D, WordBreakClass::ALetter}, + {0x093E, 0x094F, WordBreakClass::Extend}, + {0x0950, 0x0950, WordBreakClass::ALetter}, + {0x0951, 0x0957, WordBreakClass::Extend}, + {0x0958, 0x0961, WordBreakClass::ALetter}, + {0x0962, 0x0963, WordBreakClass::Extend}, + {0x0966, 0x096F, WordBreakClass::Numeric}, + {0x0971, 0x0980, WordBreakClass::ALetter}, + {0x0981, 0x0983, WordBreakClass::Extend}, + {0x0985, 0x098C, WordBreakClass::ALetter}, + {0x098F, 0x0990, WordBreakClass::ALetter}, + {0x0993, 0x09A8, WordBreakClass::ALetter}, + {0x09AA, 0x09B0, WordBreakClass::ALetter}, + {0x09B2, 0x09B2, WordBreakClass::ALetter}, + {0x09B6, 0x09B9, WordBreakClass::ALetter}, + {0x09BC, 0x09BC, WordBreakClass::Extend}, + {0x09BD, 0x09BD, WordBreakClass::ALetter}, + {0x09BE, 0x09C4, WordBreakClass::Extend}, + {0x09C7, 0x09C8, WordBreakClass::Extend}, + {0x09CB, 0x09CD, WordBreakClass::Extend}, + {0x09CE, 0x09CE, WordBreakClass::ALetter}, + {0x09D7, 0x09D7, WordBreakClass::Extend}, + {0x09DC, 0x09DD, WordBreakClass::ALetter}, + {0x09DF, 0x09E1, WordBreakClass::ALetter}, + {0x09E2, 0x09E3, WordBreakClass::Extend}, + {0x09E6, 0x09EF, WordBreakClass::Numeric}, + {0x09F0, 0x09F1, WordBreakClass::ALetter}, + {0x09FC, 0x09FC, WordBreakClass::ALetter}, + {0x09FE, 0x09FE, WordBreakClass::Extend}, + {0x0A01, 0x0A03, WordBreakClass::Extend}, + {0x0A05, 0x0A0A, WordBreakClass::ALetter}, + {0x0A0F, 0x0A10, WordBreakClass::ALetter}, + {0x0A13, 0x0A28, WordBreakClass::ALetter}, + {0x0A2A, 0x0A30, WordBreakClass::ALetter}, + {0x0A32, 0x0A33, WordBreakClass::ALetter}, + {0x0A35, 0x0A36, WordBreakClass::ALetter}, + {0x0A38, 0x0A39, WordBreakClass::ALetter}, + {0x0A3C, 0x0A3C, WordBreakClass::Extend}, + {0x0A3E, 0x0A42, WordBreakClass::Extend}, + {0x0A47, 0x0A48, WordBreakClass::Extend}, + {0x0A4B, 0x0A4D, WordBreakClass::Extend}, + {0x0A51, 0x0A51, WordBreakClass::Extend}, + {0x0A59, 0x0A5C, WordBreakClass::ALetter}, + {0x0A5E, 0x0A5E, WordBreakClass::ALetter}, + {0x0A66, 0x0A6F, WordBreakClass::Numeric}, + {0x0A70, 0x0A71, WordBreakClass::Extend}, + {0x0A72, 0x0A74, WordBreakClass::ALetter}, + {0x0A75, 0x0A75, WordBreakClass::Extend}, + {0x0A81, 0x0A83, WordBreakClass::Extend}, + {0x0A85, 0x0A8D, WordBreakClass::ALetter}, + {0x0A8F, 0x0A91, WordBreakClass::ALetter}, + {0x0A93, 0x0AA8, WordBreakClass::ALetter}, + {0x0AAA, 0x0AB0, WordBreakClass::ALetter}, + {0x0AB2, 0x0AB3, WordBreakClass::ALetter}, + {0x0AB5, 0x0AB9, WordBreakClass::ALetter}, + {0x0ABC, 0x0ABC, WordBreakClass::Extend}, + {0x0ABD, 0x0ABD, WordBreakClass::ALetter}, + {0x0ABE, 0x0AC5, WordBreakClass::Extend}, + {0x0AC7, 0x0AC9, WordBreakClass::Extend}, + {0x0ACB, 0x0ACD, WordBreakClass::Extend}, + {0x0AD0, 0x0AD0, WordBreakClass::ALetter}, + {0x0AE0, 0x0AE1, WordBreakClass::ALetter}, + {0x0AE2, 0x0AE3, WordBreakClass::Extend}, + {0x0AE6, 0x0AEF, WordBreakClass::Numeric}, + {0x0AF9, 0x0AF9, WordBreakClass::ALetter}, + {0x0AFA, 0x0AFF, WordBreakClass::Extend}, + {0x0B01, 0x0B03, WordBreakClass::Extend}, + {0x0B05, 0x0B0C, WordBreakClass::ALetter}, + {0x0B0F, 0x0B10, WordBreakClass::ALetter}, + {0x0B13, 0x0B28, WordBreakClass::ALetter}, + {0x0B2A, 0x0B30, WordBreakClass::ALetter}, + {0x0B32, 0x0B33, WordBreakClass::ALetter}, + {0x0B35, 0x0B39, WordBreakClass::ALetter}, + {0x0B3C, 0x0B3C, WordBreakClass::Extend}, + {0x0B3D, 0x0B3D, WordBreakClass::ALetter}, + {0x0B3E, 0x0B44, WordBreakClass::Extend}, + {0x0B47, 0x0B48, WordBreakClass::Extend}, + {0x0B4B, 0x0B4D, WordBreakClass::Extend}, + {0x0B55, 0x0B57, WordBreakClass::Extend}, + {0x0B5C, 0x0B5D, WordBreakClass::ALetter}, + {0x0B5F, 0x0B61, WordBreakClass::ALetter}, + {0x0B62, 0x0B63, WordBreakClass::Extend}, + {0x0B66, 0x0B6F, WordBreakClass::Numeric}, + {0x0B71, 0x0B71, WordBreakClass::ALetter}, + {0x0B82, 0x0B82, WordBreakClass::Extend}, + {0x0B83, 0x0B83, WordBreakClass::ALetter}, + {0x0B85, 0x0B8A, WordBreakClass::ALetter}, + {0x0B8E, 0x0B90, WordBreakClass::ALetter}, + {0x0B92, 0x0B95, WordBreakClass::ALetter}, + {0x0B99, 0x0B9A, WordBreakClass::ALetter}, + {0x0B9C, 0x0B9C, WordBreakClass::ALetter}, + {0x0B9E, 0x0B9F, WordBreakClass::ALetter}, + {0x0BA3, 0x0BA4, WordBreakClass::ALetter}, + {0x0BA8, 0x0BAA, WordBreakClass::ALetter}, + {0x0BAE, 0x0BB9, WordBreakClass::ALetter}, + {0x0BBE, 0x0BC2, WordBreakClass::Extend}, + {0x0BC6, 0x0BC8, WordBreakClass::Extend}, + {0x0BCA, 0x0BCD, WordBreakClass::Extend}, + {0x0BD0, 0x0BD0, WordBreakClass::ALetter}, + {0x0BD7, 0x0BD7, WordBreakClass::Extend}, + {0x0BE6, 0x0BEF, WordBreakClass::Numeric}, + {0x0C00, 0x0C04, WordBreakClass::Extend}, + {0x0C05, 0x0C0C, WordBreakClass::ALetter}, + {0x0C0E, 0x0C10, WordBreakClass::ALetter}, + {0x0C12, 0x0C28, WordBreakClass::ALetter}, + {0x0C2A, 0x0C39, WordBreakClass::ALetter}, + {0x0C3C, 0x0C3C, WordBreakClass::Extend}, + {0x0C3D, 0x0C3D, WordBreakClass::ALetter}, + {0x0C3E, 0x0C44, WordBreakClass::Extend}, + {0x0C46, 0x0C48, WordBreakClass::Extend}, + {0x0C4A, 0x0C4D, WordBreakClass::Extend}, + {0x0C55, 0x0C56, WordBreakClass::Extend}, + {0x0C58, 0x0C5A, WordBreakClass::ALetter}, + {0x0C5C, 0x0C5D, WordBreakClass::ALetter}, + {0x0C60, 0x0C61, WordBreakClass::ALetter}, + {0x0C62, 0x0C63, WordBreakClass::Extend}, + {0x0C66, 0x0C6F, WordBreakClass::Numeric}, + {0x0C80, 0x0C80, WordBreakClass::ALetter}, + {0x0C81, 0x0C83, WordBreakClass::Extend}, + {0x0C85, 0x0C8C, WordBreakClass::ALetter}, + {0x0C8E, 0x0C90, WordBreakClass::ALetter}, + {0x0C92, 0x0CA8, WordBreakClass::ALetter}, + {0x0CAA, 0x0CB3, WordBreakClass::ALetter}, + {0x0CB5, 0x0CB9, WordBreakClass::ALetter}, + {0x0CBC, 0x0CBC, WordBreakClass::Extend}, + {0x0CBD, 0x0CBD, WordBreakClass::ALetter}, + {0x0CBE, 0x0CC4, WordBreakClass::Extend}, + {0x0CC6, 0x0CC8, WordBreakClass::Extend}, + {0x0CCA, 0x0CCD, WordBreakClass::Extend}, + {0x0CD5, 0x0CD6, WordBreakClass::Extend}, + {0x0CDC, 0x0CDE, WordBreakClass::ALetter}, + {0x0CE0, 0x0CE1, WordBreakClass::ALetter}, + {0x0CE2, 0x0CE3, WordBreakClass::Extend}, + {0x0CE6, 0x0CEF, WordBreakClass::Numeric}, + {0x0CF1, 0x0CF2, WordBreakClass::ALetter}, + {0x0CF3, 0x0CF3, WordBreakClass::Extend}, + {0x0D00, 0x0D03, WordBreakClass::Extend}, + {0x0D04, 0x0D0C, WordBreakClass::ALetter}, + {0x0D0E, 0x0D10, WordBreakClass::ALetter}, + {0x0D12, 0x0D3A, WordBreakClass::ALetter}, + {0x0D3B, 0x0D3C, WordBreakClass::Extend}, + {0x0D3D, 0x0D3D, WordBreakClass::ALetter}, + {0x0D3E, 0x0D44, WordBreakClass::Extend}, + {0x0D46, 0x0D48, WordBreakClass::Extend}, + {0x0D4A, 0x0D4D, WordBreakClass::Extend}, + {0x0D4E, 0x0D4E, WordBreakClass::ALetter}, + {0x0D54, 0x0D56, WordBreakClass::ALetter}, + {0x0D57, 0x0D57, WordBreakClass::Extend}, + {0x0D5F, 0x0D61, WordBreakClass::ALetter}, + {0x0D62, 0x0D63, WordBreakClass::Extend}, + {0x0D66, 0x0D6F, WordBreakClass::Numeric}, + {0x0D7A, 0x0D7F, WordBreakClass::ALetter}, + {0x0D81, 0x0D83, WordBreakClass::Extend}, + {0x0D85, 0x0D96, WordBreakClass::ALetter}, + {0x0D9A, 0x0DB1, WordBreakClass::ALetter}, + {0x0DB3, 0x0DBB, WordBreakClass::ALetter}, + {0x0DBD, 0x0DBD, WordBreakClass::ALetter}, + {0x0DC0, 0x0DC6, WordBreakClass::ALetter}, + {0x0DCA, 0x0DCA, WordBreakClass::Extend}, + {0x0DCF, 0x0DD4, WordBreakClass::Extend}, + {0x0DD6, 0x0DD6, WordBreakClass::Extend}, + {0x0DD8, 0x0DDF, WordBreakClass::Extend}, + {0x0DE6, 0x0DEF, WordBreakClass::Numeric}, + {0x0DF2, 0x0DF3, WordBreakClass::Extend}, + {0x0E31, 0x0E31, WordBreakClass::Extend}, + {0x0E34, 0x0E3A, WordBreakClass::Extend}, + {0x0E47, 0x0E4E, WordBreakClass::Extend}, + {0x0E50, 0x0E59, WordBreakClass::Numeric}, + {0x0EB1, 0x0EB1, WordBreakClass::Extend}, + {0x0EB4, 0x0EBC, WordBreakClass::Extend}, + {0x0EC8, 0x0ECE, WordBreakClass::Extend}, + {0x0ED0, 0x0ED9, WordBreakClass::Numeric}, + {0x0F00, 0x0F00, WordBreakClass::ALetter}, + {0x0F18, 0x0F19, WordBreakClass::Extend}, + {0x0F20, 0x0F29, WordBreakClass::Numeric}, + {0x0F35, 0x0F35, WordBreakClass::Extend}, + {0x0F37, 0x0F37, WordBreakClass::Extend}, + {0x0F39, 0x0F39, WordBreakClass::Extend}, + {0x0F3E, 0x0F3F, WordBreakClass::Extend}, + {0x0F40, 0x0F47, WordBreakClass::ALetter}, + {0x0F49, 0x0F6C, WordBreakClass::ALetter}, + {0x0F71, 0x0F84, WordBreakClass::Extend}, + {0x0F86, 0x0F87, WordBreakClass::Extend}, + {0x0F88, 0x0F8C, WordBreakClass::ALetter}, + {0x0F8D, 0x0F97, WordBreakClass::Extend}, + {0x0F99, 0x0FBC, WordBreakClass::Extend}, + {0x0FC6, 0x0FC6, WordBreakClass::Extend}, + {0x102B, 0x103E, WordBreakClass::Extend}, + {0x1040, 0x1049, WordBreakClass::Numeric}, + {0x1056, 0x1059, WordBreakClass::Extend}, + {0x105E, 0x1060, WordBreakClass::Extend}, + {0x1062, 0x1064, WordBreakClass::Extend}, + {0x1067, 0x106D, WordBreakClass::Extend}, + {0x1071, 0x1074, WordBreakClass::Extend}, + {0x1082, 0x108D, WordBreakClass::Extend}, + {0x108F, 0x108F, WordBreakClass::Extend}, + {0x1090, 0x1099, WordBreakClass::Numeric}, + {0x109A, 0x109D, WordBreakClass::Extend}, + {0x10A0, 0x10C5, WordBreakClass::ALetter}, + {0x10C7, 0x10C7, WordBreakClass::ALetter}, + {0x10CD, 0x10CD, WordBreakClass::ALetter}, + {0x10D0, 0x10FA, WordBreakClass::ALetter}, + {0x10FC, 0x1248, WordBreakClass::ALetter}, + {0x124A, 0x124D, WordBreakClass::ALetter}, + {0x1250, 0x1256, WordBreakClass::ALetter}, + {0x1258, 0x1258, WordBreakClass::ALetter}, + {0x125A, 0x125D, WordBreakClass::ALetter}, + {0x1260, 0x1288, WordBreakClass::ALetter}, + {0x128A, 0x128D, WordBreakClass::ALetter}, + {0x1290, 0x12B0, WordBreakClass::ALetter}, + {0x12B2, 0x12B5, WordBreakClass::ALetter}, + {0x12B8, 0x12BE, WordBreakClass::ALetter}, + {0x12C0, 0x12C0, WordBreakClass::ALetter}, + {0x12C2, 0x12C5, WordBreakClass::ALetter}, + {0x12C8, 0x12D6, WordBreakClass::ALetter}, + {0x12D8, 0x1310, WordBreakClass::ALetter}, + {0x1312, 0x1315, WordBreakClass::ALetter}, + {0x1318, 0x135A, WordBreakClass::ALetter}, + {0x135D, 0x135F, WordBreakClass::Extend}, + {0x1380, 0x138F, WordBreakClass::ALetter}, + {0x13A0, 0x13F5, WordBreakClass::ALetter}, + {0x13F8, 0x13FD, WordBreakClass::ALetter}, + {0x1401, 0x166C, WordBreakClass::ALetter}, + {0x166F, 0x167F, WordBreakClass::ALetter}, + {0x1680, 0x1680, WordBreakClass::WSegSpace}, + {0x1681, 0x169A, WordBreakClass::ALetter}, + {0x16A0, 0x16EA, WordBreakClass::ALetter}, + {0x16EE, 0x16F8, WordBreakClass::ALetter}, + {0x1700, 0x1711, WordBreakClass::ALetter}, + {0x1712, 0x1715, WordBreakClass::Extend}, + {0x171F, 0x1731, WordBreakClass::ALetter}, + {0x1732, 0x1734, WordBreakClass::Extend}, + {0x1740, 0x1751, WordBreakClass::ALetter}, + {0x1752, 0x1753, WordBreakClass::Extend}, + {0x1760, 0x176C, WordBreakClass::ALetter}, + {0x176E, 0x1770, WordBreakClass::ALetter}, + {0x1772, 0x1773, WordBreakClass::Extend}, + {0x17B4, 0x17D3, WordBreakClass::Extend}, + {0x17DD, 0x17DD, WordBreakClass::Extend}, + {0x17E0, 0x17E9, WordBreakClass::Numeric}, + {0x180B, 0x180D, WordBreakClass::Extend}, + {0x180E, 0x180E, WordBreakClass::Format}, + {0x180F, 0x180F, WordBreakClass::Extend}, + {0x1810, 0x1819, WordBreakClass::Numeric}, + {0x1820, 0x1878, WordBreakClass::ALetter}, + {0x1880, 0x1884, WordBreakClass::ALetter}, + {0x1885, 0x1886, WordBreakClass::Extend}, + {0x1887, 0x18A8, WordBreakClass::ALetter}, + {0x18A9, 0x18A9, WordBreakClass::Extend}, + {0x18AA, 0x18AA, WordBreakClass::ALetter}, + {0x18B0, 0x18F5, WordBreakClass::ALetter}, + {0x1900, 0x191E, WordBreakClass::ALetter}, + {0x1920, 0x192B, WordBreakClass::Extend}, + {0x1930, 0x193B, WordBreakClass::Extend}, + {0x1946, 0x194F, WordBreakClass::Numeric}, + {0x19D0, 0x19DA, WordBreakClass::Numeric}, + {0x1A00, 0x1A16, WordBreakClass::ALetter}, + {0x1A17, 0x1A1B, WordBreakClass::Extend}, + {0x1A55, 0x1A5E, WordBreakClass::Extend}, + {0x1A60, 0x1A7C, WordBreakClass::Extend}, + {0x1A7F, 0x1A7F, WordBreakClass::Extend}, + {0x1A80, 0x1A89, WordBreakClass::Numeric}, + {0x1A90, 0x1A99, WordBreakClass::Numeric}, + {0x1AB0, 0x1ADD, WordBreakClass::Extend}, + {0x1AE0, 0x1AEB, WordBreakClass::Extend}, + {0x1B00, 0x1B04, WordBreakClass::Extend}, + {0x1B05, 0x1B33, WordBreakClass::ALetter}, + {0x1B34, 0x1B44, WordBreakClass::Extend}, + {0x1B45, 0x1B4C, WordBreakClass::ALetter}, + {0x1B50, 0x1B59, WordBreakClass::Numeric}, + {0x1B6B, 0x1B73, WordBreakClass::Extend}, + {0x1B80, 0x1B82, WordBreakClass::Extend}, + {0x1B83, 0x1BA0, WordBreakClass::ALetter}, + {0x1BA1, 0x1BAD, WordBreakClass::Extend}, + {0x1BAE, 0x1BAF, WordBreakClass::ALetter}, + {0x1BB0, 0x1BB9, WordBreakClass::Numeric}, + {0x1BBA, 0x1BE5, WordBreakClass::ALetter}, + {0x1BE6, 0x1BF3, WordBreakClass::Extend}, + {0x1C00, 0x1C23, WordBreakClass::ALetter}, + {0x1C24, 0x1C37, WordBreakClass::Extend}, + {0x1C40, 0x1C49, WordBreakClass::Numeric}, + {0x1C4D, 0x1C4F, WordBreakClass::ALetter}, + {0x1C50, 0x1C59, WordBreakClass::Numeric}, + {0x1C5A, 0x1C7D, WordBreakClass::ALetter}, + {0x1C80, 0x1C8A, WordBreakClass::ALetter}, + {0x1C90, 0x1CBA, WordBreakClass::ALetter}, + {0x1CBD, 0x1CBF, WordBreakClass::ALetter}, + {0x1CD0, 0x1CD2, WordBreakClass::Extend}, + {0x1CD4, 0x1CE8, WordBreakClass::Extend}, + {0x1CE9, 0x1CEC, WordBreakClass::ALetter}, + {0x1CED, 0x1CED, WordBreakClass::Extend}, + {0x1CEE, 0x1CF3, WordBreakClass::ALetter}, + {0x1CF4, 0x1CF4, WordBreakClass::Extend}, + {0x1CF5, 0x1CF6, WordBreakClass::ALetter}, + {0x1CF7, 0x1CF9, WordBreakClass::Extend}, + {0x1CFA, 0x1CFA, WordBreakClass::ALetter}, + {0x1D00, 0x1DBF, WordBreakClass::ALetter}, + {0x1DC0, 0x1DFF, WordBreakClass::Extend}, + {0x1E00, 0x1F15, WordBreakClass::ALetter}, + {0x1F18, 0x1F1D, WordBreakClass::ALetter}, + {0x1F20, 0x1F45, WordBreakClass::ALetter}, + {0x1F48, 0x1F4D, WordBreakClass::ALetter}, + {0x1F50, 0x1F57, WordBreakClass::ALetter}, + {0x1F59, 0x1F59, WordBreakClass::ALetter}, + {0x1F5B, 0x1F5B, WordBreakClass::ALetter}, + {0x1F5D, 0x1F5D, WordBreakClass::ALetter}, + {0x1F5F, 0x1F7D, WordBreakClass::ALetter}, + {0x1F80, 0x1FB4, WordBreakClass::ALetter}, + {0x1FB6, 0x1FBC, WordBreakClass::ALetter}, + {0x1FBE, 0x1FBE, WordBreakClass::ALetter}, + {0x1FC2, 0x1FC4, WordBreakClass::ALetter}, + {0x1FC6, 0x1FCC, WordBreakClass::ALetter}, + {0x1FD0, 0x1FD3, WordBreakClass::ALetter}, + {0x1FD6, 0x1FDB, WordBreakClass::ALetter}, + {0x1FE0, 0x1FEC, WordBreakClass::ALetter}, + {0x1FF2, 0x1FF4, WordBreakClass::ALetter}, + {0x1FF6, 0x1FFC, WordBreakClass::ALetter}, + {0x2000, 0x2006, WordBreakClass::WSegSpace}, + {0x2008, 0x200A, WordBreakClass::WSegSpace}, + {0x200C, 0x200C, WordBreakClass::Extend}, + {0x200D, 0x200D, WordBreakClass::ZWJ}, + {0x200E, 0x200F, WordBreakClass::Format}, + {0x2018, 0x2019, WordBreakClass::MidNumLet}, + {0x2024, 0x2024, WordBreakClass::MidNumLet}, + {0x2027, 0x2027, WordBreakClass::MidLetter}, + {0x2028, 0x2029, WordBreakClass::Newline}, + {0x202A, 0x202E, WordBreakClass::Format}, + {0x202F, 0x202F, WordBreakClass::ExtendNumLet}, + {0x203F, 0x2040, WordBreakClass::ExtendNumLet}, + {0x2044, 0x2044, WordBreakClass::MidNum}, + {0x2054, 0x2054, WordBreakClass::ExtendNumLet}, + {0x205F, 0x205F, WordBreakClass::WSegSpace}, + {0x2060, 0x2064, WordBreakClass::Format}, + {0x2066, 0x206F, WordBreakClass::Format}, + {0x2071, 0x2071, WordBreakClass::ALetter}, + {0x207F, 0x207F, WordBreakClass::ALetter}, + {0x2090, 0x209C, WordBreakClass::ALetter}, + {0x20D0, 0x20F0, WordBreakClass::Extend}, + {0x2102, 0x2102, WordBreakClass::ALetter}, + {0x2107, 0x2107, WordBreakClass::ALetter}, + {0x210A, 0x2113, WordBreakClass::ALetter}, + {0x2115, 0x2115, WordBreakClass::ALetter}, + {0x2119, 0x211D, WordBreakClass::ALetter}, + {0x2124, 0x2124, WordBreakClass::ALetter}, + {0x2126, 0x2126, WordBreakClass::ALetter}, + {0x2128, 0x2128, WordBreakClass::ALetter}, + {0x212A, 0x212D, WordBreakClass::ALetter}, + {0x212F, 0x2139, WordBreakClass::ALetter}, + {0x213C, 0x213F, WordBreakClass::ALetter}, + {0x2145, 0x2149, WordBreakClass::ALetter}, + {0x214E, 0x214E, WordBreakClass::ALetter}, + {0x2160, 0x2188, WordBreakClass::ALetter}, + {0x24B6, 0x24E9, WordBreakClass::ALetter}, + {0x2C00, 0x2CE4, WordBreakClass::ALetter}, + {0x2CEB, 0x2CEE, WordBreakClass::ALetter}, + {0x2CEF, 0x2CF1, WordBreakClass::Extend}, + {0x2CF2, 0x2CF3, WordBreakClass::ALetter}, + {0x2D00, 0x2D25, WordBreakClass::ALetter}, + {0x2D27, 0x2D27, WordBreakClass::ALetter}, + {0x2D2D, 0x2D2D, WordBreakClass::ALetter}, + {0x2D30, 0x2D67, WordBreakClass::ALetter}, + {0x2D6F, 0x2D6F, WordBreakClass::ALetter}, + {0x2D7F, 0x2D7F, WordBreakClass::Extend}, + {0x2D80, 0x2D96, WordBreakClass::ALetter}, + {0x2DA0, 0x2DA6, WordBreakClass::ALetter}, + {0x2DA8, 0x2DAE, WordBreakClass::ALetter}, + {0x2DB0, 0x2DB6, WordBreakClass::ALetter}, + {0x2DB8, 0x2DBE, WordBreakClass::ALetter}, + {0x2DC0, 0x2DC6, WordBreakClass::ALetter}, + {0x2DC8, 0x2DCE, WordBreakClass::ALetter}, + {0x2DD0, 0x2DD6, WordBreakClass::ALetter}, + {0x2DD8, 0x2DDE, WordBreakClass::ALetter}, + {0x2DE0, 0x2DFF, WordBreakClass::Extend}, + {0x2E2F, 0x2E2F, WordBreakClass::ALetter}, + {0x3000, 0x3000, WordBreakClass::WSegSpace}, + {0x3005, 0x3005, WordBreakClass::ALetter}, + {0x302A, 0x302F, WordBreakClass::Extend}, + {0x3031, 0x3035, WordBreakClass::Katakana}, + {0x303B, 0x303C, WordBreakClass::ALetter}, + {0x3099, 0x309A, WordBreakClass::Extend}, + {0x309B, 0x309C, WordBreakClass::Katakana}, + {0x30A0, 0x30FA, WordBreakClass::Katakana}, + {0x30FC, 0x30FF, WordBreakClass::Katakana}, + {0x3105, 0x312F, WordBreakClass::ALetter}, + {0x3131, 0x318E, WordBreakClass::ALetter}, + {0x31A0, 0x31BF, WordBreakClass::ALetter}, + {0x31F0, 0x31FF, WordBreakClass::Katakana}, + {0x32D0, 0x32FE, WordBreakClass::Katakana}, + {0x3300, 0x3357, WordBreakClass::Katakana}, + {0xA000, 0xA48C, WordBreakClass::ALetter}, + {0xA4D0, 0xA4FD, WordBreakClass::ALetter}, + {0xA500, 0xA60C, WordBreakClass::ALetter}, + {0xA610, 0xA61F, WordBreakClass::ALetter}, + {0xA620, 0xA629, WordBreakClass::Numeric}, + {0xA62A, 0xA62B, WordBreakClass::ALetter}, + {0xA640, 0xA66E, WordBreakClass::ALetter}, + {0xA66F, 0xA672, WordBreakClass::Extend}, + {0xA674, 0xA67D, WordBreakClass::Extend}, + {0xA67F, 0xA69D, WordBreakClass::ALetter}, + {0xA69E, 0xA69F, WordBreakClass::Extend}, + {0xA6A0, 0xA6EF, WordBreakClass::ALetter}, + {0xA6F0, 0xA6F1, WordBreakClass::Extend}, + {0xA708, 0xA7DC, WordBreakClass::ALetter}, + {0xA7F1, 0xA801, WordBreakClass::ALetter}, + {0xA802, 0xA802, WordBreakClass::Extend}, + {0xA803, 0xA805, WordBreakClass::ALetter}, + {0xA806, 0xA806, WordBreakClass::Extend}, + {0xA807, 0xA80A, WordBreakClass::ALetter}, + {0xA80B, 0xA80B, WordBreakClass::Extend}, + {0xA80C, 0xA822, WordBreakClass::ALetter}, + {0xA823, 0xA827, WordBreakClass::Extend}, + {0xA82C, 0xA82C, WordBreakClass::Extend}, + {0xA840, 0xA873, WordBreakClass::ALetter}, + {0xA880, 0xA881, WordBreakClass::Extend}, + {0xA882, 0xA8B3, WordBreakClass::ALetter}, + {0xA8B4, 0xA8C5, WordBreakClass::Extend}, + {0xA8D0, 0xA8D9, WordBreakClass::Numeric}, + {0xA8E0, 0xA8F1, WordBreakClass::Extend}, + {0xA8F2, 0xA8F7, WordBreakClass::ALetter}, + {0xA8FB, 0xA8FB, WordBreakClass::ALetter}, + {0xA8FD, 0xA8FE, WordBreakClass::ALetter}, + {0xA8FF, 0xA8FF, WordBreakClass::Extend}, + {0xA900, 0xA909, WordBreakClass::Numeric}, + {0xA90A, 0xA925, WordBreakClass::ALetter}, + {0xA926, 0xA92D, WordBreakClass::Extend}, + {0xA930, 0xA946, WordBreakClass::ALetter}, + {0xA947, 0xA953, WordBreakClass::Extend}, + {0xA960, 0xA97C, WordBreakClass::ALetter}, + {0xA980, 0xA983, WordBreakClass::Extend}, + {0xA984, 0xA9B2, WordBreakClass::ALetter}, + {0xA9B3, 0xA9C0, WordBreakClass::Extend}, + {0xA9CF, 0xA9CF, WordBreakClass::ALetter}, + {0xA9D0, 0xA9D9, WordBreakClass::Numeric}, + {0xA9E5, 0xA9E5, WordBreakClass::Extend}, + {0xA9F0, 0xA9F9, WordBreakClass::Numeric}, + {0xAA00, 0xAA28, WordBreakClass::ALetter}, + {0xAA29, 0xAA36, WordBreakClass::Extend}, + {0xAA40, 0xAA42, WordBreakClass::ALetter}, + {0xAA43, 0xAA43, WordBreakClass::Extend}, + {0xAA44, 0xAA4B, WordBreakClass::ALetter}, + {0xAA4C, 0xAA4D, WordBreakClass::Extend}, + {0xAA50, 0xAA59, WordBreakClass::Numeric}, + {0xAA7B, 0xAA7D, WordBreakClass::Extend}, + {0xAAB0, 0xAAB0, WordBreakClass::Extend}, + {0xAAB2, 0xAAB4, WordBreakClass::Extend}, + {0xAAB7, 0xAAB8, WordBreakClass::Extend}, + {0xAABE, 0xAABF, WordBreakClass::Extend}, + {0xAAC1, 0xAAC1, WordBreakClass::Extend}, + {0xAAE0, 0xAAEA, WordBreakClass::ALetter}, + {0xAAEB, 0xAAEF, WordBreakClass::Extend}, + {0xAAF2, 0xAAF4, WordBreakClass::ALetter}, + {0xAAF5, 0xAAF6, WordBreakClass::Extend}, + {0xAB01, 0xAB06, WordBreakClass::ALetter}, + {0xAB09, 0xAB0E, WordBreakClass::ALetter}, + {0xAB11, 0xAB16, WordBreakClass::ALetter}, + {0xAB20, 0xAB26, WordBreakClass::ALetter}, + {0xAB28, 0xAB2E, WordBreakClass::ALetter}, + {0xAB30, 0xAB69, WordBreakClass::ALetter}, + {0xAB70, 0xABE2, WordBreakClass::ALetter}, + {0xABE3, 0xABEA, WordBreakClass::Extend}, + {0xABEC, 0xABED, WordBreakClass::Extend}, + {0xABF0, 0xABF9, WordBreakClass::Numeric}, + {0xAC00, 0xD7A3, WordBreakClass::ALetter}, + {0xD7B0, 0xD7C6, WordBreakClass::ALetter}, + {0xD7CB, 0xD7FB, WordBreakClass::ALetter}, + {0xFB00, 0xFB06, WordBreakClass::ALetter}, + {0xFB13, 0xFB17, WordBreakClass::ALetter}, + {0xFB1D, 0xFB1D, WordBreakClass::HebrewLetter}, + {0xFB1E, 0xFB1E, WordBreakClass::Extend}, + {0xFB1F, 0xFB28, WordBreakClass::HebrewLetter}, + {0xFB2A, 0xFB36, WordBreakClass::HebrewLetter}, + {0xFB38, 0xFB3C, WordBreakClass::HebrewLetter}, + {0xFB3E, 0xFB3E, WordBreakClass::HebrewLetter}, + {0xFB40, 0xFB41, WordBreakClass::HebrewLetter}, + {0xFB43, 0xFB44, WordBreakClass::HebrewLetter}, + {0xFB46, 0xFB4F, WordBreakClass::HebrewLetter}, + {0xFB50, 0xFBB1, WordBreakClass::ALetter}, + {0xFBD3, 0xFD3D, WordBreakClass::ALetter}, + {0xFD50, 0xFD8F, WordBreakClass::ALetter}, + {0xFD92, 0xFDC7, WordBreakClass::ALetter}, + {0xFDF0, 0xFDFB, WordBreakClass::ALetter}, + {0xFE00, 0xFE0F, WordBreakClass::Extend}, + {0xFE13, 0xFE13, WordBreakClass::MidLetter}, + {0xFE20, 0xFE2F, WordBreakClass::Extend}, + {0xFE33, 0xFE34, WordBreakClass::ExtendNumLet}, + {0xFE4D, 0xFE4F, WordBreakClass::ExtendNumLet}, + {0xFE50, 0xFE50, WordBreakClass::MidNum}, + {0xFE52, 0xFE52, WordBreakClass::MidNumLet}, + {0xFE54, 0xFE54, WordBreakClass::MidNum}, + {0xFE55, 0xFE55, WordBreakClass::MidLetter}, + {0xFE70, 0xFE74, WordBreakClass::ALetter}, + {0xFE76, 0xFEFC, WordBreakClass::ALetter}, + {0xFEFF, 0xFEFF, WordBreakClass::Format}, + {0xFF07, 0xFF07, WordBreakClass::MidNumLet}, + {0xFF0C, 0xFF0C, WordBreakClass::MidNum}, + {0xFF0E, 0xFF0E, WordBreakClass::MidNumLet}, + {0xFF10, 0xFF19, WordBreakClass::Numeric}, + {0xFF1A, 0xFF1A, WordBreakClass::MidLetter}, + {0xFF1B, 0xFF1B, WordBreakClass::MidNum}, + {0xFF21, 0xFF3A, WordBreakClass::ALetter}, + {0xFF3F, 0xFF3F, WordBreakClass::ExtendNumLet}, + {0xFF41, 0xFF5A, WordBreakClass::ALetter}, + {0xFF66, 0xFF9D, WordBreakClass::Katakana}, + {0xFF9E, 0xFF9F, WordBreakClass::Extend}, + {0xFFA0, 0xFFBE, WordBreakClass::ALetter}, + {0xFFC2, 0xFFC7, WordBreakClass::ALetter}, + {0xFFCA, 0xFFCF, WordBreakClass::ALetter}, + {0xFFD2, 0xFFD7, WordBreakClass::ALetter}, + {0xFFDA, 0xFFDC, WordBreakClass::ALetter}, + {0xFFF9, 0xFFFB, WordBreakClass::Format}, + {0x10000, 0x1000B, WordBreakClass::ALetter}, + {0x1000D, 0x10026, WordBreakClass::ALetter}, + {0x10028, 0x1003A, WordBreakClass::ALetter}, + {0x1003C, 0x1003D, WordBreakClass::ALetter}, + {0x1003F, 0x1004D, WordBreakClass::ALetter}, + {0x10050, 0x1005D, WordBreakClass::ALetter}, + {0x10080, 0x100FA, WordBreakClass::ALetter}, + {0x10140, 0x10174, WordBreakClass::ALetter}, + {0x101FD, 0x101FD, WordBreakClass::Extend}, + {0x10280, 0x1029C, WordBreakClass::ALetter}, + {0x102A0, 0x102D0, WordBreakClass::ALetter}, + {0x102E0, 0x102E0, WordBreakClass::Extend}, + {0x10300, 0x1031F, WordBreakClass::ALetter}, + {0x1032D, 0x1034A, WordBreakClass::ALetter}, + {0x10350, 0x10375, WordBreakClass::ALetter}, + {0x10376, 0x1037A, WordBreakClass::Extend}, + {0x10380, 0x1039D, WordBreakClass::ALetter}, + {0x103A0, 0x103C3, WordBreakClass::ALetter}, + {0x103C8, 0x103CF, WordBreakClass::ALetter}, + {0x103D1, 0x103D5, WordBreakClass::ALetter}, + {0x10400, 0x1049D, WordBreakClass::ALetter}, + {0x104A0, 0x104A9, WordBreakClass::Numeric}, + {0x104B0, 0x104D3, WordBreakClass::ALetter}, + {0x104D8, 0x104FB, WordBreakClass::ALetter}, + {0x10500, 0x10527, WordBreakClass::ALetter}, + {0x10530, 0x10563, WordBreakClass::ALetter}, + {0x10570, 0x1057A, WordBreakClass::ALetter}, + {0x1057C, 0x1058A, WordBreakClass::ALetter}, + {0x1058C, 0x10592, WordBreakClass::ALetter}, + {0x10594, 0x10595, WordBreakClass::ALetter}, + {0x10597, 0x105A1, WordBreakClass::ALetter}, + {0x105A3, 0x105B1, WordBreakClass::ALetter}, + {0x105B3, 0x105B9, WordBreakClass::ALetter}, + {0x105BB, 0x105BC, WordBreakClass::ALetter}, + {0x105C0, 0x105F3, WordBreakClass::ALetter}, + {0x10600, 0x10736, WordBreakClass::ALetter}, + {0x10740, 0x10755, WordBreakClass::ALetter}, + {0x10760, 0x10767, WordBreakClass::ALetter}, + {0x10780, 0x10785, WordBreakClass::ALetter}, + {0x10787, 0x107B0, WordBreakClass::ALetter}, + {0x107B2, 0x107BA, WordBreakClass::ALetter}, + {0x10800, 0x10805, WordBreakClass::ALetter}, + {0x10808, 0x10808, WordBreakClass::ALetter}, + {0x1080A, 0x10835, WordBreakClass::ALetter}, + {0x10837, 0x10838, WordBreakClass::ALetter}, + {0x1083C, 0x1083C, WordBreakClass::ALetter}, + {0x1083F, 0x10855, WordBreakClass::ALetter}, + {0x10860, 0x10876, WordBreakClass::ALetter}, + {0x10880, 0x1089E, WordBreakClass::ALetter}, + {0x108E0, 0x108F2, WordBreakClass::ALetter}, + {0x108F4, 0x108F5, WordBreakClass::ALetter}, + {0x10900, 0x10915, WordBreakClass::ALetter}, + {0x10920, 0x10939, WordBreakClass::ALetter}, + {0x10940, 0x10959, WordBreakClass::ALetter}, + {0x10980, 0x109B7, WordBreakClass::ALetter}, + {0x109BE, 0x109BF, WordBreakClass::ALetter}, + {0x10A00, 0x10A00, WordBreakClass::ALetter}, + {0x10A01, 0x10A03, WordBreakClass::Extend}, + {0x10A05, 0x10A06, WordBreakClass::Extend}, + {0x10A0C, 0x10A0F, WordBreakClass::Extend}, + {0x10A10, 0x10A13, WordBreakClass::ALetter}, + {0x10A15, 0x10A17, WordBreakClass::ALetter}, + {0x10A19, 0x10A35, WordBreakClass::ALetter}, + {0x10A38, 0x10A3A, WordBreakClass::Extend}, + {0x10A3F, 0x10A3F, WordBreakClass::Extend}, + {0x10A60, 0x10A7C, WordBreakClass::ALetter}, + {0x10A80, 0x10A9C, WordBreakClass::ALetter}, + {0x10AC0, 0x10AC7, WordBreakClass::ALetter}, + {0x10AC9, 0x10AE4, WordBreakClass::ALetter}, + {0x10AE5, 0x10AE6, WordBreakClass::Extend}, + {0x10B00, 0x10B35, WordBreakClass::ALetter}, + {0x10B40, 0x10B55, WordBreakClass::ALetter}, + {0x10B60, 0x10B72, WordBreakClass::ALetter}, + {0x10B80, 0x10B91, WordBreakClass::ALetter}, + {0x10C00, 0x10C48, WordBreakClass::ALetter}, + {0x10C80, 0x10CB2, WordBreakClass::ALetter}, + {0x10CC0, 0x10CF2, WordBreakClass::ALetter}, + {0x10D00, 0x10D23, WordBreakClass::ALetter}, + {0x10D24, 0x10D27, WordBreakClass::Extend}, + {0x10D30, 0x10D39, WordBreakClass::Numeric}, + {0x10D40, 0x10D49, WordBreakClass::Numeric}, + {0x10D4A, 0x10D65, WordBreakClass::ALetter}, + {0x10D69, 0x10D6D, WordBreakClass::Extend}, + {0x10D6F, 0x10D85, WordBreakClass::ALetter}, + {0x10E80, 0x10EA9, WordBreakClass::ALetter}, + {0x10EAB, 0x10EAC, WordBreakClass::Extend}, + {0x10EB0, 0x10EB1, WordBreakClass::ALetter}, + {0x10EC2, 0x10EC7, WordBreakClass::ALetter}, + {0x10EFA, 0x10EFF, WordBreakClass::Extend}, + {0x10F00, 0x10F1C, WordBreakClass::ALetter}, + {0x10F27, 0x10F27, WordBreakClass::ALetter}, + {0x10F30, 0x10F45, WordBreakClass::ALetter}, + {0x10F46, 0x10F50, WordBreakClass::Extend}, + {0x10F70, 0x10F81, WordBreakClass::ALetter}, + {0x10F82, 0x10F85, WordBreakClass::Extend}, + {0x10FB0, 0x10FC4, WordBreakClass::ALetter}, + {0x10FE0, 0x10FF6, WordBreakClass::ALetter}, + {0x11000, 0x11002, WordBreakClass::Extend}, + {0x11003, 0x11037, WordBreakClass::ALetter}, + {0x11038, 0x11046, WordBreakClass::Extend}, + {0x11066, 0x1106F, WordBreakClass::Numeric}, + {0x11070, 0x11070, WordBreakClass::Extend}, + {0x11071, 0x11072, WordBreakClass::ALetter}, + {0x11073, 0x11074, WordBreakClass::Extend}, + {0x11075, 0x11075, WordBreakClass::ALetter}, + {0x1107F, 0x11082, WordBreakClass::Extend}, + {0x11083, 0x110AF, WordBreakClass::ALetter}, + {0x110B0, 0x110BA, WordBreakClass::Extend}, + {0x110BD, 0x110BD, WordBreakClass::Numeric}, + {0x110C2, 0x110C2, WordBreakClass::Extend}, + {0x110CD, 0x110CD, WordBreakClass::Numeric}, + {0x110D0, 0x110E8, WordBreakClass::ALetter}, + {0x110F0, 0x110F9, WordBreakClass::Numeric}, + {0x11100, 0x11102, WordBreakClass::Extend}, + {0x11103, 0x11126, WordBreakClass::ALetter}, + {0x11127, 0x11134, WordBreakClass::Extend}, + {0x11136, 0x1113F, WordBreakClass::Numeric}, + {0x11144, 0x11144, WordBreakClass::ALetter}, + {0x11145, 0x11146, WordBreakClass::Extend}, + {0x11147, 0x11147, WordBreakClass::ALetter}, + {0x11150, 0x11172, WordBreakClass::ALetter}, + {0x11173, 0x11173, WordBreakClass::Extend}, + {0x11176, 0x11176, WordBreakClass::ALetter}, + {0x11180, 0x11182, WordBreakClass::Extend}, + {0x11183, 0x111B2, WordBreakClass::ALetter}, + {0x111B3, 0x111C0, WordBreakClass::Extend}, + {0x111C1, 0x111C4, WordBreakClass::ALetter}, + {0x111C9, 0x111CC, WordBreakClass::Extend}, + {0x111CE, 0x111CF, WordBreakClass::Extend}, + {0x111D0, 0x111D9, WordBreakClass::Numeric}, + {0x111DA, 0x111DA, WordBreakClass::ALetter}, + {0x111DC, 0x111DC, WordBreakClass::ALetter}, + {0x11200, 0x11211, WordBreakClass::ALetter}, + {0x11213, 0x1122B, WordBreakClass::ALetter}, + {0x1122C, 0x11237, WordBreakClass::Extend}, + {0x1123E, 0x1123E, WordBreakClass::Extend}, + {0x1123F, 0x11240, WordBreakClass::ALetter}, + {0x11241, 0x11241, WordBreakClass::Extend}, + {0x11280, 0x11286, WordBreakClass::ALetter}, + {0x11288, 0x11288, WordBreakClass::ALetter}, + {0x1128A, 0x1128D, WordBreakClass::ALetter}, + {0x1128F, 0x1129D, WordBreakClass::ALetter}, + {0x1129F, 0x112A8, WordBreakClass::ALetter}, + {0x112B0, 0x112DE, WordBreakClass::ALetter}, + {0x112DF, 0x112EA, WordBreakClass::Extend}, + {0x112F0, 0x112F9, WordBreakClass::Numeric}, + {0x11300, 0x11303, WordBreakClass::Extend}, + {0x11305, 0x1130C, WordBreakClass::ALetter}, + {0x1130F, 0x11310, WordBreakClass::ALetter}, + {0x11313, 0x11328, WordBreakClass::ALetter}, + {0x1132A, 0x11330, WordBreakClass::ALetter}, + {0x11332, 0x11333, WordBreakClass::ALetter}, + {0x11335, 0x11339, WordBreakClass::ALetter}, + {0x1133B, 0x1133C, WordBreakClass::Extend}, + {0x1133D, 0x1133D, WordBreakClass::ALetter}, + {0x1133E, 0x11344, WordBreakClass::Extend}, + {0x11347, 0x11348, WordBreakClass::Extend}, + {0x1134B, 0x1134D, WordBreakClass::Extend}, + {0x11350, 0x11350, WordBreakClass::ALetter}, + {0x11357, 0x11357, WordBreakClass::Extend}, + {0x1135D, 0x11361, WordBreakClass::ALetter}, + {0x11362, 0x11363, WordBreakClass::Extend}, + {0x11366, 0x1136C, WordBreakClass::Extend}, + {0x11370, 0x11374, WordBreakClass::Extend}, + {0x11380, 0x11389, WordBreakClass::ALetter}, + {0x1138B, 0x1138B, WordBreakClass::ALetter}, + {0x1138E, 0x1138E, WordBreakClass::ALetter}, + {0x11390, 0x113B5, WordBreakClass::ALetter}, + {0x113B7, 0x113B7, WordBreakClass::ALetter}, + {0x113B8, 0x113C0, WordBreakClass::Extend}, + {0x113C2, 0x113C2, WordBreakClass::Extend}, + {0x113C5, 0x113C5, WordBreakClass::Extend}, + {0x113C7, 0x113CA, WordBreakClass::Extend}, + {0x113CC, 0x113D0, WordBreakClass::Extend}, + {0x113D1, 0x113D1, WordBreakClass::ALetter}, + {0x113D2, 0x113D2, WordBreakClass::Extend}, + {0x113D3, 0x113D3, WordBreakClass::ALetter}, + {0x113E1, 0x113E2, WordBreakClass::Extend}, + {0x11400, 0x11434, WordBreakClass::ALetter}, + {0x11435, 0x11446, WordBreakClass::Extend}, + {0x11447, 0x1144A, WordBreakClass::ALetter}, + {0x11450, 0x11459, WordBreakClass::Numeric}, + {0x1145E, 0x1145E, WordBreakClass::Extend}, + {0x1145F, 0x11461, WordBreakClass::ALetter}, + {0x11480, 0x114AF, WordBreakClass::ALetter}, + {0x114B0, 0x114C3, WordBreakClass::Extend}, + {0x114C4, 0x114C5, WordBreakClass::ALetter}, + {0x114C7, 0x114C7, WordBreakClass::ALetter}, + {0x114D0, 0x114D9, WordBreakClass::Numeric}, + {0x11580, 0x115AE, WordBreakClass::ALetter}, + {0x115AF, 0x115B5, WordBreakClass::Extend}, + {0x115B8, 0x115C0, WordBreakClass::Extend}, + {0x115D8, 0x115DB, WordBreakClass::ALetter}, + {0x115DC, 0x115DD, WordBreakClass::Extend}, + {0x11600, 0x1162F, WordBreakClass::ALetter}, + {0x11630, 0x11640, WordBreakClass::Extend}, + {0x11644, 0x11644, WordBreakClass::ALetter}, + {0x11650, 0x11659, WordBreakClass::Numeric}, + {0x11680, 0x116AA, WordBreakClass::ALetter}, + {0x116AB, 0x116B7, WordBreakClass::Extend}, + {0x116B8, 0x116B8, WordBreakClass::ALetter}, + {0x116C0, 0x116C9, WordBreakClass::Numeric}, + {0x116D0, 0x116E3, WordBreakClass::Numeric}, + {0x1171D, 0x1172B, WordBreakClass::Extend}, + {0x11730, 0x11739, WordBreakClass::Numeric}, + {0x11800, 0x1182B, WordBreakClass::ALetter}, + {0x1182C, 0x1183A, WordBreakClass::Extend}, + {0x118A0, 0x118DF, WordBreakClass::ALetter}, + {0x118E0, 0x118E9, WordBreakClass::Numeric}, + {0x118FF, 0x11906, WordBreakClass::ALetter}, + {0x11909, 0x11909, WordBreakClass::ALetter}, + {0x1190C, 0x11913, WordBreakClass::ALetter}, + {0x11915, 0x11916, WordBreakClass::ALetter}, + {0x11918, 0x1192F, WordBreakClass::ALetter}, + {0x11930, 0x11935, WordBreakClass::Extend}, + {0x11937, 0x11938, WordBreakClass::Extend}, + {0x1193B, 0x1193E, WordBreakClass::Extend}, + {0x1193F, 0x1193F, WordBreakClass::ALetter}, + {0x11940, 0x11940, WordBreakClass::Extend}, + {0x11941, 0x11941, WordBreakClass::ALetter}, + {0x11942, 0x11943, WordBreakClass::Extend}, + {0x11950, 0x11959, WordBreakClass::Numeric}, + {0x119A0, 0x119A7, WordBreakClass::ALetter}, + {0x119AA, 0x119D0, WordBreakClass::ALetter}, + {0x119D1, 0x119D7, WordBreakClass::Extend}, + {0x119DA, 0x119E0, WordBreakClass::Extend}, + {0x119E1, 0x119E1, WordBreakClass::ALetter}, + {0x119E3, 0x119E3, WordBreakClass::ALetter}, + {0x119E4, 0x119E4, WordBreakClass::Extend}, + {0x11A00, 0x11A00, WordBreakClass::ALetter}, + {0x11A01, 0x11A0A, WordBreakClass::Extend}, + {0x11A0B, 0x11A32, WordBreakClass::ALetter}, + {0x11A33, 0x11A39, WordBreakClass::Extend}, + {0x11A3A, 0x11A3A, WordBreakClass::ALetter}, + {0x11A3B, 0x11A3E, WordBreakClass::Extend}, + {0x11A47, 0x11A47, WordBreakClass::Extend}, + {0x11A50, 0x11A50, WordBreakClass::ALetter}, + {0x11A51, 0x11A5B, WordBreakClass::Extend}, + {0x11A5C, 0x11A89, WordBreakClass::ALetter}, + {0x11A8A, 0x11A99, WordBreakClass::Extend}, + {0x11A9D, 0x11A9D, WordBreakClass::ALetter}, + {0x11AB0, 0x11AF8, WordBreakClass::ALetter}, + {0x11B60, 0x11B67, WordBreakClass::Extend}, + {0x11BC0, 0x11BE0, WordBreakClass::ALetter}, + {0x11BF0, 0x11BF9, WordBreakClass::Numeric}, + {0x11C00, 0x11C08, WordBreakClass::ALetter}, + {0x11C0A, 0x11C2E, WordBreakClass::ALetter}, + {0x11C2F, 0x11C36, WordBreakClass::Extend}, + {0x11C38, 0x11C3F, WordBreakClass::Extend}, + {0x11C40, 0x11C40, WordBreakClass::ALetter}, + {0x11C50, 0x11C59, WordBreakClass::Numeric}, + {0x11C72, 0x11C8F, WordBreakClass::ALetter}, + {0x11C92, 0x11CA7, WordBreakClass::Extend}, + {0x11CA9, 0x11CB6, WordBreakClass::Extend}, + {0x11D00, 0x11D06, WordBreakClass::ALetter}, + {0x11D08, 0x11D09, WordBreakClass::ALetter}, + {0x11D0B, 0x11D30, WordBreakClass::ALetter}, + {0x11D31, 0x11D36, WordBreakClass::Extend}, + {0x11D3A, 0x11D3A, WordBreakClass::Extend}, + {0x11D3C, 0x11D3D, WordBreakClass::Extend}, + {0x11D3F, 0x11D45, WordBreakClass::Extend}, + {0x11D46, 0x11D46, WordBreakClass::ALetter}, + {0x11D47, 0x11D47, WordBreakClass::Extend}, + {0x11D50, 0x11D59, WordBreakClass::Numeric}, + {0x11D60, 0x11D65, WordBreakClass::ALetter}, + {0x11D67, 0x11D68, WordBreakClass::ALetter}, + {0x11D6A, 0x11D89, WordBreakClass::ALetter}, + {0x11D8A, 0x11D8E, WordBreakClass::Extend}, + {0x11D90, 0x11D91, WordBreakClass::Extend}, + {0x11D93, 0x11D97, WordBreakClass::Extend}, + {0x11D98, 0x11D98, WordBreakClass::ALetter}, + {0x11DA0, 0x11DA9, WordBreakClass::Numeric}, + {0x11DB0, 0x11DDB, WordBreakClass::ALetter}, + {0x11DE0, 0x11DE9, WordBreakClass::Numeric}, + {0x11EE0, 0x11EF2, WordBreakClass::ALetter}, + {0x11EF3, 0x11EF6, WordBreakClass::Extend}, + {0x11F00, 0x11F01, WordBreakClass::Extend}, + {0x11F02, 0x11F02, WordBreakClass::ALetter}, + {0x11F03, 0x11F03, WordBreakClass::Extend}, + {0x11F04, 0x11F10, WordBreakClass::ALetter}, + {0x11F12, 0x11F33, WordBreakClass::ALetter}, + {0x11F34, 0x11F3A, WordBreakClass::Extend}, + {0x11F3E, 0x11F42, WordBreakClass::Extend}, + {0x11F50, 0x11F59, WordBreakClass::Numeric}, + {0x11F5A, 0x11F5A, WordBreakClass::Extend}, + {0x11FB0, 0x11FB0, WordBreakClass::ALetter}, + {0x12000, 0x12399, WordBreakClass::ALetter}, + {0x12400, 0x1246E, WordBreakClass::ALetter}, + {0x12480, 0x12543, WordBreakClass::ALetter}, + {0x12F90, 0x12FF0, WordBreakClass::ALetter}, + {0x13000, 0x1342F, WordBreakClass::ALetter}, + {0x13430, 0x1343F, WordBreakClass::Format}, + {0x13440, 0x13440, WordBreakClass::Extend}, + {0x13441, 0x13446, WordBreakClass::ALetter}, + {0x13447, 0x13455, WordBreakClass::Extend}, + {0x13460, 0x143FA, WordBreakClass::ALetter}, + {0x14400, 0x14646, WordBreakClass::ALetter}, + {0x16100, 0x1611D, WordBreakClass::ALetter}, + {0x1611E, 0x1612F, WordBreakClass::Extend}, + {0x16130, 0x16139, WordBreakClass::Numeric}, + {0x16800, 0x16A38, WordBreakClass::ALetter}, + {0x16A40, 0x16A5E, WordBreakClass::ALetter}, + {0x16A60, 0x16A69, WordBreakClass::Numeric}, + {0x16A70, 0x16ABE, WordBreakClass::ALetter}, + {0x16AC0, 0x16AC9, WordBreakClass::Numeric}, + {0x16AD0, 0x16AED, WordBreakClass::ALetter}, + {0x16AF0, 0x16AF4, WordBreakClass::Extend}, + {0x16B00, 0x16B2F, WordBreakClass::ALetter}, + {0x16B30, 0x16B36, WordBreakClass::Extend}, + {0x16B40, 0x16B43, WordBreakClass::ALetter}, + {0x16B50, 0x16B59, WordBreakClass::Numeric}, + {0x16B63, 0x16B77, WordBreakClass::ALetter}, + {0x16B7D, 0x16B8F, WordBreakClass::ALetter}, + {0x16D40, 0x16D6C, WordBreakClass::ALetter}, + {0x16D70, 0x16D79, WordBreakClass::Numeric}, + {0x16E40, 0x16E7F, WordBreakClass::ALetter}, + {0x16EA0, 0x16EB8, WordBreakClass::ALetter}, + {0x16EBB, 0x16ED3, WordBreakClass::ALetter}, + {0x16F00, 0x16F4A, WordBreakClass::ALetter}, + {0x16F4F, 0x16F4F, WordBreakClass::Extend}, + {0x16F50, 0x16F50, WordBreakClass::ALetter}, + {0x16F51, 0x16F87, WordBreakClass::Extend}, + {0x16F8F, 0x16F92, WordBreakClass::Extend}, + {0x16F93, 0x16F9F, WordBreakClass::ALetter}, + {0x16FE0, 0x16FE1, WordBreakClass::ALetter}, + {0x16FE3, 0x16FE3, WordBreakClass::ALetter}, + {0x16FE4, 0x16FE4, WordBreakClass::Extend}, + {0x16FF0, 0x16FF1, WordBreakClass::Extend}, + {0x1AFF0, 0x1AFF3, WordBreakClass::Katakana}, + {0x1AFF5, 0x1AFFB, WordBreakClass::Katakana}, + {0x1AFFD, 0x1AFFE, WordBreakClass::Katakana}, + {0x1B000, 0x1B000, WordBreakClass::Katakana}, + {0x1B120, 0x1B122, WordBreakClass::Katakana}, + {0x1B155, 0x1B155, WordBreakClass::Katakana}, + {0x1B164, 0x1B167, WordBreakClass::Katakana}, + {0x1BC00, 0x1BC6A, WordBreakClass::ALetter}, + {0x1BC70, 0x1BC7C, WordBreakClass::ALetter}, + {0x1BC80, 0x1BC88, WordBreakClass::ALetter}, + {0x1BC90, 0x1BC99, WordBreakClass::ALetter}, + {0x1BC9D, 0x1BC9E, WordBreakClass::Extend}, + {0x1BCA0, 0x1BCA3, WordBreakClass::Format}, + {0x1CCF0, 0x1CCF9, WordBreakClass::Numeric}, + {0x1CF00, 0x1CF2D, WordBreakClass::Extend}, + {0x1CF30, 0x1CF46, WordBreakClass::Extend}, + {0x1D165, 0x1D169, WordBreakClass::Extend}, + {0x1D16D, 0x1D172, WordBreakClass::Extend}, + {0x1D173, 0x1D17A, WordBreakClass::Format}, + {0x1D17B, 0x1D182, WordBreakClass::Extend}, + {0x1D185, 0x1D18B, WordBreakClass::Extend}, + {0x1D1AA, 0x1D1AD, WordBreakClass::Extend}, + {0x1D242, 0x1D244, WordBreakClass::Extend}, + {0x1D400, 0x1D454, WordBreakClass::ALetter}, + {0x1D456, 0x1D49C, WordBreakClass::ALetter}, + {0x1D49E, 0x1D49F, WordBreakClass::ALetter}, + {0x1D4A2, 0x1D4A2, WordBreakClass::ALetter}, + {0x1D4A5, 0x1D4A6, WordBreakClass::ALetter}, + {0x1D4A9, 0x1D4AC, WordBreakClass::ALetter}, + {0x1D4AE, 0x1D4B9, WordBreakClass::ALetter}, + {0x1D4BB, 0x1D4BB, WordBreakClass::ALetter}, + {0x1D4BD, 0x1D4C3, WordBreakClass::ALetter}, + {0x1D4C5, 0x1D505, WordBreakClass::ALetter}, + {0x1D507, 0x1D50A, WordBreakClass::ALetter}, + {0x1D50D, 0x1D514, WordBreakClass::ALetter}, + {0x1D516, 0x1D51C, WordBreakClass::ALetter}, + {0x1D51E, 0x1D539, WordBreakClass::ALetter}, + {0x1D53B, 0x1D53E, WordBreakClass::ALetter}, + {0x1D540, 0x1D544, WordBreakClass::ALetter}, + {0x1D546, 0x1D546, WordBreakClass::ALetter}, + {0x1D54A, 0x1D550, WordBreakClass::ALetter}, + {0x1D552, 0x1D6A5, WordBreakClass::ALetter}, + {0x1D6A8, 0x1D6C0, WordBreakClass::ALetter}, + {0x1D6C2, 0x1D6DA, WordBreakClass::ALetter}, + {0x1D6DC, 0x1D6FA, WordBreakClass::ALetter}, + {0x1D6FC, 0x1D714, WordBreakClass::ALetter}, + {0x1D716, 0x1D734, WordBreakClass::ALetter}, + {0x1D736, 0x1D74E, WordBreakClass::ALetter}, + {0x1D750, 0x1D76E, WordBreakClass::ALetter}, + {0x1D770, 0x1D788, WordBreakClass::ALetter}, + {0x1D78A, 0x1D7A8, WordBreakClass::ALetter}, + {0x1D7AA, 0x1D7C2, WordBreakClass::ALetter}, + {0x1D7C4, 0x1D7CB, WordBreakClass::ALetter}, + {0x1D7CE, 0x1D7FF, WordBreakClass::Numeric}, + {0x1DA00, 0x1DA36, WordBreakClass::Extend}, + {0x1DA3B, 0x1DA6C, WordBreakClass::Extend}, + {0x1DA75, 0x1DA75, WordBreakClass::Extend}, + {0x1DA84, 0x1DA84, WordBreakClass::Extend}, + {0x1DA9B, 0x1DA9F, WordBreakClass::Extend}, + {0x1DAA1, 0x1DAAF, WordBreakClass::Extend}, + {0x1DF00, 0x1DF1E, WordBreakClass::ALetter}, + {0x1DF25, 0x1DF2A, WordBreakClass::ALetter}, + {0x1E000, 0x1E006, WordBreakClass::Extend}, + {0x1E008, 0x1E018, WordBreakClass::Extend}, + {0x1E01B, 0x1E021, WordBreakClass::Extend}, + {0x1E023, 0x1E024, WordBreakClass::Extend}, + {0x1E026, 0x1E02A, WordBreakClass::Extend}, + {0x1E030, 0x1E06D, WordBreakClass::ALetter}, + {0x1E08F, 0x1E08F, WordBreakClass::Extend}, + {0x1E100, 0x1E12C, WordBreakClass::ALetter}, + {0x1E130, 0x1E136, WordBreakClass::Extend}, + {0x1E137, 0x1E13D, WordBreakClass::ALetter}, + {0x1E140, 0x1E149, WordBreakClass::Numeric}, + {0x1E14E, 0x1E14E, WordBreakClass::ALetter}, + {0x1E290, 0x1E2AD, WordBreakClass::ALetter}, + {0x1E2AE, 0x1E2AE, WordBreakClass::Extend}, + {0x1E2C0, 0x1E2EB, WordBreakClass::ALetter}, + {0x1E2EC, 0x1E2EF, WordBreakClass::Extend}, + {0x1E2F0, 0x1E2F9, WordBreakClass::Numeric}, + {0x1E4D0, 0x1E4EB, WordBreakClass::ALetter}, + {0x1E4EC, 0x1E4EF, WordBreakClass::Extend}, + {0x1E4F0, 0x1E4F9, WordBreakClass::Numeric}, + {0x1E5D0, 0x1E5ED, WordBreakClass::ALetter}, + {0x1E5EE, 0x1E5EF, WordBreakClass::Extend}, + {0x1E5F0, 0x1E5F0, WordBreakClass::ALetter}, + {0x1E5F1, 0x1E5FA, WordBreakClass::Numeric}, + {0x1E6C0, 0x1E6DE, WordBreakClass::ALetter}, + {0x1E6E0, 0x1E6E2, WordBreakClass::ALetter}, + {0x1E6E3, 0x1E6E3, WordBreakClass::Extend}, + {0x1E6E4, 0x1E6E5, WordBreakClass::ALetter}, + {0x1E6E6, 0x1E6E6, WordBreakClass::Extend}, + {0x1E6E7, 0x1E6ED, WordBreakClass::ALetter}, + {0x1E6EE, 0x1E6EF, WordBreakClass::Extend}, + {0x1E6F0, 0x1E6F4, WordBreakClass::ALetter}, + {0x1E6F5, 0x1E6F5, WordBreakClass::Extend}, + {0x1E6FE, 0x1E6FF, WordBreakClass::ALetter}, + {0x1E7E0, 0x1E7E6, WordBreakClass::ALetter}, + {0x1E7E8, 0x1E7EB, WordBreakClass::ALetter}, + {0x1E7ED, 0x1E7EE, WordBreakClass::ALetter}, + {0x1E7F0, 0x1E7FE, WordBreakClass::ALetter}, + {0x1E800, 0x1E8C4, WordBreakClass::ALetter}, + {0x1E8D0, 0x1E8D6, WordBreakClass::Extend}, + {0x1E900, 0x1E943, WordBreakClass::ALetter}, + {0x1E944, 0x1E94A, WordBreakClass::Extend}, + {0x1E94B, 0x1E94B, WordBreakClass::ALetter}, + {0x1E950, 0x1E959, WordBreakClass::Numeric}, + {0x1EE00, 0x1EE03, WordBreakClass::ALetter}, + {0x1EE05, 0x1EE1F, WordBreakClass::ALetter}, + {0x1EE21, 0x1EE22, WordBreakClass::ALetter}, + {0x1EE24, 0x1EE24, WordBreakClass::ALetter}, + {0x1EE27, 0x1EE27, WordBreakClass::ALetter}, + {0x1EE29, 0x1EE32, WordBreakClass::ALetter}, + {0x1EE34, 0x1EE37, WordBreakClass::ALetter}, + {0x1EE39, 0x1EE39, WordBreakClass::ALetter}, + {0x1EE3B, 0x1EE3B, WordBreakClass::ALetter}, + {0x1EE42, 0x1EE42, WordBreakClass::ALetter}, + {0x1EE47, 0x1EE47, WordBreakClass::ALetter}, + {0x1EE49, 0x1EE49, WordBreakClass::ALetter}, + {0x1EE4B, 0x1EE4B, WordBreakClass::ALetter}, + {0x1EE4D, 0x1EE4F, WordBreakClass::ALetter}, + {0x1EE51, 0x1EE52, WordBreakClass::ALetter}, + {0x1EE54, 0x1EE54, WordBreakClass::ALetter}, + {0x1EE57, 0x1EE57, WordBreakClass::ALetter}, + {0x1EE59, 0x1EE59, WordBreakClass::ALetter}, + {0x1EE5B, 0x1EE5B, WordBreakClass::ALetter}, + {0x1EE5D, 0x1EE5D, WordBreakClass::ALetter}, + {0x1EE5F, 0x1EE5F, WordBreakClass::ALetter}, + {0x1EE61, 0x1EE62, WordBreakClass::ALetter}, + {0x1EE64, 0x1EE64, WordBreakClass::ALetter}, + {0x1EE67, 0x1EE6A, WordBreakClass::ALetter}, + {0x1EE6C, 0x1EE72, WordBreakClass::ALetter}, + {0x1EE74, 0x1EE77, WordBreakClass::ALetter}, + {0x1EE79, 0x1EE7C, WordBreakClass::ALetter}, + {0x1EE7E, 0x1EE7E, WordBreakClass::ALetter}, + {0x1EE80, 0x1EE89, WordBreakClass::ALetter}, + {0x1EE8B, 0x1EE9B, WordBreakClass::ALetter}, + {0x1EEA1, 0x1EEA3, WordBreakClass::ALetter}, + {0x1EEA5, 0x1EEA9, WordBreakClass::ALetter}, + {0x1EEAB, 0x1EEBB, WordBreakClass::ALetter}, + {0x1F130, 0x1F149, WordBreakClass::ALetter}, + {0x1F150, 0x1F169, WordBreakClass::ALetter}, + {0x1F170, 0x1F189, WordBreakClass::ALetter}, + {0x1F1E6, 0x1F1FF, WordBreakClass::RegionalIndicator}, + {0x1F3FB, 0x1F3FF, WordBreakClass::Extend}, + {0x1FBF0, 0x1FBF9, WordBreakClass::Numeric}, + {0xE0001, 0xE0001, WordBreakClass::Format}, + {0xE0020, 0xE007F, WordBreakClass::Extend}, + {0xE0100, 0xE01EF, WordBreakClass::Extend}, +}; + +constexpr UnicodeClassRange kScriptClassRanges[] = { + {0x1100, 0x11FF, WordBreakClass::Hangul}, + {0x2E80, 0x2E99, WordBreakClass::Ideographic}, + {0x2E9B, 0x2EF3, WordBreakClass::Ideographic}, + {0x2F00, 0x2FD5, WordBreakClass::Ideographic}, + {0x3005, 0x3005, WordBreakClass::Ideographic}, + {0x3007, 0x3007, WordBreakClass::Ideographic}, + {0x3021, 0x3029, WordBreakClass::Ideographic}, + {0x302E, 0x302F, WordBreakClass::Hangul}, + {0x3038, 0x303B, WordBreakClass::Ideographic}, + {0x3041, 0x3096, WordBreakClass::Hiragana}, + {0x309D, 0x309F, WordBreakClass::Hiragana}, + {0x3131, 0x318E, WordBreakClass::Hangul}, + {0x3200, 0x321E, WordBreakClass::Hangul}, + {0x3260, 0x327E, WordBreakClass::Hangul}, + {0x3400, 0x4DBF, WordBreakClass::Ideographic}, + {0x4E00, 0x9FFF, WordBreakClass::Ideographic}, + {0xA960, 0xA97C, WordBreakClass::Hangul}, + {0xAC00, 0xD7A3, WordBreakClass::Hangul}, + {0xD7B0, 0xD7C6, WordBreakClass::Hangul}, + {0xD7CB, 0xD7FB, WordBreakClass::Hangul}, + {0xF900, 0xFA6D, WordBreakClass::Ideographic}, + {0xFA70, 0xFAD9, WordBreakClass::Ideographic}, + {0xFFA0, 0xFFBE, WordBreakClass::Hangul}, + {0xFFC2, 0xFFC7, WordBreakClass::Hangul}, + {0xFFCA, 0xFFCF, WordBreakClass::Hangul}, + {0xFFD2, 0xFFD7, WordBreakClass::Hangul}, + {0xFFDA, 0xFFDC, WordBreakClass::Hangul}, + {0x16FE2, 0x16FE3, WordBreakClass::Ideographic}, + {0x16FF0, 0x16FF6, WordBreakClass::Ideographic}, + {0x1B001, 0x1B11F, WordBreakClass::Hiragana}, + {0x1B132, 0x1B132, WordBreakClass::Hiragana}, + {0x1B150, 0x1B152, WordBreakClass::Hiragana}, + {0x1F200, 0x1F200, WordBreakClass::Hiragana}, + {0x20000, 0x2A6DF, WordBreakClass::Ideographic}, + {0x2A700, 0x2B81D, WordBreakClass::Ideographic}, + {0x2B820, 0x2CEAD, WordBreakClass::Ideographic}, + {0x2CEB0, 0x2EBE0, WordBreakClass::Ideographic}, + {0x2EBF0, 0x2EE5D, WordBreakClass::Ideographic}, + {0x2F800, 0x2FA1D, WordBreakClass::Ideographic}, + {0x30000, 0x3134A, WordBreakClass::Ideographic}, + {0x31350, 0x33479, WordBreakClass::Ideographic}, +}; + +constexpr UnicodeRange kExtendedPictographicRanges[] = { + {0x00A9, 0x00A9}, + {0x00AE, 0x00AE}, + {0x203C, 0x203C}, + {0x2049, 0x2049}, + {0x2122, 0x2122}, + {0x2139, 0x2139}, + {0x2194, 0x2199}, + {0x21A9, 0x21AA}, + {0x231A, 0x231B}, + {0x2328, 0x2328}, + {0x23CF, 0x23CF}, + {0x23E9, 0x23F3}, + {0x23F8, 0x23FA}, + {0x24C2, 0x24C2}, + {0x25AA, 0x25AB}, + {0x25B6, 0x25B6}, + {0x25C0, 0x25C0}, + {0x25FB, 0x25FE}, + {0x2600, 0x2604}, + {0x260E, 0x260E}, + {0x2611, 0x2611}, + {0x2614, 0x2615}, + {0x2618, 0x2618}, + {0x261D, 0x261D}, + {0x2620, 0x2620}, + {0x2622, 0x2623}, + {0x2626, 0x2626}, + {0x262A, 0x262A}, + {0x262E, 0x262F}, + {0x2638, 0x263A}, + {0x2640, 0x2640}, + {0x2642, 0x2642}, + {0x2648, 0x2653}, + {0x265F, 0x2660}, + {0x2663, 0x2663}, + {0x2665, 0x2666}, + {0x2668, 0x2668}, + {0x267B, 0x267B}, + {0x267E, 0x267F}, + {0x2692, 0x2697}, + {0x2699, 0x2699}, + {0x269B, 0x269C}, + {0x26A0, 0x26A1}, + {0x26A7, 0x26A7}, + {0x26AA, 0x26AB}, + {0x26B0, 0x26B1}, + {0x26BD, 0x26BE}, + {0x26C4, 0x26C5}, + {0x26C8, 0x26C8}, + {0x26CE, 0x26CF}, + {0x26D1, 0x26D1}, + {0x26D3, 0x26D4}, + {0x26E9, 0x26EA}, + {0x26F0, 0x26F5}, + {0x26F7, 0x26FA}, + {0x26FD, 0x26FD}, + {0x2702, 0x2702}, + {0x2705, 0x2705}, + {0x2708, 0x270D}, + {0x270F, 0x270F}, + {0x2712, 0x2712}, + {0x2714, 0x2714}, + {0x2716, 0x2716}, + {0x271D, 0x271D}, + {0x2721, 0x2721}, + {0x2728, 0x2728}, + {0x2733, 0x2734}, + {0x2744, 0x2744}, + {0x2747, 0x2747}, + {0x274C, 0x274C}, + {0x274E, 0x274E}, + {0x2753, 0x2755}, + {0x2757, 0x2757}, + {0x2763, 0x2764}, + {0x2795, 0x2797}, + {0x27A1, 0x27A1}, + {0x27B0, 0x27B0}, + {0x27BF, 0x27BF}, + {0x2934, 0x2935}, + {0x2B05, 0x2B07}, + {0x2B1B, 0x2B1C}, + {0x2B50, 0x2B50}, + {0x2B55, 0x2B55}, + {0x3030, 0x3030}, + {0x303D, 0x303D}, + {0x3297, 0x3297}, + {0x3299, 0x3299}, + {0x1F004, 0x1F004}, + {0x1F02C, 0x1F02F}, + {0x1F094, 0x1F09F}, + {0x1F0AF, 0x1F0B0}, + {0x1F0C0, 0x1F0C0}, + {0x1F0CF, 0x1F0D0}, + {0x1F0F6, 0x1F0FF}, + {0x1F170, 0x1F171}, + {0x1F17E, 0x1F17F}, + {0x1F18E, 0x1F18E}, + {0x1F191, 0x1F19A}, + {0x1F1AE, 0x1F1E5}, + {0x1F201, 0x1F20F}, + {0x1F21A, 0x1F21A}, + {0x1F22F, 0x1F22F}, + {0x1F232, 0x1F23A}, + {0x1F23C, 0x1F23F}, + {0x1F249, 0x1F25F}, + {0x1F266, 0x1F321}, + {0x1F324, 0x1F393}, + {0x1F396, 0x1F397}, + {0x1F399, 0x1F39B}, + {0x1F39E, 0x1F3F0}, + {0x1F3F3, 0x1F3F5}, + {0x1F3F7, 0x1F3FA}, + {0x1F400, 0x1F4FD}, + {0x1F4FF, 0x1F53D}, + {0x1F549, 0x1F54E}, + {0x1F550, 0x1F567}, + {0x1F56F, 0x1F570}, + {0x1F573, 0x1F57A}, + {0x1F587, 0x1F587}, + {0x1F58A, 0x1F58D}, + {0x1F590, 0x1F590}, + {0x1F595, 0x1F596}, + {0x1F5A4, 0x1F5A5}, + {0x1F5A8, 0x1F5A8}, + {0x1F5B1, 0x1F5B2}, + {0x1F5BC, 0x1F5BC}, + {0x1F5C2, 0x1F5C4}, + {0x1F5D1, 0x1F5D3}, + {0x1F5DC, 0x1F5DE}, + {0x1F5E1, 0x1F5E1}, + {0x1F5E3, 0x1F5E3}, + {0x1F5E8, 0x1F5E8}, + {0x1F5EF, 0x1F5EF}, + {0x1F5F3, 0x1F5F3}, + {0x1F5FA, 0x1F64F}, + {0x1F680, 0x1F6C5}, + {0x1F6CB, 0x1F6D2}, + {0x1F6D5, 0x1F6E5}, + {0x1F6E9, 0x1F6E9}, + {0x1F6EB, 0x1F6F0}, + {0x1F6F3, 0x1F6FF}, + {0x1F7DA, 0x1F7FF}, + {0x1F80C, 0x1F80F}, + {0x1F848, 0x1F84F}, + {0x1F85A, 0x1F85F}, + {0x1F888, 0x1F88F}, + {0x1F8AE, 0x1F8AF}, + {0x1F8BC, 0x1F8BF}, + {0x1F8C2, 0x1F8CF}, + {0x1F8D9, 0x1F8FF}, + {0x1F90C, 0x1F93A}, + {0x1F93C, 0x1F945}, + {0x1F947, 0x1F9FF}, + {0x1FA58, 0x1FA5F}, + {0x1FA6E, 0x1FAFF}, + {0x1FC00, 0x1FFFD}, +}; + +constexpr UnicodeRange kEmojiModifierBaseRanges[] = { + {0x261D, 0x261D}, + {0x26F9, 0x26F9}, + {0x270A, 0x270D}, + {0x1F385, 0x1F385}, + {0x1F3C2, 0x1F3C4}, + {0x1F3C7, 0x1F3C7}, + {0x1F3CA, 0x1F3CC}, + {0x1F442, 0x1F443}, + {0x1F446, 0x1F450}, + {0x1F466, 0x1F478}, + {0x1F47C, 0x1F47C}, + {0x1F481, 0x1F483}, + {0x1F485, 0x1F487}, + {0x1F48F, 0x1F48F}, + {0x1F491, 0x1F491}, + {0x1F4AA, 0x1F4AA}, + {0x1F574, 0x1F575}, + {0x1F57A, 0x1F57A}, + {0x1F590, 0x1F590}, + {0x1F595, 0x1F596}, + {0x1F645, 0x1F647}, + {0x1F64B, 0x1F64F}, + {0x1F6A3, 0x1F6A3}, + {0x1F6B4, 0x1F6B6}, + {0x1F6C0, 0x1F6C0}, + {0x1F6CC, 0x1F6CC}, + {0x1F90C, 0x1F90C}, + {0x1F90F, 0x1F90F}, + {0x1F918, 0x1F91F}, + {0x1F926, 0x1F926}, + {0x1F930, 0x1F939}, + {0x1F93C, 0x1F93E}, + {0x1F977, 0x1F977}, + {0x1F9B5, 0x1F9B6}, + {0x1F9B8, 0x1F9B9}, + {0x1F9BB, 0x1F9BB}, + {0x1F9CD, 0x1F9CF}, + {0x1F9D1, 0x1F9DD}, + {0x1FAC3, 0x1FAC5}, + {0x1FAF0, 0x1FAF8}, +}; + +constexpr UnicodeRange kEmojiModifierRanges[] = { + {0x1F3FB, 0x1F3FF}, +}; + +constexpr UnicodeRange kLineBreakComplexContextRanges[] = { + {0x0E01, 0x0E3A}, + {0x0E40, 0x0E4E}, + {0x0E81, 0x0E82}, + {0x0E84, 0x0E84}, + {0x0E86, 0x0E8A}, + {0x0E8C, 0x0EA3}, + {0x0EA5, 0x0EA5}, + {0x0EA7, 0x0EBD}, + {0x0EC0, 0x0EC4}, + {0x0EC6, 0x0EC6}, + {0x0EC8, 0x0ECE}, + {0x0EDC, 0x0EDF}, + {0x1000, 0x103F}, + {0x1050, 0x108F}, + {0x109A, 0x109F}, + {0x1780, 0x17D3}, + {0x17D7, 0x17D7}, + {0x17DC, 0x17DD}, + {0x1950, 0x196D}, + {0x1970, 0x1974}, + {0x1980, 0x19AB}, + {0x19B0, 0x19C9}, + {0x19DE, 0x19DF}, + {0x1A20, 0x1A5E}, + {0x1A60, 0x1A7C}, + {0x1AA0, 0x1AAD}, + {0xA9E0, 0xA9EF}, + {0xA9FA, 0xA9FE}, + {0xAA60, 0xAAC2}, + {0xAADB, 0xAADF}, + {0x11700, 0x1171A}, + {0x1171D, 0x1172B}, + {0x1173A, 0x1173B}, + {0x1173F, 0x11746}, +}; + +// clang-format on diff --git a/tests/db/index/column/fts_column/standard_tokenizer_test.cc b/tests/db/index/column/fts_column/standard_tokenizer_test.cc index 9a9e07c..7d8fb57 100644 --- a/tests/db/index/column/fts_column/standard_tokenizer_test.cc +++ b/tests/db/index/column/fts_column/standard_tokenizer_test.cc @@ -20,6 +20,15 @@ using namespace zvec::fts; +static std::vector token_texts(const std::vector &tokens) { + std::vector texts; + texts.reserve(tokens.size()); + for (const auto &token : tokens) { + texts.push_back(token.text); + } + return texts; +} + class StandardTokenizerTest : public ::testing::Test { protected: void SetUp() override { @@ -47,7 +56,7 @@ TEST_F(StandardTokenizerTest, SimpleAsciiWords) { } TEST_F(StandardTokenizerTest, PunctuationAsDelimiter) { - auto tokens = tokenize("hello,world.test"); + auto tokens = tokenize("hello,world!test"); ASSERT_EQ(tokens.size(), 3u); EXPECT_EQ(tokens[0].text, "hello"); EXPECT_EQ(tokens[1].text, "world"); @@ -71,6 +80,19 @@ TEST_F(StandardTokenizerTest, OnlyDelimiters) { EXPECT_TRUE(tokens.empty()); } +TEST_F(StandardTokenizerTest, MalformedUtf8BreaksTokens) { + std::string text = "ab"; + text.push_back(static_cast(0xFF)); + text += "cd"; + + auto tokens = tokenize(text); + ASSERT_EQ(tokens.size(), 2u); + EXPECT_EQ(tokens[0].text, "ab"); + EXPECT_EQ(tokens[0].offset, 0u); + EXPECT_EQ(tokens[1].text, "cd"); + EXPECT_EQ(tokens[1].offset, 3u); +} + TEST_F(StandardTokenizerTest, OffsetAndPosition) { auto tokens = tokenize(" hello world"); ASSERT_EQ(tokens.size(), 2u); @@ -92,11 +114,11 @@ TEST_F(StandardTokenizerTest, AccentedLatin) { TEST_F(StandardTokenizerTest, MarksContinueButDoNotStartTokens) { // e + U+0301 keeps the combining mark with the base letter. - // Standalone U+0301 and the heart variation selector are not indexed. + // Standalone U+0301 and U+FE0F are not indexed. auto tokens = tokenize( "e\xCC\x81 " "\xCC\x81 " - "\xE2\x9D\xA4\xEF\xB8\x8F"); + "\xEF\xB8\x8F"); ASSERT_EQ(tokens.size(), 1u); EXPECT_EQ(tokens[0].text, "e\xCC\x81"); } @@ -182,6 +204,13 @@ TEST_F(StandardTokenizerTest, CJKCompatibilitySupplement) { EXPECT_EQ(tokens[0].text, "\xF0\xAF\xA0\x80"); } +TEST_F(StandardTokenizerTest, CJKSingleCharKeepsTrailingMarks) { + auto tokens = tokenize("\xE4\xB8\xAD\xEF\xB8\x80\xE6\x96\x87"); + ASSERT_EQ(tokens.size(), 2u); + EXPECT_EQ(tokens[0].text, "\xE4\xB8\xAD\xEF\xB8\x80"); + EXPECT_EQ(tokens[1].text, "\xE6\x96\x87"); +} + // --- Mixed scripts --- TEST_F(StandardTokenizerTest, MixedLatinAndCJK) { @@ -279,3 +308,178 @@ TEST_F(StandardTokenizerTest, MaxTokenLengthCountsCodepointsNotBytes) { EXPECT_EQ(tokens3[0].text, "caf"); EXPECT_EQ(tokens3[1].text, "\xC3\xA9"); } + +TEST_F(StandardTokenizerTest, MaxTokenLengthDropsConnectorOnlySplitSegments) { + FtsIndexParams params3; + params3.tokenizer_name = "standard"; + params3.filters.clear(); + params3.extra_params = R"({"max_token_length":3})"; + auto pipeline3 = TokenizerFactory::create(params3); + ASSERT_NE(pipeline3, nullptr); + auto tokens3 = pipeline3->process("dog's"); + ASSERT_EQ(tokens3.size(), 2u); + EXPECT_EQ(tokens3[0].text, "dog"); + EXPECT_EQ(tokens3[1].text, "s"); + + FtsIndexParams params1; + params1.tokenizer_name = "standard"; + params1.filters.clear(); + params1.extra_params = R"({"max_token_length":1})"; + auto pipeline1 = TokenizerFactory::create(params1); + ASSERT_NE(pipeline1, nullptr); + auto leading = pipeline1->process("_lead"); + std::vector expected_leading = {"l", "e", "a", "d"}; + EXPECT_EQ(token_texts(leading), expected_leading); + + auto internal = pipeline1->process("abc__def"); + std::vector expected_internal = {"a", "b", "c", "d", "e", "f"}; + EXPECT_EQ(token_texts(internal), expected_internal); +} + +TEST_F(StandardTokenizerTest, IntraWordPunctuation) { + auto tokens = tokenize( + "dog's 3.14 1,000 example.com hello,world host:port a:b " + "host:9200"); + std::vector expected = { + "dog's", "3.14", "1,000", "example.com", "hello", + "world", "host:port", "a:b", "host", "9200"}; + EXPECT_EQ(token_texts(tokens), expected); +} + +TEST_F(StandardTokenizerTest, ExtendNumLetConnectsWordsAndNumbers) { + auto tokens = tokenize("foo_bar v1_2 _lead __123 _"); + std::vector expected = {"foo_bar", "v1_2", "_lead", "__123"}; + EXPECT_EQ(token_texts(tokens), expected); +} + +TEST_F(StandardTokenizerTest, EmojiZwjSequence) { + auto tokens = tokenize( + "\xF0\x9F\x91\xA9\xE2\x80\x8D\xF0\x9F\x92\xBB " + "\xE2\x9D\xA4\xEF\xB8\x8F"); + ASSERT_EQ(tokens.size(), 2u); + EXPECT_EQ(tokens[0].text, "\xF0\x9F\x91\xA9\xE2\x80\x8D\xF0\x9F\x92\xBB"); + EXPECT_EQ(tokens[1].text, "\xE2\x9D\xA4\xEF\xB8\x8F"); +} + +TEST_F(StandardTokenizerTest, EmojiKeycapSequences) { + auto tokens = tokenize( + "1\xEF\xB8\x8F\xE2\x83\xA3 " + "#\xE2\x83\xA3 " + "*\xEF\xB8\x8F\xE2\x83\xA3"); + ASSERT_EQ(tokens.size(), 3u); + EXPECT_EQ(tokens[0].text, "1\xEF\xB8\x8F\xE2\x83\xA3"); + EXPECT_EQ(tokens[1].text, "#\xE2\x83\xA3"); + EXPECT_EQ(tokens[2].text, "*\xEF\xB8\x8F\xE2\x83\xA3"); +} + +TEST_F(StandardTokenizerTest, EmojiModifierSequences) { + auto tokens = tokenize( + "\xF0\x9F\x91\x8D\xF0\x9F\x8F\xBD " + "\xE2\x98\x9D\xEF\xB8\x8F\xF0\x9F\x8F\xBB " + "\xF0\x9F\x8F\xBD"); + ASSERT_EQ(tokens.size(), 3u); + EXPECT_EQ(tokens[0].text, "\xF0\x9F\x91\x8D\xF0\x9F\x8F\xBD"); + EXPECT_EQ(tokens[1].text, "\xE2\x98\x9D\xEF\xB8\x8F\xF0\x9F\x8F\xBB"); + EXPECT_EQ(tokens[2].text, "\xF0\x9F\x8F\xBD"); +} + +TEST_F(StandardTokenizerTest, EmojiModifierInsideZwjSequence) { + auto tokens = + tokenize("\xF0\x9F\x91\xA9\xF0\x9F\x8F\xBD\xE2\x80\x8D\xF0\x9F\x92\xBB"); + ASSERT_EQ(tokens.size(), 1u); + EXPECT_EQ(tokens[0].text, + "\xF0\x9F\x91\xA9\xF0\x9F\x8F\xBD\xE2\x80\x8D\xF0\x9F\x92\xBB"); +} + +TEST_F(StandardTokenizerTest, RegionalIndicatorPairs) { + auto tokens = tokenize( + "\xF0\x9F\x87\xBA\xF0\x9F\x87\xB8" + "\xF0\x9F\x87\xA8\xF0\x9F\x87\xA6" + "\xF0\x9F\x87\xAF"); + ASSERT_EQ(tokens.size(), 3u); + EXPECT_EQ(tokens[0].text, "\xF0\x9F\x87\xBA\xF0\x9F\x87\xB8"); + EXPECT_EQ(tokens[1].text, "\xF0\x9F\x87\xA8\xF0\x9F\x87\xA6"); + EXPECT_EQ(tokens[2].text, "\xF0\x9F\x87\xAF"); +} + +TEST_F(StandardTokenizerTest, RegionalIndicatorPairsIgnoreExtendAndZwj) { + auto tokens = tokenize( + "\xF0\x9F\x87\xA6\xCC\x88\xF0\x9F\x87\xA7 " + "\xF0\x9F\x87\xA6\xE2\x80\x8D\xF0\x9F\x87\xA7\xF0\x9F\x87\xA8"); + ASSERT_EQ(tokens.size(), 3u); + EXPECT_EQ(tokens[0].text, "\xF0\x9F\x87\xA6\xCC\x88\xF0\x9F\x87\xA7"); + EXPECT_EQ(tokens[1].text, "\xF0\x9F\x87\xA6\xE2\x80\x8D\xF0\x9F\x87\xA7"); + EXPECT_EQ(tokens[2].text, "\xF0\x9F\x87\xA8"); +} + +TEST_F(StandardTokenizerTest, MinimalWb3cZwjExtendedPictographic) { + auto tokens = tokenize( + "\xE2\x80\x8D\xF0\x9F\x9B\x91 " + "a\xE2\x80\x8D\xF0\x9F\x9B\x91 " + "\xE2\x80\x8D\xE2\x93\x82"); + ASSERT_EQ(tokens.size(), 3u); + EXPECT_EQ(tokens[0].text, "\xE2\x80\x8D\xF0\x9F\x9B\x91"); + EXPECT_EQ(tokens[1].text, "a\xE2\x80\x8D\xF0\x9F\x9B\x91"); + EXPECT_EQ(tokens[2].text, "\xE2\x80\x8D\xE2\x93\x82"); +} + +TEST_F(StandardTokenizerTest, HiraganaTokensAreSingleCharacters) { + auto tokens = tokenize("\xE3\x81\x8B\xE3\x82\x99\xE3\x81\xAA"); + ASSERT_EQ(tokens.size(), 2u); + EXPECT_EQ(tokens[0].text, "\xE3\x81\x8B\xE3\x82\x99"); + EXPECT_EQ(tokens[1].text, "\xE3\x81\xAA"); +} + +TEST_F(StandardTokenizerTest, JapaneseKoreanAndSoutheastAsianScripts) { + auto tokens = tokenize( + "\xE3\x81\xB2\xE3\x82\x89\xE3\x81\x8C\xE3\x81\xAA " + "\xE3\x82\xAB\xE3\x82\xBF\xE3\x82\xAB\xE3\x83\x8A " + "\xED\x95\x9C\xEA\xB5\xAD " + "\xE0\xB9\x84\xE0\xB8\x97\xE0\xB8\xA2 " + "\xE1\x80\x99\xE1\x80\x94"); + ASSERT_EQ(tokens.size(), 8u); + EXPECT_EQ(tokens[0].text, "\xE3\x81\xB2"); + EXPECT_EQ(tokens[1].text, "\xE3\x82\x89"); + EXPECT_EQ(tokens[2].text, "\xE3\x81\x8C"); + EXPECT_EQ(tokens[3].text, "\xE3\x81\xAA"); + EXPECT_EQ(tokens[4].text, "\xE3\x82\xAB\xE3\x82\xBF\xE3\x82\xAB\xE3\x83\x8A"); + EXPECT_EQ(tokens[5].text, "\xED\x95\x9C\xEA\xB5\xAD"); + EXPECT_EQ(tokens[6].text, "\xE0\xB9\x84\xE0\xB8\x97\xE0\xB8\xA2"); + EXPECT_EQ(tokens[7].text, "\xE1\x80\x99\xE1\x80\x94"); +} + +TEST_F(StandardTokenizerTest, SoutheastAsianMarksContinueButDoNotStartTokens) { + auto tokens = tokenize( + "\xE0\xB8\x81\xE0\xB8\xB1 " + "\xE0\xB8\xB1"); + ASSERT_EQ(tokens.size(), 1u); + EXPECT_EQ(tokens[0].text, "\xE0\xB8\x81\xE0\xB8\xB1"); +} + +TEST_F(StandardTokenizerTest, HangulSymbolsOutsideWordClassAreIgnored) { + auto tokens = tokenize("\xE3\x89\xA0 \xED\x95\x9C\xEA\xB5\xAD"); + ASSERT_EQ(tokens.size(), 1u); + EXPECT_EQ(tokens[0].text, "\xED\x95\x9C\xEA\xB5\xAD"); +} + +TEST_F(StandardTokenizerTest, HebrewSingleQuoteStaysWithLetter) { + auto tokens = tokenize("\xD7\x90' \xD7\x90\"\xD7\x91"); + ASSERT_EQ(tokens.size(), 2u); + EXPECT_EQ(tokens[0].text, "\xD7\x90'"); + EXPECT_EQ(tokens[1].text, "\xD7\x90\"\xD7\x91"); +} + +TEST(StandardTokenizerConfigTest, MaxTokenLengthValidation) { + FtsIndexParams params; + params.tokenizer_name = "standard"; + params.filters.clear(); + + params.extra_params = R"({"max_token_length":0})"; + EXPECT_EQ(TokenizerFactory::create(params), nullptr); + + params.extra_params = R"({"max_token_length":1048577})"; + EXPECT_EQ(TokenizerFactory::create(params), nullptr); + + params.extra_params = R"({"max_token_length":1})"; + EXPECT_NE(TokenizerFactory::create(params), nullptr); +} diff --git a/tests/db/sqlengine/fts_parser_test.cc b/tests/db/sqlengine/fts_parser_test.cc index 4e909e9..798c0bf 100644 --- a/tests/db/sqlengine/fts_parser_test.cc +++ b/tests/db/sqlengine/fts_parser_test.cc @@ -27,10 +27,9 @@ namespace zvec::fts { class FtsParserTest : public ::testing::Test { protected: void SetUp() override { - // Standard tokenizer + lowercase filter: ASCII tests behave the same as - // the previous whitespace split (alnum runs become tokens, delimiters - // get dropped) while CJK tests can exercise the per-character tokens - // standard produces from non-alnum bytes. + // Standard tokenizer + lowercase filter. These parser tests cover + // punctuation that standard still treats as delimiters, while CJK tests + // exercise the per-character tokens standard produces for ideographs. FtsIndexParams params; params.tokenizer_name = "standard"; params.filters = {"lowercase"}; @@ -100,8 +99,8 @@ TEST_F(FtsParserTest, SingleTermNumeric) { TEST_F(FtsParserTest, SingleTermWithHyphen) { // The lexer's REGULAR_ID rule keeps hyphenated text as one token, but the - // standard tokenizer on the parser side splits non-alphanumerics. With the - // default OR operator the term decomposes into Or[full, text] so query + // standard tokenizer on the parser side splits this hyphen delimiter. With + // the default OR operator the term decomposes into Or[full, text] so query // segmentation matches the index segmentation. auto ast = parse("full-text"); ASSERT_NE(ast, nullptr); @@ -112,6 +111,12 @@ TEST_F(FtsParserTest, SingleTermWithHyphen) { EXPECT_EQ(as_term(*or_node.children[1]).term, "text"); } +TEST_F(FtsParserTest, BareColonQueryIsFieldPrefixSyntax) { + auto ast = parse("host:port"); + EXPECT_EQ(ast, nullptr); + EXPECT_EQ(err_msg(), "field-prefixed queries are not supported"); +} + // ============================================================ // Must (+) and must_not (-/NOT) modifiers // ============================================================ @@ -754,7 +759,7 @@ TEST_F(FtsParserTest, MultiTokenBareTermPreservesMustModifier) { TEST_F(FtsParserTest, PhraseTokensRunThroughPipeline) { // The phrase body is tokenized exactly like document text. With the - // standard tokenizer, mixed delimiters between alnum runs collapse so + // standard tokenizer, comma and exclamation delimiters collapse so // "machine, learning!" becomes ["machine", "learning"]. auto ast = parse("\"machine, learning!\""); ASSERT_NE(ast, nullptr); @@ -765,6 +770,15 @@ TEST_F(FtsParserTest, PhraseTokensRunThroughPipeline) { EXPECT_EQ(phrase.terms[1], "learning"); } +TEST_F(FtsParserTest, PhraseCanSearchLiteralColonToken) { + auto ast = parse("\"host:port\""); + ASSERT_NE(ast, nullptr); + ASSERT_EQ(ast->type(), FtsNodeType::PHRASE); + const auto &phrase = as_phrase(*ast); + ASSERT_EQ(phrase.terms.size(), 1u); + EXPECT_EQ(phrase.terms[0], "host:port"); +} + TEST_F(FtsParserTest, PhraseLowercaseFilterApplies) { // The lowercase filter is part of the pipeline so phrase tokens come back // lowercased even when the input mixed case.