Files
zvec/tests/db/sqlengine/CMakeLists.txt
T
egolearner 02bfb31cf5 feat: add fts support (#408)
Add BM25-based full-text search with CJK (jieba) tokenization, supporting
  query_string and match_string syntax, phrase queries, boolean operators
  (AND/OR/NOT/MUST), and hybrid retrieval with existing vector search.

  ## Core
  - BitPacked posting format with block-max WAND pruning
  - Tokenizer pipeline: jieba (cut/cut_for_search/hmm/full), whitespace,
    lowercase, with extensible pipeline composition
  - Query parser: boolean operators, phrase queries, field scoping,
    boost, MUST(+) modifier inside OR (ES query_string semantics)
  - AST rewriter: dedup repeated terms with linear boost aggregation,
    flatten same-type composites, canonicalize OR-with-must_not into AND
    wrapper, empty-node propagation, contradiction detection
  - FTS reduce/merge integrated into Optimize compaction
  - Multi-segment score-descending sort
  - Auto-register bundled jieba dict on SDK import

  ## Performance
  - Block-max WAND with cached block_max_info_for (single binary search)
  - AVX2/SSE bitpacked encoding with cross-arch scalar fallback
  - MultiGet for batch posting retrieval and phrase position verification
  - HashSkipList memtable for posting writes
  - PinnableSlice zero-copy reads
  - Filter pushdown into composite iterators (Disjunction/Conjunction/Phrase)
  - Candidate-driven (brute-force) evaluation for selective invert filters
  - Precomputed BM25 IDF weights, cached SIMD dispatch pointers
  - Shortest-list anchor for phrase position matching
  - Single-open per-term posting iterator

  ## Query
  - Tokenize query terms through the same pipeline as indexing
  - EmptyNode for zero-token queries (all stop-words / punctuation)
  - Backslash unescape after lexing in query parser
  - Schema allows collections without vector fields (FTS-only use case)
  - Create/Drop Index validates supported index types
  - FTS fields disallowed in SQL filter expressions

  ## Bindings
  - C API: fts query params, brute-force ratio config
  - Python SDK: FTS search, jieba dict auto-registration

  ## Internals
  - Bypass cppjieba::Jieba to drop KeywordExtractor (~12MB fewer required files)
  - Hide tokenizer pipeline from public header (Pimpl-style FtsState)
  - ListColumnFamilies to avoid double-open on segment load
  - Reorganized fts_column into tokenizer/, posting/, iterator/ subdirs
2026-06-01 15:02:54 +08:00

44 lines
1.1 KiB
CMake

include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
if(APPLE)
set(APPLE_FRAMEWORK_LIBS
-framework CoreFoundation
-framework CoreGraphics
-framework CoreData
-framework CoreText
-framework Security
-framework Foundation
-Wl,-U,_MallocExtension_ReleaseFreeMemory
-Wl,-U,_ProfilerStart
-Wl,-U,_ProfilerStop
-Wl,-U,_RegisterThriftProtocol
)
endif()
file(GLOB ALL_TEST_SRCS *_test.cc)
foreach(CC_SRCS ${ALL_TEST_SRCS})
get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
cc_gmock(
NAME ${CC_TARGET} STRICT
LIBS zvec_common
zvec_proto
zvec_sqlengine
zvec_db
zvec_ailego
core_metric
core_utility
core_quantizer
core_knn_hnsw core_knn_hnsw_sparse sparsehash
core_knn_flat core_knn_flat_sparse core_knn_ivf
core_knn_hnsw_rabitq core_mix_reducer
${CMAKE_THREAD_LIBS_INIT}
${CMAKE_DL_LIBS}
SRCS ${CC_SRCS}
INCS . ${PROJECT_ROOT_DIR}/src ${PROJECT_ROOT_DIR}/src/db ${PROJECT_ROOT_DIR}/src/db/common
LDFLAGS ${APPLE_FRAMEWORK_LIBS}
)
cc_test_suite(zvec_sqlengine ${CC_TARGET})
endforeach()