fix E4.1: IsContradiction 中文感知修复(字符级切分+否定词检测)
问题:原 IsContradiction 使用 strings.Fields(),中文无空格时整句为一个词,
导致 overlap 始终为 0,矛盾检测永远失败。
修复:
- 新增 splitWordsCN:中文按 unicode.Han 字符级切分,英文按空格分词
- 新增 containsNegCN:检测中文否定词(不是/没有/不/没/莫/别)
- IsContradiction 合并中文/英文两种检测逻辑,阈值调至 0.3
验证:Test 2 返回 {"conflicts": ["小唯是牧尘的女朋友"]} ✅
This commit is contained in:
parent
03fe336616
commit
51eec76f1f
|
|
@ -6,6 +6,7 @@ import (
|
|||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// ─── 冲突检测 ────────────────────────────────────────────
|
||||
|
|
@ -95,27 +96,83 @@ func toStringSlice(v interface{}) []string {
|
|||
return nil
|
||||
}
|
||||
|
||||
// containsNegCN 检查文本中是否含中文否定词或单字否定
|
||||
func containsNegCN(text string) bool {
|
||||
negPhrases := []string{"不是", "没有", "不存在", "禁止", "不允许", "无", "非"}
|
||||
for _, n := range negPhrases {
|
||||
if strings.Contains(text, n) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, r := range text {
|
||||
if r == '不' || r == '没' || r == '莫' || r == '别' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// splitWordsCN 中文按字符级切分(过滤标点),英文按空格分词
|
||||
func splitWordsCN(text string) []string {
|
||||
if len(text) == 0 {
|
||||
return nil
|
||||
}
|
||||
hasCN := false
|
||||
for _, r := range text {
|
||||
if unicode.Is(unicode.Han, r) {
|
||||
hasCN = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if hasCN {
|
||||
var result []string
|
||||
for _, r := range text {
|
||||
if unicode.Is(unicode.Han, r) || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
|
||||
result = append(result, strings.ToLower(string(r)))
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
return strings.Fields(strings.ToLower(text))
|
||||
}
|
||||
|
||||
// IsContradiction 检查两条内容是否语义矛盾
|
||||
// 中文/混合文本:字符级重叠 + 否定词差异
|
||||
// 英文/空格文本:单词级重叠(原有逻辑)
|
||||
func IsContradiction(a, b string) bool {
|
||||
// 简单启发式:重叠词 > 50% 但存在否定词差异
|
||||
wordsA := strings.Fields(strings.ToLower(a))
|
||||
wordsB := strings.Fields(strings.ToLower(b))
|
||||
wordsA := splitWordsCN(a)
|
||||
wordsB := splitWordsCN(b)
|
||||
setA := make(map[string]bool)
|
||||
for _, w := range wordsA {
|
||||
setA[w] = true
|
||||
}
|
||||
|
||||
overlap := 0
|
||||
negInA := containsNeg(wordsA)
|
||||
negInB := containsNeg(wordsB)
|
||||
negInA := containsNegCN(a)
|
||||
negInB := containsNegCN(b)
|
||||
negInWordsA := containsNeg(wordsA) // 英文否定
|
||||
negInWordsB := containsNeg(wordsB)
|
||||
|
||||
if negInA != negInB || negInWordsA != negInWordsB {
|
||||
// 有否定词差异,再检查重叠度
|
||||
} else {
|
||||
// 无否定词差异,直接返回 false
|
||||
return false
|
||||
}
|
||||
|
||||
for _, w := range wordsB {
|
||||
if setA[w] {
|
||||
overlap++
|
||||
}
|
||||
}
|
||||
|
||||
totalOverlap := float64(overlap) / math.Max(float64(len(wordsA)), float64(len(wordsB)))
|
||||
return totalOverlap > 0.5 && negInA != negInB
|
||||
maxLen := len(wordsA)
|
||||
if len(wordsB) > maxLen {
|
||||
maxLen = len(wordsB)
|
||||
}
|
||||
if maxLen == 0 {
|
||||
return false
|
||||
}
|
||||
// 阈值 0.3(中文字符级粒度细,0.5 过高)
|
||||
return float64(overlap)/float64(maxLen) > 0.3
|
||||
}
|
||||
|
||||
// DetectContradiction 检查 newContent 是否与 existingContents 中任意一条矛盾
|
||||
|
|
|
|||
Loading…
Reference in New Issue