feat(graph): add graph cleanup + nl_query + navigate grouped format
- Add CleanupNoiseNodes() to GraphStore interface + SQLiteGraphStore impl - Add /api/v1/graph/cleanup endpoint (dry_run + execute) - Add /api/v1/graph/nl_query endpoint for natural language graph queries - Improve /api/v1/graph/navigate: add grouped_by_relation, suggestions, normalized_entity - Add CleanupNoiseNodes stubs to InMemoryGraph and FileGraph BREAKING: navigate response now includes grouped_by_relation and suggestions
This commit is contained in:
小唯
parent
79b994ce4b
commit
a76b6d3afa
|
|
@ -69,6 +69,7 @@ func (ga *GraphAPI) Query(w http.ResponseWriter, r *http.Request) {
|
|||
// 1. 单实体 BFS: {"entity": "...", "max_hops": 2}
|
||||
// 2. 双向 BFS: {"source": "...", "target": "...", "max_hops": 3}
|
||||
// 3. 关系过滤 BFS: {"entity": "...", "max_hops": 2, "relation_filter": ["related_to", "uses"]}（E1.5）
|
||||
// 返回新增 grouped_by_relation 字段，按关系类型分组，便于阅读
|
||||
func (ga *GraphAPI) Navigate(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Entity string `json:"entity"`
|
||||
|
|
@ -114,12 +115,53 @@ func (ga *GraphAPI) Navigate(w http.ResponseWriter, r *http.Request) {
|
|||
respondError(w, 500, "navigate failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// 新增：按 relation 分组，更直观
|
||||
grouped := make(map[string][]map[string]interface{})
|
||||
for _, p := range paths {
|
||||
rel := p["relation"].(string)
|
||||
if rel == "" {
|
||||
rel = "_unknown"
|
||||
}
|
||||
grouped[rel] = append(grouped[rel], map[string]interface{}{
|
||||
"from": stripPrefix(p["from"].(string)),
|
||||
"to": stripPrefix(p["to"].(string)),
|
||||
"weight": p["weight"],
|
||||
"hop": p["hop"],
|
||||
})
|
||||
}
|
||||
|
||||
// 新增：相关实体建议（从 path 提取去重的 to 节点，跳过 ep_ 开头的）
|
||||
suggestions := []string{}
|
||||
seen := make(map[string]bool)
|
||||
for _, p := range paths {
|
||||
to := stripPrefix(p["to"].(string))
|
||||
if !seen[to] && !strings.HasPrefix(p["to"].(string), "ep_") {
|
||||
seen[to] = true
|
||||
suggestions = append(suggestions, to)
|
||||
}
|
||||
}
|
||||
if len(suggestions) > 20 {
|
||||
suggestions = suggestions[:20]
|
||||
}
|
||||
|
||||
respond(w, 200, map[string]interface{}{
|
||||
"paths": paths, "entity": req.Entity, "count": len(paths),
|
||||
"relation_filter": req.RelationFilter,
|
||||
"paths": paths,
|
||||
"grouped_by_relation": grouped,
|
||||
"entity": req.Entity,
|
||||
"normalized_entity": entity,
|
||||
"count": len(paths),
|
||||
"relation_count": len(grouped),
|
||||
"suggestions": suggestions,
|
||||
"relation_filter": req.RelationFilter,
|
||||
})
|
||||
}
|
||||
|
||||
// stripPrefix 去掉节点 ID 的 n_ 前缀，用于可读展示
|
||||
func stripPrefix(id string) string {
|
||||
return strings.TrimPrefix(id, "n_")
|
||||
}
|
||||
|
||||
// normalizeEntity 规整实体名：去特殊字符 + n_前缀，保留中文和 Unicode 字符
|
||||
func normalizeEntity(entity string) string {
|
||||
if strings.HasPrefix(entity, "n_") {
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import (
|
|||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -416,6 +417,131 @@ func NewServer() http.Handler {
|
|||
})
|
||||
})
|
||||
|
||||
// 自然语言图谱查询 /api/v1/graph/nl_query
|
||||
// 输入: {"query": "织忆和牧尘是什么关系"}
|
||||
// 返回: 自然语言关系描述
|
||||
mux.HandleFunc("/api/v1/graph/nl_query", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "POST" {
|
||||
http.Error(w, "POST only", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Query string `json:"query"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
query := strings.TrimSpace(req.Query)
|
||||
if query == "" {
|
||||
http.Error(w, "query is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// 1. 从 query 中提取可能的实体名（去掉问号、语气词）
|
||||
cleanQuery := strings.TrimSuffix(strings.TrimSuffix(query, "？"), "?")
|
||||
cleanQuery = strings.TrimPrefix(cleanQuery, "请问")
|
||||
cleanQuery = strings.TrimPrefix(cleanQuery, "请说")
|
||||
cleanQuery = strings.TrimSpace(cleanQuery)
|
||||
|
||||
// 2. 尝试识别实体对（"A和B的关系"、"A与B"）
|
||||
var entityA, entityB string
|
||||
// 模式1: "A和B是什么关系" / "A和B有什么关系" / "A和B的关系"
|
||||
hasAnd := strings.Contains(cleanQuery, "和") || strings.Contains(cleanQuery, "与")
|
||||
if hasAnd && (strings.Contains(cleanQuery, "关系") || strings.Contains(cleanQuery, "关系")) {
|
||||
delim := "和"
|
||||
if strings.Contains(cleanQuery, "与") {
|
||||
delim = "与"
|
||||
}
|
||||
parts := strings.Split(cleanQuery, delim)
|
||||
if len(parts) == 2 {
|
||||
entityA = strings.TrimSpace(parts[0])
|
||||
rawB := strings.TrimSpace(parts[1])
|
||||
// 去掉句末的问句模式得到 entityB
|
||||
entityB = strings.TrimSuffix(rawB, "是什么关系")
|
||||
entityB = strings.TrimSuffix(entityB, "有什么关系")
|
||||
entityB = strings.TrimSuffix(entityB, "的")
|
||||
entityB = strings.TrimSuffix(entityB, "是什么")
|
||||
entityB = strings.TrimSpace(entityB)
|
||||
// 如果去掉问句后为空（如"织忆和牧尘" -> entityB=""），则原始 query 可能是"A和B"
|
||||
if entityB == "" {
|
||||
entityB = rawB
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var result map[string]interface{}
|
||||
if entityA != "" && entityB != "" {
|
||||
// 双向查询：A → B 的关系
|
||||
paths, _ := graphStore.Navigate(normalizeEntity(entityA), 2, "", nil)
|
||||
relB, _ := graphStore.Navigate(normalizeEntity(entityB), 2, "", nil)
|
||||
// 找 A → B 的直接边
|
||||
var directPath map[string]interface{}
|
||||
for _, p := range paths {
|
||||
if strings.TrimPrefix(p["to"].(string), "n_") == normalizeEntity(entityB) {
|
||||
directPath = p
|
||||
break
|
||||
}
|
||||
}
|
||||
result = map[string]interface{}{
|
||||
"query": query,
|
||||
"entity_a": entityA,
|
||||
"entity_b": entityB,
|
||||
"direct_path": directPath,
|
||||
"paths_from_a": len(paths),
|
||||
"paths_from_b": len(relB),
|
||||
"summary": fmt.Sprintf("%s 与 %s 存在 %d 条关联路径", entityA, entityB, len(paths)),
|
||||
}
|
||||
} else {
|
||||
// 单实体查询：找 entity 的关系网
|
||||
entity := cleanQuery
|
||||
paths, _ := graphStore.Navigate(normalizeEntity(entity), 2, "", nil)
|
||||
// 按 relation 分组
|
||||
relCounts := make(map[string]int)
|
||||
for _, p := range paths {
|
||||
rel := p["relation"].(string)
|
||||
if rel == "" {
|
||||
rel = "unknown"
|
||||
}
|
||||
relCounts[rel]++
|
||||
}
|
||||
// 生成自然语言描述
|
||||
var rels []string
|
||||
for rel, cnt := range relCounts {
|
||||
rels = append(rels, fmt.Sprintf("%s(%d条)", rel, cnt))
|
||||
}
|
||||
sort.Strings(rels)
|
||||
result = map[string]interface{}{
|
||||
"query": query,
|
||||
"entity": entity,
|
||||
"total_paths": len(paths),
|
||||
"relation_summary": rels,
|
||||
"summary": fmt.Sprintf("%s 共有 %d 条关联，分布在 %d 种关系类型", entity, len(paths), len(relCounts)),
|
||||
}
|
||||
}
|
||||
respondJSON(w, 200, result)
|
||||
})
|
||||
|
||||
// 图谱脏数据清理 /api/v1/graph/cleanup
|
||||
mux.HandleFunc("/api/v1/graph/cleanup", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "POST" && r.Method != "GET" {
|
||||
http.Error(w, "GET or POST only", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
dryRun := r.URL.Query().Get("dry_run") != "false" // 默认为 dryRun
|
||||
count, ids, err := graphStore.CleanupNoiseNodes(dryRun)
|
||||
if err != nil {
|
||||
http.Error(w, "cleanup failed: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
respondJSON(w, 200, map[string]interface{}{
|
||||
"dry_run": dryRun,
|
||||
"removed": count,
|
||||
"node_ids": ids,
|
||||
"message": fmt.Sprintf("Found %d noise nodes", count),
|
||||
})
|
||||
})
|
||||
|
||||
// 静态文件服务（知识图谱可视化 HTML）
|
||||
staticDir := os.Getenv("STATIC_DIR")
|
||||
if staticDir == "" {
|
||||
|
|
|
|||
|
|
@ -435,6 +435,13 @@ func (fg *FileGraph) Prune(minWeight float64) {
|
|||
fg.save()
|
||||
}
|
||||
|
||||
func (fg *FileGraph) CleanupNoiseNodes(dryRun bool) (int, []string, error) {
|
||||
fg.mu.Lock()
|
||||
defer fg.mu.Unlock()
|
||||
// FileGraph 不需要脏数据清理（已迁移到 SQLite）
|
||||
return 0, nil, nil
|
||||
}
|
||||
|
||||
// ─── 图谱扩展 ────────────────────────────────────────────
|
||||
|
||||
func (fg *FileGraph) ExpandFromResults(results []models.RecallResult, namespace string, maxHops int) []models.RecallResult {
|
||||
|
|
|
|||
|
|
@ -322,6 +322,13 @@ func (g *InMemoryGraph) Prune(minWeight float64) {
|
|||
}
|
||||
}
|
||||
|
||||
func (g *InMemoryGraph) CleanupNoiseNodes(dryRun bool) (int, []string, error) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
// InMemoryGraph 不需要脏数据清理（测试用）
|
||||
return 0, nil, nil
|
||||
}
|
||||
|
||||
// Query 按实体和关系查询
|
||||
func (g *InMemoryGraph) Query(entity, relation, namespace string) []map[string]interface{} {
|
||||
g.mu.RLock()
|
||||
|
|
|
|||
|
|
@ -549,6 +549,75 @@ func (gs *SQLiteGraphStore) Prune(minWeight float64) {
|
|||
execSQL(gs.db, `DELETE FROM graph_nodes WHERE id NOT IN (SELECT DISTINCT source FROM graph_edges UNION SELECT DISTINCT target FROM graph_edges)`)
|
||||
}
|
||||
|
||||
// CleanupNoiseNodes 删除名称含编码噪音的节点（如 "n_fts=517," "n_fts（sqlite）+"）
|
||||
// dryRun=true 时只检查不删除，返回预检结果
|
||||
func (gs *SQLiteGraphStore) CleanupNoiseNodes(dryRun bool) (int, []string, error) {
|
||||
gs.mu.Lock()
|
||||
defer gs.mu.Unlock()
|
||||
|
||||
// 噪音模式：节点名含 SQL 残片、编码错误符号
|
||||
noisePatterns := []string{
|
||||
"fts=",
|
||||
"fts（",
|
||||
"fts(",
|
||||
"）",
|
||||
"（sqlite",
|
||||
"__",
|
||||
}
|
||||
|
||||
// 检查节点名是否含噪音
|
||||
findNoise := func(name string) bool {
|
||||
for _, pat := range noisePatterns {
|
||||
if strings.Contains(name, pat) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// 括号不匹配检测
|
||||
open := 0
|
||||
for _, ch := range name {
|
||||
if ch == '(' || ch == '（' {
|
||||
open++
|
||||
} else if ch == ')' || ch == '）' {
|
||||
open--
|
||||
}
|
||||
}
|
||||
if open != 0 {
|
||||
return true // 括号不匹配
|
||||
}
|
||||
// 节点名含逗号/等号残片（如 "n_fts=517,"）
|
||||
if strings.HasSuffix(name, ",") || strings.HasSuffix(name, "=") {
|
||||
return true
|
||||
}
|
||||
// 含 %23 %3D 等 URL 编码残留
|
||||
if strings.Contains(name, "%") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// 查询所有节点，找出噪音节点
|
||||
rows := queryRows(gs.db, "SELECT id, name FROM graph_nodes")
|
||||
var noiseIDs []string
|
||||
for _, row := range rows {
|
||||
id := row["id"].(string)
|
||||
name := row["name"].(string)
|
||||
if findNoise(name) {
|
||||
noiseIDs = append(noiseIDs, id)
|
||||
}
|
||||
}
|
||||
|
||||
if dryRun || len(noiseIDs) == 0 {
|
||||
return len(noiseIDs), noiseIDs, nil
|
||||
}
|
||||
|
||||
// 删除噪音节点的关联边，再删节点
|
||||
for _, nid := range noiseIDs {
|
||||
execSQL(gs.db, fmt.Sprintf("DELETE FROM graph_edges WHERE source = '%s' OR target = '%s'", escape(nid), escape(nid)))
|
||||
execSQL(gs.db, fmt.Sprintf("DELETE FROM graph_nodes WHERE id = '%s'", escape(nid)))
|
||||
}
|
||||
return len(noiseIDs), noiseIDs, nil
|
||||
}
|
||||
|
||||
func (gs *SQLiteGraphStore) GetGraph(namespace string, limit int) ([]map[string]interface{}, []map[string]interface{}) {
|
||||
nodes := []map[string]interface{}{}
|
||||
edges := []map[string]interface{}{}
|
||||
|
|
|
|||
|
|
@ -40,4 +40,8 @@ type GraphStore interface {
|
|||
|
||||
// 导出完整图谱（供可视化），limit≤0 时不限制
|
||||
GetGraph(namespace string, limit int) (nodes []map[string]interface{}, edges []map[string]interface{})
|
||||
|
||||
// 清理图谱脏数据：删除名称含编码噪音的节点（如 fts=、括号不匹配等）
|
||||
// 返回被删除的节点数和节点 ID 列表
|
||||
CleanupNoiseNodes(dryRun bool) (int, []string, error)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue