diff --git a/apps/desktop/src/components/editor/QueryEditor.vue b/apps/desktop/src/components/editor/QueryEditor.vue index 76489372a..1f8171123 100644 --- a/apps/desktop/src/components/editor/QueryEditor.vue +++ b/apps/desktop/src/components/editor/QueryEditor.vue @@ -29,6 +29,9 @@ import { shouldAutoOpenSqlCompletion, extractCteDefinitions, } from "@/lib/sqlCompletion"; +import { sqlCompletionContextFromSemantic } from "@/lib/sqlSemanticCompletion"; +import { buildSqlSemanticModel } from "@/lib/sqlSemanticModel"; +import { mergeSqlSemanticReferenceAnalysis, resolveSqlSemanticNavigationTarget } from "@/lib/sqlSemanticReferences"; import { buildElasticsearchCompletionItemsFromContext, getElasticsearchCompletionContext, getElasticsearchCompletionResultValidFor, shouldAutoOpenElasticsearchCompletion, type ElasticsearchCompletionItem } from "@/lib/elasticsearchCompletion"; import { buildMongoCompletionItemsFromContext, getMongoCompletionContext, getMongoCompletionResultValidFor, shouldAutoOpenMongoCompletion, type MongoCompletionItem } from "@/lib/mongoCompletion"; import { extractIdentifierAt, isSqlKeyword, matchTable, splitQualifiedIdentifier } from "@/lib/sqlNavigation"; @@ -78,6 +81,8 @@ const props = defineProps<{ }>(); const COMPLETION_REMOTE_LATENCY_BUDGET_MS = 120; +// Internal rollback switch: flip to false to route completion, diagnostics, and navigation through the legacy SQL context path. +const SEMANTIC_SQL_COMPLETION_ENABLED = true; const emit = defineEmits<{ "update:modelValue": [value: string]; @@ -992,24 +997,29 @@ async function resolveSqlHoverTooltip(currentView: EditorViewType, pos: number) if (!range) return null; const identifier = range.text; - const parts = identifier.split("."); + const parts = splitQualifiedIdentifier(identifier); const name = parts[parts.length - 1] ?? identifier; const qualifier = parts.length > 1 ? parts[parts.length - 2] : undefined; + const semanticModel = SEMANTIC_SQL_COMPLETION_ENABLED ? buildSqlSemanticModel(sql, pos, { databaseType: props.databaseType, dialect: props.dialect }) : null; + const semanticTarget = semanticModel ? resolveSqlSemanticNavigationTarget(semanticModel, parts) : null; + const semanticQualifierIsRowSource = !!qualifier && !!semanticTarget && (semanticTarget.alias?.toLowerCase() === qualifier.toLowerCase() || semanticTarget.source.name.toLowerCase() === qualifier.toLowerCase()); + const tableLookupName = semanticTarget && !semanticQualifierIsRowSource ? semanticTarget.name : name; + const qualifiedTableLookup = semanticTarget?.schema ? `${semanticTarget.schema}.${semanticTarget.name}` : identifier; try { if (cachedTables.length === 0) { cachedTables = usesLocalOnlyCompletionMetadata() - ? connectionStore.lookupLocalCompletionTables(props.connectionId, props.database, name, MAX_COMPLETION_TABLES, props.schema) - : await connectionStore.listCompletionTables(props.connectionId, props.database, name, MAX_COMPLETION_TABLES, props.schema); + ? connectionStore.lookupLocalCompletionTables(props.connectionId, props.database, tableLookupName, MAX_COMPLETION_TABLES, props.schema) + : await connectionStore.listCompletionTables(props.connectionId, props.database, tableLookupName, MAX_COMPLETION_TABLES, props.schema); } - let table = matchTable(identifier, cachedTables) ?? matchTable(name, cachedTables); + let table = matchTable(qualifiedTableLookup, cachedTables) ?? matchTable(tableLookupName, cachedTables) ?? matchTable(identifier, cachedTables) ?? matchTable(name, cachedTables); if (!table && !usesLocalOnlyCompletionMetadata()) { - const hoverTables = await connectionStore.listCompletionTables(props.connectionId, props.database, name, MAX_COMPLETION_TABLES, props.schema); + const hoverTables = await connectionStore.listCompletionTables(props.connectionId, props.database, tableLookupName, MAX_COMPLETION_TABLES, semanticTarget?.schema ?? props.schema); cachedTables = [...cachedTables, ...hoverTables]; - table = matchTable(identifier, hoverTables) ?? matchTable(name, hoverTables); + table = matchTable(qualifiedTableLookup, hoverTables) ?? matchTable(tableLookupName, hoverTables) ?? matchTable(identifier, hoverTables) ?? matchTable(name, hoverTables); } - if (table && (!qualifier || table.schema?.toLowerCase() === qualifier.toLowerCase() || table.name === name)) { + if (table && !semanticQualifierIsRowSource && (!qualifier || table.schema?.toLowerCase() === qualifier.toLowerCase() || table.name === name)) { return { pos: range.from, end: range.to, @@ -1019,12 +1029,21 @@ async function resolveSqlHoverTooltip(currentView: EditorViewType, pos: number) }; } - const context = getSqlCompletionContext(sql, pos); + const legacyContext = getSqlCompletionContext(sql, pos); + const context = semanticModel ? sqlCompletionContextFromSemantic(semanticModel, legacyContext) : legacyContext; const candidates = qualifier ? context.referencedTables.filter((rt) => rt.alias?.toLowerCase() === qualifier.toLowerCase() || rt.name.toLowerCase() === qualifier.toLowerCase()) : context.referencedTables; for (const refTable of candidates) { - await ensureColumnsForTable(refTable); - const columns = cachedColumnsByTable.get(completionCacheKey(refTable)) ?? []; + const columns: SqlCompletionColumn[] = + refTable.columns?.map((columnName) => ({ + name: columnName, + table: refTable.name, + ...(refTable.schema ? { schema: refTable.schema } : {}), + })) ?? []; + if (columns.length === 0) { + await ensureColumnsForTable(refTable); + columns.push(...(cachedColumnsByTable.get(completionCacheKey(refTable)) ?? [])); + } const column = columns.find((col) => col.name.toLowerCase() === name.toLowerCase()); if (!column) continue; return { @@ -1164,6 +1183,10 @@ async function enrichSemanticDiagnosticTables(tables: SqlTableReference[]): Prom const enriched: SqlTableReference[] = []; const missingTables = new Set(); for (const table of tables) { + if (isStatementLocalSemanticTable(table)) { + enriched.push(table); + continue; + } try { const match = await findExactSemanticDiagnosticTable(table); if (!match) missingTables.add(tableReferenceKey(table)); @@ -1180,6 +1203,7 @@ async function ensureColumnsForSemanticDiagnostics(tables: SqlTableReference[]): const seen = new Set(); const targets: SqlTableReference[] = []; for (const table of tables) { + if (isStatementLocalSemanticTable(table)) continue; const tableWithInlineColumns = table as SqlTableReference & { columns?: string[] }; if (tableWithInlineColumns.columns && tableWithInlineColumns.columns.length > 0) continue; const cacheKey = completionCacheKey(table); @@ -1204,6 +1228,11 @@ async function ensureColumnsForSemanticDiagnostics(tables: SqlTableReference[]): return missingTables; } +function isStatementLocalSemanticTable(table: SqlTableReference): boolean { + const kind = (table as SqlTableReference & { semanticSourceKind?: string }).semanticSourceKind; + return kind === "cte" || kind === "subquery" || kind === "table_function"; +} + async function refreshSemanticDiagnostics(options: { preserveOutsideRanges?: boolean } = {}) { const currentView = view.value; const runId = ++semanticDiagnosticRunId; @@ -1256,12 +1285,15 @@ async function refreshSemanticDiagnostics(options: { preserveOutsideRanges?: boo const analysis = await api.analyzeSqlReferences(range.sql, props.formatDialect ?? props.dialect ?? "generic"); if (runId !== semanticDiagnosticRunId) return; - const { tables, missingTables } = await enrichSemanticDiagnosticTables(analysis.tables); + const semanticCursor = Math.max(0, Math.min(currentView.state.selection.main.head - range.from, range.sql.length)); + const semanticModel = SEMANTIC_SQL_COMPLETION_ENABLED ? buildSqlSemanticModel(range.sql, semanticCursor, { databaseType: props.databaseType, dialect: props.dialect }) : null; + const semanticAnalysis = semanticModel ? mergeSqlSemanticReferenceAnalysis(analysis, semanticModel) : analysis; + const { tables, missingTables } = await enrichSemanticDiagnosticTables(semanticAnalysis.tables); const columnMetadataMissingTables = await ensureColumnsForSemanticDiagnostics(tables); for (const tableKey of columnMetadataMissingTables) missingTables.add(tableKey); if (runId !== semanticDiagnosticRunId) return; - const enrichedAnalysis: SqlReferenceAnalysis = { ...analysis, tables }; + const enrichedAnalysis: SqlReferenceAnalysis = { ...semanticAnalysis, tables }; nextDiagnostics.push( ...offsetSqlSemanticDiagnostics( buildSqlSemanticDiagnostics(enrichedAnalysis, { @@ -1562,7 +1594,10 @@ async function provideMongoCompletions(currentState: import("@codemirror/state") }; } -async function provideSqlCompletions(currentState: import("@codemirror/state").EditorState, position: number, explicit: boolean) { +async function provideSqlCompletions(context: CompletionContext) { + const currentState = context.state; + const position = context.pos; + const explicit = context.explicit; if (imeCompositionActive || view.value?.compositionStarted || view.value?.composing) return null; if (!props.connectionId) return null; const fullDoc = currentState.doc.toString(); @@ -1585,7 +1620,8 @@ async function provideSqlCompletions(currentState: import("@codemirror/state").E if (isSqlCompletionSuppressedContext(fullDoc, position)) return null; if (!explicit && !shouldAutoOpenSqlCompletion(fullDoc, position)) return null; - const completionContext = getSqlCompletionContext(fullDoc, position); + const legacyCompletionContext = getSqlCompletionContext(fullDoc, position); + const completionContext = SEMANTIC_SQL_COMPLETION_ENABLED ? sqlCompletionContextFromSemantic(buildSqlSemanticModel(fullDoc, position, { databaseType: props.databaseType, dialect: props.dialect }), legacyCompletionContext) : legacyCompletionContext; if (!hasDatabase) { const items = buildSqlCompletionItemsFromContext(completionContext, { @@ -1643,6 +1679,9 @@ async function provideSqlCompletions(currentState: import("@codemirror/state").E // This prevents wasted backend calls during rapid typing while still // showing table/column names in the first popup. return new Promise>((resolve) => { + context.addEventListener("abort", () => { + if (epoch === completionEpoch) completionEpoch++; + }); completionDebounceTimer = setTimeout(async () => { completionDebounceTimer = null; if (epoch !== completionEpoch) { @@ -2250,7 +2289,7 @@ onMounted(async () => { buildSqlCompletionExtension = () => autocompletion({ activateOnTyping: true, - override: [async (context: CompletionContext) => provideSqlCompletions(context.state, context.pos, context.explicit)], + override: [async (context: CompletionContext) => provideSqlCompletions(context)], }); const dialect = createDbxCodeMirrorSqlDialect(langSql, props.dialect); diff --git a/apps/desktop/src/lib/__tests__/sqlSemanticCompletion.spec.ts b/apps/desktop/src/lib/__tests__/sqlSemanticCompletion.spec.ts new file mode 100644 index 000000000..8b88c5415 --- /dev/null +++ b/apps/desktop/src/lib/__tests__/sqlSemanticCompletion.spec.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vitest"; +import { buildSqlCompletionItemsFromContext, getSqlCompletionContext, type SqlCompletionColumn, type SqlCompletionProviderInput } from "@/lib/sqlCompletion"; +import { sqlCompletionContextFromSemantic, sqlSemanticLocalColumnsByTable } from "@/lib/sqlSemanticCompletion"; +import { buildSqlSemanticModel } from "@/lib/sqlSemanticModel"; +import { sqlFixtureCursor } from "@/lib/sqlSemanticFixtures"; +import type { DatabaseType } from "@/types/database"; + +function mergeColumns(...maps: Array | undefined>): Map { + const merged = new Map(); + for (const map of maps) { + for (const [key, columns] of map ?? []) merged.set(key, columns); + } + return merged; +} + +function semanticCompletion(markedSql: string, input: Partial = {}, options: { databaseType?: DatabaseType; dialect?: "mysql" | "postgres" | "sqlserver" } = {}) { + const { sql, cursor } = sqlFixtureCursor(markedSql); + const model = buildSqlSemanticModel(sql, cursor, options); + const context = sqlCompletionContextFromSemantic(model, getSqlCompletionContext(sql, cursor)); + const columnsByTable = mergeColumns(sqlSemanticLocalColumnsByTable(model), input.columnsByTable); + const items = buildSqlCompletionItemsFromContext(context, { + tables: input.tables ?? [], + objects: input.objects ?? [], + columnsByTable, + foreignKeysByTable: input.foreignKeysByTable, + schemas: input.schemas, + translations: input.translations, + snippets: input.snippets, + dialect: options.dialect, + databaseType: options.databaseType, + keywordCase: input.keywordCase, + autoAliasTables: input.autoAliasTables, + }); + return { sql, cursor, model, context, items }; +} + +describe("semantic SQL completion candidates", () => { + it("keeps alias-qualified column completion scoped to one row source", () => { + const columnsByTable = new Map([ + ["users", ["id", "name", "email"].map((name) => ({ name, table: "users" }))], + ["orders", ["id", "total"].map((name) => ({ name, table: "orders" }))], + ]); + + const { items } = semanticCompletion("SELECT * FROM users u JOIN orders o ON o.user_id = u.id WHERE u.|", { columnsByTable }); + + expect(items.filter((item) => item.type === "column").map((item) => item.label)).toEqual(["id", "name", "email"]); + }); + + it("uses CTE projected columns without remote metadata", () => { + const { items, context } = semanticCompletion("WITH recent_orders(id, total) AS (SELECT id, total FROM orders) SELECT * FROM recent_orders ro WHERE ro.|"); + + expect(context.exclusiveColumnSuggestions).toBe(true); + expect(items.filter((item) => item.type === "column").map((item) => item.label)).toEqual(["id", "total"]); + }); + + it("uses subquery projected columns without remote metadata", () => { + const { items } = semanticCompletion("SELECT * FROM (SELECT id, name AS user_name FROM users) sq WHERE sq.|"); + + expect(items.filter((item) => item.type === "column").map((item) => item.label)).toEqual(["id", "user_name"]); + }); + + it("expands alias star from only the qualified row source", () => { + const columnsByTable = new Map([ + ["users", ["id", "name"].map((name) => ({ name, table: "users" }))], + ["orders", ["id", "total"].map((name) => ({ name, table: "orders" }))], + ]); + + const { context, items } = semanticCompletion("SELECT u.*| FROM users u JOIN orders o ON o.user_id = u.id", { columnsByTable }); + const star = items.find((item) => item.label === "* \u2192 columns"); + + expect(context.qualifier).toBe("u"); + expect(star?.apply).toBe("id, u.name"); + }); + + it("generates collision-free table aliases from semantic row sources", () => { + const { items } = semanticCompletion("SELECT * FROM order_items oi JOIN ord|", { + tables: [{ name: "order_items", type: "table" }], + autoAliasTables: true, + }); + + expect(items.find((item) => item.label === "order_items")?.apply).toBe("order_items AS oi2"); + }); + + it("preserves dialect-aware identifier quoting in apply text", () => { + const columnsByTable = new Map([["Order Details", [{ name: "User Name", table: "Order Details" }]]]); + + const { items } = semanticCompletion('SELECT od."User| FROM "Order Details" od', { columnsByTable }, { databaseType: "postgres", dialect: "postgres" }); + + expect(items.find((item) => item.label === "User Name")?.apply).toBe('"User Name"'); + }); +}); diff --git a/apps/desktop/src/lib/__tests__/sqlSemanticDbeaverReferences.md b/apps/desktop/src/lib/__tests__/sqlSemanticDbeaverReferences.md new file mode 100644 index 000000000..88016b568 --- /dev/null +++ b/apps/desktop/src/lib/__tests__/sqlSemanticDbeaverReferences.md @@ -0,0 +1,28 @@ +# SQL Semantic Completion DBeaver References + +This change uses the local DBeaver checkout at `/Users/skyler/VsCodeProjects/dbeaver` as the behavioral reference for semantic SQL completion. + +Key files reviewed: + +- `plugins/org.jkiss.dbeaver.ui.editors.sql/src/org/jkiss/dbeaver/ui/editors/sql/syntax/SQLCompletionProcessor.java` + - Selects semantic, legacy, or combined completion paths and handles asynchronous proposal jobs. +- `plugins/org.jkiss.dbeaver.model.sql/src/org/jkiss/dbeaver/model/sql/semantics/completion/SQLQueryCompletionContext.java` + - Builds completion proposals from syntax inspection, lexical scope, row-source context, and cursor offset. +- `plugins/org.jkiss.dbeaver.model.sql/src/org/jkiss/dbeaver/model/sql/semantics/context/SQLQueryRowsSourceContext.java` + - Tracks table sources, aliases, dynamic CTE sources, unresolved sources, and known-source collections. +- `plugins/org.jkiss.dbeaver.model.sql/src/org/jkiss/dbeaver/model/sql/semantics/completion/SQLQueryCompletionAnalyzer.java` + - Converts semantic completion items into editor proposals with replacement ranges, descriptions, images, and scoring. + +DBX intentionally implements a smaller frontend semantic model first. The immediate goal is not DBeaver parser parity; it is to move cursor intent, row-source resolution, CTE/subquery handling, and fallback confidence into one reusable layer before routing completion, diagnostics, and navigation through it. + +## Completion Assistant Field Audit + +The semantic completion scopes added in this change fit the existing DBX completion assistant and item-builder fields: + +- Table/schema/catalog lookup maps to `suggestTables`, `qualifier`, `qualifierParts`, `schemas`, and existing table metadata lookup methods. +- Routine/package lookup maps to `suggestRoutines`, `exclusiveRoutineSuggestions`, `qualifier`, and existing completion object lookup methods. +- Alias, CTE, subquery, INSERT target, UPDATE target, join, and star column lookup maps to `referencedTables`, `columns`, `insertTable`, `updateTarget`, `deleteTarget`, `onStar`, and `columnsByTable`. +- Projection aliases map to `prioritizeSelectAliases` and `selectAliases`. +- Fallback confidence maps to the existing legacy `getSqlCompletionContext()` path without requiring broad all-column scans. + +No backend/API field gap was found for the current semantic scopes. The implementation therefore keeps the Java/Rust assistant protocol unchanged and limits new work to frontend semantic context, conversion, and tests. diff --git a/apps/desktop/src/lib/__tests__/sqlSemanticModel.spec.ts b/apps/desktop/src/lib/__tests__/sqlSemanticModel.spec.ts new file mode 100644 index 000000000..01979cec8 --- /dev/null +++ b/apps/desktop/src/lib/__tests__/sqlSemanticModel.spec.ts @@ -0,0 +1,156 @@ +import { describe, expect, it } from "vitest"; +import { sqlSemanticCompletionScope, sqlSemanticLocalColumnsByTable, sqlSemanticProjectionAliasColumns } from "@/lib/sqlSemanticCompletion"; +import { SQL_SEMANTIC_BASELINE_FIXTURES, sqlFixtureCursor } from "@/lib/sqlSemanticFixtures"; +import { buildSqlSemanticModel } from "@/lib/sqlSemanticModel"; + +describe("sqlSemanticModel baseline fixtures", () => { + for (const fixture of SQL_SEMANTIC_BASELINE_FIXTURES) { + it(fixture.name, () => { + const { sql, cursor } = sqlFixtureCursor(fixture.sql); + const model = buildSqlSemanticModel(sql, cursor, { databaseType: fixture.databaseType }); + const scope = sqlSemanticCompletionScope(model); + + expect(model.statement.kind).toBe(fixture.expected.statementKind); + expect(model.cursorIntent.kind).toBe(fixture.expected.cursorKind); + expect(scope.kind).toBe(fixture.expected.completionScope); + expect(model.cursorIntent.prefix).toBe(fixture.expected.prefix); + expect(model.cursorIntent.qualifierParts).toEqual(fixture.expected.qualifierParts ?? []); + expect(model.cursorIntent.confidence).toBe(fixture.expected.confidence); + + for (const expectedSource of fixture.expected.rowSources ?? []) { + const expectedObject = Object.fromEntries(Object.entries(expectedSource).filter(([, value]) => value !== undefined)); + expect(model.rowSources).toEqual(expect.arrayContaining([expect.objectContaining(expectedObject)])); + } + + if (fixture.expected.completionLabels) { + const labels = [...sqlSemanticLocalColumnsByTable(model).values()].flat().map((column) => column.name); + expect(labels).toEqual(expect.arrayContaining(fixture.expected.completionLabels)); + } + }); + } + + it("does not mix row sources from inactive statements", () => { + const { sql, cursor } = sqlFixtureCursor("select * from users u; select * from orders o where o.|"); + const model = buildSqlSemanticModel(sql, cursor); + + expect(model.rowSources.some((source) => source.name === "orders")).toBe(true); + expect(model.rowSources.some((source) => source.name === "users")).toBe(false); + }); + + it("does not expose CTE body tables as outer query row sources", () => { + const { sql, cursor } = sqlFixtureCursor("WITH recent_orders AS (SELECT id FROM orders) SELECT * FROM recent_orders ro WHERE ro.|"); + const model = buildSqlSemanticModel(sql, cursor); + + expect(model.rowSources.some((source) => source.name === "recent_orders")).toBe(true); + expect(model.rowSources.some((source) => source.name === "orders")).toBe(false); + }); + + it("does not expose subquery body tables as outer query row sources", () => { + const { sql, cursor } = sqlFixtureCursor("SELECT * FROM (SELECT id FROM users) sq WHERE sq.|"); + const model = buildSqlSemanticModel(sql, cursor); + + expect(model.rowSources).toEqual(expect.arrayContaining([expect.objectContaining({ name: "sq", kind: "subquery" })])); + expect(model.rowSources.some((source) => source.name === "users")).toBe(false); + }); + + it("suppresses completion inside string literals without metadata scope", () => { + const { sql, cursor } = sqlFixtureCursor("SELECT 'u.|' FROM users"); + const model = buildSqlSemanticModel(sql, cursor); + const scope = sqlSemanticCompletionScope(model); + + expect(model.cursorIntent.kind).toBe("suppressed"); + expect(scope.useRemoteMetadata).toBe(false); + }); + + it("classifies table references after comma-separated table lists", () => { + const { sql, cursor } = sqlFixtureCursor("SELECT * FROM users u, ord|"); + const model = buildSqlSemanticModel(sql, cursor); + const scope = sqlSemanticCompletionScope(model); + + expect(model.cursorIntent.kind).toBe("table"); + expect(model.cursorIntent.prefix).toBe("ord"); + expect(scope.kind).toBe("table"); + }); + + it("classifies alias-qualified star with replacement range", () => { + const { sql, cursor } = sqlFixtureCursor("SELECT u.*| FROM users u"); + const model = buildSqlSemanticModel(sql, cursor); + + expect(model.cursorIntent.kind).toBe("star"); + expect(model.cursorIntent.prefix).toBe("*"); + expect(model.cursorIntent.qualifierParts).toEqual(["u"]); + expect(sql.slice(model.cursorIntent.replacementRange.start, model.cursorIntent.replacementRange.end)).toBe("*"); + }); + + it("returns low-confidence keyword fallback for unknown SQL", () => { + const { sql, cursor } = sqlFixtureCursor("explain analyze |"); + const model = buildSqlSemanticModel(sql, cursor); + const scope = sqlSemanticCompletionScope(model); + + expect(model.cursorIntent.kind).toBe("keyword"); + expect(model.cursorIntent.confidence).toBe("low"); + expect(scope.useRemoteMetadata).toBe(false); + }); + + it("exposes PostgreSQL projection aliases in ORDER BY but not WHERE", () => { + const orderBy = sqlFixtureCursor("select total_amount as total from orders order by to|"); + const where = sqlFixtureCursor("select total_amount as total from orders where to|"); + + expect(sqlSemanticProjectionAliasColumns(buildSqlSemanticModel(orderBy.sql, orderBy.cursor, { databaseType: "postgres" })).map((column) => column.name)).toContain("total"); + expect(sqlSemanticProjectionAliasColumns(buildSqlSemanticModel(where.sql, where.cursor, { databaseType: "postgres" })).map((column) => column.name)).not.toContain("total"); + }); + + it("exposes MySQL projection aliases in GROUP BY and HAVING", () => { + const groupBy = sqlFixtureCursor("select total_amount as total from orders group by to|"); + const having = sqlFixtureCursor("select total_amount as total from orders having to|"); + + expect(sqlSemanticProjectionAliasColumns(buildSqlSemanticModel(groupBy.sql, groupBy.cursor, { databaseType: "mysql" })).map((column) => column.name)).toContain("total"); + expect(sqlSemanticProjectionAliasColumns(buildSqlSemanticModel(having.sql, having.cursor, { databaseType: "mysql" })).map((column) => column.name)).toContain("total"); + }); + + it("keeps dialect-specific identifier normalization and qualifier scopes", () => { + const sqlServer = sqlFixtureCursor("SELECT * FROM [dbo].[Users] u WHERE u.|"); + const postgres = sqlFixtureCursor('SELECT total AS "Order Total" FROM "Sales"."Orders" o ORDER BY "Order|'); + const mysql = sqlFixtureCursor("SELECT * FROM `analytics`.`events` e WHERE e.|"); + const sqlite = sqlFixtureCursor("SELECT * FROM main.users u WHERE u.|"); + + expect(buildSqlSemanticModel(sqlServer.sql, sqlServer.cursor, { databaseType: "sqlserver" }).rowSources[0]).toEqual(expect.objectContaining({ name: "Users", qualifierParts: ["dbo"], alias: "u" })); + expect(sqlSemanticProjectionAliasColumns(buildSqlSemanticModel(postgres.sql, postgres.cursor, { databaseType: "postgres" })).map((column) => column.name)).toContain("Order Total"); + expect(buildSqlSemanticModel(mysql.sql, mysql.cursor, { databaseType: "mysql" }).rowSources[0]).toEqual(expect.objectContaining({ name: "events", qualifierParts: ["analytics"], alias: "e" })); + expect(buildSqlSemanticModel(sqlite.sql, sqlite.cursor, { databaseType: "sqlite" }).rowSources[0]).toEqual(expect.objectContaining({ name: "users", qualifierParts: ["main"], alias: "u" })); + }); + + it("covers SQL Server case-insensitive bracket and multi-part qualifier contexts", () => { + const { sql, cursor } = sqlFixtureCursor("SELECT * FROM [ServerOne].[AppDb].[dbo].[Users] U WHERE u.na|"); + const model = buildSqlSemanticModel(sql, cursor, { databaseType: "sqlserver" }); + + expect(model.rowSources[0]).toEqual(expect.objectContaining({ name: "Users", qualifierParts: ["ServerOne", "AppDb", "dbo"], alias: "U" })); + expect(model.cursorIntent.kind).toBe("alias_column"); + expect(model.cursorIntent.qualifierParts).toEqual(["u"]); + }); + + it("covers PostgreSQL lower-case folding with CTEs and ORDER BY projection aliases", () => { + const { sql, cursor } = sqlFixtureCursor("WITH RecentOrders AS (SELECT id, total FROM orders) SELECT total AS total_alias FROM RecentOrders ro ORDER BY total_|"); + const model = buildSqlSemanticModel(sql, cursor, { databaseType: "postgres" }); + + expect(model.rowSources).toEqual(expect.arrayContaining([expect.objectContaining({ name: "recentorders", alias: "ro", columns: ["id", "total"] })])); + expect(sqlSemanticProjectionAliasColumns(model).map((column) => column.name)).toContain("total_alias"); + }); + + it("covers MySQL database-qualified backticks and projection alias visibility", () => { + const groupBy = sqlFixtureCursor("SELECT amount AS total FROM `analytics`.`events` e GROUP BY to|"); + const where = sqlFixtureCursor("SELECT amount AS total FROM `analytics`.`events` e WHERE to|"); + + expect(buildSqlSemanticModel(groupBy.sql, groupBy.cursor, { databaseType: "mysql" }).rowSources[0]).toEqual(expect.objectContaining({ name: "events", qualifierParts: ["analytics"], alias: "e" })); + expect(sqlSemanticProjectionAliasColumns(buildSqlSemanticModel(groupBy.sql, groupBy.cursor, { databaseType: "mysql" })).map((column) => column.name)).toContain("total"); + expect(sqlSemanticProjectionAliasColumns(buildSqlSemanticModel(where.sql, where.cursor, { databaseType: "mysql" })).map((column) => column.name)).not.toContain("total"); + }); + + it("covers SQLite and DuckDB schema-light local row-source behavior", () => { + const sqlite = sqlFixtureCursor("SELECT * FROM main.users u WHERE u.|"); + const duckdb = sqlFixtureCursor("SELECT * FROM read_csv('users.csv') csv WHERE csv.|"); + + expect(sqlSemanticCompletionScope(buildSqlSemanticModel(sqlite.sql, sqlite.cursor, { databaseType: "sqlite" })).useRemoteMetadata).toBe(true); + expect(buildSqlSemanticModel(duckdb.sql, duckdb.cursor, { databaseType: "duckdb" }).rowSources[0]).toEqual(expect.objectContaining({ kind: "table_function", name: "csv", alias: "csv" })); + }); +}); diff --git a/apps/desktop/src/lib/__tests__/sqlSemanticReferences.spec.ts b/apps/desktop/src/lib/__tests__/sqlSemanticReferences.spec.ts new file mode 100644 index 000000000..5561573ab --- /dev/null +++ b/apps/desktop/src/lib/__tests__/sqlSemanticReferences.spec.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; +import { buildSqlSemanticDiagnostics } from "@/lib/sqlSemanticDiagnostics"; +import { buildSqlSemanticModel } from "@/lib/sqlSemanticModel"; +import { sqlFixtureCursor } from "@/lib/sqlSemanticFixtures"; +import { resolveSqlSemanticNavigationTarget, mergeSqlSemanticReferenceAnalysis, sqlSemanticCompletionReferenceTables, sqlSemanticTableReferences } from "@/lib/sqlSemanticReferences"; +import { splitQualifiedIdentifier } from "@/lib/sqlNavigation"; +import type { SqlReferenceAnalysis } from "@/types/database"; + +const span = (startColumn: number, endColumn: number) => ({ + start_line: 1, + start_column: startColumn, + end_line: 1, + end_column: endColumn, +}); + +describe("sqlSemanticReferences shared consumers", () => { + it("feeds CTE projected columns to diagnostics without physical table metadata", () => { + const { sql, cursor } = sqlFixtureCursor("WITH recent_orders(id, total) AS (SELECT id, total FROM orders) SELECT ro.missing FROM recent_orders ro WHERE ro.|"); + const model = buildSqlSemanticModel(sql, cursor); + const analysis: SqlReferenceAnalysis = { + tables: [], + columns: [{ name: "missing", qualifier: "ro", span: span(sql.indexOf("missing") + 1, sql.indexOf("missing") + "missing".length) }], + }; + + const diagnostics = buildSqlSemanticDiagnostics(mergeSqlSemanticReferenceAnalysis(analysis, model), { + tables: [], + columnsByTable: new Map(), + sql, + }); + + expect(diagnostics.map((diagnostic) => diagnostic.message)).toEqual(["Unknown column ro.missing"]); + }); + + it("resolves navigation targets from subquery aliases and projected columns", () => { + const { sql, cursor } = sqlFixtureCursor("SELECT sq.user_| FROM (SELECT id, name AS user_name FROM users) sq"); + const model = buildSqlSemanticModel(sql, cursor); + const target = resolveSqlSemanticNavigationTarget(model, splitQualifiedIdentifier("sq.user_name")); + + expect(target).toEqual(expect.objectContaining({ name: "sq", alias: "sq", columns: ["id", "user_name"] })); + expect(target?.source.kind).toBe("subquery"); + }); + + it("keeps completion diagnostics and navigation row-source resolution consistent", () => { + const { sql, cursor } = sqlFixtureCursor("WITH recent_orders(id, total) AS (SELECT id, total FROM orders) SELECT * FROM recent_orders ro WHERE ro.|"); + const model = buildSqlSemanticModel(sql, cursor); + const completionRefs = sqlSemanticCompletionReferenceTables(model); + const diagnosticRefs = sqlSemanticTableReferences(model); + const navigationTarget = resolveSqlSemanticNavigationTarget(model, ["ro", "total"]); + + expect(completionRefs).toEqual(expect.arrayContaining([expect.objectContaining({ name: "recent_orders", alias: "ro", columns: ["id", "total"] })])); + expect(diagnosticRefs).toEqual(expect.arrayContaining([expect.objectContaining({ name: "recent_orders", alias: "ro", columns: ["id", "total"] })])); + expect(navigationTarget).toEqual(expect.objectContaining({ name: "recent_orders", alias: "ro", columns: ["id", "total"] })); + }); +}); diff --git a/apps/desktop/src/lib/__tests__/sqlSemanticTokens.spec.ts b/apps/desktop/src/lib/__tests__/sqlSemanticTokens.spec.ts new file mode 100644 index 000000000..8407e3df5 --- /dev/null +++ b/apps/desktop/src/lib/__tests__/sqlSemanticTokens.spec.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { findActiveSqlStatementSpan, isSuppressedSqlSemanticContext, tokenizeSqlSemantic, unquoteSqlSemanticIdentifier } from "@/lib/sqlSemanticTokens"; + +describe("sqlSemanticTokens", () => { + it("tokenizes comments, strings, quoted identifiers, brackets, and backticks", () => { + const sql = "select [User Name], `order`, \"Mixed\" from users -- comment\nwhere name = 'it''s ok'"; + const tokens = tokenizeSqlSemantic(sql); + + expect(tokens.some((token) => token.kind === "quoted_identifier" && unquoteSqlSemanticIdentifier(token) === "User Name")).toBe(true); + expect(tokens.some((token) => token.kind === "quoted_identifier" && unquoteSqlSemanticIdentifier(token) === "order")).toBe(true); + expect(tokens.some((token) => token.kind === "quoted_identifier" && unquoteSqlSemanticIdentifier(token) === "Mixed")).toBe(true); + expect(tokens.some((token) => token.kind === "comment" && token.text.includes("comment"))).toBe(true); + expect(tokens.some((token) => token.kind === "string" && token.text === "'it''s ok'")).toBe(true); + }); + + it("marks comments and string literals as suppressed contexts", () => { + const sql = "select * from users -- user."; + const tokens = tokenizeSqlSemantic(sql); + + expect(isSuppressedSqlSemanticContext(tokens, sql.length)).toBe(true); + expect(isSuppressedSqlSemanticContext(tokens, "select * from users".length)).toBe(false); + }); + + it("finds active statement spans across semicolon-separated scripts", () => { + const sql = "select * from users;\nselect * from orders where id = 1;"; + const cursor = sql.indexOf("orders"); + const span = findActiveSqlStatementSpan(sql, tokenizeSqlSemantic(sql), cursor); + + expect(sql.slice(span.start, span.end)).toBe("select * from orders where id = 1"); + }); +}); diff --git a/apps/desktop/src/lib/sqlCompletion.ts b/apps/desktop/src/lib/sqlCompletion.ts index 08d36ce3c..21d147389 100644 --- a/apps/desktop/src/lib/sqlCompletion.ts +++ b/apps/desktop/src/lib/sqlCompletion.ts @@ -1282,7 +1282,7 @@ class SqlCompletionProvider { } if (context.onStar) { - const starItem = buildStarExpansionItem(this.input.columnsByTable, this.t, this.dialect); + const starItem = buildStarExpansionItem(context, this.input.columnsByTable, this.t, this.dialect); if (starItem) this.items.push(starItem); } @@ -2692,22 +2692,16 @@ function buildPreferredKeywordItems(prefix: string, keywords: string[], keywordC })); } -function buildStarExpansionItem(columnsByTable: Map, t?: SqlCompletionTranslations, dialect?: "mysql" | "postgres" | "sqlserver"): SqlCompletionItem | null { - const allColumns: string[] = []; - const seen = new Set(); - for (const [, cols] of columnsByTable) { - for (const col of cols) { - if (seen.has(col.name)) continue; - seen.add(col.name); - allColumns.push(quoteSqlIdentifier(col.name, dialect)); - } - } - if (allColumns.length === 0) return null; - const expansion = allColumns.join(", "); +function buildStarExpansionItem(context: SqlCompletionContext, columnsByTable: Map, t?: SqlCompletionTranslations, dialect?: "mysql" | "postgres" | "sqlserver"): SqlCompletionItem | null { + const columns = context.qualifier ? referencedTablesForSelectAllColumns(context).flatMap((ref) => columnsForSelectAllReferencedTable(ref, columnsByTable)) : [...columnsByTable.values()].flat(); + const uniqueColumns = uniqueColumnsByName(columns); + if (uniqueColumns.length === 0) return null; + // `alias.*` replaces only the `*`, so the first column must continue the already typed `alias.`. + const expansion = context.qualifier ? buildSelectAllColumnExpansion(uniqueColumns, context.qualifier, true, dialect) : uniqueColumns.map((column) => quoteSqlIdentifier(column.name, dialect)).join(", "); return { label: "* → columns", type: "snippet" as const, - detail: `${(t?.starExpansionColumns ?? "{count} columns").replace("{count}", String(allColumns.length))}: ${expansion.length > 60 ? expansion.slice(0, 57) + "..." : expansion}`, + detail: `${(t?.starExpansionColumns ?? "{count} columns").replace("{count}", String(uniqueColumns.length))}: ${expansion.length > 60 ? expansion.slice(0, 57) + "..." : expansion}`, apply: expansion, boost: 1900, }; diff --git a/apps/desktop/src/lib/sqlSemanticCompletion.ts b/apps/desktop/src/lib/sqlSemanticCompletion.ts new file mode 100644 index 000000000..50abe408b --- /dev/null +++ b/apps/desktop/src/lib/sqlSemanticCompletion.ts @@ -0,0 +1,208 @@ +import type { SqlCompletionColumn, SqlCompletionContext, SqlCompletionReferencedTable } from "@/lib/sqlCompletion"; +import { SQL_SEMANTIC_DIALECTS } from "@/lib/sqlSemanticDialect"; +import type { SqlSemanticModel, SqlSemanticRowSource } from "@/lib/sqlSemanticTypes"; + +export type SqlSemanticCompletionScopeKind = "keyword" | "table" | "schema" | "catalog" | "routine" | "columns" | "local"; + +export interface SqlSemanticCompletionScope { + kind: SqlSemanticCompletionScopeKind; + prefix: string; + qualifierParts: string[]; + targetSource?: SqlSemanticRowSource; + useRemoteMetadata: boolean; + fallbackReason?: string; +} + +export function sqlSemanticReferencedTables(model: SqlSemanticModel): SqlCompletionReferencedTable[] { + return model.rowSources + .filter((source) => source.kind !== "unknown") + .map((source) => ({ + name: source.name, + schema: source.qualifierParts[source.qualifierParts.length - 1], + alias: source.alias, + columns: source.columns, + })); +} + +export function sqlSemanticLocalColumnsByTable(model: SqlSemanticModel): Map { + const columnsByTable = new Map(); + for (const source of model.rowSources) { + if (!source.columns?.length) continue; + columnsByTable.set( + source.name, + source.columns.map((name) => ({ + name, + table: source.name, + schema: source.qualifierParts[source.qualifierParts.length - 1], + })), + ); + } + return columnsByTable; +} + +function activeProjectionAliasClause(model: SqlSemanticModel): "where" | "groupBy" | "having" | "orderBy" | null { + const words = model.tokens.filter((token) => token.span.end <= model.cursor && token.kind === "word").map((token) => token.normalized); + for (let index = words.length - 1; index >= 0; index -= 1) { + const word = words[index]; + const previous = words[index - 1]; + if (word === "by" && previous === "order") return "orderBy"; + if (word === "by" && previous === "group") return "groupBy"; + if (word === "having") return "having"; + if (word === "where") return "where"; + if (word === "from" || word === "join" || word === "select") return null; + } + return null; +} + +export function sqlSemanticProjectionAliasColumns(model: SqlSemanticModel): SqlCompletionColumn[] { + const clause = activeProjectionAliasClause(model); + if (!clause) return []; + const adapter = SQL_SEMANTIC_DIALECTS[model.dialectId] ?? SQL_SEMANTIC_DIALECTS.generic; + if (!adapter.projectionAliasVisibility[clause]) return []; + return model.projections + .filter((projection) => projection.name) + .map((projection) => ({ + name: projection.name, + table: "__projection__", + comment: "Projection alias", + })); +} + +export function sqlSemanticCompletionScope(model: SqlSemanticModel): SqlSemanticCompletionScope { + const intent = model.cursorIntent; + const targetSource = intent.targetSourceId ? model.rowSources.find((source) => source.id === intent.targetSourceId) : undefined; + switch (intent.kind) { + case "table": + case "delete_target": + return { + kind: "table", + prefix: intent.prefix, + qualifierParts: intent.qualifierParts, + useRemoteMetadata: intent.confidence !== "low", + fallbackReason: intent.fallbackReason, + }; + case "schema": + return { + kind: "schema", + prefix: intent.prefix, + qualifierParts: intent.qualifierParts, + useRemoteMetadata: intent.confidence !== "low", + fallbackReason: intent.fallbackReason, + }; + case "catalog": + return { + kind: "catalog", + prefix: intent.prefix, + qualifierParts: intent.qualifierParts, + useRemoteMetadata: intent.confidence !== "low", + fallbackReason: intent.fallbackReason, + }; + case "routine": + return { + kind: "routine", + prefix: intent.prefix, + qualifierParts: intent.qualifierParts, + useRemoteMetadata: intent.confidence !== "low", + fallbackReason: intent.fallbackReason, + }; + case "column": + case "alias_column": + case "insert_column": + case "update_column": + case "join_condition": + case "star": + return { + kind: "columns", + prefix: intent.prefix, + qualifierParts: intent.qualifierParts, + targetSource, + useRemoteMetadata: intent.confidence !== "low" && (!!targetSource || model.rowSources.length > 0), + fallbackReason: intent.fallbackReason, + }; + case "suppressed": + return { + kind: "local", + prefix: intent.prefix, + qualifierParts: [], + useRemoteMetadata: false, + fallbackReason: intent.fallbackReason ?? "suppressed", + }; + case "keyword": + return { + kind: "keyword", + prefix: intent.prefix, + qualifierParts: intent.qualifierParts, + useRemoteMetadata: false, + fallbackReason: intent.fallbackReason, + }; + } +} + +function semanticContextKind(model: SqlSemanticModel): SqlCompletionContext["contextKind"] { + switch (model.cursorIntent.kind) { + case "table": + case "schema": + case "catalog": + case "delete_target": + return "table"; + case "routine": + return "routine"; + case "alias_column": + return "alias_column"; + case "insert_column": + case "update_column": + case "column": + case "star": + return "column"; + case "join_condition": + return "join"; + case "keyword": + case "suppressed": + return "keyword"; + } +} + +function semanticMutationTarget(model: SqlSemanticModel): SqlSemanticRowSource | undefined { + const targetId = model.cursorIntent.targetSourceId; + return targetId ? model.rowSources.find((source) => source.id === targetId) : model.rowSources.find((source) => source.kind === "mutation_target"); +} + +export function sqlCompletionContextFromSemantic(model: SqlSemanticModel, base: SqlCompletionContext): SqlCompletionContext { + if (model.cursorIntent.confidence === "low" || model.cursorIntent.kind === "suppressed") { + return base; + } + + const scope = sqlSemanticCompletionScope(model); + const qualifier = model.cursorIntent.qualifierParts.length > 0 ? model.cursorIntent.qualifierParts.join(".") : undefined; + const referencedTables = sqlSemanticReferencedTables(model); + const mutationTarget = semanticMutationTarget(model); + const mutationSchema = mutationTarget?.qualifierParts[mutationTarget.qualifierParts.length - 1]; + const suggestTables = scope.kind === "table" || scope.kind === "schema" || scope.kind === "catalog"; + const suggestColumns = scope.kind === "columns"; + const suggestRoutines = scope.kind === "routine"; + const projectionAliases = sqlSemanticProjectionAliasColumns(model).map((column) => column.name); + + return { + ...base, + prefix: model.cursorIntent.prefix, + qualifier, + qualifierParts: model.cursorIntent.qualifierParts.length > 0 ? [...model.cursorIntent.qualifierParts] : undefined, + suggestTables, + suggestColumns, + suggestKeywords: scope.kind === "keyword" || (!suggestTables && !suggestColumns && !suggestRoutines), + suggestRoutines, + suggestJoinConditions: model.cursorIntent.kind === "join_condition", + exclusiveTableSuggestions: suggestTables, + exclusiveColumnSuggestions: model.cursorIntent.kind === "alias_column" || model.cursorIntent.kind === "insert_column" || model.cursorIntent.kind === "update_column", + exclusiveRoutineSuggestions: suggestRoutines, + prioritizeSelectAliases: base.prioritizeSelectAliases || projectionAliases.length > 0, + selectAliases: projectionAliases.length > 0 ? projectionAliases : base.selectAliases, + referencedTables: referencedTables.length > 0 ? referencedTables : base.referencedTables, + insertTable: model.cursorIntent.kind === "insert_column" ? mutationTarget?.name : base.insertTable, + insertSchema: model.cursorIntent.kind === "insert_column" ? mutationSchema : base.insertSchema, + updateTarget: model.cursorIntent.kind === "update_column" && mutationTarget ? { table: mutationTarget.name, schema: mutationSchema } : base.updateTarget, + deleteTarget: model.cursorIntent.kind === "delete_target" && mutationTarget ? { table: mutationTarget.name, schema: mutationSchema } : base.deleteTarget, + onStar: model.cursorIntent.kind === "star" || base.onStar, + contextKind: semanticContextKind(model), + }; +} diff --git a/apps/desktop/src/lib/sqlSemanticDiagnostics.ts b/apps/desktop/src/lib/sqlSemanticDiagnostics.ts index 7377d0f76..23c98eb46 100644 --- a/apps/desktop/src/lib/sqlSemanticDiagnostics.ts +++ b/apps/desktop/src/lib/sqlSemanticDiagnostics.ts @@ -276,6 +276,14 @@ function statementIndexAt(sql: string, offset: number): number { } function columnsForTable(table: SqlTableReference, columnsByTable: Map, loadedColumnTables?: Set): SqlCompletionColumn[] | null { + const inlineColumns = (table as SqlTableReference & { columns?: string[] }).columns; + if (inlineColumns && inlineColumns.length > 0) { + return inlineColumns.map((name) => ({ + name, + table: table.name, + schema: table.schema ?? undefined, + })); + } const keys = table.schema ? [`${table.schema}.${table.name}`, table.name] : [table.name, ...keysWithTableName(columnsByTable, table.name)]; for (const key of keys) { const normalizedKey = normalizeName(key); diff --git a/apps/desktop/src/lib/sqlSemanticDialect.ts b/apps/desktop/src/lib/sqlSemanticDialect.ts new file mode 100644 index 000000000..8220a6519 --- /dev/null +++ b/apps/desktop/src/lib/sqlSemanticDialect.ts @@ -0,0 +1,172 @@ +import type { DatabaseType } from "@/types/database"; + +export interface SqlSemanticProjectionAliasVisibility { + where: boolean; + groupBy: boolean; + having: boolean; + orderBy: boolean; +} + +export interface SqlSemanticDialectAdapter { + id: string; + identifierQuotes: Array<{ open: string; close: string }>; + supportsAsForTableAlias: boolean; + projectionAliasVisibility: SqlSemanticProjectionAliasVisibility; + normalizeIdentifier(identifier: string, quoted?: boolean): string; + quoteIdentifier(identifier: string): string; + qualifierRole(parts: string[], context: "table" | "column" | "routine"): "catalog" | "schema" | "table" | "package" | "unknown"; +} + +function quoteWith(identifier: string, quote: string): string { + return `${quote}${identifier.replaceAll(quote, quote + quote)}${quote}`; +} + +function defaultNormalize(identifier: string): string { + return identifier; +} + +function lowerUnquoted(identifier: string, quoted?: boolean): string { + return quoted ? identifier : identifier.toLowerCase(); +} + +function upperUnquoted(identifier: string, quoted?: boolean): string { + return quoted ? identifier : identifier.toUpperCase(); +} + +const defaultProjectionAliasVisibility: SqlSemanticProjectionAliasVisibility = { + where: false, + groupBy: false, + having: false, + orderBy: true, +}; + +function roleForGenericQualifier(parts: string[], context: "table" | "column" | "routine"): "catalog" | "schema" | "table" | "package" | "unknown" { + if (parts.length <= 0) return "unknown"; + if (context === "column") return parts.length >= 2 ? "table" : "table"; + if (context === "routine") return parts.length >= 2 ? "package" : "schema"; + if (parts.length >= 2) return "schema"; + return "schema"; +} + +export const SQL_SEMANTIC_DIALECTS: Record = { + generic: { + id: "generic", + identifierQuotes: [{ open: '"', close: '"' }], + supportsAsForTableAlias: true, + projectionAliasVisibility: defaultProjectionAliasVisibility, + normalizeIdentifier: defaultNormalize, + quoteIdentifier: (identifier) => quoteWith(identifier, '"'), + qualifierRole: roleForGenericQualifier, + }, + postgres: { + id: "postgres", + identifierQuotes: [{ open: '"', close: '"' }], + supportsAsForTableAlias: true, + projectionAliasVisibility: defaultProjectionAliasVisibility, + normalizeIdentifier: lowerUnquoted, + quoteIdentifier: (identifier) => quoteWith(identifier, '"'), + qualifierRole: roleForGenericQualifier, + }, + mysql: { + id: "mysql", + identifierQuotes: [ + { open: "`", close: "`" }, + { open: '"', close: '"' }, + ], + supportsAsForTableAlias: true, + projectionAliasVisibility: { where: false, groupBy: true, having: true, orderBy: true }, + normalizeIdentifier: defaultNormalize, + quoteIdentifier: (identifier) => quoteWith(identifier, "`"), + qualifierRole(parts, context) { + if (context === "column") return parts.length >= 2 ? "table" : "table"; + if (context === "routine") return parts.length >= 2 ? "package" : "schema"; + return parts.length >= 1 ? "schema" : "unknown"; + }, + }, + sqlserver: { + id: "sqlserver", + identifierQuotes: [ + { open: "[", close: "]" }, + { open: '"', close: '"' }, + ], + supportsAsForTableAlias: true, + projectionAliasVisibility: defaultProjectionAliasVisibility, + normalizeIdentifier: defaultNormalize, + quoteIdentifier: (identifier) => `[${identifier.replaceAll("]", "]]")}]`, + qualifierRole(parts, context) { + if (context === "column") return parts.length >= 2 ? "table" : "table"; + if (context === "routine") return parts.length >= 2 ? "package" : "schema"; + if (parts.length >= 2) return "schema"; + return "schema"; + }, + }, + sqlite: { + id: "sqlite", + identifierQuotes: [ + { open: '"', close: '"' }, + { open: "`", close: "`" }, + { open: "[", close: "]" }, + ], + supportsAsForTableAlias: true, + projectionAliasVisibility: { where: false, groupBy: true, having: true, orderBy: true }, + normalizeIdentifier: defaultNormalize, + quoteIdentifier: (identifier) => quoteWith(identifier, '"'), + qualifierRole: roleForGenericQualifier, + }, + duckdb: { + id: "duckdb", + identifierQuotes: [{ open: '"', close: '"' }], + supportsAsForTableAlias: true, + projectionAliasVisibility: defaultProjectionAliasVisibility, + normalizeIdentifier: defaultNormalize, + quoteIdentifier: (identifier) => quoteWith(identifier, '"'), + qualifierRole: roleForGenericQualifier, + }, + oracle: { + id: "oracle", + identifierQuotes: [{ open: '"', close: '"' }], + supportsAsForTableAlias: false, + projectionAliasVisibility: defaultProjectionAliasVisibility, + normalizeIdentifier: upperUnquoted, + quoteIdentifier: (identifier) => quoteWith(identifier, '"'), + qualifierRole(parts, context) { + if (context === "column") return parts.length >= 2 ? "table" : "table"; + if (context === "routine") return parts.length >= 2 ? "package" : "schema"; + return parts.length >= 1 ? "schema" : "unknown"; + }, + }, +}; + +export function sqlSemanticDialectFor(options: { databaseType?: DatabaseType; dialect?: "mysql" | "postgres" | "sqlserver" }): SqlSemanticDialectAdapter { + if (options.dialect && SQL_SEMANTIC_DIALECTS[options.dialect]) return SQL_SEMANTIC_DIALECTS[options.dialect]; + switch (options.databaseType) { + case "postgres": + case "redshift": + case "opengauss": + case "gaussdb": + case "highgo": + return SQL_SEMANTIC_DIALECTS.postgres; + case "mysql": + case "doris": + case "starrocks": + return SQL_SEMANTIC_DIALECTS.mysql; + case "sqlserver": + return SQL_SEMANTIC_DIALECTS.sqlserver; + case "sqlite": + case "rqlite": + case "turso": + return SQL_SEMANTIC_DIALECTS.sqlite; + case "duckdb": + return SQL_SEMANTIC_DIALECTS.duckdb; + case "oracle": + case "oceanbase-oracle": + case "dameng": + case "kingbase": + case "vastbase": + case "goldendb": + case "yashandb": + return SQL_SEMANTIC_DIALECTS.oracle; + default: + return SQL_SEMANTIC_DIALECTS.generic; + } +} diff --git a/apps/desktop/src/lib/sqlSemanticFixtures.ts b/apps/desktop/src/lib/sqlSemanticFixtures.ts new file mode 100644 index 000000000..1ecb28a7f --- /dev/null +++ b/apps/desktop/src/lib/sqlSemanticFixtures.ts @@ -0,0 +1,168 @@ +import type { SqlSemanticCompletionScopeKind } from "@/lib/sqlSemanticCompletion"; +import type { SqlSemanticConfidence, SqlSemanticCursorKind, SqlSemanticStatementKind } from "@/lib/sqlSemanticTypes"; + +export interface SqlSemanticFixture { + name: string; + sql: string; + databaseType?: "postgres" | "mysql" | "sqlserver" | "sqlite" | "duckdb" | "oracle"; + expected: { + statementKind: SqlSemanticStatementKind; + cursorKind: SqlSemanticCursorKind; + completionScope: SqlSemanticCompletionScopeKind; + prefix: string; + qualifierParts?: string[]; + confidence: SqlSemanticConfidence; + rowSources?: Array<{ name: string; alias?: string; kind?: string; columns?: string[] }>; + completionLabels?: string[]; + }; +} + +export function sqlFixtureCursor(input: string): { sql: string; cursor: number } { + const cursor = input.indexOf("|"); + if (cursor < 0) { + throw new Error("SQL semantic fixture is missing a | cursor marker"); + } + return { + sql: input.slice(0, cursor) + input.slice(cursor + 1), + cursor, + }; +} + +export const SQL_SEMANTIC_BASELINE_FIXTURES: SqlSemanticFixture[] = [ + { + name: "select alias column", + sql: "SELECT * FROM users u JOIN orders o ON o.user_id = u.id WHERE u.|", + expected: { + statementKind: "select", + cursorKind: "alias_column", + completionScope: "columns", + prefix: "", + qualifierParts: ["u"], + confidence: "high", + rowSources: [ + { name: "users", alias: "u", kind: "table" }, + { name: "orders", alias: "o", kind: "table" }, + ], + }, + }, + { + name: "with cte table source", + sql: "WITH recent_orders(id, total) AS (SELECT id, total FROM orders) SELECT * FROM recent_orders ro WHERE ro.|", + expected: { + statementKind: "select", + cursorKind: "alias_column", + completionScope: "columns", + prefix: "", + qualifierParts: ["ro"], + confidence: "high", + rowSources: [{ name: "recent_orders", alias: "ro", kind: "cte", columns: ["id", "total"] }], + completionLabels: ["id", "total"], + }, + }, + { + name: "subquery projection columns", + sql: "SELECT * FROM (SELECT id, name AS user_name FROM users) sq WHERE sq.|", + expected: { + statementKind: "select", + cursorKind: "alias_column", + completionScope: "columns", + prefix: "", + qualifierParts: ["sq"], + confidence: "high", + rowSources: [{ name: "sq", alias: "sq", kind: "subquery", columns: ["id", "user_name"] }], + completionLabels: ["id", "user_name"], + }, + }, + { + name: "insert target columns", + sql: "INSERT INTO dbo.Users (|", + databaseType: "sqlserver", + expected: { + statementKind: "insert", + cursorKind: "insert_column", + completionScope: "columns", + prefix: "", + qualifierParts: [], + confidence: "medium", + rowSources: [{ name: "Users", kind: "mutation_target" }], + }, + }, + { + name: "update set columns", + sql: "UPDATE dbo.Users SET |", + databaseType: "sqlserver", + expected: { + statementKind: "update", + cursorKind: "update_column", + completionScope: "columns", + prefix: "", + qualifierParts: [], + confidence: "medium", + rowSources: [{ name: "Users", kind: "mutation_target" }], + }, + }, + { + name: "call routine", + sql: "CALL app.refresh_|", + databaseType: "postgres", + expected: { + statementKind: "call", + cursorKind: "routine", + completionScope: "routine", + prefix: "refresh_", + qualifierParts: ["app"], + confidence: "high", + }, + }, + { + name: "delete target", + sql: "DELETE FROM audit_events ae WHERE ae.|", + expected: { + statementKind: "delete", + cursorKind: "alias_column", + completionScope: "columns", + prefix: "", + qualifierParts: ["ae"], + confidence: "high", + rowSources: [{ name: "audit_events", alias: "ae", kind: "mutation_target" }], + }, + }, + { + name: "table function alias", + sql: "SELECT * FROM JSON_TABLE(payload, '$' COLUMNS(id INT PATH '$.id')) jt WHERE jt.|", + databaseType: "oracle", + expected: { + statementKind: "select", + cursorKind: "alias_column", + completionScope: "columns", + prefix: "", + qualifierParts: ["JT"], + confidence: "high", + rowSources: [{ name: "JT", alias: "JT", kind: "table_function" }], + }, + }, + { + name: "schema qualified table", + sql: "SELECT * FROM reporting.|", + databaseType: "postgres", + expected: { + statementKind: "select", + cursorKind: "table", + completionScope: "table", + prefix: "", + qualifierParts: ["reporting"], + confidence: "medium", + }, + }, + { + name: "comment suppressed", + sql: "SELECT * FROM users -- u.|", + expected: { + statementKind: "select", + cursorKind: "suppressed", + completionScope: "local", + prefix: "", + confidence: "high", + }, + }, +]; diff --git a/apps/desktop/src/lib/sqlSemanticModel.ts b/apps/desktop/src/lib/sqlSemanticModel.ts new file mode 100644 index 000000000..2bf4d21e1 --- /dev/null +++ b/apps/desktop/src/lib/sqlSemanticModel.ts @@ -0,0 +1,580 @@ +import { sqlSemanticDialectFor, type SqlSemanticDialectAdapter } from "@/lib/sqlSemanticDialect"; +import { findActiveSqlStatementSpan, isSuppressedSqlSemanticContext, tokenIsIdentifier, tokenizeSqlSemantic, unquoteSqlSemanticIdentifier } from "@/lib/sqlSemanticTokens"; +import type { + SqlSemanticBuildOptions, + SqlSemanticClauseSpans, + SqlSemanticCursorIntent, + SqlSemanticIdentifierPart, + SqlSemanticModel, + SqlSemanticProjection, + SqlSemanticQualifiedName, + SqlSemanticRowSource, + SqlSemanticScope, + SqlSemanticSpan, + SqlSemanticStatement, + SqlSemanticStatementKind, + SqlSemanticToken, +} from "@/lib/sqlSemanticTypes"; + +const TABLE_INTRODUCERS = new Set(["from", "join", "update", "into", "using", "apply"]); +const JOIN_MODIFIERS = new Set(["left", "right", "inner", "outer", "cross", "full", "natural"]); +const CLAUSE_BOUNDARIES = new Set(["where", "group", "having", "order", "limit", "offset", "union", "intersect", "except", "on", "set", "values", "returning"]); +const ALIAS_BLACKLIST = new Set([...CLAUSE_BOUNDARIES, "join", "left", "right", "inner", "outer", "cross", "full", "natural", "as", "select", "from"]); + +interface ParseState { + dialect: SqlSemanticDialectAdapter; + tokens: SqlSemanticToken[]; + statement: SqlSemanticStatement; + cteSources: SqlSemanticRowSource[]; +} + +interface TrailingIdentifier { + prefix: string; + replacementRange: SqlSemanticSpan; + qualifierParts: string[]; +} + +function significantTokens(tokens: readonly SqlSemanticToken[]): SqlSemanticToken[] { + return tokens.filter((item) => item.kind !== "comment"); +} + +function tokenTextAt(sql: string, span: SqlSemanticSpan): string { + return sql.slice(span.start, span.end); +} + +function firstWord(tokens: readonly SqlSemanticToken[]): string { + return tokens.find((item) => item.kind === "word")?.normalized ?? ""; +} + +function statementKind(tokens: readonly SqlSemanticToken[]): SqlSemanticStatementKind { + const word = firstWord(tokens); + if (word === "with" || word === "select") return "select"; + if (word === "insert") return "insert"; + if (word === "update") return "update"; + if (word === "delete") return "delete"; + if (word === "call" || word === "exec" || word === "execute") return "call"; + return "unknown"; +} + +function identifierPart(tokenValue: SqlSemanticToken, dialect: SqlSemanticDialectAdapter): SqlSemanticIdentifierPart { + const quoted = tokenValue.kind === "quoted_identifier"; + const raw = unquoteSqlSemanticIdentifier(tokenValue); + return { + raw: tokenValue.text, + name: dialect.normalizeIdentifier(raw, quoted), + span: tokenValue.span, + quote: tokenValue.quote, + }; +} + +function readQualifiedName(tokens: readonly SqlSemanticToken[], startIndex: number, dialect: SqlSemanticDialectAdapter): { name: SqlSemanticQualifiedName; nextIndex: number } | null { + const parts: SqlSemanticIdentifierPart[] = []; + let index = startIndex; + while (index < tokens.length) { + const current = tokens[index]; + if (!tokenIsIdentifier(current)) break; + parts.push(identifierPart(current, dialect)); + if (tokens[index + 1]?.text !== ".") { + index += 1; + break; + } + index += 2; + } + if (parts.length === 0) return null; + return { + name: { + parts, + span: { start: parts[0]?.span.start ?? tokens[startIndex]?.span.start ?? 0, end: parts[parts.length - 1]?.span.end ?? tokens[startIndex]?.span.end ?? 0 }, + }, + nextIndex: index, + }; +} + +function sourceNameFromQualifiedName(name: SqlSemanticQualifiedName): { name: string; qualifierParts: string[] } { + const parts = name.parts.map((part) => part.name); + return { + name: parts[parts.length - 1] ?? "", + qualifierParts: parts.slice(0, -1), + }; +} + +function findMatchingParenToken(tokens: readonly SqlSemanticToken[], openIndex: number): number { + if (tokens[openIndex]?.text !== "(") return -1; + const startDepth = tokens[openIndex]?.depth ?? 0; + for (let index = openIndex + 1; index < tokens.length; index += 1) { + const item = tokens[index]; + if (item?.text === ")" && item.depth === startDepth) return index; + } + return -1; +} + +function splitTopLevelByComma(tokens: readonly SqlSemanticToken[]): SqlSemanticToken[][] { + const groups: SqlSemanticToken[][] = []; + let current: SqlSemanticToken[] = []; + const baseDepth = tokens.reduce((min, item) => Math.min(min, item.depth), Number.POSITIVE_INFINITY); + for (const item of tokens) { + if (item.text === "," && item.depth === baseDepth) { + groups.push(current); + current = []; + } else { + current.push(item); + } + } + if (current.length > 0) groups.push(current); + return groups; +} + +function projectionNameFromTokens(tokens: readonly SqlSemanticToken[], dialect: SqlSemanticDialectAdapter): SqlSemanticProjection | null { + const useful = tokens.filter((item) => item.kind !== "comment"); + if (useful.length === 0) return null; + let asIndex = -1; + for (let index = useful.length - 1; index >= 0; index -= 1) { + if (useful[index]?.kind === "word" && useful[index]?.normalized === "as") { + asIndex = index; + break; + } + } + const aliasToken = asIndex >= 0 ? useful[asIndex + 1] : undefined; + if (tokenIsIdentifier(aliasToken)) { + const name = identifierPart(aliasToken, dialect).name; + return { + name, + alias: name, + aliasSpan: aliasToken.span, + sourceExpression: useful.map((item) => item.text).join(" "), + span: { start: useful[0]?.span.start ?? 0, end: useful[useful.length - 1]?.span.end ?? 0 }, + }; + } + const lastIdentifier = [...useful].reverse().find(tokenIsIdentifier); + if (!lastIdentifier) return null; + const name = identifierPart(lastIdentifier, dialect).name; + return { + name, + sourceExpression: useful.map((item) => item.text).join(" "), + span: { start: useful[0]?.span.start ?? 0, end: useful[useful.length - 1]?.span.end ?? 0 }, + }; +} + +function parseSelectProjections(tokens: readonly SqlSemanticToken[], dialect: SqlSemanticDialectAdapter): SqlSemanticProjection[] { + const baseDepth = tokens.reduce((min, item) => Math.min(min, item.depth), Number.POSITIVE_INFINITY); + const selectDepth = Number.isFinite(baseDepth) ? baseDepth : 0; + const selectIndex = tokens.findIndex((item) => item.depth === selectDepth && item.kind === "word" && item.normalized === "select"); + if (selectIndex < 0) return []; + let fromIndex = tokens.findIndex((item, index) => index > selectIndex && item.depth === selectDepth && item.kind === "word" && item.normalized === "from"); + if (fromIndex < 0) fromIndex = tokens.length; + const projectionTokens = tokens.slice(selectIndex + 1, fromIndex); + return splitTopLevelByComma(projectionTokens) + .map((group) => projectionNameFromTokens(group, dialect)) + .filter((projection): projection is SqlSemanticProjection => projection != null && projection.name !== "*"); +} + +function parseCteSources(state: ParseState): SqlSemanticRowSource[] { + const tokens = state.tokens; + const first = tokens.findIndex((item) => item.kind === "word" && item.normalized === "with"); + if (first < 0) return []; + const sources: SqlSemanticRowSource[] = []; + let index = first + 1; + if (tokens[index]?.normalized === "recursive") index += 1; + + while (index < tokens.length) { + while (tokens[index]?.text === ",") index += 1; + const nameToken = tokens[index]; + if (!tokenIsIdentifier(nameToken)) break; + const namePart = identifierPart(nameToken, state.dialect); + index += 1; + + const explicitColumns: string[] = []; + if (tokens[index]?.text === "(") { + const close = findMatchingParenToken(tokens, index); + if (close > index) { + for (const part of splitTopLevelByComma(tokens.slice(index + 1, close))) { + const identifier = part.find(tokenIsIdentifier); + if (identifier) explicitColumns.push(identifierPart(identifier, state.dialect).name); + } + index = close + 1; + } + } + + if (tokens[index]?.normalized === "as") index += 1; + if (tokens[index]?.text !== "(") break; + const bodyOpen = index; + const bodyClose = findMatchingParenToken(tokens, bodyOpen); + const safeBodyClose = bodyClose < 0 ? tokens.length - 1 : bodyClose; + const bodyTokens = tokens.slice(bodyOpen + 1, safeBodyClose); + const bodyColumns = explicitColumns.length > 0 ? explicitColumns : parseSelectProjections(bodyTokens, state.dialect).map((projection) => projection.name); + sources.push({ + id: `cte:${namePart.name}:${sources.length}`, + kind: "cte", + name: namePart.name, + qualifierParts: [], + sourceSpan: { start: nameToken.span.start, end: tokens[safeBodyClose]?.span.end ?? nameToken.span.end }, + columns: bodyColumns, + }); + index = safeBodyClose + 1; + + if (tokens[index]?.text !== ",") break; + index += 1; + } + return sources; +} + +function aliasAfter(tokens: readonly SqlSemanticToken[], index: number, dialect: SqlSemanticDialectAdapter): { alias?: string; aliasSpan?: SqlSemanticSpan; nextIndex: number } { + let cursor = index; + if (tokens[cursor]?.kind === "word" && tokens[cursor]?.normalized === "as") cursor += 1; + const aliasToken = tokens[cursor]; + if (tokenIsIdentifier(aliasToken)) { + const alias = identifierPart(aliasToken, dialect).name; + if (!ALIAS_BLACKLIST.has(alias.toLowerCase())) { + return { alias, aliasSpan: aliasToken.span, nextIndex: cursor + 1 }; + } + } + return { nextIndex: index }; +} + +function parseSubquerySource(state: ParseState, openIndex: number, introducer: string, sourceIndex: number): { source: SqlSemanticRowSource; nextIndex: number } | null { + const close = findMatchingParenToken(state.tokens, openIndex); + if (close < 0) return null; + const alias = aliasAfter(state.tokens, close + 1, state.dialect); + if (!alias.alias) return null; + const bodyTokens = state.tokens.slice(openIndex + 1, close); + const columns = parseSelectProjections(bodyTokens, state.dialect).map((projection) => projection.name); + return { + source: { + id: `${introducer}:subquery:${sourceIndex}`, + kind: "subquery", + name: alias.alias, + qualifierParts: [], + alias: alias.alias, + aliasSpan: alias.aliasSpan, + sourceSpan: { start: state.tokens[openIndex]?.span.start ?? 0, end: alias.aliasSpan?.end ?? state.tokens[close]?.span.end ?? 0 }, + columns, + }, + nextIndex: alias.nextIndex, + }; +} + +function parseTableFunctionSource(state: ParseState, nameIndex: number, introducer: string, sourceIndex: number): { source: SqlSemanticRowSource; nextIndex: number } | null { + const nameToken = state.tokens[nameIndex]; + if (!nameToken || nameToken.kind !== "word" || !["table", "xmltable", "json_table", "the", "read_csv", "read_parquet", "read_json", "unnest"].includes(nameToken.normalized)) return null; + if (state.tokens[nameIndex + 1]?.text !== "(") return null; + const close = findMatchingParenToken(state.tokens, nameIndex + 1); + const safeClose = close < 0 ? nameIndex + 1 : close; + const alias = aliasAfter(state.tokens, safeClose + 1, state.dialect); + const sourceName = alias.alias ?? nameToken.normalized; + return { + source: { + id: `${introducer}:table_function:${sourceIndex}`, + kind: "table_function", + name: sourceName, + qualifierParts: [], + alias: alias.alias, + aliasSpan: alias.aliasSpan, + sourceSpan: { start: nameToken.span.start, end: alias.aliasSpan?.end ?? state.tokens[safeClose]?.span.end ?? nameToken.span.end }, + unresolved: close < 0, + }, + nextIndex: alias.nextIndex, + }; +} + +function parseTableSource(state: ParseState, nameIndex: number, introducer: string, sourceIndex: number): { source: SqlSemanticRowSource; nextIndex: number } | null { + const qualified = readQualifiedName(state.tokens, nameIndex, state.dialect); + if (!qualified) return null; + const { name, qualifierParts } = sourceNameFromQualifiedName(qualified.name); + const alias = aliasAfter(state.tokens, qualified.nextIndex, state.dialect); + const cte = state.cteSources.find((source) => source.name.toLowerCase() === name.toLowerCase()); + const kind = cte ? "cte" : introducer === "update" || introducer === "into" || (state.statement.kind === "delete" && introducer === "from") ? "mutation_target" : "table"; + const source: SqlSemanticRowSource = { + id: `${introducer}:${name}:${sourceIndex}`, + kind, + name, + qualifiedName: qualified.name, + qualifierParts, + alias: alias.alias, + aliasSpan: alias.aliasSpan, + sourceSpan: { start: qualified.name.span.start, end: alias.aliasSpan?.end ?? qualified.name.span.end }, + columns: cte?.columns, + metadataTarget: { + schema: qualifierParts[qualifierParts.length - 1], + table: name, + }, + }; + return { source, nextIndex: alias.nextIndex }; +} + +function parseRowSources(state: ParseState): SqlSemanticRowSource[] { + const sources: SqlSemanticRowSource[] = [...state.cteSources]; + const rootDepth = state.tokens.reduce((min, item) => Math.min(min, item.depth), Number.POSITIVE_INFINITY); + const sourceDepth = Number.isFinite(rootDepth) ? rootDepth : 0; + for (let index = 0; index < state.tokens.length; index += 1) { + const item = state.tokens[index]; + if (!item || item.kind !== "word") continue; + if (item.depth !== sourceDepth) continue; + const normalized = item.normalized; + if (!TABLE_INTRODUCERS.has(normalized)) continue; + if (JOIN_MODIFIERS.has(normalized)) continue; + let target = index + 1; + while (JOIN_MODIFIERS.has(state.tokens[target]?.normalized ?? "")) target += 1; + if (state.tokens[target]?.text === "(") { + const subquery = parseSubquerySource(state, target, normalized, sources.length); + if (subquery) { + sources.push(subquery.source); + index = subquery.nextIndex - 1; + } + continue; + } + const tableFunction = parseTableFunctionSource(state, target, normalized, sources.length); + if (tableFunction) { + sources.push(tableFunction.source); + index = tableFunction.nextIndex - 1; + continue; + } + const table = parseTableSource(state, target, normalized, sources.length); + if (table) { + sources.push(table.source); + index = table.nextIndex - 1; + } + } + return dedupeSources(sources); +} + +function dedupeSources(sources: SqlSemanticRowSource[]): SqlSemanticRowSource[] { + const seen = new Set(); + const result: SqlSemanticRowSource[] = []; + for (const source of sources) { + const key = `${source.kind}:${source.name}:${source.alias ?? ""}:${source.sourceSpan.start}`; + if (seen.has(key)) continue; + seen.add(key); + result.push(source); + } + return result; +} + +function clauseSpans(tokens: readonly SqlSemanticToken[]): SqlSemanticClauseSpans { + const spans: SqlSemanticClauseSpans = {}; + const depth = tokens[0]?.depth ?? 0; + for (let index = 0; index < tokens.length; index += 1) { + const item = tokens[index]; + if (!item || item.depth !== depth || item.kind !== "word") continue; + const next = tokens[index + 1]; + const start = item.span.start; + const end = next?.span.start ?? tokens[tokens.length - 1]?.span.end ?? item.span.end; + if (item.normalized === "select") spans.select = { start, end }; + if (item.normalized === "from") spans.from = { start, end }; + if (item.normalized === "where") spans.where = { start, end }; + if (item.normalized === "having") spans.having = { start, end }; + if (item.normalized === "limit") spans.limit = { start, end }; + if (item.normalized === "group" && next?.normalized === "by") spans.groupBy = { start, end: next.span.end }; + if (item.normalized === "order" && next?.normalized === "by") spans.orderBy = { start, end: next.span.end }; + if (item.normalized === "set") spans.updateSet = { start, end }; + } + return spans; +} + +function trailingIdentifier(tokens: readonly SqlSemanticToken[], cursor: number, dialect: SqlSemanticDialectAdapter): TrailingIdentifier { + const before = tokens.filter((item) => item.span.start < cursor && item.kind !== "comment" && item.kind !== "string"); + const last = before[before.length - 1]; + if (!last) return { prefix: "", replacementRange: { start: cursor, end: cursor }, qualifierParts: [] }; + if (last.span.end < cursor && last.text !== ".") { + return { prefix: "", replacementRange: { start: cursor, end: cursor }, qualifierParts: [] }; + } + + let prefix = ""; + let replacementRange: SqlSemanticSpan = { start: cursor, end: cursor }; + let index = before.length - 1; + if (tokenIsIdentifier(last) && cursor <= last.span.end) { + const rawPrefix = tokenTextAt(last.text, { start: 0, end: Math.max(0, cursor - last.span.start) }); + prefix = last.kind === "quoted_identifier" ? unquoteSqlSemanticIdentifier({ ...last, text: rawPrefix.endsWith(last.quote ?? "") ? rawPrefix : rawPrefix + (last.quote === "[" ? "]" : (last.quote ?? "")) }) : rawPrefix; + replacementRange = { start: last.span.start, end: cursor }; + index -= 1; + if (before[index]?.text === ".") index -= 1; + } else if (last.text === ".") { + index -= 1; + } + + const qualifierParts: string[] = []; + while (index >= 0) { + const identifier = before[index]; + if (!tokenIsIdentifier(identifier)) break; + qualifierParts.unshift(identifierPart(identifier, dialect).name); + if (before[index - 1]?.text !== ".") break; + index -= 2; + } + + return { prefix, replacementRange, qualifierParts }; +} + +function previousWord(tokens: readonly SqlSemanticToken[], cursor: number): string { + const before = tokens.filter((item) => item.span.end <= cursor && item.kind === "word"); + return before[before.length - 1]?.normalized ?? ""; +} + +function wordBeforePosition(tokens: readonly SqlSemanticToken[], position: number): string { + const before = tokens.filter((item) => item.span.end <= position && item.kind === "word"); + return before[before.length - 1]?.normalized ?? ""; +} + +function isTableListContinuation(tokens: readonly SqlSemanticToken[], position: number): boolean { + const before = tokens.filter((item) => item.span.end <= position && item.kind !== "comment"); + const commaIndex = before.length - 1; + const comma = before[commaIndex]; + if (comma?.text !== ",") return false; + const depth = comma.depth; + for (let index = commaIndex - 1; index >= 0; index -= 1) { + const token = before[index]; + if (!token || token.depth !== depth || token.kind !== "word") continue; + if (TABLE_INTRODUCERS.has(token.normalized) || token.normalized === "from" || token.normalized === "join") return true; + if (CLAUSE_BOUNDARIES.has(token.normalized) || token.normalized === "select") return false; + } + return false; +} + +function hasWordBefore(tokens: readonly SqlSemanticToken[], cursor: number, word: string): boolean { + return tokens.some((item) => item.span.end <= cursor && item.kind === "word" && item.normalized === word); +} + +function isBeforeWord(tokens: readonly SqlSemanticToken[], cursor: number, word: string): boolean { + const target = tokens.find((item) => item.span.start >= cursor && item.kind === "word" && item.normalized === word); + return !!target; +} + +function sourceForQualifier(sources: readonly SqlSemanticRowSource[], qualifierParts: readonly string[]): SqlSemanticRowSource | undefined { + const qualifier = qualifierParts[qualifierParts.length - 1]?.toLowerCase(); + if (!qualifier) return undefined; + return sources.find((source) => source.alias?.toLowerCase() === qualifier || source.name.toLowerCase() === qualifier); +} + +function starQualifierParts(before: readonly SqlSemanticToken[], starIndex: number, dialect: SqlSemanticDialectAdapter): string[] { + let index = starIndex - 1; + if (before[index]?.text === ".") index -= 1; + const qualifierParts: string[] = []; + while (index >= 0) { + const identifier = before[index]; + if (!tokenIsIdentifier(identifier)) break; + qualifierParts.unshift(identifierPart(identifier, dialect).name); + if (before[index - 1]?.text !== ".") break; + index -= 2; + } + return qualifierParts; +} + +function buildCursorIntent(tokens: readonly SqlSemanticToken[], cursor: number, sources: readonly SqlSemanticRowSource[], dialect: SqlSemanticDialectAdapter, suppressed: boolean, kind: SqlSemanticStatementKind): SqlSemanticCursorIntent { + if (suppressed) { + return { kind: "suppressed", prefix: "", replacementRange: { start: cursor, end: cursor }, qualifierParts: [], expectedObjectKinds: [], confidence: "high", fallbackReason: "comment_or_string" }; + } + const trailing = trailingIdentifier(tokens, cursor, dialect); + const previous = previousWord(tokens, cursor); + const targetSource = sourceForQualifier(sources, trailing.qualifierParts); + const before = tokens.filter((item) => item.span.end <= cursor); + const last = before[before.length - 1]; + const wordBeforeReplacement = wordBeforePosition(tokens, trailing.replacementRange.start); + const tableListContinuation = isTableListContinuation(tokens, trailing.replacementRange.start); + + if (last?.text === "*" || trailing.prefix === "*") { + const starIndex = last?.text === "*" ? before.length - 1 : -1; + const qualifierParts = starIndex >= 0 ? starQualifierParts(before, starIndex, dialect) : trailing.qualifierParts; + const starTarget = sourceForQualifier(sources, qualifierParts); + const replacementRange = last?.text === "*" ? { start: last.span.start, end: Math.min(cursor, last.span.end) } : trailing.replacementRange; + return { kind: "star", prefix: "*", replacementRange, qualifierParts, targetSourceId: starTarget?.id, expectedObjectKinds: ["column"], confidence: "high" }; + } + + if (kind === "call") { + return { kind: "routine", prefix: trailing.prefix, replacementRange: trailing.replacementRange, qualifierParts: trailing.qualifierParts, expectedObjectKinds: ["routine", "procedure", "function"], confidence: "high" }; + } + + if (kind === "insert" && hasWordBefore(tokens, cursor, "into") && !hasWordBefore(tokens, cursor, "values")) { + const mutationTarget = sources.find((source) => source.kind === "mutation_target"); + return { + kind: "insert_column", + prefix: trailing.prefix, + replacementRange: trailing.replacementRange, + qualifierParts: trailing.qualifierParts, + targetSourceId: mutationTarget?.id, + expectedObjectKinds: ["column"], + confidence: mutationTarget ? "medium" : "low", + fallbackReason: mutationTarget ? undefined : "unresolved_insert_target", + }; + } + + if ( + trailing.qualifierParts.length > 0 && + (previous === "from" || previous === "join" || TABLE_INTRODUCERS.has(previous) || TABLE_INTRODUCERS.has(wordBeforeReplacement) || (!!targetSource && !targetSource.alias && TABLE_INTRODUCERS.has(wordBeforePosition(tokens, targetSource.sourceSpan.start)))) + ) { + const role = dialect.qualifierRole(trailing.qualifierParts, "table"); + return { kind: role === "catalog" ? "catalog" : "table", prefix: trailing.prefix, replacementRange: trailing.replacementRange, qualifierParts: trailing.qualifierParts, expectedObjectKinds: ["table", "view"], confidence: "medium" }; + } + + if (trailing.qualifierParts.length > 0 && targetSource) { + return { kind: "alias_column", prefix: trailing.prefix, replacementRange: trailing.replacementRange, qualifierParts: trailing.qualifierParts, targetSourceId: targetSource.id, expectedObjectKinds: ["column"], confidence: "high" }; + } + + if (TABLE_INTRODUCERS.has(previous) || JOIN_MODIFIERS.has(previous) || previous === "from" || previous === "join" || tableListContinuation) { + return { kind: previous === "join" ? "table" : "table", prefix: trailing.prefix, replacementRange: trailing.replacementRange, qualifierParts: trailing.qualifierParts, expectedObjectKinds: ["table", "view"], confidence: "high" }; + } + + if (previous === "call" || previous === "exec" || previous === "execute") { + return { kind: "routine", prefix: trailing.prefix, replacementRange: trailing.replacementRange, qualifierParts: trailing.qualifierParts, expectedObjectKinds: ["routine", "procedure", "function"], confidence: "high" }; + } + + if (previous === "set") { + return { + kind: "update_column", + prefix: trailing.prefix, + replacementRange: trailing.replacementRange, + qualifierParts: trailing.qualifierParts, + expectedObjectKinds: ["column"], + confidence: sources.length > 0 ? "medium" : "low", + fallbackReason: sources.length > 0 ? undefined : "unresolved_update_target", + }; + } + + if (["where", "on", "and", "or", "having", "by", "select"].includes(previous) && sources.length > 0 && !isBeforeWord(tokens, cursor, "from")) { + return { kind: previous === "on" ? "join_condition" : "column", prefix: trailing.prefix, replacementRange: trailing.replacementRange, qualifierParts: trailing.qualifierParts, expectedObjectKinds: ["column"], confidence: "medium" }; + } + + return { kind: "keyword", prefix: trailing.prefix, replacementRange: trailing.replacementRange, qualifierParts: trailing.qualifierParts, expectedObjectKinds: [], confidence: "low", fallbackReason: "keyword_context" }; +} + +function buildScope(statement: SqlSemanticStatement, rowSources: SqlSemanticRowSource[], projections: SqlSemanticProjection[], tokens: SqlSemanticToken[]): SqlSemanticScope { + return { + id: "root", + kind: statement.kind, + span: statement.span, + rowSources, + projections, + clauseSpans: clauseSpans(tokens), + }; +} + +export function buildSqlSemanticModel(sql: string, cursor: number, options: SqlSemanticBuildOptions = {}): SqlSemanticModel { + const safeCursor = Math.max(0, Math.min(cursor, sql.length)); + const dialect = sqlSemanticDialectFor(options); + const allTokens = tokenizeSqlSemantic(sql); + const statementSpan = findActiveSqlStatementSpan(sql, allTokens, safeCursor); + const tokens = significantTokens(allTokens.filter((item) => item.span.end > statementSpan.start && item.span.start < statementSpan.end)); + const kind = statementKind(tokens); + const statement: SqlSemanticStatement = { + kind, + span: statementSpan, + text: sql.slice(statementSpan.start, statementSpan.end), + }; + const suppressed = isSuppressedSqlSemanticContext(allTokens, safeCursor); + const parseState: ParseState = { dialect, tokens, statement, cteSources: [] }; + parseState.cteSources = parseCteSources(parseState); + const rowSources = parseRowSources(parseState); + const projections = parseSelectProjections(tokens, dialect); + const cursorIntent = buildCursorIntent(tokens, safeCursor, rowSources, dialect, suppressed, kind); + const scopes = [buildScope(statement, rowSources, projections, tokens)]; + return { + databaseType: options.databaseType, + dialectId: dialect.id, + sql, + cursor: safeCursor, + statement, + tokens: allTokens, + scopes, + rowSources, + projections, + cursorIntent, + diagnostics: [], + }; +} diff --git a/apps/desktop/src/lib/sqlSemanticReferences.ts b/apps/desktop/src/lib/sqlSemanticReferences.ts new file mode 100644 index 000000000..a39a7738f --- /dev/null +++ b/apps/desktop/src/lib/sqlSemanticReferences.ts @@ -0,0 +1,154 @@ +import type { SqlCompletionReferencedTable } from "@/lib/sqlCompletion"; +import type { SqlSemanticModel, SqlSemanticRowSource, SqlSemanticSpan } from "@/lib/sqlSemanticTypes"; +import type { SqlReferenceAnalysis, SqlTableReference, SqlTextSpan } from "@/types/database"; + +export type SqlSemanticTableReference = SqlTableReference & { + columns?: string[]; + semanticSourceId?: string; + semanticSourceKind?: SqlSemanticRowSource["kind"]; +}; + +export interface SqlSemanticNavigationTarget { + name: string; + schema?: string; + alias?: string; + columns?: string[]; + source: SqlSemanticRowSource; +} + +function offsetToLineColumn(sql: string, offset: number): { line: number; column: number } { + const safeOffset = Math.max(0, Math.min(offset, sql.length)); + let line = 1; + let lineStart = 0; + for (let index = 0; index < safeOffset; index += 1) { + if (sql[index] === "\n") { + line += 1; + lineStart = index + 1; + } + } + return { line, column: safeOffset - lineStart }; +} + +function semanticSpanToSqlTextSpan(sql: string, span: SqlSemanticSpan): SqlTextSpan { + const start = offsetToLineColumn(sql, span.start); + const end = offsetToLineColumn(sql, Math.max(span.end, span.start)); + return { + start_line: start.line, + start_column: start.column + 1, + end_line: end.line, + end_column: Math.max(end.column, start.column + 1), + }; +} + +function normalized(value: string | null | undefined): string { + let result = value ?? ""; + while (result && '`"['.includes(result[0])) result = result.slice(1); + while (result && '`"]'.includes(result[result.length - 1])) result = result.slice(0, -1); + return result.toLowerCase(); +} + +function sourceSchema(source: SqlSemanticRowSource): string | undefined { + return source.metadataTarget?.schema ?? source.qualifierParts[source.qualifierParts.length - 1]; +} + +function sourceTableName(source: SqlSemanticRowSource): string { + return source.metadataTarget?.table ?? source.name; +} + +function sourceReferenceKey(source: SqlSemanticTableReference): string { + return [normalized(source.schema), normalized(source.name), normalized(source.alias), source.span.start_line, source.span.start_column].join(":"); +} + +function existingScopeId(analysis: SqlReferenceAnalysis): number { + return analysis.tables[0]?.scope_id ?? analysis.columns[0]?.scope_id ?? 0; +} + +export function sqlSemanticTableReferences(model: SqlSemanticModel, scopeId = 0): SqlSemanticTableReference[] { + if (model.cursorIntent.kind === "suppressed" || model.cursorIntent.confidence === "low") return []; + return model.rowSources + .filter((source) => source.kind !== "unknown") + .map((source) => { + const span = source.qualifiedName?.span ?? source.aliasSpan ?? source.sourceSpan; + const schema = sourceSchema(source); + return { + name: sourceTableName(source), + schema, + alias: source.alias, + span: semanticSpanToSqlTextSpan(model.sql, span), + scope_id: scopeId, + columns: source.columns?.length ? [...source.columns] : undefined, + semanticSourceId: source.id, + semanticSourceKind: source.kind, + }; + }); +} + +export function mergeSqlSemanticReferenceAnalysis(analysis: SqlReferenceAnalysis, model: SqlSemanticModel): SqlReferenceAnalysis { + const semanticTables = sqlSemanticTableReferences(model, existingScopeId(analysis)); + if (semanticTables.length === 0) return analysis; + + const merged = new Map(); + for (const table of analysis.tables as SqlSemanticTableReference[]) { + merged.set(sourceReferenceKey(table), table); + } + + for (const table of semanticTables) { + const existing = [...merged.values()].find((candidate) => { + if (normalized(candidate.name) !== normalized(table.name)) return false; + if (candidate.alias && table.alias && normalized(candidate.alias) !== normalized(table.alias)) return false; + if (candidate.schema && table.schema && normalized(candidate.schema) !== normalized(table.schema)) return false; + return true; + }); + if (existing) { + existing.schema = existing.schema ?? table.schema; + existing.alias = existing.alias ?? table.alias; + existing.columns = existing.columns ?? table.columns; + existing.semanticSourceId = existing.semanticSourceId ?? table.semanticSourceId; + } else { + merged.set(sourceReferenceKey(table), table); + } + } + + return { + ...analysis, + tables: [...merged.values()], + }; +} + +export function sqlSemanticCompletionReferenceTables(model: SqlSemanticModel): SqlCompletionReferencedTable[] { + return sqlSemanticTableReferences(model).map((table) => ({ + name: table.name, + schema: table.schema ?? undefined, + alias: table.alias ?? undefined, + columns: table.columns, + })); +} + +export function resolveSqlSemanticNavigationTarget(model: SqlSemanticModel, identifierParts: readonly string[]): SqlSemanticNavigationTarget | null { + if (model.cursorIntent.kind === "suppressed" || model.cursorIntent.confidence === "low" || identifierParts.length === 0) return null; + const normalizedParts = identifierParts.map(normalized).filter(Boolean); + const name = normalizedParts[normalizedParts.length - 1]; + const qualifier = normalizedParts.length > 1 ? normalizedParts[normalizedParts.length - 2] : undefined; + + const source = + model.rowSources.find((candidate) => { + const candidateName = normalized(sourceTableName(candidate)); + const candidateAlias = normalized(candidate.alias); + const candidateSchema = normalized(sourceSchema(candidate)); + if (qualifier) { + if (candidateAlias && candidateAlias === qualifier) return true; + return candidateName === name && (!candidateSchema || candidateSchema === qualifier); + } + return candidateName === name || candidateAlias === name; + }) ?? null; + + if (!source) return null; + const schema = sourceSchema(source); + return { + name: sourceTableName(source), + schema, + alias: source.alias, + columns: source.columns, + source, + }; +} diff --git a/apps/desktop/src/lib/sqlSemanticTokens.ts b/apps/desktop/src/lib/sqlSemanticTokens.ts new file mode 100644 index 000000000..02dbbd0b7 --- /dev/null +++ b/apps/desktop/src/lib/sqlSemanticTokens.ts @@ -0,0 +1,163 @@ +import type { SqlSemanticSpan, SqlSemanticToken } from "@/lib/sqlSemanticTypes"; + +const WORD_START = /[A-Za-z_@$#]/; +const WORD_PART = /[A-Za-z0-9_@$#]/; + +function token(kind: SqlSemanticToken["kind"], text: string, start: number, end: number, depth: number, quote?: string): SqlSemanticToken { + return { + kind, + text, + normalized: kind === "word" ? text.toLowerCase() : text, + span: { start, end }, + depth, + quote, + }; +} + +function readQuoted(input: string, start: number, open: string, close: string): number { + let index = start + open.length; + while (index < input.length) { + if (input.startsWith(close, index)) { + if (input.startsWith(close + close, index)) { + index += close.length * 2; + continue; + } + return index + close.length; + } + index += 1; + } + return input.length; +} + +export function tokenizeSqlSemantic(input: string): SqlSemanticToken[] { + const tokens: SqlSemanticToken[] = []; + let index = 0; + let depth = 0; + + while (index < input.length) { + const start = index; + const ch = input[index] ?? ""; + const next = input[index + 1] ?? ""; + + if (/\s/.test(ch)) { + index += 1; + continue; + } + + if (ch === "-" && next === "-") { + index += 2; + while (index < input.length && input[index] !== "\n" && input[index] !== "\r") index += 1; + tokens.push(token("comment", input.slice(start, index), start, index, depth)); + continue; + } + + if (ch === "#") { + index += 1; + while (index < input.length && input[index] !== "\n" && input[index] !== "\r") index += 1; + tokens.push(token("comment", input.slice(start, index), start, index, depth)); + continue; + } + + if (ch === "/" && next === "*") { + index += 2; + while (index < input.length && !(input[index] === "*" && input[index + 1] === "/")) index += 1; + index = Math.min(input.length, index + (index < input.length ? 2 : 0)); + tokens.push(token("comment", input.slice(start, index), start, index, depth)); + continue; + } + + if (ch === "'") { + index = readQuoted(input, start, "'", "'"); + tokens.push(token("string", input.slice(start, index), start, index, depth, "'")); + continue; + } + + if (ch === '"') { + index = readQuoted(input, start, '"', '"'); + tokens.push(token("quoted_identifier", input.slice(start, index), start, index, depth, '"')); + continue; + } + + if (ch === "`") { + index = readQuoted(input, start, "`", "`"); + tokens.push(token("quoted_identifier", input.slice(start, index), start, index, depth, "`")); + continue; + } + + if (ch === "[") { + index = readQuoted(input, start, "[", "]"); + tokens.push(token("quoted_identifier", input.slice(start, index), start, index, depth, "[")); + continue; + } + + if (ch === ":" || ch === "?") { + index += 1; + while (index < input.length && WORD_PART.test(input[index] ?? "")) index += 1; + tokens.push(token("parameter", input.slice(start, index), start, index, depth)); + continue; + } + + if (/[0-9]/.test(ch)) { + index += 1; + while (index < input.length && /[0-9.]/.test(input[index] ?? "")) index += 1; + tokens.push(token("number", input.slice(start, index), start, index, depth)); + continue; + } + + if (WORD_START.test(ch)) { + index += 1; + while (index < input.length && WORD_PART.test(input[index] ?? "")) index += 1; + tokens.push(token("word", input.slice(start, index), start, index, depth)); + continue; + } + + if ("(),.;*".includes(ch)) { + if (ch === ")") depth = Math.max(0, depth - 1); + tokens.push(token("punctuation", ch, start, start + 1, depth)); + if (ch === "(") depth += 1; + index += 1; + continue; + } + + index += 1; + tokens.push(token("operator", ch, start, index, depth)); + } + + return tokens; +} + +export function tokenContainsPosition(tokenValue: SqlSemanticToken, position: number): boolean { + return tokenValue.span.start <= position && position <= tokenValue.span.end; +} + +export function isSuppressedSqlSemanticContext(tokens: readonly SqlSemanticToken[], cursor: number): boolean { + return tokens.some((item) => (item.kind === "comment" || item.kind === "string") && item.span.start < cursor && cursor <= item.span.end); +} + +export function findActiveSqlStatementSpan(sql: string, tokens: readonly SqlSemanticToken[], cursor: number): SqlSemanticSpan { + let start = 0; + let end = sql.length; + for (const item of tokens) { + if (item.kind !== "punctuation" || item.text !== ";" || item.depth !== 0) continue; + if (item.span.end <= cursor) start = item.span.end; + if (item.span.start >= cursor) { + end = item.span.start; + break; + } + } + + while (start < end && /\s/.test(sql[start] ?? "")) start += 1; + while (end > start && /\s/.test(sql[end - 1] ?? "")) end -= 1; + return { start, end }; +} + +export function unquoteSqlSemanticIdentifier(tokenValue: SqlSemanticToken): string { + if (tokenValue.kind !== "quoted_identifier") return tokenValue.text; + if (tokenValue.quote === "[") return tokenValue.text.slice(1, -1).replaceAll("]]", "]"); + const quote = tokenValue.quote ?? tokenValue.text[0] ?? ""; + return tokenValue.text.slice(1, -1).replaceAll(quote + quote, quote); +} + +export function tokenIsIdentifier(tokenValue: SqlSemanticToken | undefined): tokenValue is SqlSemanticToken { + return !!tokenValue && (tokenValue.kind === "word" || tokenValue.kind === "quoted_identifier"); +} diff --git a/apps/desktop/src/lib/sqlSemanticTypes.ts b/apps/desktop/src/lib/sqlSemanticTypes.ts new file mode 100644 index 000000000..9c6d46cc9 --- /dev/null +++ b/apps/desktop/src/lib/sqlSemanticTypes.ts @@ -0,0 +1,130 @@ +import type { DatabaseType } from "@/types/database"; + +export interface SqlSemanticSpan { + start: number; + end: number; +} + +export type SqlSemanticConfidence = "high" | "medium" | "low"; + +export type SqlSemanticStatementKind = "select" | "insert" | "update" | "delete" | "call" | "unknown"; + +export type SqlSemanticTokenKind = "word" | "quoted_identifier" | "string" | "number" | "comment" | "parameter" | "punctuation" | "operator"; + +export interface SqlSemanticToken { + kind: SqlSemanticTokenKind; + text: string; + normalized: string; + span: SqlSemanticSpan; + depth: number; + quote?: string; +} + +export interface SqlSemanticIdentifierPart { + raw: string; + name: string; + span: SqlSemanticSpan; + quote?: string; +} + +export interface SqlSemanticQualifiedName { + parts: SqlSemanticIdentifierPart[]; + span: SqlSemanticSpan; +} + +export type SqlSemanticRowSourceKind = "table" | "cte" | "subquery" | "table_function" | "mutation_target" | "unknown"; + +export interface SqlSemanticMetadataTarget { + database?: string; + schema?: string; + table?: string; + packageName?: string; +} + +export interface SqlSemanticProjection { + name: string; + sourceExpression: string; + span: SqlSemanticSpan; + alias?: string; + aliasSpan?: SqlSemanticSpan; +} + +export interface SqlSemanticRowSource { + id: string; + kind: SqlSemanticRowSourceKind; + name: string; + qualifiedName?: SqlSemanticQualifiedName; + qualifierParts: string[]; + alias?: string; + aliasSpan?: SqlSemanticSpan; + sourceSpan: SqlSemanticSpan; + columns?: string[]; + metadataTarget?: SqlSemanticMetadataTarget; + unresolved?: boolean; +} + +export interface SqlSemanticClauseSpans { + select?: SqlSemanticSpan; + from?: SqlSemanticSpan; + where?: SqlSemanticSpan; + groupBy?: SqlSemanticSpan; + having?: SqlSemanticSpan; + orderBy?: SqlSemanticSpan; + limit?: SqlSemanticSpan; + insertColumns?: SqlSemanticSpan; + updateSet?: SqlSemanticSpan; +} + +export interface SqlSemanticScope { + id: string; + kind: SqlSemanticStatementKind | "subquery" | "cte"; + span: SqlSemanticSpan; + parentId?: string; + rowSources: SqlSemanticRowSource[]; + projections: SqlSemanticProjection[]; + clauseSpans: SqlSemanticClauseSpans; +} + +export type SqlSemanticCursorKind = "table" | "schema" | "catalog" | "routine" | "column" | "alias_column" | "insert_column" | "update_column" | "delete_target" | "join_condition" | "star" | "keyword" | "suppressed"; + +export interface SqlSemanticCursorIntent { + kind: SqlSemanticCursorKind; + prefix: string; + replacementRange: SqlSemanticSpan; + qualifierParts: string[]; + targetSourceId?: string; + expectedObjectKinds: Array<"database" | "schema" | "table" | "view" | "routine" | "procedure" | "function" | "column">; + confidence: SqlSemanticConfidence; + fallbackReason?: string; +} + +export interface SqlSemanticStatement { + kind: SqlSemanticStatementKind; + span: SqlSemanticSpan; + text: string; +} + +export interface SqlSemanticDiagnostic { + message: string; + span: SqlSemanticSpan; + severity: "info" | "warning" | "error"; +} + +export interface SqlSemanticModel { + databaseType?: DatabaseType; + dialectId: string; + sql: string; + cursor: number; + statement: SqlSemanticStatement; + tokens: SqlSemanticToken[]; + scopes: SqlSemanticScope[]; + rowSources: SqlSemanticRowSource[]; + projections: SqlSemanticProjection[]; + cursorIntent: SqlSemanticCursorIntent; + diagnostics: SqlSemanticDiagnostic[]; +} + +export interface SqlSemanticBuildOptions { + databaseType?: DatabaseType; + dialect?: "mysql" | "postgres" | "sqlserver"; +} diff --git a/packages/app-tests/sqlCompletionPerformance.test.ts b/packages/app-tests/sqlCompletionPerformance.test.ts index 2a30cc52e..06686397c 100644 --- a/packages/app-tests/sqlCompletionPerformance.test.ts +++ b/packages/app-tests/sqlCompletionPerformance.test.ts @@ -1,7 +1,9 @@ import assert from "node:assert/strict"; import { test } from "vitest"; import { createPinia, setActivePinia } from "pinia"; -import { buildSqlCompletionItems, recordCompletionSelection } from "../../apps/desktop/src/lib/sqlCompletion.ts"; +import { buildSqlCompletionItems, buildSqlCompletionItemsFromContext, getSqlCompletionContext, recordCompletionSelection } from "../../apps/desktop/src/lib/sqlCompletion.ts"; +import { sqlCompletionContextFromSemantic } from "../../apps/desktop/src/lib/sqlSemanticCompletion.ts"; +import { buildSqlSemanticModel } from "../../apps/desktop/src/lib/sqlSemanticModel.ts"; import { useConnectionStore } from "../../apps/desktop/src/stores/connectionStore.ts"; import type { ConnectionConfig, TableInfo } from "../../apps/desktop/src/types/database.ts"; @@ -70,6 +72,36 @@ test("large table catalogs produce bounded SQL completion items", () => { assert.equal(tableItems[0].label, "customer_event_0000"); }); +test("semantic completion parses only the active large-script statement within bounds", () => { + const inactiveStatements = Array.from({ length: 400 }, (_, index) => `SELECT * FROM archived_${index} WHERE id = ${index};`).join("\n"); + const activeStatement = "SELECT * FROM active_orders ao JOIN active_users au ON au.id = ao.user_id WHERE au.na"; + const sql = `${inactiveStatements}\n${activeStatement}`; + const cursor = sql.length; + const tables = Array.from({ length: 2500 }, (_, index) => ({ + name: `active_${String(index).padStart(4, "0")}`, + schema: "public", + type: "table" as const, + })); + const columnsByTable = new Map([ + ["active_users", ["id", "name", "email"].map((name) => ({ name, table: "active_users" }))], + ["active_orders", ["id", "user_id", "total"].map((name) => ({ name, table: "active_orders" }))], + ]); + + const startedAt = performance.now(); + const model = buildSqlSemanticModel(sql, cursor, { databaseType: "postgres" }); + const elapsedMs = performance.now() - startedAt; + const context = sqlCompletionContextFromSemantic(model, getSqlCompletionContext(sql, cursor)); + const items = buildSqlCompletionItemsFromContext(context, { tables, columnsByTable, dialect: "postgres" }); + + assert.equal(model.statement.text, activeStatement); + assert.ok(elapsedMs < 200, `semantic model build took ${elapsedMs.toFixed(1)}ms`); + assert.ok(items.length <= 200); + assert.deepEqual( + items.filter((item) => item.type === "column").map((item) => item.label), + ["name"], + ); +}); + test("recently selected completion items receive a ranking boost", () => { const tables = [ { name: "customer_accounts", schema: "public", type: "table" as const },