From f07509b5ad784f3b06b7dc3d171e83203a21b730 Mon Sep 17 00:00:00 2001 From: onenewcode Date: Thu, 23 Jul 2026 17:06:23 +0800 Subject: [PATCH] fix(elasticsearch): correct totals and CAT JSON output --- .../common/ElasticsearchJsonResponsePanel.vue | 2 +- .../components/document/DocumentBrowser.vue | 164 ++++++- apps/desktop/src/components/grid/DataGrid.vue | 20 +- apps/desktop/src/i18n/locales/en.ts | 1 + apps/desktop/src/i18n/locales/es.ts | 1 + apps/desktop/src/i18n/locales/it.ts | 1 + apps/desktop/src/i18n/locales/ja.ts | 1 + apps/desktop/src/i18n/locales/pt-BR.ts | 1 + apps/desktop/src/i18n/locales/zh-CN.ts | 1 + apps/desktop/src/i18n/locales/zh-TW.ts | 1 + .../elasticsearchDocumentTotals.spec.ts | 45 ++ .../elasticsearchJsonResponse.spec.ts | 19 + apps/desktop/src/lib/backend/api.ts | 2 + apps/desktop/src/lib/backend/http.ts | 7 +- apps/desktop/src/lib/backend/tauri.ts | 14 +- .../document/elasticsearchDocumentTotals.ts | 49 +++ .../elasticsearchJsonResponse.ts | 5 +- crates/dbx-core/src/db/document_result.rs | 50 +++ .../dbx-core/src/db/elasticsearch_driver.rs | 408 +++++++++++++++++- crates/dbx-core/src/db/mod.rs | 1 + crates/dbx-core/src/db/mongo_driver.rs | 47 +- crates/dbx-core/src/db/vector_driver.rs | 8 +- crates/dbx-core/src/document_ops.rs | 22 +- crates/dbx-web/src/main.rs | 4 + crates/dbx-web/src/routes/document_store.rs | 29 +- src-tauri/src/commands/document_cmd.rs | 21 +- src-tauri/src/lib.rs | 1 + 27 files changed, 855 insertions(+), 70 deletions(-) create mode 100644 apps/desktop/src/lib/__tests__/document/elasticsearchDocumentTotals.spec.ts create mode 100644 apps/desktop/src/lib/__tests__/elasticsearchJsonResponse.spec.ts create mode 100644 apps/desktop/src/lib/document/elasticsearchDocumentTotals.ts create mode 100644 crates/dbx-core/src/db/document_result.rs diff --git a/apps/desktop/src/components/common/ElasticsearchJsonResponsePanel.vue b/apps/desktop/src/components/common/ElasticsearchJsonResponsePanel.vue index 73e24c199..4e54f73af 100644 --- a/apps/desktop/src/components/common/ElasticsearchJsonResponsePanel.vue +++ b/apps/desktop/src/components/common/ElasticsearchJsonResponsePanel.vue @@ -111,7 +111,7 @@ onMounted(() => {
- 
{{ body }}
+ 
{{ body }}
diff --git a/apps/desktop/src/components/document/DocumentBrowser.vue b/apps/desktop/src/components/document/DocumentBrowser.vue index b9176f349..ba35ab7b4 100644 --- a/apps/desktop/src/components/document/DocumentBrowser.vue +++ b/apps/desktop/src/components/document/DocumentBrowser.vue @@ -16,6 +16,7 @@ import * as api from "@/lib/backend/api"; import { useConnectionStore } from "@/stores/connectionStore"; import { clampSearchSplitWidth } from "@/lib/dataGrid/dataGridSearchSplit"; import { documentViewerFontStyle } from "@/lib/document/documentViewerFontStyle"; +import { clampDocumentPage, documentPageRequestLimit, resetElasticsearchDocumentTotals, resolveElasticsearchDocumentTotals } from "@/lib/document/elasticsearchDocumentTotals"; import { arrayObjectAncestorPathForDocumentField, buildDocumentFilterCondition, @@ -84,7 +85,9 @@ type ViewMode = "document" | "table"; const documents = ref([]); const copyDocuments = ref([]); const lastGridColumns = ref([]); -const total = ref(0); +const total = ref(undefined); +const totalIsExact = ref(true); +const paginationTotal = ref(undefined); const loading = ref(false); const documentLoadExecutionId = ref(""); const documentLoadCancelling = ref(false); @@ -123,15 +126,26 @@ const tableFindPaneWidth = ref(null); const isResizingTableSearchSplit = ref(false); let tableSearchSplitStartX = 0; let tableSearchSplitStartWidth = 0; +let elasticsearchCountKey: string | null = null; +let elasticsearchExactTotal: number | undefined; +let elasticsearchPaginationLowerBound: number | undefined; +let elasticsearchCountExecutionId = ""; +let elasticsearchCountGeneration = 0; const documentStoreProvider = computed(() => documentStoreProviderFor(props.databaseType)); +const pageTotal = computed(() => paginationTotal.value ?? total.value ?? 0); +const documentRequestLimit = computed(() => { + if (documentStoreProvider.value.kind !== "elasticsearch" || paginationTotal.value === undefined) return pageSize.value; + return documentPageRequestLimit(page.value, pageSize.value, paginationTotal.value); +}); + const tableFindPaneStyle = computed(() => { if (tableFindPaneWidth.value == null) return {}; return { flex: `0 0 ${tableFindPaneWidth.value}px` }; }); const documentFontStyle = computed(() => documentViewerFontStyle(settingsStore.editorSettings)); const documentStoreLabels = computed(() => ({ - documentsLabel: documentStoreProvider.value.documentsLabel({ total: total.value, t }), + documentsLabel: documentStoreProvider.value.documentsLabel({ total: total.value ?? 0, t }), queryPreview: documentQueryPreview.value, })); @@ -436,7 +450,7 @@ const documentQueryPreview = computed(() => { filterJson: filter, sortJson: sortInput.value.trim(), skip: page.value * pageSize.value, - limit: pageSize.value, + limit: documentRequestLimit.value, }); }); @@ -580,6 +594,7 @@ async function gridSave(changes: DocumentGridChanges) { await api.documentInsertDocument(props.connectionId, props.database, props.collection, JSON.stringify(doc)); } + if (isEs) resetElasticsearchTotals({ preservePaginationTotal: true }); await load(); } @@ -684,6 +699,100 @@ function startDocumentLoadingTimer() { }, 100); } +function elasticsearchCountFilterKey(filter: string | undefined): string { + return JSON.stringify([props.connectionId, props.database, props.collection, filter ?? ""]); +} + +function cancelElasticsearchCount() { + elasticsearchCountGeneration++; + const executionId = elasticsearchCountExecutionId; + elasticsearchCountExecutionId = ""; + if (executionId) void api.cancelQuery(executionId); +} + +function resetElasticsearchTotals(options: { preservePaginationTotal?: boolean } = {}) { + const nextTotals = resetElasticsearchDocumentTotals(paginationTotal.value, options.preservePaginationTotal); + cancelElasticsearchCount(); + elasticsearchCountKey = null; + elasticsearchExactTotal = undefined; + elasticsearchPaginationLowerBound = undefined; + paginationTotal.value = nextTotals.paginationTotal; + total.value = nextTotals.total; + totalIsExact.value = nextTotals.totalIsExact; +} + +function clampPageToPaginationTotal(): boolean { + const cap = paginationTotal.value; + if (cap === undefined) return false; + const nextPage = clampDocumentPage(page.value, pageSize.value, cap); + if (page.value === nextPage) return false; + page.value = nextPage; + return true; +} + +function startElasticsearchExactCount(filter: string | undefined) { + if (elasticsearchCountExecutionId || elasticsearchExactTotal !== undefined || !elasticsearchCountKey) return; + const key = elasticsearchCountKey; + const executionId = uuid(); + const generation = elasticsearchCountGeneration; + elasticsearchCountExecutionId = executionId; + + void api + .elasticsearchCountDocuments(props.connectionId, props.collection, filter, executionId) + .then((exactCount) => { + if (generation !== elasticsearchCountGeneration || key !== elasticsearchCountKey || executionId !== elasticsearchCountExecutionId || !Number.isFinite(exactCount) || exactCount < 0) { + return; + } + elasticsearchExactTotal = exactCount; + const totals = resolveElasticsearchDocumentTotals(elasticsearchPaginationLowerBound ?? exactCount, false, exactCount); + total.value = totals.total; + totalIsExact.value = totals.totalIsExact; + paginationTotal.value = totals.paginationTotal; + if (clampPageToPaginationTotal()) void load(); + }) + .catch(() => { + // The lower-bound result remains truthful when a background count fails. + }) + .finally(() => { + if (generation === elasticsearchCountGeneration && executionId === elasticsearchCountExecutionId) { + elasticsearchCountExecutionId = ""; + } + }); +} + +function applyElasticsearchSearchTotal(searchTotal: number, isExact: boolean, filter: string | undefined) { + const key = elasticsearchCountFilterKey(filter); + if (key !== elasticsearchCountKey) { + cancelElasticsearchCount(); + elasticsearchCountKey = key; + elasticsearchExactTotal = undefined; + elasticsearchPaginationLowerBound = undefined; + } + + elasticsearchPaginationLowerBound = searchTotal; + const totals = resolveElasticsearchDocumentTotals(searchTotal, isExact, elasticsearchExactTotal); + if (isExact) { + cancelElasticsearchCount(); + elasticsearchExactTotal = searchTotal; + total.value = totals.total; + totalIsExact.value = totals.totalIsExact; + paginationTotal.value = totals.paginationTotal; + return; + } + + if (elasticsearchExactTotal !== undefined) { + total.value = totals.total; + totalIsExact.value = totals.totalIsExact; + paginationTotal.value = totals.paginationTotal; + return; + } + + total.value = totals.total; + totalIsExact.value = totals.totalIsExact; + paginationTotal.value = totals.paginationTotal; + startElasticsearchExactCount(filter); +} + async function load() { if (documentLoadExecutionId.value) void api.cancelQuery(documentLoadExecutionId.value); const executionId = uuid(); @@ -696,8 +805,12 @@ async function load() { const previousSelectedId = previousSelectedIdx === null ? null : documentIdentity(documents.value[previousSelectedIdx]); try { const filter = currentDocumentFilter(); + if (documentStoreProvider.value.kind === "elasticsearch" && elasticsearchCountKey !== null && elasticsearchCountKey !== elasticsearchCountFilterKey(filter)) { + resetElasticsearchTotals(); + } const sort = currentDocumentSortJson(sortInput.value); - const result = await api.documentFindDocuments(props.connectionId, props.database, props.collection, page.value * pageSize.value, pageSize.value, filter, undefined, sort, executionId); + const skip = page.value * pageSize.value; + const result = await api.documentFindDocuments(props.connectionId, props.database, props.collection, skip, documentRequestLimit.value, filter, undefined, sort, executionId); if (documentLoadExecutionId.value !== executionId) return; const nextDocuments = documentStoreProvider.value.kind === "elasticsearch" && result.raw_documents?.length === result.documents.length @@ -722,7 +835,14 @@ async function load() { } lastGridColumns.value = [...keySet]; } - total.value = result.total; + if (documentStoreProvider.value.kind === "elasticsearch") { + applyElasticsearchSearchTotal(result.total, result.total_is_exact !== false, filter); + } else { + cancelElasticsearchCount(); + total.value = result.total; + totalIsExact.value = true; + paginationTotal.value = result.total; + } syncSelectedDocumentAfterLoad(previousSelectedIdx, previousSelectedId); } catch (e: unknown) { if (documentLoadExecutionId.value === executionId) error.value = e instanceof Error ? e.message : String(e); @@ -736,6 +856,11 @@ async function load() { } } +async function refreshDocuments() { + if (documentStoreProvider.value.kind === "elasticsearch") resetElasticsearchTotals({ preservePaginationTotal: true }); + await load(); +} + async function cancelDocumentLoad() { const executionId = documentLoadExecutionId.value; if (!executionId || documentLoadCancelling.value) return; @@ -754,20 +879,22 @@ async function cancelDocumentLoad() { function applyFilter() { page.value = 0; - load(); + if (documentStoreProvider.value.kind === "elasticsearch") resetElasticsearchTotals(); + void load(); } function paginate(offset: number, limit: number) { const normalizedLimit = normalizeResultPageSize(limit, pageSize.value); pageSize.value = normalizedLimit; - page.value = Math.floor(Math.max(0, offset) / normalizedLimit); - load(); + const requestedPage = Math.floor(Math.max(0, offset) / normalizedLimit); + page.value = clampDocumentPage(requestedPage, normalizedLimit, paginationTotal.value); + void load(); } function onSort(column: string, _columnIndex: number, direction: "asc" | "desc" | null) { sortInput.value = documentStoreProvider.value.sortInputForColumn(column, direction); page.value = 0; - load(); + void load(); } function asRecord(value: unknown): JsonRecord { @@ -1129,6 +1256,7 @@ async function saveDoc() { isNew.value = false; documentEditMode.value = "fields"; editFields.value = []; + if (kind === "elasticsearch") resetElasticsearchTotals({ preservePaginationTotal: true }); await load(); if (selectedIdx.value !== null && documents.value[selectedIdx.value]) { editJson.value = stringifyDocumentStoreValue(documents.value[selectedIdx.value], documentStoreProvider.value.kind, 2); @@ -1151,6 +1279,7 @@ async function applyDeleteDoc(idx: number) { selectedIdx.value = null; editJson.value = ""; } + if (documentStoreProvider.value.kind === "elasticsearch") resetElasticsearchTotals({ preservePaginationTotal: true }); await load(); } catch (e: unknown) { error.value = e instanceof Error ? e.message : String(e); @@ -1176,13 +1305,13 @@ async function confirmDelete() { function prevPage() { if (page.value <= 0) return; page.value--; - load(); + void load(); } function nextPage() { - if ((page.value + 1) * pageSize.value >= total.value) return; + if ((page.value + 1) * pageSize.value >= pageTotal.value) return; page.value++; - load(); + void load(); } function docPreview(doc: JsonRecord): string { @@ -1274,6 +1403,7 @@ onMounted(async () => { onBeforeUnmount(() => { window.removeEventListener("pointerdown", handleDocumentBrowserPointerDown, true); if (documentLoadExecutionId.value) void api.cancelQuery(documentLoadExecutionId.value); + cancelElasticsearchCount(); stopDocumentLoadingTimer(); endTableSearchSplitResize(); }); @@ -1339,14 +1469,14 @@ defineExpose({ focusSearch }); {{ documentStoreLabels.documentsLabel }} - +
- {{ page + 1 }} / {{ Math.max(1, Math.ceil(total / pageSize)) }} -
@@ -1446,8 +1576,10 @@ defineExpose({ focusSearch }); :page-offset="page * pageSize" :page-limit="pageSize" :total-row-count="total" + :total-row-count-is-exact="totalIsExact" + :pagination-total-row-count="pageTotal" @sort="onSort" - @reload="load" + @reload="refreshDocuments" @paginate="(offset: number, limit: number) => paginate(offset, limit)" >