fix(mongodb): preserve int64 document filters

This commit is contained in:
t8y2 2026-07-03 17:39:48 +08:00
parent 5c79c26954
commit 55aa0f8624
6 changed files with 182 additions and 42 deletions

View File

@ -124,6 +124,18 @@ class MongoAgentTest {
assertEquals(null, MongoAgent.documentOrNull(params, "sort"));
}
@Test
void documentParametersParseExtendedJsonLongFilters() {
JsonObject params = new JsonObject();
params.addProperty("filter", "{\"processInfoId\":{\"$numberLong\":\"2048938405781032962\"},\"snowflake\":{\"$numberLong\":\"9007199254740993\"}}");
Document filter = MongoAgent.documentOrNull(params, "filter");
assertNotNull(filter);
assertEquals(2_048_938_405_781_032_962L, filter.get("processInfoId"));
assertEquals(9_007_199_254_740_993L, filter.get("snowflake"));
}
@Test
void serverVersionMethodIsRecognizedOverJsonRpc() {
String response = MongoAgent.handleRequest(

View File

@ -210,7 +210,7 @@ function resetDocumentFilterBuilder() {
}
function currentDocumentFilter(): string | undefined {
return currentDocumentFilterJson(filterInput.value, appliedDocumentFilter.value);
return currentDocumentFilterJson(filterInput.value, appliedDocumentFilter.value, documentStoreProvider.value.kind);
}
const documentQueryPreview = computed(() => {
@ -230,7 +230,12 @@ const documentQueryPreview = computed(() => {
});
async function applyDocumentStructuredFilters() {
const items = documentFilterRules.value.map((rule) => ({ rule, condition: buildDocumentFilterCondition(rule) })).filter((item): item is { rule: DocumentFilterRule; condition: Record<string, unknown> } => !!item.condition);
const items = documentFilterRules.value
.map((rule) => ({
rule,
condition: buildDocumentFilterCondition(rule, { kind: documentStoreProvider.value.kind }),
}))
.filter((item): item is { rule: DocumentFilterRule; condition: Record<string, unknown> } => !!item.condition);
const structured = combineDocumentFilterConditions(
items.map((item) => item.condition),
items.map((item) => item.rule),

View File

@ -1,6 +1,7 @@
import type { ComposerTranslation } from "vue-i18n";
import type { DatabaseType } from "@/types/database";
import { quoteUnquotedObjectKeys } from "@/lib/mongoShellCommand";
import { formatMongoShellLiteral } from "@/lib/mongoDocumentValues";
export type DocumentStoreKind = "mongodb" | "elasticsearch";
export type DocumentFilterMode = "equals" | "not-equals" | "like" | "not-like" | "greater-than" | "less-than" | "is-null" | "is-not-null";
@ -48,14 +49,24 @@ const mongoDocumentProvider: DocumentStoreProvider = {
documentsLabel: ({ total, t }) => t("mongo.documents", { count: total }),
queryPreview: ({ collection, filterJson, sortJson, skip, limit }) => {
const collectionRef = `db.getCollection(${JSON.stringify(collection)})`;
const parts = [`${collectionRef}.find(${filterJson || "{}"})`];
if (sortJson?.trim()) parts.push(`.sort(${sortJson.trim()})`);
const parts = [`${collectionRef}.find(${mongoShellPreviewLiteral(filterJson || "{}")})`];
if (sortJson?.trim()) parts.push(`.sort(${mongoShellPreviewLiteral(sortJson)})`);
parts.push(`.skip(${skip}).limit(${limit})`);
return parts.join("");
},
sortInputForColumn: (column, direction) => (direction ? JSON.stringify({ [column]: direction === "asc" ? 1 : -1 }) : ""),
};
function mongoShellPreviewLiteral(json: string): string {
const trimmed = json.trim();
if (!trimmed) return "{}";
try {
return formatMongoShellLiteral(JSON.parse(trimmed));
} catch {
return trimmed;
}
}
const elasticsearchDocumentProvider: DocumentStoreProvider = {
kind: "elasticsearch",
filterInputLabel: "filter",
@ -86,19 +97,24 @@ export function documentFilterModeNeedsValue(mode: DocumentFilterMode): boolean
return mode !== "is-null" && mode !== "is-not-null";
}
export function buildDocumentFilterCondition(rule: DocumentFilterRule): Record<string, unknown> | null {
type DocumentFilterParseOptions = {
kind?: DocumentStoreKind;
};
export function buildDocumentFilterCondition(rule: DocumentFilterRule, options: DocumentFilterParseOptions = {}): Record<string, unknown> | null {
if (!rule.fieldName) return null;
if (documentFilterModeNeedsValue(rule.mode) && !rule.rawValue.trim()) return null;
const value = documentFilterModeNeedsValue(rule.mode) ? parseDocumentFilterValue(rule.rawValue) : null;
const value = documentFilterModeNeedsValue(rule.mode) ? parseDocumentFilterValue(rule.rawValue, options) : null;
const textValue = documentFilterModeNeedsValue(rule.mode) ? String(parseDocumentFilterValue(rule.rawValue)) : "";
switch (rule.mode) {
case "equals":
return { [rule.fieldName]: value };
case "not-equals":
return { [rule.fieldName]: { $ne: value } };
case "like":
return { [rule.fieldName]: { $regex: String(value), $options: "i" } };
return { [rule.fieldName]: { $regex: textValue, $options: "i" } };
case "not-like":
return { [rule.fieldName]: { $not: { $regex: String(value), $options: "i" } } };
return { [rule.fieldName]: { $not: { $regex: textValue, $options: "i" } } };
case "greater-than":
return { [rule.fieldName]: { $gt: value } };
case "less-than":
@ -121,32 +137,109 @@ export function combineDocumentFilterConditions(conditions: Record<string, unkno
}
const MAX_SAFE_BIGINT = 9007199254740991n;
const MIN_BSON_INT64 = -9223372036854775808n;
const MAX_BSON_INT64 = 9223372036854775807n;
function parseJsonPreservingLargeIntegers(json: string): unknown {
const safe = json.replace(/([\[:,\s]\s*?)(-?\d+)(\s*[,\]\}])/g, (_match, before, num, after) => {
try {
const n = BigInt(num);
if (n > MAX_SAFE_BIGINT || n < -MAX_SAFE_BIGINT) {
return `${before}"${num}"${after}`;
}
} catch {
/* not a valid integer */
}
return _match;
});
return JSON.parse(safe);
function parseJsonPreservingLargeIntegers(json: string, options: DocumentFilterParseOptions = {}): unknown {
return JSON.parse(rewriteUnsafeIntegerTokens(json, options));
}
export function parseDocumentFilterInput(input: string): Record<string, unknown> {
function rewriteUnsafeIntegerTokens(json: string, options: DocumentFilterParseOptions): string {
let output = "";
let i = 0;
while (i < json.length) {
const ch = json[i];
if (ch === '"') {
const start = i;
i++;
while (i < json.length) {
const current = json[i++];
if (current === "\\") {
i++;
} else if (current === '"') {
break;
}
}
output += json.slice(start, i);
continue;
}
if (ch === "-" || isDigit(ch)) {
const start = i;
let end = i;
if (json[end] === "-") end++;
if (!isDigit(json[end])) {
output += ch;
i++;
continue;
}
if (json[end] === "0") {
end++;
} else {
while (isDigit(json[end])) end++;
}
let decimalOrExponent = false;
if (json[end] === ".") {
decimalOrExponent = true;
end++;
while (isDigit(json[end])) end++;
}
if (json[end] === "e" || json[end] === "E") {
decimalOrExponent = true;
end++;
if (json[end] === "+" || json[end] === "-") end++;
while (isDigit(json[end])) end++;
}
const token = json.slice(start, end);
if (!decimalOrExponent) {
try {
const n = BigInt(token);
if (n > MAX_SAFE_BIGINT || n < -MAX_SAFE_BIGINT) {
output += unsafeIntegerReplacement(token, n, options);
i = end;
continue;
}
} catch {
/* not a valid integer */
}
}
output += token;
i = end;
continue;
}
output += ch;
i++;
}
return output;
}
function unsafeIntegerReplacement(token: string, value: bigint, options: DocumentFilterParseOptions): string {
if (options.kind === "mongodb" && value >= MIN_BSON_INT64 && value <= MAX_BSON_INT64) {
// MongoDB int64 filters must use Extended JSON so JS Number never rounds snowflake-style IDs.
return `{"$numberLong":${JSON.stringify(token)}}`;
}
return JSON.stringify(token);
}
function isDigit(value: string | undefined): boolean {
return value !== undefined && value >= "0" && value <= "9";
}
export function parseDocumentFilterInput(input: string, options: DocumentFilterParseOptions = {}): Record<string, unknown> {
const trimmed = input.trim();
if (!trimmed) return {};
const safe = quoteUnquotedObjectKeys(trimmed);
const parsed = parseJsonPreservingLargeIntegers(safe);
const parsed = parseJsonPreservingLargeIntegers(safe, options);
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : {};
}
export function currentDocumentFilterJson(input: string, structured: Record<string, unknown> | null): string | undefined {
const manual = parseDocumentFilterInput(input);
export function currentDocumentFilterJson(input: string, structured: Record<string, unknown> | null, kind?: DocumentStoreKind): string | undefined {
const manual = parseDocumentFilterInput(input, { kind });
const filter = structured ? (Object.keys(manual).length ? { $and: [manual, structured] } : structured) : manual;
return Object.keys(filter).length ? JSON.stringify(filter) : undefined;
}
@ -156,11 +249,11 @@ export function currentDocumentSortJson(input: string): string | undefined {
return Object.keys(sort).length ? JSON.stringify(sort) : undefined;
}
function parseDocumentFilterValue(raw: string): unknown {
function parseDocumentFilterValue(raw: string, options: DocumentFilterParseOptions = {}): unknown {
const trimmed = raw.trim();
if (!trimmed) return "";
try {
return parseJsonPreservingLargeIntegers(trimmed);
return parseJsonPreservingLargeIntegers(trimmed, options);
} catch {
return trimmed;
}

View File

@ -98,6 +98,9 @@ export function formatMongoShellLiteral(value: unknown): string {
if (keys.length === 1 && typeof object.$oid === "string" && MONGO_OBJECT_ID_PATTERN.test(object.$oid)) {
return `ObjectId(${JSON.stringify(object.$oid)})`;
}
if (keys.length === 1 && typeof object.$numberLong === "string") {
return `NumberLong(${JSON.stringify(object.$numberLong)})`;
}
return `{${keys.map((key) => `${JSON.stringify(key)}:${formatMongoShellLiteral(object[key])}`).join(",")}}`;
}
return JSON.stringify(String(value));

View File

@ -26,6 +26,7 @@ test("providers build store-specific query previews", () => {
assert.equal(mongo.documentsLabel({ total: 7, t }), "mongo.documents:7");
assert.equal(mongo.queryPreview({ collection: "orders", filterJson: '{"city":"长治"}', sortJson: '{"createdAt":-1}', skip: 20, limit: 10 }), 'db.getCollection("orders").find({"city":"长治"}).sort({"createdAt":-1}).skip(20).limit(10)');
assert.equal(mongo.queryPreview({ collection: "order-events", filterJson: '{"city":"长治"}', sortJson: undefined, skip: 0, limit: 100 }), 'db.getCollection("order-events").find({"city":"长治"}).skip(0).limit(100)');
assert.equal(mongo.queryPreview({ collection: "orders", filterJson: '{"snowflake":{"$numberLong":"9007199254740993"}}', sortJson: undefined, skip: 0, limit: 100 }), 'db.getCollection("orders").find({"snowflake":NumberLong("9007199254740993")}).skip(0).limit(100)');
assert.equal(elasticsearch.documentsLabel({ total: 7, t }), "Documents");
assert.equal(elasticsearch.filterInputLabel, "filter");
assert.equal(
@ -42,6 +43,34 @@ test("builds reusable document filter conditions", () => {
assert.deepEqual(buildDocumentFilterCondition(rule({ mode: "is-not-null", rawValue: "" })), { city: { $ne: null } });
});
test("preserves MongoDB int64 document filter values", () => {
const id = "2048938405781032962";
const firstUnsafeInteger = "9007199254740993";
assert.deepEqual(buildDocumentFilterCondition(rule({ fieldName: "processInfoId", rawValue: id }), { kind: "mongodb" }), {
processInfoId: { $numberLong: id },
});
assert.deepEqual(buildDocumentFilterCondition(rule({ fieldName: "snowflake", rawValue: firstUnsafeInteger }), { kind: "mongodb" }), {
snowflake: { $numberLong: firstUnsafeInteger },
});
assert.deepEqual(buildDocumentFilterCondition(rule({ fieldName: "processInfoId", mode: "greater-than", rawValue: id }), { kind: "mongodb" }), {
processInfoId: { $gt: { $numberLong: id } },
});
assert.deepEqual(buildDocumentFilterCondition(rule({ fieldName: "processInfoId", mode: "like", rawValue: id }), { kind: "mongodb" }), {
processInfoId: { $regex: id, $options: "i" },
});
assert.deepEqual(buildDocumentFilterCondition(rule({ fieldName: "processInfoId", rawValue: `"${id}"` }), { kind: "mongodb" }), {
processInfoId: id,
});
assert.equal(currentDocumentFilterJson(`{processInfoId:${id}}`, null, "mongodb"), JSON.stringify({ processInfoId: { $numberLong: id } }));
assert.equal(currentDocumentFilterJson("", { processInfoId: { $numberLong: id } }, "mongodb"), JSON.stringify({ processInfoId: { $numberLong: id } }));
});
test("keeps unsafe standalone document filter integers precise outside MongoDB", () => {
assert.deepEqual(buildDocumentFilterCondition(rule({ fieldName: "processInfoId", rawValue: "2048938405781032962" })), {
processInfoId: "2048938405781032962",
});
});
test("combines manual and structured document filters", () => {
const structured = combineDocumentFilterConditions([{ city: "长治" }, { status: "active" }], [rule({}), rule({ fieldName: "status", rawValue: "active", conjunction: "OR" })]);

View File

@ -43,23 +43,17 @@ test("builds Mongo inserts with parsed date values", () => {
});
test("builds Mongo copy inserts with ObjectId and parsed document values", () => {
assert.deepEqual(
buildMongoCopyInsertDocument(
["6743e4bfa3f6f84bc3fff6c8", "577", '{"endingBalance":{"beginningBalance":"0"},"Line":[]}', 'ISODate("2024-11-25T02:45:36.184Z")'],
["_id", "accountId", "data", "lastUpdatedDate"],
),
{
_id: { $oid: "6743e4bfa3f6f84bc3fff6c8" },
accountId: 577,
data: {
endingBalance: {
beginningBalance: "0",
},
Line: [],
assert.deepEqual(buildMongoCopyInsertDocument(["6743e4bfa3f6f84bc3fff6c8", "577", '{"endingBalance":{"beginningBalance":"0"},"Line":[]}', 'ISODate("2024-11-25T02:45:36.184Z")'], ["_id", "accountId", "data", "lastUpdatedDate"]), {
_id: { $oid: "6743e4bfa3f6f84bc3fff6c8" },
accountId: 577,
data: {
endingBalance: {
beginningBalance: "0",
},
lastUpdatedDate: { $date: "2024-11-25T02:45:36.184Z" },
Line: [],
},
);
lastUpdatedDate: { $date: "2024-11-25T02:45:36.184Z" },
});
});
test("builds Mongo copy inserts without primary keys when requested", () => {
@ -82,3 +76,7 @@ test("formats extended JSON dates as Mongo shell ISODate literals", () => {
test("formats extended JSON object ids as Mongo shell ObjectId literals", () => {
assert.equal(formatMongoShellLiteral({ $oid: "6743e4bfa3f6f84bc3fff6c8" }), 'ObjectId("6743e4bfa3f6f84bc3fff6c8")');
});
test("formats extended JSON int64 values as Mongo shell NumberLong literals", () => {
assert.equal(formatMongoShellLiteral({ snowflake: { $numberLong: "9007199254740993" } }), '{"snowflake":NumberLong("9007199254740993")}');
});