fix(mongodb): preserve int64 document filters
This commit is contained in:
parent
5c79c26954
commit
55aa0f8624
|
|
@ -124,6 +124,18 @@ class MongoAgentTest {
|
|||
assertEquals(null, MongoAgent.documentOrNull(params, "sort"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void documentParametersParseExtendedJsonLongFilters() {
|
||||
JsonObject params = new JsonObject();
|
||||
params.addProperty("filter", "{\"processInfoId\":{\"$numberLong\":\"2048938405781032962\"},\"snowflake\":{\"$numberLong\":\"9007199254740993\"}}");
|
||||
|
||||
Document filter = MongoAgent.documentOrNull(params, "filter");
|
||||
|
||||
assertNotNull(filter);
|
||||
assertEquals(2_048_938_405_781_032_962L, filter.get("processInfoId"));
|
||||
assertEquals(9_007_199_254_740_993L, filter.get("snowflake"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void serverVersionMethodIsRecognizedOverJsonRpc() {
|
||||
String response = MongoAgent.handleRequest(
|
||||
|
|
|
|||
|
|
@ -210,7 +210,7 @@ function resetDocumentFilterBuilder() {
|
|||
}
|
||||
|
||||
function currentDocumentFilter(): string | undefined {
|
||||
return currentDocumentFilterJson(filterInput.value, appliedDocumentFilter.value);
|
||||
return currentDocumentFilterJson(filterInput.value, appliedDocumentFilter.value, documentStoreProvider.value.kind);
|
||||
}
|
||||
|
||||
const documentQueryPreview = computed(() => {
|
||||
|
|
@ -230,7 +230,12 @@ const documentQueryPreview = computed(() => {
|
|||
});
|
||||
|
||||
async function applyDocumentStructuredFilters() {
|
||||
const items = documentFilterRules.value.map((rule) => ({ rule, condition: buildDocumentFilterCondition(rule) })).filter((item): item is { rule: DocumentFilterRule; condition: Record<string, unknown> } => !!item.condition);
|
||||
const items = documentFilterRules.value
|
||||
.map((rule) => ({
|
||||
rule,
|
||||
condition: buildDocumentFilterCondition(rule, { kind: documentStoreProvider.value.kind }),
|
||||
}))
|
||||
.filter((item): item is { rule: DocumentFilterRule; condition: Record<string, unknown> } => !!item.condition);
|
||||
const structured = combineDocumentFilterConditions(
|
||||
items.map((item) => item.condition),
|
||||
items.map((item) => item.rule),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import type { ComposerTranslation } from "vue-i18n";
|
||||
import type { DatabaseType } from "@/types/database";
|
||||
import { quoteUnquotedObjectKeys } from "@/lib/mongoShellCommand";
|
||||
import { formatMongoShellLiteral } from "@/lib/mongoDocumentValues";
|
||||
|
||||
export type DocumentStoreKind = "mongodb" | "elasticsearch";
|
||||
export type DocumentFilterMode = "equals" | "not-equals" | "like" | "not-like" | "greater-than" | "less-than" | "is-null" | "is-not-null";
|
||||
|
|
@ -48,14 +49,24 @@ const mongoDocumentProvider: DocumentStoreProvider = {
|
|||
documentsLabel: ({ total, t }) => t("mongo.documents", { count: total }),
|
||||
queryPreview: ({ collection, filterJson, sortJson, skip, limit }) => {
|
||||
const collectionRef = `db.getCollection(${JSON.stringify(collection)})`;
|
||||
const parts = [`${collectionRef}.find(${filterJson || "{}"})`];
|
||||
if (sortJson?.trim()) parts.push(`.sort(${sortJson.trim()})`);
|
||||
const parts = [`${collectionRef}.find(${mongoShellPreviewLiteral(filterJson || "{}")})`];
|
||||
if (sortJson?.trim()) parts.push(`.sort(${mongoShellPreviewLiteral(sortJson)})`);
|
||||
parts.push(`.skip(${skip}).limit(${limit})`);
|
||||
return parts.join("");
|
||||
},
|
||||
sortInputForColumn: (column, direction) => (direction ? JSON.stringify({ [column]: direction === "asc" ? 1 : -1 }) : ""),
|
||||
};
|
||||
|
||||
function mongoShellPreviewLiteral(json: string): string {
|
||||
const trimmed = json.trim();
|
||||
if (!trimmed) return "{}";
|
||||
try {
|
||||
return formatMongoShellLiteral(JSON.parse(trimmed));
|
||||
} catch {
|
||||
return trimmed;
|
||||
}
|
||||
}
|
||||
|
||||
const elasticsearchDocumentProvider: DocumentStoreProvider = {
|
||||
kind: "elasticsearch",
|
||||
filterInputLabel: "filter",
|
||||
|
|
@ -86,19 +97,24 @@ export function documentFilterModeNeedsValue(mode: DocumentFilterMode): boolean
|
|||
return mode !== "is-null" && mode !== "is-not-null";
|
||||
}
|
||||
|
||||
export function buildDocumentFilterCondition(rule: DocumentFilterRule): Record<string, unknown> | null {
|
||||
type DocumentFilterParseOptions = {
|
||||
kind?: DocumentStoreKind;
|
||||
};
|
||||
|
||||
export function buildDocumentFilterCondition(rule: DocumentFilterRule, options: DocumentFilterParseOptions = {}): Record<string, unknown> | null {
|
||||
if (!rule.fieldName) return null;
|
||||
if (documentFilterModeNeedsValue(rule.mode) && !rule.rawValue.trim()) return null;
|
||||
const value = documentFilterModeNeedsValue(rule.mode) ? parseDocumentFilterValue(rule.rawValue) : null;
|
||||
const value = documentFilterModeNeedsValue(rule.mode) ? parseDocumentFilterValue(rule.rawValue, options) : null;
|
||||
const textValue = documentFilterModeNeedsValue(rule.mode) ? String(parseDocumentFilterValue(rule.rawValue)) : "";
|
||||
switch (rule.mode) {
|
||||
case "equals":
|
||||
return { [rule.fieldName]: value };
|
||||
case "not-equals":
|
||||
return { [rule.fieldName]: { $ne: value } };
|
||||
case "like":
|
||||
return { [rule.fieldName]: { $regex: String(value), $options: "i" } };
|
||||
return { [rule.fieldName]: { $regex: textValue, $options: "i" } };
|
||||
case "not-like":
|
||||
return { [rule.fieldName]: { $not: { $regex: String(value), $options: "i" } } };
|
||||
return { [rule.fieldName]: { $not: { $regex: textValue, $options: "i" } } };
|
||||
case "greater-than":
|
||||
return { [rule.fieldName]: { $gt: value } };
|
||||
case "less-than":
|
||||
|
|
@ -121,32 +137,109 @@ export function combineDocumentFilterConditions(conditions: Record<string, unkno
|
|||
}
|
||||
|
||||
const MAX_SAFE_BIGINT = 9007199254740991n;
|
||||
const MIN_BSON_INT64 = -9223372036854775808n;
|
||||
const MAX_BSON_INT64 = 9223372036854775807n;
|
||||
|
||||
function parseJsonPreservingLargeIntegers(json: string): unknown {
|
||||
const safe = json.replace(/([\[:,\s]\s*?)(-?\d+)(\s*[,\]\}])/g, (_match, before, num, after) => {
|
||||
try {
|
||||
const n = BigInt(num);
|
||||
if (n > MAX_SAFE_BIGINT || n < -MAX_SAFE_BIGINT) {
|
||||
return `${before}"${num}"${after}`;
|
||||
}
|
||||
} catch {
|
||||
/* not a valid integer */
|
||||
}
|
||||
return _match;
|
||||
});
|
||||
return JSON.parse(safe);
|
||||
function parseJsonPreservingLargeIntegers(json: string, options: DocumentFilterParseOptions = {}): unknown {
|
||||
return JSON.parse(rewriteUnsafeIntegerTokens(json, options));
|
||||
}
|
||||
|
||||
export function parseDocumentFilterInput(input: string): Record<string, unknown> {
|
||||
function rewriteUnsafeIntegerTokens(json: string, options: DocumentFilterParseOptions): string {
|
||||
let output = "";
|
||||
let i = 0;
|
||||
while (i < json.length) {
|
||||
const ch = json[i];
|
||||
if (ch === '"') {
|
||||
const start = i;
|
||||
i++;
|
||||
while (i < json.length) {
|
||||
const current = json[i++];
|
||||
if (current === "\\") {
|
||||
i++;
|
||||
} else if (current === '"') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
output += json.slice(start, i);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ch === "-" || isDigit(ch)) {
|
||||
const start = i;
|
||||
let end = i;
|
||||
if (json[end] === "-") end++;
|
||||
if (!isDigit(json[end])) {
|
||||
output += ch;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (json[end] === "0") {
|
||||
end++;
|
||||
} else {
|
||||
while (isDigit(json[end])) end++;
|
||||
}
|
||||
|
||||
let decimalOrExponent = false;
|
||||
if (json[end] === ".") {
|
||||
decimalOrExponent = true;
|
||||
end++;
|
||||
while (isDigit(json[end])) end++;
|
||||
}
|
||||
if (json[end] === "e" || json[end] === "E") {
|
||||
decimalOrExponent = true;
|
||||
end++;
|
||||
if (json[end] === "+" || json[end] === "-") end++;
|
||||
while (isDigit(json[end])) end++;
|
||||
}
|
||||
|
||||
const token = json.slice(start, end);
|
||||
if (!decimalOrExponent) {
|
||||
try {
|
||||
const n = BigInt(token);
|
||||
if (n > MAX_SAFE_BIGINT || n < -MAX_SAFE_BIGINT) {
|
||||
output += unsafeIntegerReplacement(token, n, options);
|
||||
i = end;
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
/* not a valid integer */
|
||||
}
|
||||
}
|
||||
|
||||
output += token;
|
||||
i = end;
|
||||
continue;
|
||||
}
|
||||
|
||||
output += ch;
|
||||
i++;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function unsafeIntegerReplacement(token: string, value: bigint, options: DocumentFilterParseOptions): string {
|
||||
if (options.kind === "mongodb" && value >= MIN_BSON_INT64 && value <= MAX_BSON_INT64) {
|
||||
// MongoDB int64 filters must use Extended JSON so JS Number never rounds snowflake-style IDs.
|
||||
return `{"$numberLong":${JSON.stringify(token)}}`;
|
||||
}
|
||||
return JSON.stringify(token);
|
||||
}
|
||||
|
||||
function isDigit(value: string | undefined): boolean {
|
||||
return value !== undefined && value >= "0" && value <= "9";
|
||||
}
|
||||
|
||||
export function parseDocumentFilterInput(input: string, options: DocumentFilterParseOptions = {}): Record<string, unknown> {
|
||||
const trimmed = input.trim();
|
||||
if (!trimmed) return {};
|
||||
const safe = quoteUnquotedObjectKeys(trimmed);
|
||||
const parsed = parseJsonPreservingLargeIntegers(safe);
|
||||
const parsed = parseJsonPreservingLargeIntegers(safe, options);
|
||||
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : {};
|
||||
}
|
||||
|
||||
export function currentDocumentFilterJson(input: string, structured: Record<string, unknown> | null): string | undefined {
|
||||
const manual = parseDocumentFilterInput(input);
|
||||
export function currentDocumentFilterJson(input: string, structured: Record<string, unknown> | null, kind?: DocumentStoreKind): string | undefined {
|
||||
const manual = parseDocumentFilterInput(input, { kind });
|
||||
const filter = structured ? (Object.keys(manual).length ? { $and: [manual, structured] } : structured) : manual;
|
||||
return Object.keys(filter).length ? JSON.stringify(filter) : undefined;
|
||||
}
|
||||
|
|
@ -156,11 +249,11 @@ export function currentDocumentSortJson(input: string): string | undefined {
|
|||
return Object.keys(sort).length ? JSON.stringify(sort) : undefined;
|
||||
}
|
||||
|
||||
function parseDocumentFilterValue(raw: string): unknown {
|
||||
function parseDocumentFilterValue(raw: string, options: DocumentFilterParseOptions = {}): unknown {
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) return "";
|
||||
try {
|
||||
return parseJsonPreservingLargeIntegers(trimmed);
|
||||
return parseJsonPreservingLargeIntegers(trimmed, options);
|
||||
} catch {
|
||||
return trimmed;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -98,6 +98,9 @@ export function formatMongoShellLiteral(value: unknown): string {
|
|||
if (keys.length === 1 && typeof object.$oid === "string" && MONGO_OBJECT_ID_PATTERN.test(object.$oid)) {
|
||||
return `ObjectId(${JSON.stringify(object.$oid)})`;
|
||||
}
|
||||
if (keys.length === 1 && typeof object.$numberLong === "string") {
|
||||
return `NumberLong(${JSON.stringify(object.$numberLong)})`;
|
||||
}
|
||||
return `{${keys.map((key) => `${JSON.stringify(key)}:${formatMongoShellLiteral(object[key])}`).join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(String(value));
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ test("providers build store-specific query previews", () => {
|
|||
assert.equal(mongo.documentsLabel({ total: 7, t }), "mongo.documents:7");
|
||||
assert.equal(mongo.queryPreview({ collection: "orders", filterJson: '{"city":"长治"}', sortJson: '{"createdAt":-1}', skip: 20, limit: 10 }), 'db.getCollection("orders").find({"city":"长治"}).sort({"createdAt":-1}).skip(20).limit(10)');
|
||||
assert.equal(mongo.queryPreview({ collection: "order-events", filterJson: '{"city":"长治"}', sortJson: undefined, skip: 0, limit: 100 }), 'db.getCollection("order-events").find({"city":"长治"}).skip(0).limit(100)');
|
||||
assert.equal(mongo.queryPreview({ collection: "orders", filterJson: '{"snowflake":{"$numberLong":"9007199254740993"}}', sortJson: undefined, skip: 0, limit: 100 }), 'db.getCollection("orders").find({"snowflake":NumberLong("9007199254740993")}).skip(0).limit(100)');
|
||||
assert.equal(elasticsearch.documentsLabel({ total: 7, t }), "Documents");
|
||||
assert.equal(elasticsearch.filterInputLabel, "filter");
|
||||
assert.equal(
|
||||
|
|
@ -42,6 +43,34 @@ test("builds reusable document filter conditions", () => {
|
|||
assert.deepEqual(buildDocumentFilterCondition(rule({ mode: "is-not-null", rawValue: "" })), { city: { $ne: null } });
|
||||
});
|
||||
|
||||
test("preserves MongoDB int64 document filter values", () => {
|
||||
const id = "2048938405781032962";
|
||||
const firstUnsafeInteger = "9007199254740993";
|
||||
assert.deepEqual(buildDocumentFilterCondition(rule({ fieldName: "processInfoId", rawValue: id }), { kind: "mongodb" }), {
|
||||
processInfoId: { $numberLong: id },
|
||||
});
|
||||
assert.deepEqual(buildDocumentFilterCondition(rule({ fieldName: "snowflake", rawValue: firstUnsafeInteger }), { kind: "mongodb" }), {
|
||||
snowflake: { $numberLong: firstUnsafeInteger },
|
||||
});
|
||||
assert.deepEqual(buildDocumentFilterCondition(rule({ fieldName: "processInfoId", mode: "greater-than", rawValue: id }), { kind: "mongodb" }), {
|
||||
processInfoId: { $gt: { $numberLong: id } },
|
||||
});
|
||||
assert.deepEqual(buildDocumentFilterCondition(rule({ fieldName: "processInfoId", mode: "like", rawValue: id }), { kind: "mongodb" }), {
|
||||
processInfoId: { $regex: id, $options: "i" },
|
||||
});
|
||||
assert.deepEqual(buildDocumentFilterCondition(rule({ fieldName: "processInfoId", rawValue: `"${id}"` }), { kind: "mongodb" }), {
|
||||
processInfoId: id,
|
||||
});
|
||||
assert.equal(currentDocumentFilterJson(`{processInfoId:${id}}`, null, "mongodb"), JSON.stringify({ processInfoId: { $numberLong: id } }));
|
||||
assert.equal(currentDocumentFilterJson("", { processInfoId: { $numberLong: id } }, "mongodb"), JSON.stringify({ processInfoId: { $numberLong: id } }));
|
||||
});
|
||||
|
||||
test("keeps unsafe standalone document filter integers precise outside MongoDB", () => {
|
||||
assert.deepEqual(buildDocumentFilterCondition(rule({ fieldName: "processInfoId", rawValue: "2048938405781032962" })), {
|
||||
processInfoId: "2048938405781032962",
|
||||
});
|
||||
});
|
||||
|
||||
test("combines manual and structured document filters", () => {
|
||||
const structured = combineDocumentFilterConditions([{ city: "长治" }, { status: "active" }], [rule({}), rule({ fieldName: "status", rawValue: "active", conjunction: "OR" })]);
|
||||
|
||||
|
|
|
|||
|
|
@ -43,23 +43,17 @@ test("builds Mongo inserts with parsed date values", () => {
|
|||
});
|
||||
|
||||
test("builds Mongo copy inserts with ObjectId and parsed document values", () => {
|
||||
assert.deepEqual(
|
||||
buildMongoCopyInsertDocument(
|
||||
["6743e4bfa3f6f84bc3fff6c8", "577", '{"endingBalance":{"beginningBalance":"0"},"Line":[]}', 'ISODate("2024-11-25T02:45:36.184Z")'],
|
||||
["_id", "accountId", "data", "lastUpdatedDate"],
|
||||
),
|
||||
{
|
||||
_id: { $oid: "6743e4bfa3f6f84bc3fff6c8" },
|
||||
accountId: 577,
|
||||
data: {
|
||||
endingBalance: {
|
||||
beginningBalance: "0",
|
||||
},
|
||||
Line: [],
|
||||
assert.deepEqual(buildMongoCopyInsertDocument(["6743e4bfa3f6f84bc3fff6c8", "577", '{"endingBalance":{"beginningBalance":"0"},"Line":[]}', 'ISODate("2024-11-25T02:45:36.184Z")'], ["_id", "accountId", "data", "lastUpdatedDate"]), {
|
||||
_id: { $oid: "6743e4bfa3f6f84bc3fff6c8" },
|
||||
accountId: 577,
|
||||
data: {
|
||||
endingBalance: {
|
||||
beginningBalance: "0",
|
||||
},
|
||||
lastUpdatedDate: { $date: "2024-11-25T02:45:36.184Z" },
|
||||
Line: [],
|
||||
},
|
||||
);
|
||||
lastUpdatedDate: { $date: "2024-11-25T02:45:36.184Z" },
|
||||
});
|
||||
});
|
||||
|
||||
test("builds Mongo copy inserts without primary keys when requested", () => {
|
||||
|
|
@ -82,3 +76,7 @@ test("formats extended JSON dates as Mongo shell ISODate literals", () => {
|
|||
test("formats extended JSON object ids as Mongo shell ObjectId literals", () => {
|
||||
assert.equal(formatMongoShellLiteral({ $oid: "6743e4bfa3f6f84bc3fff6c8" }), 'ObjectId("6743e4bfa3f6f84bc3fff6c8")');
|
||||
});
|
||||
|
||||
test("formats extended JSON int64 values as Mongo shell NumberLong literals", () => {
|
||||
assert.equal(formatMongoShellLiteral({ snowflake: { $numberLong: "9007199254740993" } }), '{"snowflake":NumberLong("9007199254740993")}');
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue