diff --git a/package.json b/package.json index 5b583493..251ba5d8 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "emoji-mart": "5.5.2", "fast-average-color": "9.4.0", "fast-deep-equal": "3.1.3", + "gpt-tokenizer": "^2.1.2", "hast-util-from-html": "2.0.1", "hast-util-to-html": "9.0.0", "i18next-resources-to-backend": "1.2.0", @@ -157,6 +158,7 @@ "uuid": "9.0.1", "viem": "1.19.9", "wagmi": "1.4.7", + "xregexp": "^5.1.1", "zustand": "4.4.6" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bcd5e432..9769c7de 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -152,6 +152,9 @@ dependencies: fast-deep-equal: specifier: 3.1.3 version: 3.1.3 + gpt-tokenizer: + specifier: ^2.1.2 + version: 2.1.2 hast-util-from-html: specifier: 2.0.1 version: 2.0.1 @@ -386,6 +389,9 @@ dependencies: wagmi: specifier: 1.4.7 version: 1.4.7(@types/react@18.2.38)(immer@10.0.3)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.2)(viem@1.19.9)(zod@3.22.4) + xregexp: + specifier: ^5.1.1 + version: 5.1.1 zustand: specifier: 4.4.6 version: 4.4.6(@types/react@18.2.38)(immer@10.0.3)(react@18.2.0) @@ -802,15 +808,16 @@ packages: dependencies: '@babel/types': 7.23.4 - /@babel/runtime@7.23.1: - resolution: {integrity: sha512-hC2v6p8ZSI/W0HUzh3V8C5g+NwSKzKPtJwSpTjwl0o297GP9+ZLQSkdvHz46CM3LqyoXxq+5G9komY+eSqSO0g==} + /@babel/runtime-corejs3@7.23.4: + resolution: {integrity: sha512-zQyB4MJGM+rvd4pM58n26kf3xbiitw9MHzL8oLiBMKb8MCtVDfV5nDzzJWWzLMtbvKI9wN6XwJYl479qF4JluQ==} engines: {node: '>=6.9.0'} dependencies: + core-js-pure: 3.33.3 regenerator-runtime: 0.14.0 dev: false - /@babel/runtime@7.23.2: - resolution: {integrity: sha512-mM8eg4yl5D6i3lu2QKPuPH4FArvJ8KhTofbE7jwMUv9KX5mBvwPAqnV3MlyBNqdp9RyRKP6Yck8TrfYrPvX3bg==} + /@babel/runtime@7.23.4: + resolution: {integrity: sha512-2Yv65nlWnWlSpe3fXEyX5i7fx5kIKo4Qbcj+hMO0odwaneFjfXw5fdum+4yL20O0QiaHpia0cYQ9xpNMqrBwHg==} engines: {node: '>=6.9.0'} dependencies: regenerator-runtime: 0.14.0 @@ -923,12 +930,12 @@ packages: /@codemirror/lang-angular@0.1.2: resolution: {integrity: sha512-Nq7lmx9SU+JyoaRcs6SaJs7uAmW2W06HpgJVQYeZptVGNWDzDvzhjwVb/ZuG1rwTlOocY4Y9GwNOBuKCeJbKtw==} dependencies: - '@codemirror/lang-html': 6.4.6 + '@codemirror/lang-html': 6.4.7 '@codemirror/lang-javascript': 6.2.1 '@codemirror/language': 6.9.3 '@lezer/common': 1.1.1 '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 + '@lezer/lr': 1.3.14 dev: false /@codemirror/lang-cpp@6.0.2: @@ -945,25 +952,11 @@ packages: '@codemirror/language': 6.9.3 '@codemirror/state': 6.3.2 '@lezer/common': 1.1.1 - '@lezer/css': 1.1.3 + '@lezer/css': 1.1.4 transitivePeerDependencies: - '@codemirror/view' dev: false - /@codemirror/lang-html@6.4.6: - resolution: {integrity: sha512-E4C8CVupBksXvgLSme/zv31x91g06eZHSph7NczVxZW+/K+3XgJGWNT//2WLzaKSBoxpAjaOi5ZnPU1SHhjh3A==} - dependencies: - '@codemirror/autocomplete': 6.11.1(@codemirror/language@6.9.3)(@codemirror/state@6.3.2)(@codemirror/view@6.22.1)(@lezer/common@1.1.1) - '@codemirror/lang-css': 6.2.1(@codemirror/view@6.22.1) - '@codemirror/lang-javascript': 6.2.1 - '@codemirror/language': 6.9.3 - '@codemirror/state': 6.3.2 - '@codemirror/view': 6.22.1 - '@lezer/common': 1.1.1 - '@lezer/css': 1.1.3 - '@lezer/html': 1.3.6 - dev: false - /@codemirror/lang-html@6.4.7: resolution: {integrity: sha512-y9hWSSO41XlcL4uYwWyk0lEgTHcelWWfRuqmvcAmxfCs0HNWZdriWo/EU43S63SxEZpc1Hd50Itw7ktfQvfkUg==} dependencies: @@ -1010,7 +1003,7 @@ packages: '@codemirror/lang-css': 6.2.1(@codemirror/view@6.22.1) '@codemirror/language': 6.9.3 '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 + '@lezer/lr': 1.3.14 transitivePeerDependencies: - '@codemirror/view' dev: false @@ -1030,7 +1023,7 @@ packages: /@codemirror/lang-php@6.0.1: resolution: {integrity: sha512-ublojMdw/PNWa7qdN5TMsjmqkNuTBD3k6ndZ4Z0S25SBAiweFGyY68AS3xNcIOlb6DDFDvKlinLQ40vSLqf8xA==} dependencies: - '@codemirror/lang-html': 6.4.6 + '@codemirror/lang-html': 6.4.7 '@codemirror/language': 6.9.3 '@codemirror/state': 6.3.2 '@lezer/common': 1.1.1 @@ -1075,7 +1068,7 @@ packages: '@codemirror/language': 6.9.3 '@codemirror/state': 6.3.2 '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 + '@lezer/lr': 1.3.14 transitivePeerDependencies: - '@codemirror/view' - '@lezer/common' @@ -1084,12 +1077,12 @@ packages: /@codemirror/lang-vue@0.1.2: resolution: {integrity: sha512-D4YrefiRBAr+CfEIM4S3yvGSbYW+N69mttIfGMEf7diHpRbmygDxS+R/5xSqjgtkY6VO6qmUrre1GkRcWeZa9A==} dependencies: - '@codemirror/lang-html': 6.4.6 + '@codemirror/lang-html': 6.4.7 '@codemirror/lang-javascript': 6.2.1 '@codemirror/language': 6.9.3 '@lezer/common': 1.1.1 '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 + '@lezer/lr': 1.3.14 dev: false /@codemirror/lang-wast@6.0.1: @@ -1097,7 +1090,7 @@ packages: dependencies: '@codemirror/language': 6.9.3 '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 + '@lezer/lr': 1.3.14 dev: false /@codemirror/lang-xml@6.0.2(@codemirror/view@6.22.1): @@ -1118,7 +1111,7 @@ packages: '@codemirror/lang-angular': 0.1.2 '@codemirror/lang-cpp': 6.0.2 '@codemirror/lang-css': 6.2.1(@codemirror/view@6.22.1) - '@codemirror/lang-html': 6.4.6 + '@codemirror/lang-html': 6.4.7 '@codemirror/lang-java': 6.0.1 '@codemirror/lang-javascript': 6.2.1 '@codemirror/lang-json': 6.0.1 @@ -1756,7 +1749,7 @@ packages: resolution: {integrity: sha512-m4HEDZleaaCH+XgDDsPF15Ht6wTLsgDTeR3WYj9Q/k76JtWhrJjcP4+/XlG8LGT/Rol9qUfOIztXeA84ATpqPQ==} dependencies: '@babel/helper-module-imports': 7.22.15 - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 '@emotion/hash': 0.9.1 '@emotion/memoize': 0.8.1 '@emotion/serialize': 1.1.2 @@ -1795,7 +1788,7 @@ packages: '@types/react': optional: true dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 '@emotion/babel-plugin': 11.11.0 '@emotion/cache': 11.11.0 '@emotion/serialize': 1.1.2 @@ -2360,14 +2353,7 @@ packages: resolution: {integrity: sha512-eS1M3L3U2mDowoFVPG7tEp01SWu9/68Nx3HEBgLJVn3N9ku7g5S7WdFv0jzmcTipAyONYfZJ+7x4WRkfdB2Ung==} dependencies: '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 - dev: false - - /@lezer/css@1.1.3: - resolution: {integrity: sha512-SjSM4pkQnQdJDVc80LYzEaMiNy9txsFbI7HsMgeVF28NdLaAdHNtQ+kB/QqDUzRBV/75NTXjJ/R5IdC8QQGxMg==} - dependencies: - '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 + '@lezer/lr': 1.3.14 dev: false /@lezer/css@1.1.4: @@ -2383,14 +2369,6 @@ packages: '@lezer/common': 1.1.0 dev: false - /@lezer/html@1.3.6: - resolution: {integrity: sha512-Kk9HJARZTc0bAnMQUqbtuhFVsB4AnteR2BFUWfZV7L/x1H0aAKz6YabrfJ2gk/BEgjh9L3hg5O4y2IDZRBdzuQ==} - dependencies: - '@lezer/common': 1.1.1 - '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 - dev: false - /@lezer/html@1.3.7: resolution: {integrity: sha512-Wo+rZ5UjLP0VqUTyXjzgmTYRW5bvTJUFn4Uw0K3HCQjX2/+f+zRo9GLN5BCAojwHQISPvaQk8BWSv2SSKx/UcQ==} dependencies: @@ -2403,27 +2381,21 @@ packages: resolution: {integrity: sha512-POc53LHf2AuNeRXjqZbXNu88GKj0KZTjjSx0L7tYeXlrEHF+3NAQx+dEwKVuCbkl0ZMtpRy2VsDYOV7KKV0oyg==} dependencies: '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 + '@lezer/lr': 1.3.14 dev: false /@lezer/javascript@1.4.7: resolution: {integrity: sha512-OVWlK0YEi7HM+9JRWtRkir8qvcg0/kVYg2TAMHlVtl6DU1C9yK1waEOLBMztZsV/axRJxsqfJKhzYz+bxZme5g==} dependencies: '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 + '@lezer/lr': 1.3.14 dev: false /@lezer/json@1.0.1: resolution: {integrity: sha512-nkVC27qiEZEjySbi6gQRuMwa2sDu2PtfjSgz0A4QF81QyRGm3kb2YRzLcOPcTEtmcwvrX/cej7mlhbwViA4WJw==} dependencies: '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 - dev: false - - /@lezer/lr@1.3.12: - resolution: {integrity: sha512-5nwY1JzCueUdRtlMBnlf1SUi69iGCq2ABq7WQFQMkn/kxPvoACAEnTp4P17CtXxYr7WCwtYPLL2AEvxKPuF1OQ==} - dependencies: - '@lezer/common': 1.1.1 + '@lezer/lr': 1.3.14 dev: false /@lezer/lr@1.3.14: @@ -2443,35 +2415,35 @@ packages: resolution: {integrity: sha512-aqdCQJOXJ66De22vzdwnuC502hIaG9EnPK2rSi+ebXyUd+j7GAX1mRjWZOVOmf3GST1YUfUCu6WXDiEgDGOVwA==} dependencies: '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 + '@lezer/lr': 1.3.14 dev: false /@lezer/python@1.1.8: resolution: {integrity: sha512-1T/XsmeF57ijrjpC0Zmrf9YeO5mn2zC1XeSNrOnc0KB+6PgxJ5m7kWKt0CnwyS74oHQXbJxUUL+QDQJR26c1Gw==} dependencies: '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 + '@lezer/lr': 1.3.14 dev: false /@lezer/rust@1.0.1: resolution: {integrity: sha512-j+ToFKM6Wpglv3OQ4ebHYdYIMT2dh0ziCCV0rTf47AWiHOVhR0WjaKrBq+yuvDQNEhr5sxPxVI7+naJIgpqcsQ==} dependencies: '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 + '@lezer/lr': 1.3.14 dev: false /@lezer/sass@1.0.3: resolution: {integrity: sha512-n4l2nVOB7gWiGU/Cg2IVxpt2Ic9Hgfgy/7gk+p/XJibAsPXs0lSbsfGwQgwsAw9B/euYo3oS6lEFr9WytoqcZg==} dependencies: '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 + '@lezer/lr': 1.3.14 dev: false /@lezer/xml@1.0.2: resolution: {integrity: sha512-dlngsWceOtQBMuBPw5wtHpaxdPJ71aVntqjbpGkFtWsp4WtQmCnuTjQGocviymydN6M18fhj6UQX3oiEtSuY7w==} dependencies: '@lezer/highlight': 1.2.0 - '@lezer/lr': 1.3.12 + '@lezer/lr': 1.3.14 dev: false /@lit-labs/ssr-dom-shim@1.1.1: @@ -2884,13 +2856,13 @@ packages: /@radix-ui/number@1.0.0: resolution: {integrity: sha512-Ofwh/1HX69ZfJRiRBMTy7rgjAzHmwe4kW9C9Y99HTRUcYLUuVT0KESFj15rPjRgKJs20GPq8Bm5aEDJ8DuA3vA==} dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 dev: false /@radix-ui/primitive@1.0.0: resolution: {integrity: sha512-3e7rn8FDMin4CgeL7Z/49smCA3rFYY3Ha2rUQ7HRWFadS5iCRw08ZgVT1LaNTCNqgvrUiyczLflrVrF0SRQtNA==} dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 dev: false /@radix-ui/react-compose-refs@1.0.0(react@18.2.0): @@ -2898,7 +2870,7 @@ packages: peerDependencies: react: ^16.8 || ^17.0 || ^18.0 dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 react: 18.2.0 dev: false @@ -2907,7 +2879,7 @@ packages: peerDependencies: react: ^16.8 || ^17.0 || ^18.0 dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 react: 18.2.0 dev: false @@ -2916,7 +2888,7 @@ packages: peerDependencies: react: ^16.8 || ^17.0 || ^18.0 dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 react: 18.2.0 dev: false @@ -2926,7 +2898,7 @@ packages: react: ^16.8 || ^17.0 || ^18.0 react-dom: ^16.8 || ^17.0 || ^18.0 dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 '@radix-ui/react-compose-refs': 1.0.0(react@18.2.0) '@radix-ui/react-use-layout-effect': 1.0.0(react@18.2.0) react: 18.2.0 @@ -2939,7 +2911,7 @@ packages: react: ^16.8 || ^17.0 || ^18.0 react-dom: ^16.8 || ^17.0 || ^18.0 dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 '@radix-ui/react-slot': 1.0.1(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) @@ -2951,7 +2923,7 @@ packages: react: ^16.8 || ^17.0 || ^18.0 react-dom: ^16.8 || ^17.0 || ^18.0 dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 '@radix-ui/number': 1.0.0 '@radix-ui/primitive': 1.0.0 '@radix-ui/react-compose-refs': 1.0.0(react@18.2.0) @@ -2970,7 +2942,7 @@ packages: peerDependencies: react: ^16.8 || ^17.0 || ^18.0 dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 '@radix-ui/react-compose-refs': 1.0.0(react@18.2.0) react: 18.2.0 dev: false @@ -2980,7 +2952,7 @@ packages: peerDependencies: react: ^16.8 || ^17.0 || ^18.0 dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 react: 18.2.0 dev: false @@ -2989,7 +2961,7 @@ packages: peerDependencies: react: ^16.8 || ^17.0 || ^18.0 dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 react: 18.2.0 dev: false @@ -3072,7 +3044,7 @@ packages: /@solana/web3.js@1.78.5: resolution: {integrity: sha512-2ZHsDNqkKdglJQrIvJ3p2DmgS3cGnary3VJyqt9C1SPrpAtLYzcElr3xyXJOznyQTU/8AMw+GoF11lFoKbicKg==} dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 '@noble/curves': 1.2.0 '@noble/hashes': 1.3.2 '@solana/buffer-layout': 4.0.1 @@ -4332,7 +4304,7 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@babel/runtime': 7.23.1 + '@babel/runtime': 7.23.4 '@types/js-cookie': 2.2.7 ahooks-v3-count: 1.0.0 dayjs: 1.11.10 @@ -4661,7 +4633,7 @@ packages: resolution: {integrity: sha512-Cg7TFGpIr01vOQNODXOOaGz2NpCU5gl8x1qJFbb6hbZxR7XrcE2vtbAsTAbJ7/xwJtUuJEw8K8Zr/AE0LHlesg==} engines: {node: '>=10', npm: '>=6'} dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 cosmiconfig: 7.1.0 resolve: 1.22.6 dev: false @@ -5176,6 +5148,11 @@ packages: toggle-selection: 1.0.6 dev: false + /core-js-pure@3.33.3: + resolution: {integrity: sha512-taJ00IDOP+XYQEA2dAe4ESkmHt1fL8wzYDo3mRWQey8uO9UojlBFMneA65kMyxfYP7106c6LzWaq7/haDT6BCQ==} + requiresBuild: true + dev: false + /core-util-is@1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} dev: true @@ -5734,7 +5711,7 @@ packages: resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==} engines: {node: '>=0.11'} dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 dev: false /dayjs@1.11.10: @@ -5924,7 +5901,7 @@ packages: /dom-helpers@5.2.1: resolution: {integrity: sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==} dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 csstype: 3.1.2 dev: false @@ -6371,7 +6348,7 @@ packages: peerDependencies: eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8 dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 aria-query: 5.3.0 array-includes: 3.1.7 array.prototype.flatmap: 1.3.2 @@ -7069,6 +7046,12 @@ packages: dependencies: get-intrinsic: 1.2.1 + /gpt-tokenizer@2.1.2: + resolution: {integrity: sha512-HSuI5d6uey+c7x/VzQlPfCoGrfLyAc28vxWofKbjR9PJHm0AjQGSWkKw/OJnb+8S1g7nzgRsf0WH3dK+NNWYbg==} + dependencies: + rfc4648: 1.5.3 + dev: false + /graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} @@ -10594,7 +10577,7 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 react: 18.2.0 use-composed-ref: 1.3.0(react@18.2.0) use-latest: 1.2.1(@types/react@18.2.38)(react@18.2.0) @@ -10608,7 +10591,7 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 react: 18.2.0 use-composed-ref: 1.3.0(react@18.2.0) use-latest: 1.2.1(@types/react@18.2.38)(react@18.2.0) @@ -10622,7 +10605,7 @@ packages: react: '>=16.6.0' react-dom: '>=16.6.0' dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 dom-helpers: 5.2.1 loose-envify: 1.4.0 prop-types: 15.8.1 @@ -10636,7 +10619,7 @@ packages: react: '>=16.6.0' react-dom: '>=16.6.0' dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 dom-helpers: 5.2.1 loose-envify: 1.4.0 prop-types: 15.8.1 @@ -11044,6 +11027,10 @@ packages: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + /rfc4648@1.5.3: + resolution: {integrity: sha512-MjOWxM065+WswwnmNONOT+bD1nXzY9Km6u3kzvnx8F8/HXGZdz3T6e6vZJ8Q/RIMUSp/nxqjH3GwvJDy8ijeQQ==} + dev: false + /rfdc@1.3.0: resolution: {integrity: sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==} dev: true @@ -11062,7 +11049,7 @@ packages: /rpc-websockets@7.6.0: resolution: {integrity: sha512-Jgcs8q6t8Go98dEulww1x7RysgTkzpCMelVxZW4hvuyFtOGpeUz9prpr2KjUa/usqxgFCd9Tu3+yhHEP9GVmiQ==} dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 eventemitter3: 4.0.7 uuid: 8.3.2 ws: 8.14.2(bufferutil@4.0.8)(utf-8-validate@5.0.10) @@ -11644,7 +11631,7 @@ packages: /tailwind-merge@2.0.0: resolution: {integrity: sha512-WO8qghn9yhsldLSg80au+3/gY9E4hFxIvQ3qOmlpXnqpDKoMruKfi/56BbbMg6fHTQJ9QD3cc79PoWqlaQE4rw==} dependencies: - '@babel/runtime': 7.23.2 + '@babel/runtime': 7.23.4 dev: false /tailwind-scrollbar-hide@1.1.7: @@ -12837,6 +12824,12 @@ packages: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} dev: false + /xregexp@5.1.1: + resolution: {integrity: sha512-fKXeVorD+CzWvFs7VBuKTYIW63YD1e1osxwQ8caZ6o1jg6pDAbABDG54LCIq0j5cy7PjRvGIq6sef9DYPXpncg==} + dependencies: + '@babel/runtime-corejs3': 7.23.4 + dev: false + /xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} diff --git a/prisma/migrations/20231020093655_/migration.sql b/prisma/migrations/20231020093655_/migration.sql new file mode 100644 index 00000000..7054d795 --- /dev/null +++ b/prisma/migrations/20231020093655_/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "Metadata" ADD COLUMN "ai_translation" JSONB; \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma index d60ea06a..ec056ca2 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -27,6 +27,19 @@ model Metadata { // ... // } image_dimensions Json? + // Expected JSON structure for the translation field: + // { + // "zh": { + // "title": "string", + // "content": "string", + // }, + // "jp": { + // "title": "string", + // "content": "string", + // }, + // ... + // } + ai_translation Json? @@id([uri]) @@index([uri]) } diff --git a/src/app/[locale]/site/[site]/[slug]/page.tsx b/src/app/[locale]/site/[site]/[slug]/page.tsx index 1a141dab..6d20f044 100644 --- a/src/app/[locale]/site/[site]/[slug]/page.tsx +++ b/src/app/[locale]/site/[site]/[slug]/page.tsx @@ -16,6 +16,7 @@ import { toCid, toGateway } from "~/lib/ipfs-parser" import { isInRN } from "~/lib/is-in-rn" import { isOnlyContent } from "~/lib/is-only-content" import getQueryClient from "~/lib/query-client" +import { Language } from "~/lib/types" import { fetchGetPage, getSummary } from "~/queries/page.server" import { fetchGetSite } from "~/queries/site.server" @@ -28,7 +29,7 @@ export async function generateMetadata({ } }): Promise { const queryClient = getQueryClient() - + const locale = (await getLocale()) as Language const site = await fetchGetSite(params.site, queryClient) const page = await fetchGetPage( @@ -36,6 +37,7 @@ export async function generateMetadata({ characterId: site?.characterId, slug: params.slug, useStat: true, + translateTo: locale, }, queryClient, ) diff --git a/src/app/api/feed/route.ts b/src/app/api/feed/route.ts index 830762a0..2bbc08f6 100644 --- a/src/app/api/feed/route.ts +++ b/src/app/api/feed/route.ts @@ -16,6 +16,7 @@ export async function GET(req: Request) { useHTML: false, useImageDimensions: query.useImageDimensions === "true", topic: query.topic, + translateTo: query.translateTo, }) const res = new NextServerResponse() diff --git a/src/app/api/translate-note/route.ts b/src/app/api/translate-note/route.ts new file mode 100644 index 00000000..586a89af --- /dev/null +++ b/src/app/api/translate-note/route.ts @@ -0,0 +1,210 @@ +import AsyncLock from "async-lock" +import { LLMChain } from "langchain/chains" +import { OpenAI } from "langchain/llms/openai" +import { PromptTemplate } from "langchain/prompts" + +import { Metadata } from "@prisma/client" + +import { languageNames } from "~/i18n" +import { toGateway } from "~/lib/ipfs-parser" +import { llmModelSwitcherByTextLength } from "~/lib/llm-model-switcher-by-text-length" +import prisma from "~/lib/prisma.server" +import { cacheGet } from "~/lib/redis.server" +import { getQuery, NextServerResponse } from "~/lib/server-helper" +import { Language } from "~/lib/types" + +const lock = new AsyncLock() + +type ContentTranslation = { + title?: string + content?: string +} + +let translationModel4K: OpenAI | undefined +let translationModel16K: OpenAI | undefined +if (process.env.OPENAI_API_KEY) { + const options = { + openAIApiKey: process.env.OPENAI_API_KEY, + temperature: 0.2, + maxTokens: -1, + } + translationModel4K = new OpenAI({ ...options, modelName: "gpt-3.5-turbo" }) + translationModel16K = new OpenAI({ + ...options, + modelName: "gpt-3.5-turbo-16k", + }) +} + +type ChainKeyType = `${4 | 16}k_${Language}` // e.g. "4k_en" | "4k_zh" | "4k_zh-TW" | "4k_ja" | "16k_en" | "16k_zh" | "16k_zh-TW" | "16k_ja" +const translationChains = new Map() + +const getOriginalTranslation = async ( + cid: string, + targetLang: Language, + originalLang?: Language, +): Promise => { + if (!translationModel4K || !translationModel16K) return + + try { + const { title, content } = await ( + await fetch(toGateway(`ipfs://${cid}`)) + ).json() + + // If the detected language is the same as the target language, return the original content + if (originalLang === targetLang) { + console.warn( + `|__ Warn: Detected language is the same as the target language, return the original content: ${cid}, ${targetLang}`, + ) + return + } + + console.time(`fetching translation ${cid}, ${targetLang}`) + const { modelSize, tokens } = llmModelSwitcherByTextLength(content, { + includeResponse: { lang: targetLang }, + }) + + if (!modelSize) { + console.error( + `|__ Error: Content too long for translation: ${cid}, ${targetLang}. (Tokens: ${tokens})`, + ) + return + } + + let chain = translationChains.get(`${modelSize}_${targetLang}`) + + if (!chain) { + const prompt = new PromptTemplate({ + template: `Translate the following text into "${languageNames[targetLang]}" language: + {text} + Translation:`, + inputVariables: ["text"], + }) + + const translateModel = + modelSize === "4k" ? translationModel4K : translationModel16K + + chain = new LLMChain({ llm: translateModel, prompt }) + + translationChains.set(`${modelSize}_${targetLang}`, chain) + } + + const t = await chain.call({ text: title }) + const c = await chain.call({ text: content }) + + console.timeEnd(`fetching translation ${cid}, ${targetLang}`) + + return { + title: t.text, + content: c.text, + } + } catch (error) { + console.error(error) + console.timeEnd(`fetching translation ${cid}, ${targetLang}`) + } +} + +async function getTranslation({ + cid, + fromLang, + toLang = "en", +}: { + cid: string + fromLang?: Language + toLang?: Language +}): Promise { + if (fromLang === toLang) { + return undefined + } + + const translatedContent = (await cacheGet({ + key: ["translation", cid, toLang], + allowEmpty: true, + noUpdate: true, + noExpire: true, + getValueFun: async () => { + let result + + await lock.acquire(`translation_${cid}`, async () => { + const meta = await prisma.metadata.findFirst({ + where: { + uri: `ipfs://${cid}`, + }, + }) + + const key = "ai_translation" + const translations = meta?.[key as keyof Metadata] as Record< + string, + ContentTranslation + > + const translatedJson = translations?.[toLang] + + if (translatedJson) { + result = translatedJson + } else { + const newTranslation = await getOriginalTranslation(cid, toLang) + if (newTranslation) { + /** + * e.g. + * + * { + * "en": { + * "title": "title", + * "content": "content" + * }, + * "zh": { + * "title": "标题", + * "content": "内容" + * }, + * ... + * } + * + */ + const finalTranslation = { + ...translations, + [toLang]: newTranslation, + } + + if (meta) { + await prisma.metadata.update({ + where: { uri: `ipfs://${cid}` }, + data: { + [key as keyof Metadata]: finalTranslation, + }, + }) + } else { + await prisma.metadata.create({ + data: { + uri: `ipfs://${cid}`, + [key as keyof Metadata]: finalTranslation, + }, + }) + } + result = newTranslation + } + } + }) + + return result + }, + })) as ContentTranslation | undefined + + return translatedContent +} + +export async function GET(req: Request): Promise { + let { cid, fromLang, toLang } = getQuery(req) + + const res = new NextServerResponse() + + if (!cid) { + return res.status(400).send("Bad Request") + } + + return res.status(200).json({ + data: await getTranslation({ + cid, + fromLang, + toLang, + }), + }) +} diff --git a/src/components/home/HomeFeed.tsx b/src/components/home/HomeFeed.tsx index c72d26cc..0cff24d0 100644 --- a/src/components/home/HomeFeed.tsx +++ b/src/components/home/HomeFeed.tsx @@ -1,6 +1,6 @@ "use client" -import { useTranslations } from "next-intl" +import { useLocale, useTranslations } from "next-intl" import { useParams, useSearchParams } from "next/navigation" import { useEffect, useState } from "react" import { VirtuosoGrid } from "react-virtuoso" @@ -15,13 +15,14 @@ import { Tabs } from "~/components/ui/Tabs" import { Tooltip } from "~/components/ui/Tooltip" import { useIsMobileLayout } from "~/hooks/useMobileLayout" import { getStorage, setStorage } from "~/lib/storage" -import { ExpandedNote } from "~/lib/types" +import { ExpandedNote, Language } from "~/lib/types" import type { FeedType, SearchType } from "~/models/home.model" import { useGetFeed } from "~/queries/home" export const HomeFeed = ({ type }: { type?: FeedType }) => { const t = useTranslations() const searchParams = useSearchParams() + const locale = useLocale() as Language const currentCharacterId = useAccountState( (s) => s.computed.account?.characterId, @@ -43,6 +44,7 @@ export const HomeFeed = ({ type }: { type?: FeedType }) => { let feedConfig: Parameters[0] = { type, } + switch (type) { case "following": feedConfig = { @@ -77,7 +79,10 @@ export const HomeFeed = ({ type }: { type?: FeedType }) => { break } - const feed = useGetFeed(feedConfig) + const feed = useGetFeed({ + ...feedConfig, + translateTo: locale, + }) const hasFiltering = type === "latest" diff --git a/src/i18n.ts b/src/i18n.ts index fa246d85..38a7984a 100644 --- a/src/i18n.ts +++ b/src/i18n.ts @@ -1,6 +1,12 @@ import { getRequestConfig } from "next-intl/server" -export const locales = ["en", "ja", "zh", "zh-TW"] +export const locales = ["en", "ja", "zh", "zh-TW"] as const +export const languageNames: Record<(typeof locales)[number], string> = { + en: "English", + ja: "Japanese", + zh: "Chinese (Simplified)", + "zh-TW": "Chinese (Traditional)", +} export const defaultLocale = "en" export default getRequestConfig(async ({ locale }) => ({ diff --git a/src/lib/calculate-tokens.ts b/src/lib/calculate-tokens.ts new file mode 100644 index 00000000..62cf242d --- /dev/null +++ b/src/lib/calculate-tokens.ts @@ -0,0 +1,66 @@ +import { encode } from "gpt-tokenizer" + +export type ModelTypes = + | "gpt-3.5-turbo" + | "gpt-3.5-turbo-0613" + | "gpt-3.5-turbo-16k" + | "gpt-3.5-turbo-16k-0613" + | "gpt-3.5-turbo-0301" + | "gpt-4" + | "gpt-4-0314" + | "gpt-4-32k-0314" + | "gpt-4-0613" + | "gpt-4-32k-0613" + +export function calculateTokens( + messages: string, + model: ModelTypes = "gpt-3.5-turbo-0613", +) { + let tokens_per_message = 0 + let tokens_per_name = 0 + if ( + [ + "gpt-3.5-turbo-0613", + "gpt-3.5-turbo-16k-0613", + "gpt-4-0314", + "gpt-4-32k-0314", + "gpt-4-0613", + "gpt-4-32k-0613", + ].includes(model) + ) { + tokens_per_message = 3 + tokens_per_name = 1 + } else if (model == "gpt-3.5-turbo-0301") { + tokens_per_message = 4 + tokens_per_name = -1 + } else if (model.includes("gpt-3.5-turbo")) { + console.log( + "Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.", + ) + return calculateTokens(messages, "gpt-3.5-turbo-0613") + } else if (model.includes("gpt-4")) { + console.log( + "Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.", + ) + return calculateTokens(messages, "gpt-4-0613") + } else { + throw new Error( + `num_tokens_from_messages() is not implemented for model ${model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.`, + ) + } + let num_tokens = 0 + for (let i = 0; i < messages.length; i++) { + let message = messages[i] + num_tokens += tokens_per_message + // @ts-ignore + for (let key in message) { + let value = message[key] + num_tokens += encode(value).length + if (key == "name") { + num_tokens += tokens_per_name + } + } + } + num_tokens += 3 + return num_tokens +} diff --git a/src/lib/detect-lang.ts b/src/lib/detect-lang.ts new file mode 100644 index 00000000..d5e3c927 --- /dev/null +++ b/src/lib/detect-lang.ts @@ -0,0 +1,67 @@ +import XRegExp from "xregexp" + +import { Language } from "./types" + +type Thresholds = Partial> + +const _thresholds: Thresholds = { + en: 0.8, + zh: 0.3, + ja: 0.5, +} + +export function detectLanguage( + text: string, + thresholds: Thresholds = _thresholds, +): Language { + // Pre-process the text + text = text.replace(/[\d\s\p{P}]/gu, "") // Remove numbers, whitespace, and punctuation + + // split into words + const langs: Array = text + .trim() + .split(/\s+/) + .map((word) => { + return detect(word, thresholds) + }) + + // pick the lang with the most occurrences + const result = langs.reduce<{ + k: Record + max: string | null + }>( + (acc, el) => { + if (el) { + acc.k[el] = acc.k[el] ? acc.k[el] + 1 : 1 + acc.max = acc.max ? (acc.k[acc.max] < acc.k[el] ? el : acc.max) : el + } + return acc + }, + { k: {}, max: null }, + ) + + return result.max as Language + + function detect(text: string, thresholds: Thresholds): string | null { + const scores: Record = {} + const regexes: Record = { + en: XRegExp("\\p{Latin}", "gi"), + zh: XRegExp("\\p{Han}", "gi"), + ja: XRegExp("[\\p{Hiragana}\\p{Katakana}]", "gi"), + } + + for (const [lang, regex] of Object.entries(regexes)) { + const matches = XRegExp.match(text, regex) || [] + const score = matches.length / text.length + if (score && score >= thresholds[lang as keyof Thresholds]!) { + return lang + } + scores[lang] = score + } + + if (Object.keys(scores).length === 0) return null + + // pick lang with highest percentage + return Object.keys(scores).reduce((a, b) => (scores[a] > scores[b] ? a : b)) + } +} diff --git a/src/lib/expand-unit.ts b/src/lib/expand-unit.ts index bee795e6..a9f14e8f 100644 --- a/src/lib/expand-unit.ts +++ b/src/lib/expand-unit.ts @@ -5,8 +5,14 @@ import removeMarkdown from "remove-markdown" import { SCORE_API_DOMAIN, SITE_URL } from "~/lib/env" import { toCid, toGateway } from "~/lib/ipfs-parser" import readingTime from "~/lib/reading-time" -import { ExpandedCharacter, ExpandedNote, PortfolioStats } from "~/lib/types" +import { + ExpandedCharacter, + ExpandedNote, + Language, + PortfolioStats, +} from "~/lib/types" +import { detectLanguage } from "./detect-lang" import { getNoteSlug } from "./helpers" export const expandCrossbellNote = async ({ @@ -17,6 +23,7 @@ export const expandCrossbellNote = async ({ keyword, useHTML, disableAutofill, + translateTo, }: { note: NoteEntity & { metadata?: { @@ -31,6 +38,7 @@ export const expandCrossbellNote = async ({ keyword?: string useHTML?: boolean disableAutofill?: boolean + translateTo?: Language }) => { if (note.metadata?.uri && !note.metadata?.content) { note.metadata.content = await ( @@ -51,12 +59,45 @@ export const expandCrossbellNote = async ({ let rendered if (expandedNote.metadata?.content?.content) { const { renderPageContent } = await import("~/markdown") - rendered = renderPageContent(expandedNote.metadata.content.content, true) + + if (translateTo) { + try { + const processedContent = removeMarkdown( + expandedNote.metadata.content.content.replace(/```[^]+?```/g, ""), + ) + const detectedLang = detectLanguage(processedContent) + + const translation = await ( + await fetch( + `${SITE_URL}/api/translate-note?` + + new URLSearchParams({ + cid: toCid(expandedNote.metadata.uri || ""), + fromLang: detectedLang, + toLang: translateTo, + } as any), + ) + ).json() + + if (translation.data.content) { + expandedNote.metadata.content.content = translation.data + .content as string + } + + if (translation.data.title) { + expandedNote.metadata.content.title = translation.data + .title as string + } + } catch (e) { + // do nothing + } + } + + rendered = renderPageContent(expandedNote.metadata.content.content!, true) if (keyword) { const position = expandedNote.metadata.content.content .toLowerCase() .indexOf(keyword.toLowerCase()) - expandedNote.metadata.content.summary = `...${expandedNote.metadata.content.content.slice( + expandedNote.metadata.content.summary = `...${expandedNote.metadata.content.content!.slice( position - 10, position + 100, )}` diff --git a/src/lib/llm-model-switcher-by-text-length.ts b/src/lib/llm-model-switcher-by-text-length.ts new file mode 100644 index 00000000..6cef8b35 --- /dev/null +++ b/src/lib/llm-model-switcher-by-text-length.ts @@ -0,0 +1,54 @@ +import { calculateTokens } from "./calculate-tokens" +import { Language } from "./types" + +function getTolerantCount(textLength: "4k" | "16k"): number { + const ratio = 0.8 + + if (textLength == "4k") { + return 4000 * ratio + } + + return 16000 +} + +// TODO: This is a temporary solution. Because we don't know the language of the original text, we can't calculate the tokens accurately. Currently, only "Chinese to other languages" is supported. +export function llmModelSwitcherByTextLength( + text: string, + options: { + // If true, include the response in the text length calculation with the specified language + includeResponse?: { lang: Language } + } = {}, +): { + modelSize: "4k" | "16k" | undefined + tokens: number +} { + const { includeResponse } = options + const tokens = calculateTokens(text) + + const langRatioMapping: { [key in Language]: number } = { + en: 1.6, + ja: 1.25, + zh: 1, + "zh-TW": 1, + } + + const finalCount = + tokens / (includeResponse ? langRatioMapping[includeResponse.lang] : 1) + + if (finalCount < getTolerantCount("4k")) { + return { + modelSize: "4k", + tokens, + } + } else if (finalCount < getTolerantCount("16k")) { + return { + modelSize: "16k", + tokens, + } + } + + return { + modelSize: undefined, + tokens, + } +} diff --git a/src/lib/types.ts b/src/lib/types.ts index 6cf62609..739180e8 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -1,5 +1,7 @@ import type { CharacterEntity, NoteEntity } from "crossbell" +export type Language = "en" | "zh" | "zh-TW" | "ja" + export type Site = { id: string name: string diff --git a/src/models/home.model.ts b/src/models/home.model.ts index 353192de..19f0d49c 100644 --- a/src/models/home.model.ts +++ b/src/models/home.model.ts @@ -6,7 +6,7 @@ import { gql } from "@urql/core" import countCharacters from "~/lib/character-count" import { expandCrossbellNote } from "~/lib/expand-unit" -import { ExpandedNote } from "~/lib/types" +import { ExpandedNote, Language } from "~/lib/types" import { client } from "~/queries/graphql" import filter from "../../data/filter.json" @@ -37,6 +37,7 @@ export async function getFeed({ useHTML, useImageDimensions, topic, + translateTo, }: { type?: FeedType cursor?: string @@ -49,6 +50,7 @@ export async function getFeed({ useHTML?: boolean useImageDimensions?: boolean topic?: string + translateTo?: Language }) { if (type === "search" && !searchKeyword) { type = "latest" @@ -144,6 +146,7 @@ export async function getFeed({ useScore: true, useHTML, useImageDimensions, + translateTo, }), ), ) @@ -239,6 +242,7 @@ export async function getFeed({ note: page, useHTML, useImageDimensions, + translateTo, }) if (expand.toNote) { if (expand.toNote.toNote) { @@ -256,6 +260,7 @@ export async function getFeed({ note: expand.toNote, useHTML, useImageDimensions, + translateTo, }) delete expand.toNote.toNote } @@ -327,6 +332,7 @@ export async function getFeed({ useStat: true, useHTML, useImageDimensions, + translateTo, }), ), ) @@ -366,6 +372,7 @@ export async function getFeed({ note: page, useHTML, useImageDimensions, + translateTo, }), ), ) @@ -473,6 +480,7 @@ export async function getFeed({ note: page, useHTML, useImageDimensions, + translateTo, }), ), ) @@ -549,6 +557,7 @@ export async function getFeed({ note: page, useHTML, useImageDimensions, + translateTo, }), ), ) @@ -654,6 +663,7 @@ export async function getFeed({ useHTML, useStat: true, useImageDimensions, + translateTo, }) return expand }, @@ -780,6 +790,7 @@ export async function getFeed({ useHTML, useStat: true, useImageDimensions, + translateTo, }) return expand }, @@ -825,6 +836,7 @@ export async function getFeed({ keyword: searchKeyword, useHTML, useImageDimensions, + translateTo, }), ), ) @@ -863,6 +875,7 @@ export async function getFeed({ useScore: true, useHTML, useImageDimensions, + translateTo, }), ), ) diff --git a/src/models/page.model.ts b/src/models/page.model.ts index 8e561094..08043882 100644 --- a/src/models/page.model.ts +++ b/src/models/page.model.ts @@ -22,6 +22,7 @@ import { checkSlugReservedWords } from "~/lib/slug-reserved-words" import { getKeys, getStorage } from "~/lib/storage" import { ExpandedNote, + Language, NoteType, PagesSortTypes, PageVisibilityEnum, @@ -421,6 +422,7 @@ export async function getPage(input: { noteId?: number handle?: string // In order to be compatible with old drafts disableAutofill?: boolean + translateTo?: Language }) { const mustLocal = input.slug?.startsWith("!local-") && !input.noteId @@ -572,6 +574,7 @@ export async function getPage(input: { note: page, useStat: input.useStat, disableAutofill: input.disableAutofill, + translateTo: input.translateTo, }) } diff --git a/src/queries/page.server.ts b/src/queries/page.server.ts index ad60ef4f..0211c3c4 100644 --- a/src/queries/page.server.ts +++ b/src/queries/page.server.ts @@ -11,6 +11,7 @@ import { defaultLocale, locales } from "~/i18n" import { toGateway } from "~/lib/ipfs-parser" import prisma from "~/lib/prisma.server" import { cacheGet } from "~/lib/redis.server" +import { Language } from "~/lib/types" import * as pageModel from "~/models/page.model" export const fetchGetPage = async ( @@ -31,6 +32,7 @@ export const fetchGetPage = async ( useStat: input.useStat, noteId: input.noteId, handle: input.handle, + translateTo: input.translateTo, }), }) as Promise> }) @@ -147,7 +149,7 @@ export async function getSummary({ noUpdate: true, noExpire: true, getValueFun: async () => { - if (locales.includes(lang)) { + if (locales.includes(lang as Language)) { let result await lock.acquire(cid, async () => { const meta = await prisma.metadata.findFirst({