Merge branch 'dev' into feat/add_image_dimension_field
This commit is contained in:
Caspian
commit
25f1c6ea73
|
|
@ -79,6 +79,7 @@
|
|||
"emoji-mart": "5.5.2",
|
||||
"fast-average-color": "9.4.0",
|
||||
"fast-deep-equal": "3.1.3",
|
||||
"gpt-tokenizer": "^2.1.2",
|
||||
"hast-util-from-html": "2.0.1",
|
||||
"hast-util-to-html": "9.0.0",
|
||||
"i18next-resources-to-backend": "1.2.0",
|
||||
|
|
@ -157,6 +158,7 @@
|
|||
"uuid": "9.0.1",
|
||||
"viem": "1.19.9",
|
||||
"wagmi": "1.4.7",
|
||||
"xregexp": "^5.1.1",
|
||||
"zustand": "4.4.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
|
|||
159
pnpm-lock.yaml
159
pnpm-lock.yaml
|
|
@ -152,6 +152,9 @@ dependencies:
|
|||
fast-deep-equal:
|
||||
specifier: 3.1.3
|
||||
version: 3.1.3
|
||||
gpt-tokenizer:
|
||||
specifier: ^2.1.2
|
||||
version: 2.1.2
|
||||
hast-util-from-html:
|
||||
specifier: 2.0.1
|
||||
version: 2.0.1
|
||||
|
|
@ -386,6 +389,9 @@ dependencies:
|
|||
wagmi:
|
||||
specifier: 1.4.7
|
||||
version: 1.4.7(@types/react@18.2.38)(immer@10.0.3)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.2)(viem@1.19.9)(zod@3.22.4)
|
||||
xregexp:
|
||||
specifier: ^5.1.1
|
||||
version: 5.1.1
|
||||
zustand:
|
||||
specifier: 4.4.6
|
||||
version: 4.4.6(@types/react@18.2.38)(immer@10.0.3)(react@18.2.0)
|
||||
|
|
@ -802,15 +808,16 @@ packages:
|
|||
dependencies:
|
||||
'@babel/types': 7.23.4
|
||||
|
||||
/@babel/runtime@7.23.1:
|
||||
resolution: {integrity: sha512-hC2v6p8ZSI/W0HUzh3V8C5g+NwSKzKPtJwSpTjwl0o297GP9+ZLQSkdvHz46CM3LqyoXxq+5G9komY+eSqSO0g==}
|
||||
/@babel/runtime-corejs3@7.23.4:
|
||||
resolution: {integrity: sha512-zQyB4MJGM+rvd4pM58n26kf3xbiitw9MHzL8oLiBMKb8MCtVDfV5nDzzJWWzLMtbvKI9wN6XwJYl479qF4JluQ==}
|
||||
engines: {node: '>=6.9.0'}
|
||||
dependencies:
|
||||
core-js-pure: 3.33.3
|
||||
regenerator-runtime: 0.14.0
|
||||
dev: false
|
||||
|
||||
/@babel/runtime@7.23.2:
|
||||
resolution: {integrity: sha512-mM8eg4yl5D6i3lu2QKPuPH4FArvJ8KhTofbE7jwMUv9KX5mBvwPAqnV3MlyBNqdp9RyRKP6Yck8TrfYrPvX3bg==}
|
||||
/@babel/runtime@7.23.4:
|
||||
resolution: {integrity: sha512-2Yv65nlWnWlSpe3fXEyX5i7fx5kIKo4Qbcj+hMO0odwaneFjfXw5fdum+4yL20O0QiaHpia0cYQ9xpNMqrBwHg==}
|
||||
engines: {node: '>=6.9.0'}
|
||||
dependencies:
|
||||
regenerator-runtime: 0.14.0
|
||||
|
|
@ -923,12 +930,12 @@ packages:
|
|||
/@codemirror/lang-angular@0.1.2:
|
||||
resolution: {integrity: sha512-Nq7lmx9SU+JyoaRcs6SaJs7uAmW2W06HpgJVQYeZptVGNWDzDvzhjwVb/ZuG1rwTlOocY4Y9GwNOBuKCeJbKtw==}
|
||||
dependencies:
|
||||
'@codemirror/lang-html': 6.4.6
|
||||
'@codemirror/lang-html': 6.4.7
|
||||
'@codemirror/lang-javascript': 6.2.1
|
||||
'@codemirror/language': 6.9.3
|
||||
'@lezer/common': 1.1.1
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
'@lezer/lr': 1.3.14
|
||||
dev: false
|
||||
|
||||
/@codemirror/lang-cpp@6.0.2:
|
||||
|
|
@ -945,25 +952,11 @@ packages:
|
|||
'@codemirror/language': 6.9.3
|
||||
'@codemirror/state': 6.3.2
|
||||
'@lezer/common': 1.1.1
|
||||
'@lezer/css': 1.1.3
|
||||
'@lezer/css': 1.1.4
|
||||
transitivePeerDependencies:
|
||||
- '@codemirror/view'
|
||||
dev: false
|
||||
|
||||
/@codemirror/lang-html@6.4.6:
|
||||
resolution: {integrity: sha512-E4C8CVupBksXvgLSme/zv31x91g06eZHSph7NczVxZW+/K+3XgJGWNT//2WLzaKSBoxpAjaOi5ZnPU1SHhjh3A==}
|
||||
dependencies:
|
||||
'@codemirror/autocomplete': 6.11.1(@codemirror/language@6.9.3)(@codemirror/state@6.3.2)(@codemirror/view@6.22.1)(@lezer/common@1.1.1)
|
||||
'@codemirror/lang-css': 6.2.1(@codemirror/view@6.22.1)
|
||||
'@codemirror/lang-javascript': 6.2.1
|
||||
'@codemirror/language': 6.9.3
|
||||
'@codemirror/state': 6.3.2
|
||||
'@codemirror/view': 6.22.1
|
||||
'@lezer/common': 1.1.1
|
||||
'@lezer/css': 1.1.3
|
||||
'@lezer/html': 1.3.6
|
||||
dev: false
|
||||
|
||||
/@codemirror/lang-html@6.4.7:
|
||||
resolution: {integrity: sha512-y9hWSSO41XlcL4uYwWyk0lEgTHcelWWfRuqmvcAmxfCs0HNWZdriWo/EU43S63SxEZpc1Hd50Itw7ktfQvfkUg==}
|
||||
dependencies:
|
||||
|
|
@ -1010,7 +1003,7 @@ packages:
|
|||
'@codemirror/lang-css': 6.2.1(@codemirror/view@6.22.1)
|
||||
'@codemirror/language': 6.9.3
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
'@lezer/lr': 1.3.14
|
||||
transitivePeerDependencies:
|
||||
- '@codemirror/view'
|
||||
dev: false
|
||||
|
|
@ -1030,7 +1023,7 @@ packages:
|
|||
/@codemirror/lang-php@6.0.1:
|
||||
resolution: {integrity: sha512-ublojMdw/PNWa7qdN5TMsjmqkNuTBD3k6ndZ4Z0S25SBAiweFGyY68AS3xNcIOlb6DDFDvKlinLQ40vSLqf8xA==}
|
||||
dependencies:
|
||||
'@codemirror/lang-html': 6.4.6
|
||||
'@codemirror/lang-html': 6.4.7
|
||||
'@codemirror/language': 6.9.3
|
||||
'@codemirror/state': 6.3.2
|
||||
'@lezer/common': 1.1.1
|
||||
|
|
@ -1075,7 +1068,7 @@ packages:
|
|||
'@codemirror/language': 6.9.3
|
||||
'@codemirror/state': 6.3.2
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
'@lezer/lr': 1.3.14
|
||||
transitivePeerDependencies:
|
||||
- '@codemirror/view'
|
||||
- '@lezer/common'
|
||||
|
|
@ -1084,12 +1077,12 @@ packages:
|
|||
/@codemirror/lang-vue@0.1.2:
|
||||
resolution: {integrity: sha512-D4YrefiRBAr+CfEIM4S3yvGSbYW+N69mttIfGMEf7diHpRbmygDxS+R/5xSqjgtkY6VO6qmUrre1GkRcWeZa9A==}
|
||||
dependencies:
|
||||
'@codemirror/lang-html': 6.4.6
|
||||
'@codemirror/lang-html': 6.4.7
|
||||
'@codemirror/lang-javascript': 6.2.1
|
||||
'@codemirror/language': 6.9.3
|
||||
'@lezer/common': 1.1.1
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
'@lezer/lr': 1.3.14
|
||||
dev: false
|
||||
|
||||
/@codemirror/lang-wast@6.0.1:
|
||||
|
|
@ -1097,7 +1090,7 @@ packages:
|
|||
dependencies:
|
||||
'@codemirror/language': 6.9.3
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
'@lezer/lr': 1.3.14
|
||||
dev: false
|
||||
|
||||
/@codemirror/lang-xml@6.0.2(@codemirror/view@6.22.1):
|
||||
|
|
@ -1118,7 +1111,7 @@ packages:
|
|||
'@codemirror/lang-angular': 0.1.2
|
||||
'@codemirror/lang-cpp': 6.0.2
|
||||
'@codemirror/lang-css': 6.2.1(@codemirror/view@6.22.1)
|
||||
'@codemirror/lang-html': 6.4.6
|
||||
'@codemirror/lang-html': 6.4.7
|
||||
'@codemirror/lang-java': 6.0.1
|
||||
'@codemirror/lang-javascript': 6.2.1
|
||||
'@codemirror/lang-json': 6.0.1
|
||||
|
|
@ -1756,7 +1749,7 @@ packages:
|
|||
resolution: {integrity: sha512-m4HEDZleaaCH+XgDDsPF15Ht6wTLsgDTeR3WYj9Q/k76JtWhrJjcP4+/XlG8LGT/Rol9qUfOIztXeA84ATpqPQ==}
|
||||
dependencies:
|
||||
'@babel/helper-module-imports': 7.22.15
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
'@emotion/hash': 0.9.1
|
||||
'@emotion/memoize': 0.8.1
|
||||
'@emotion/serialize': 1.1.2
|
||||
|
|
@ -1795,7 +1788,7 @@ packages:
|
|||
'@types/react':
|
||||
optional: true
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
'@emotion/babel-plugin': 11.11.0
|
||||
'@emotion/cache': 11.11.0
|
||||
'@emotion/serialize': 1.1.2
|
||||
|
|
@ -2360,14 +2353,7 @@ packages:
|
|||
resolution: {integrity: sha512-eS1M3L3U2mDowoFVPG7tEp01SWu9/68Nx3HEBgLJVn3N9ku7g5S7WdFv0jzmcTipAyONYfZJ+7x4WRkfdB2Ung==}
|
||||
dependencies:
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
dev: false
|
||||
|
||||
/@lezer/css@1.1.3:
|
||||
resolution: {integrity: sha512-SjSM4pkQnQdJDVc80LYzEaMiNy9txsFbI7HsMgeVF28NdLaAdHNtQ+kB/QqDUzRBV/75NTXjJ/R5IdC8QQGxMg==}
|
||||
dependencies:
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
'@lezer/lr': 1.3.14
|
||||
dev: false
|
||||
|
||||
/@lezer/css@1.1.4:
|
||||
|
|
@ -2383,14 +2369,6 @@ packages:
|
|||
'@lezer/common': 1.1.0
|
||||
dev: false
|
||||
|
||||
/@lezer/html@1.3.6:
|
||||
resolution: {integrity: sha512-Kk9HJARZTc0bAnMQUqbtuhFVsB4AnteR2BFUWfZV7L/x1H0aAKz6YabrfJ2gk/BEgjh9L3hg5O4y2IDZRBdzuQ==}
|
||||
dependencies:
|
||||
'@lezer/common': 1.1.1
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
dev: false
|
||||
|
||||
/@lezer/html@1.3.7:
|
||||
resolution: {integrity: sha512-Wo+rZ5UjLP0VqUTyXjzgmTYRW5bvTJUFn4Uw0K3HCQjX2/+f+zRo9GLN5BCAojwHQISPvaQk8BWSv2SSKx/UcQ==}
|
||||
dependencies:
|
||||
|
|
@ -2403,27 +2381,21 @@ packages:
|
|||
resolution: {integrity: sha512-POc53LHf2AuNeRXjqZbXNu88GKj0KZTjjSx0L7tYeXlrEHF+3NAQx+dEwKVuCbkl0ZMtpRy2VsDYOV7KKV0oyg==}
|
||||
dependencies:
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
'@lezer/lr': 1.3.14
|
||||
dev: false
|
||||
|
||||
/@lezer/javascript@1.4.7:
|
||||
resolution: {integrity: sha512-OVWlK0YEi7HM+9JRWtRkir8qvcg0/kVYg2TAMHlVtl6DU1C9yK1waEOLBMztZsV/axRJxsqfJKhzYz+bxZme5g==}
|
||||
dependencies:
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
'@lezer/lr': 1.3.14
|
||||
dev: false
|
||||
|
||||
/@lezer/json@1.0.1:
|
||||
resolution: {integrity: sha512-nkVC27qiEZEjySbi6gQRuMwa2sDu2PtfjSgz0A4QF81QyRGm3kb2YRzLcOPcTEtmcwvrX/cej7mlhbwViA4WJw==}
|
||||
dependencies:
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
dev: false
|
||||
|
||||
/@lezer/lr@1.3.12:
|
||||
resolution: {integrity: sha512-5nwY1JzCueUdRtlMBnlf1SUi69iGCq2ABq7WQFQMkn/kxPvoACAEnTp4P17CtXxYr7WCwtYPLL2AEvxKPuF1OQ==}
|
||||
dependencies:
|
||||
'@lezer/common': 1.1.1
|
||||
'@lezer/lr': 1.3.14
|
||||
dev: false
|
||||
|
||||
/@lezer/lr@1.3.14:
|
||||
|
|
@ -2443,35 +2415,35 @@ packages:
|
|||
resolution: {integrity: sha512-aqdCQJOXJ66De22vzdwnuC502hIaG9EnPK2rSi+ebXyUd+j7GAX1mRjWZOVOmf3GST1YUfUCu6WXDiEgDGOVwA==}
|
||||
dependencies:
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
'@lezer/lr': 1.3.14
|
||||
dev: false
|
||||
|
||||
/@lezer/python@1.1.8:
|
||||
resolution: {integrity: sha512-1T/XsmeF57ijrjpC0Zmrf9YeO5mn2zC1XeSNrOnc0KB+6PgxJ5m7kWKt0CnwyS74oHQXbJxUUL+QDQJR26c1Gw==}
|
||||
dependencies:
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
'@lezer/lr': 1.3.14
|
||||
dev: false
|
||||
|
||||
/@lezer/rust@1.0.1:
|
||||
resolution: {integrity: sha512-j+ToFKM6Wpglv3OQ4ebHYdYIMT2dh0ziCCV0rTf47AWiHOVhR0WjaKrBq+yuvDQNEhr5sxPxVI7+naJIgpqcsQ==}
|
||||
dependencies:
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
'@lezer/lr': 1.3.14
|
||||
dev: false
|
||||
|
||||
/@lezer/sass@1.0.3:
|
||||
resolution: {integrity: sha512-n4l2nVOB7gWiGU/Cg2IVxpt2Ic9Hgfgy/7gk+p/XJibAsPXs0lSbsfGwQgwsAw9B/euYo3oS6lEFr9WytoqcZg==}
|
||||
dependencies:
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
'@lezer/lr': 1.3.14
|
||||
dev: false
|
||||
|
||||
/@lezer/xml@1.0.2:
|
||||
resolution: {integrity: sha512-dlngsWceOtQBMuBPw5wtHpaxdPJ71aVntqjbpGkFtWsp4WtQmCnuTjQGocviymydN6M18fhj6UQX3oiEtSuY7w==}
|
||||
dependencies:
|
||||
'@lezer/highlight': 1.2.0
|
||||
'@lezer/lr': 1.3.12
|
||||
'@lezer/lr': 1.3.14
|
||||
dev: false
|
||||
|
||||
/@lit-labs/ssr-dom-shim@1.1.1:
|
||||
|
|
@ -2884,13 +2856,13 @@ packages:
|
|||
/@radix-ui/number@1.0.0:
|
||||
resolution: {integrity: sha512-Ofwh/1HX69ZfJRiRBMTy7rgjAzHmwe4kW9C9Y99HTRUcYLUuVT0KESFj15rPjRgKJs20GPq8Bm5aEDJ8DuA3vA==}
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
dev: false
|
||||
|
||||
/@radix-ui/primitive@1.0.0:
|
||||
resolution: {integrity: sha512-3e7rn8FDMin4CgeL7Z/49smCA3rFYY3Ha2rUQ7HRWFadS5iCRw08ZgVT1LaNTCNqgvrUiyczLflrVrF0SRQtNA==}
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
dev: false
|
||||
|
||||
/@radix-ui/react-compose-refs@1.0.0(react@18.2.0):
|
||||
|
|
@ -2898,7 +2870,7 @@ packages:
|
|||
peerDependencies:
|
||||
react: ^16.8 || ^17.0 || ^18.0
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
react: 18.2.0
|
||||
dev: false
|
||||
|
||||
|
|
@ -2907,7 +2879,7 @@ packages:
|
|||
peerDependencies:
|
||||
react: ^16.8 || ^17.0 || ^18.0
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
react: 18.2.0
|
||||
dev: false
|
||||
|
||||
|
|
@ -2916,7 +2888,7 @@ packages:
|
|||
peerDependencies:
|
||||
react: ^16.8 || ^17.0 || ^18.0
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
react: 18.2.0
|
||||
dev: false
|
||||
|
||||
|
|
@ -2926,7 +2898,7 @@ packages:
|
|||
react: ^16.8 || ^17.0 || ^18.0
|
||||
react-dom: ^16.8 || ^17.0 || ^18.0
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
'@radix-ui/react-compose-refs': 1.0.0(react@18.2.0)
|
||||
'@radix-ui/react-use-layout-effect': 1.0.0(react@18.2.0)
|
||||
react: 18.2.0
|
||||
|
|
@ -2939,7 +2911,7 @@ packages:
|
|||
react: ^16.8 || ^17.0 || ^18.0
|
||||
react-dom: ^16.8 || ^17.0 || ^18.0
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
'@radix-ui/react-slot': 1.0.1(react@18.2.0)
|
||||
react: 18.2.0
|
||||
react-dom: 18.2.0(react@18.2.0)
|
||||
|
|
@ -2951,7 +2923,7 @@ packages:
|
|||
react: ^16.8 || ^17.0 || ^18.0
|
||||
react-dom: ^16.8 || ^17.0 || ^18.0
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
'@radix-ui/number': 1.0.0
|
||||
'@radix-ui/primitive': 1.0.0
|
||||
'@radix-ui/react-compose-refs': 1.0.0(react@18.2.0)
|
||||
|
|
@ -2970,7 +2942,7 @@ packages:
|
|||
peerDependencies:
|
||||
react: ^16.8 || ^17.0 || ^18.0
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
'@radix-ui/react-compose-refs': 1.0.0(react@18.2.0)
|
||||
react: 18.2.0
|
||||
dev: false
|
||||
|
|
@ -2980,7 +2952,7 @@ packages:
|
|||
peerDependencies:
|
||||
react: ^16.8 || ^17.0 || ^18.0
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
react: 18.2.0
|
||||
dev: false
|
||||
|
||||
|
|
@ -2989,7 +2961,7 @@ packages:
|
|||
peerDependencies:
|
||||
react: ^16.8 || ^17.0 || ^18.0
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
react: 18.2.0
|
||||
dev: false
|
||||
|
||||
|
|
@ -3072,7 +3044,7 @@ packages:
|
|||
/@solana/web3.js@1.78.5:
|
||||
resolution: {integrity: sha512-2ZHsDNqkKdglJQrIvJ3p2DmgS3cGnary3VJyqt9C1SPrpAtLYzcElr3xyXJOznyQTU/8AMw+GoF11lFoKbicKg==}
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
'@noble/curves': 1.2.0
|
||||
'@noble/hashes': 1.3.2
|
||||
'@solana/buffer-layout': 4.0.1
|
||||
|
|
@ -4332,7 +4304,7 @@ packages:
|
|||
peerDependencies:
|
||||
react: ^16.8.0 || ^17.0.0 || ^18.0.0
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.1
|
||||
'@babel/runtime': 7.23.4
|
||||
'@types/js-cookie': 2.2.7
|
||||
ahooks-v3-count: 1.0.0
|
||||
dayjs: 1.11.10
|
||||
|
|
@ -4661,7 +4633,7 @@ packages:
|
|||
resolution: {integrity: sha512-Cg7TFGpIr01vOQNODXOOaGz2NpCU5gl8x1qJFbb6hbZxR7XrcE2vtbAsTAbJ7/xwJtUuJEw8K8Zr/AE0LHlesg==}
|
||||
engines: {node: '>=10', npm: '>=6'}
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
cosmiconfig: 7.1.0
|
||||
resolve: 1.22.6
|
||||
dev: false
|
||||
|
|
@ -5176,6 +5148,11 @@ packages:
|
|||
toggle-selection: 1.0.6
|
||||
dev: false
|
||||
|
||||
/core-js-pure@3.33.3:
|
||||
resolution: {integrity: sha512-taJ00IDOP+XYQEA2dAe4ESkmHt1fL8wzYDo3mRWQey8uO9UojlBFMneA65kMyxfYP7106c6LzWaq7/haDT6BCQ==}
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
|
||||
/core-util-is@1.0.3:
|
||||
resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==}
|
||||
dev: true
|
||||
|
|
@ -5734,7 +5711,7 @@ packages:
|
|||
resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==}
|
||||
engines: {node: '>=0.11'}
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
dev: false
|
||||
|
||||
/dayjs@1.11.10:
|
||||
|
|
@ -5924,7 +5901,7 @@ packages:
|
|||
/dom-helpers@5.2.1:
|
||||
resolution: {integrity: sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==}
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
csstype: 3.1.2
|
||||
dev: false
|
||||
|
||||
|
|
@ -6371,7 +6348,7 @@ packages:
|
|||
peerDependencies:
|
||||
eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
aria-query: 5.3.0
|
||||
array-includes: 3.1.7
|
||||
array.prototype.flatmap: 1.3.2
|
||||
|
|
@ -7069,6 +7046,12 @@ packages:
|
|||
dependencies:
|
||||
get-intrinsic: 1.2.1
|
||||
|
||||
/gpt-tokenizer@2.1.2:
|
||||
resolution: {integrity: sha512-HSuI5d6uey+c7x/VzQlPfCoGrfLyAc28vxWofKbjR9PJHm0AjQGSWkKw/OJnb+8S1g7nzgRsf0WH3dK+NNWYbg==}
|
||||
dependencies:
|
||||
rfc4648: 1.5.3
|
||||
dev: false
|
||||
|
||||
/graceful-fs@4.2.11:
|
||||
resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}
|
||||
|
||||
|
|
@ -10594,7 +10577,7 @@ packages:
|
|||
peerDependencies:
|
||||
react: ^16.8.0 || ^17.0.0 || ^18.0.0
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
react: 18.2.0
|
||||
use-composed-ref: 1.3.0(react@18.2.0)
|
||||
use-latest: 1.2.1(@types/react@18.2.38)(react@18.2.0)
|
||||
|
|
@ -10608,7 +10591,7 @@ packages:
|
|||
peerDependencies:
|
||||
react: ^16.8.0 || ^17.0.0 || ^18.0.0
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
react: 18.2.0
|
||||
use-composed-ref: 1.3.0(react@18.2.0)
|
||||
use-latest: 1.2.1(@types/react@18.2.38)(react@18.2.0)
|
||||
|
|
@ -10622,7 +10605,7 @@ packages:
|
|||
react: '>=16.6.0'
|
||||
react-dom: '>=16.6.0'
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
dom-helpers: 5.2.1
|
||||
loose-envify: 1.4.0
|
||||
prop-types: 15.8.1
|
||||
|
|
@ -10636,7 +10619,7 @@ packages:
|
|||
react: '>=16.6.0'
|
||||
react-dom: '>=16.6.0'
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
dom-helpers: 5.2.1
|
||||
loose-envify: 1.4.0
|
||||
prop-types: 15.8.1
|
||||
|
|
@ -11044,6 +11027,10 @@ packages:
|
|||
resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==}
|
||||
engines: {iojs: '>=1.0.0', node: '>=0.10.0'}
|
||||
|
||||
/rfc4648@1.5.3:
|
||||
resolution: {integrity: sha512-MjOWxM065+WswwnmNONOT+bD1nXzY9Km6u3kzvnx8F8/HXGZdz3T6e6vZJ8Q/RIMUSp/nxqjH3GwvJDy8ijeQQ==}
|
||||
dev: false
|
||||
|
||||
/rfdc@1.3.0:
|
||||
resolution: {integrity: sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==}
|
||||
dev: true
|
||||
|
|
@ -11062,7 +11049,7 @@ packages:
|
|||
/rpc-websockets@7.6.0:
|
||||
resolution: {integrity: sha512-Jgcs8q6t8Go98dEulww1x7RysgTkzpCMelVxZW4hvuyFtOGpeUz9prpr2KjUa/usqxgFCd9Tu3+yhHEP9GVmiQ==}
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
eventemitter3: 4.0.7
|
||||
uuid: 8.3.2
|
||||
ws: 8.14.2(bufferutil@4.0.8)(utf-8-validate@5.0.10)
|
||||
|
|
@ -11644,7 +11631,7 @@ packages:
|
|||
/tailwind-merge@2.0.0:
|
||||
resolution: {integrity: sha512-WO8qghn9yhsldLSg80au+3/gY9E4hFxIvQ3qOmlpXnqpDKoMruKfi/56BbbMg6fHTQJ9QD3cc79PoWqlaQE4rw==}
|
||||
dependencies:
|
||||
'@babel/runtime': 7.23.2
|
||||
'@babel/runtime': 7.23.4
|
||||
dev: false
|
||||
|
||||
/tailwind-scrollbar-hide@1.1.7:
|
||||
|
|
@ -12837,6 +12824,12 @@ packages:
|
|||
resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==}
|
||||
dev: false
|
||||
|
||||
/xregexp@5.1.1:
|
||||
resolution: {integrity: sha512-fKXeVorD+CzWvFs7VBuKTYIW63YD1e1osxwQ8caZ6o1jg6pDAbABDG54LCIq0j5cy7PjRvGIq6sef9DYPXpncg==}
|
||||
dependencies:
|
||||
'@babel/runtime-corejs3': 7.23.4
|
||||
dev: false
|
||||
|
||||
/xtend@4.0.2:
|
||||
resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==}
|
||||
engines: {node: '>=0.4'}
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
ALTER TABLE "Metadata" ADD COLUMN "ai_translation" JSONB;
|
||||
|
|
@ -27,6 +27,19 @@ model Metadata {
|
|||
// ...
|
||||
// }
|
||||
image_dimensions Json?
|
||||
// Expected JSON structure for the translation field:
|
||||
// {
|
||||
// "zh": {
|
||||
// "title": "string",
|
||||
// "content": "string",
|
||||
// },
|
||||
// "jp": {
|
||||
// "title": "string",
|
||||
// "content": "string",
|
||||
// },
|
||||
// ...
|
||||
// }
|
||||
ai_translation Json?
|
||||
@@id([uri])
|
||||
@@index([uri])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import { toCid, toGateway } from "~/lib/ipfs-parser"
|
|||
import { isInRN } from "~/lib/is-in-rn"
|
||||
import { isOnlyContent } from "~/lib/is-only-content"
|
||||
import getQueryClient from "~/lib/query-client"
|
||||
import { Language } from "~/lib/types"
|
||||
import { fetchGetPage, getSummary } from "~/queries/page.server"
|
||||
import { fetchGetSite } from "~/queries/site.server"
|
||||
|
||||
|
|
@ -28,7 +29,7 @@ export async function generateMetadata({
|
|||
}
|
||||
}): Promise<Metadata> {
|
||||
const queryClient = getQueryClient()
|
||||
|
||||
const locale = (await getLocale()) as Language
|
||||
const site = await fetchGetSite(params.site, queryClient)
|
||||
|
||||
const page = await fetchGetPage(
|
||||
|
|
@ -36,6 +37,7 @@ export async function generateMetadata({
|
|||
characterId: site?.characterId,
|
||||
slug: params.slug,
|
||||
useStat: true,
|
||||
translateTo: locale,
|
||||
},
|
||||
queryClient,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ export async function GET(req: Request) {
|
|||
useHTML: false,
|
||||
useImageDimensions: query.useImageDimensions === "true",
|
||||
topic: query.topic,
|
||||
translateTo: query.translateTo,
|
||||
})
|
||||
|
||||
const res = new NextServerResponse()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,210 @@
|
|||
import AsyncLock from "async-lock"
|
||||
import { LLMChain } from "langchain/chains"
|
||||
import { OpenAI } from "langchain/llms/openai"
|
||||
import { PromptTemplate } from "langchain/prompts"
|
||||
|
||||
import { Metadata } from "@prisma/client"
|
||||
|
||||
import { languageNames } from "~/i18n"
|
||||
import { toGateway } from "~/lib/ipfs-parser"
|
||||
import { llmModelSwitcherByTextLength } from "~/lib/llm-model-switcher-by-text-length"
|
||||
import prisma from "~/lib/prisma.server"
|
||||
import { cacheGet } from "~/lib/redis.server"
|
||||
import { getQuery, NextServerResponse } from "~/lib/server-helper"
|
||||
import { Language } from "~/lib/types"
|
||||
|
||||
const lock = new AsyncLock()
|
||||
|
||||
type ContentTranslation = {
|
||||
title?: string
|
||||
content?: string
|
||||
}
|
||||
|
||||
let translationModel4K: OpenAI | undefined
|
||||
let translationModel16K: OpenAI | undefined
|
||||
if (process.env.OPENAI_API_KEY) {
|
||||
const options = {
|
||||
openAIApiKey: process.env.OPENAI_API_KEY,
|
||||
temperature: 0.2,
|
||||
maxTokens: -1,
|
||||
}
|
||||
translationModel4K = new OpenAI({ ...options, modelName: "gpt-3.5-turbo" })
|
||||
translationModel16K = new OpenAI({
|
||||
...options,
|
||||
modelName: "gpt-3.5-turbo-16k",
|
||||
})
|
||||
}
|
||||
|
||||
type ChainKeyType = `${4 | 16}k_${Language}` // e.g. "4k_en" | "4k_zh" | "4k_zh-TW" | "4k_ja" | "16k_en" | "16k_zh" | "16k_zh-TW" | "16k_ja"
|
||||
const translationChains = new Map<ChainKeyType, LLMChain>()
|
||||
|
||||
const getOriginalTranslation = async (
|
||||
cid: string,
|
||||
targetLang: Language,
|
||||
originalLang?: Language,
|
||||
): Promise<ContentTranslation | undefined> => {
|
||||
if (!translationModel4K || !translationModel16K) return
|
||||
|
||||
try {
|
||||
const { title, content } = await (
|
||||
await fetch(toGateway(`ipfs://${cid}`))
|
||||
).json()
|
||||
|
||||
// If the detected language is the same as the target language, return the original content
|
||||
if (originalLang === targetLang) {
|
||||
console.warn(
|
||||
`|__ Warn: Detected language is the same as the target language, return the original content: ${cid}, ${targetLang}`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
console.time(`fetching translation ${cid}, ${targetLang}`)
|
||||
const { modelSize, tokens } = llmModelSwitcherByTextLength(content, {
|
||||
includeResponse: { lang: targetLang },
|
||||
})
|
||||
|
||||
if (!modelSize) {
|
||||
console.error(
|
||||
`|__ Error: Content too long for translation: ${cid}, ${targetLang}. (Tokens: ${tokens})`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
let chain = translationChains.get(`${modelSize}_${targetLang}`)
|
||||
|
||||
if (!chain) {
|
||||
const prompt = new PromptTemplate({
|
||||
template: `Translate the following text into "${languageNames[targetLang]}" language:
|
||||
{text}
|
||||
Translation:`,
|
||||
inputVariables: ["text"],
|
||||
})
|
||||
|
||||
const translateModel =
|
||||
modelSize === "4k" ? translationModel4K : translationModel16K
|
||||
|
||||
chain = new LLMChain({ llm: translateModel, prompt })
|
||||
|
||||
translationChains.set(`${modelSize}_${targetLang}`, chain)
|
||||
}
|
||||
|
||||
const t = await chain.call({ text: title })
|
||||
const c = await chain.call({ text: content })
|
||||
|
||||
console.timeEnd(`fetching translation ${cid}, ${targetLang}`)
|
||||
|
||||
return {
|
||||
title: t.text,
|
||||
content: c.text,
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
console.timeEnd(`fetching translation ${cid}, ${targetLang}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function getTranslation({
|
||||
cid,
|
||||
fromLang,
|
||||
toLang = "en",
|
||||
}: {
|
||||
cid: string
|
||||
fromLang?: Language
|
||||
toLang?: Language
|
||||
}): Promise<ContentTranslation | undefined> {
|
||||
if (fromLang === toLang) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const translatedContent = (await cacheGet({
|
||||
key: ["translation", cid, toLang],
|
||||
allowEmpty: true,
|
||||
noUpdate: true,
|
||||
noExpire: true,
|
||||
getValueFun: async () => {
|
||||
let result
|
||||
|
||||
await lock.acquire(`translation_${cid}`, async () => {
|
||||
const meta = await prisma.metadata.findFirst({
|
||||
where: {
|
||||
uri: `ipfs://${cid}`,
|
||||
},
|
||||
})
|
||||
|
||||
const key = "ai_translation"
|
||||
const translations = meta?.[key as keyof Metadata] as Record<
|
||||
string,
|
||||
ContentTranslation
|
||||
>
|
||||
const translatedJson = translations?.[toLang]
|
||||
|
||||
if (translatedJson) {
|
||||
result = translatedJson
|
||||
} else {
|
||||
const newTranslation = await getOriginalTranslation(cid, toLang)
|
||||
if (newTranslation) {
|
||||
/**
|
||||
* e.g.
|
||||
*
|
||||
* {
|
||||
* "en": {
|
||||
* "title": "title",
|
||||
* "content": "content"
|
||||
* },
|
||||
* "zh": {
|
||||
* "title": "标题",
|
||||
* "content": "内容"
|
||||
* },
|
||||
* ...
|
||||
* }
|
||||
*
|
||||
*/
|
||||
const finalTranslation = {
|
||||
...translations,
|
||||
[toLang]: newTranslation,
|
||||
}
|
||||
|
||||
if (meta) {
|
||||
await prisma.metadata.update({
|
||||
where: { uri: `ipfs://${cid}` },
|
||||
data: {
|
||||
[key as keyof Metadata]: finalTranslation,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
await prisma.metadata.create({
|
||||
data: {
|
||||
uri: `ipfs://${cid}`,
|
||||
[key as keyof Metadata]: finalTranslation,
|
||||
},
|
||||
})
|
||||
}
|
||||
result = newTranslation
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return result
|
||||
},
|
||||
})) as ContentTranslation | undefined
|
||||
|
||||
return translatedContent
|
||||
}
|
||||
|
||||
export async function GET(req: Request): Promise<Response> {
|
||||
let { cid, fromLang, toLang } = getQuery(req)
|
||||
|
||||
const res = new NextServerResponse()
|
||||
|
||||
if (!cid) {
|
||||
return res.status(400).send("Bad Request")
|
||||
}
|
||||
|
||||
return res.status(200).json({
|
||||
data: await getTranslation({
|
||||
cid,
|
||||
fromLang,
|
||||
toLang,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
"use client"
|
||||
|
||||
import { useTranslations } from "next-intl"
|
||||
import { useLocale, useTranslations } from "next-intl"
|
||||
import { useParams, useSearchParams } from "next/navigation"
|
||||
import { useEffect, useState } from "react"
|
||||
import { VirtuosoGrid } from "react-virtuoso"
|
||||
|
|
@ -15,13 +15,14 @@ import { Tabs } from "~/components/ui/Tabs"
|
|||
import { Tooltip } from "~/components/ui/Tooltip"
|
||||
import { useIsMobileLayout } from "~/hooks/useMobileLayout"
|
||||
import { getStorage, setStorage } from "~/lib/storage"
|
||||
import { ExpandedNote } from "~/lib/types"
|
||||
import { ExpandedNote, Language } from "~/lib/types"
|
||||
import type { FeedType, SearchType } from "~/models/home.model"
|
||||
import { useGetFeed } from "~/queries/home"
|
||||
|
||||
export const HomeFeed = ({ type }: { type?: FeedType }) => {
|
||||
const t = useTranslations()
|
||||
const searchParams = useSearchParams()
|
||||
const locale = useLocale() as Language
|
||||
|
||||
const currentCharacterId = useAccountState(
|
||||
(s) => s.computed.account?.characterId,
|
||||
|
|
@ -43,6 +44,7 @@ export const HomeFeed = ({ type }: { type?: FeedType }) => {
|
|||
let feedConfig: Parameters<typeof useGetFeed>[0] = {
|
||||
type,
|
||||
}
|
||||
|
||||
switch (type) {
|
||||
case "following":
|
||||
feedConfig = {
|
||||
|
|
@ -77,7 +79,10 @@ export const HomeFeed = ({ type }: { type?: FeedType }) => {
|
|||
break
|
||||
}
|
||||
|
||||
const feed = useGetFeed(feedConfig)
|
||||
const feed = useGetFeed({
|
||||
...feedConfig,
|
||||
translateTo: locale,
|
||||
})
|
||||
|
||||
const hasFiltering = type === "latest"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,12 @@
|
|||
import { getRequestConfig } from "next-intl/server"
|
||||
|
||||
export const locales = ["en", "ja", "zh", "zh-TW"]
|
||||
export const locales = ["en", "ja", "zh", "zh-TW"] as const
|
||||
export const languageNames: Record<(typeof locales)[number], string> = {
|
||||
en: "English",
|
||||
ja: "Japanese",
|
||||
zh: "Chinese (Simplified)",
|
||||
"zh-TW": "Chinese (Traditional)",
|
||||
}
|
||||
export const defaultLocale = "en"
|
||||
|
||||
export default getRequestConfig(async ({ locale }) => ({
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
import { encode } from "gpt-tokenizer"
|
||||
|
||||
export type ModelTypes =
|
||||
| "gpt-3.5-turbo"
|
||||
| "gpt-3.5-turbo-0613"
|
||||
| "gpt-3.5-turbo-16k"
|
||||
| "gpt-3.5-turbo-16k-0613"
|
||||
| "gpt-3.5-turbo-0301"
|
||||
| "gpt-4"
|
||||
| "gpt-4-0314"
|
||||
| "gpt-4-32k-0314"
|
||||
| "gpt-4-0613"
|
||||
| "gpt-4-32k-0613"
|
||||
|
||||
export function calculateTokens(
|
||||
messages: string,
|
||||
model: ModelTypes = "gpt-3.5-turbo-0613",
|
||||
) {
|
||||
let tokens_per_message = 0
|
||||
let tokens_per_name = 0
|
||||
if (
|
||||
[
|
||||
"gpt-3.5-turbo-0613",
|
||||
"gpt-3.5-turbo-16k-0613",
|
||||
"gpt-4-0314",
|
||||
"gpt-4-32k-0314",
|
||||
"gpt-4-0613",
|
||||
"gpt-4-32k-0613",
|
||||
].includes(model)
|
||||
) {
|
||||
tokens_per_message = 3
|
||||
tokens_per_name = 1
|
||||
} else if (model == "gpt-3.5-turbo-0301") {
|
||||
tokens_per_message = 4
|
||||
tokens_per_name = -1
|
||||
} else if (model.includes("gpt-3.5-turbo")) {
|
||||
console.log(
|
||||
"Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.",
|
||||
)
|
||||
return calculateTokens(messages, "gpt-3.5-turbo-0613")
|
||||
} else if (model.includes("gpt-4")) {
|
||||
console.log(
|
||||
"Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.",
|
||||
)
|
||||
return calculateTokens(messages, "gpt-4-0613")
|
||||
} else {
|
||||
throw new Error(
|
||||
`num_tokens_from_messages() is not implemented for model ${model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.`,
|
||||
)
|
||||
}
|
||||
let num_tokens = 0
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
let message = messages[i]
|
||||
num_tokens += tokens_per_message
|
||||
// @ts-ignore
|
||||
for (let key in message) {
|
||||
let value = message[key]
|
||||
num_tokens += encode(value).length
|
||||
if (key == "name") {
|
||||
num_tokens += tokens_per_name
|
||||
}
|
||||
}
|
||||
}
|
||||
num_tokens += 3
|
||||
return num_tokens
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
import XRegExp from "xregexp"
|
||||
|
||||
import { Language } from "./types"
|
||||
|
||||
type Thresholds = Partial<Record<Language, number>>
|
||||
|
||||
const _thresholds: Thresholds = {
|
||||
en: 0.8,
|
||||
zh: 0.3,
|
||||
ja: 0.5,
|
||||
}
|
||||
|
||||
export function detectLanguage(
|
||||
text: string,
|
||||
thresholds: Thresholds = _thresholds,
|
||||
): Language {
|
||||
// Pre-process the text
|
||||
text = text.replace(/[\d\s\p{P}]/gu, "") // Remove numbers, whitespace, and punctuation
|
||||
|
||||
// split into words
|
||||
const langs: Array<string | null> = text
|
||||
.trim()
|
||||
.split(/\s+/)
|
||||
.map((word) => {
|
||||
return detect(word, thresholds)
|
||||
})
|
||||
|
||||
// pick the lang with the most occurrences
|
||||
const result = langs.reduce<{
|
||||
k: Record<string, number>
|
||||
max: string | null
|
||||
}>(
|
||||
(acc, el) => {
|
||||
if (el) {
|
||||
acc.k[el] = acc.k[el] ? acc.k[el] + 1 : 1
|
||||
acc.max = acc.max ? (acc.k[acc.max] < acc.k[el] ? el : acc.max) : el
|
||||
}
|
||||
return acc
|
||||
},
|
||||
{ k: {}, max: null },
|
||||
)
|
||||
|
||||
return result.max as Language
|
||||
|
||||
function detect(text: string, thresholds: Thresholds): string | null {
|
||||
const scores: Record<string, number> = {}
|
||||
const regexes: Record<string, RegExp> = {
|
||||
en: XRegExp("\\p{Latin}", "gi"),
|
||||
zh: XRegExp("\\p{Han}", "gi"),
|
||||
ja: XRegExp("[\\p{Hiragana}\\p{Katakana}]", "gi"),
|
||||
}
|
||||
|
||||
for (const [lang, regex] of Object.entries(regexes)) {
|
||||
const matches = XRegExp.match(text, regex) || []
|
||||
const score = matches.length / text.length
|
||||
if (score && score >= thresholds[lang as keyof Thresholds]!) {
|
||||
return lang
|
||||
}
|
||||
scores[lang] = score
|
||||
}
|
||||
|
||||
if (Object.keys(scores).length === 0) return null
|
||||
|
||||
// pick lang with highest percentage
|
||||
return Object.keys(scores).reduce((a, b) => (scores[a] > scores[b] ? a : b))
|
||||
}
|
||||
}
|
||||
|
|
@ -5,8 +5,14 @@ import removeMarkdown from "remove-markdown"
|
|||
import { SCORE_API_DOMAIN, SITE_URL } from "~/lib/env"
|
||||
import { toCid, toGateway } from "~/lib/ipfs-parser"
|
||||
import readingTime from "~/lib/reading-time"
|
||||
import { ExpandedCharacter, ExpandedNote, PortfolioStats } from "~/lib/types"
|
||||
import {
|
||||
ExpandedCharacter,
|
||||
ExpandedNote,
|
||||
Language,
|
||||
PortfolioStats,
|
||||
} from "~/lib/types"
|
||||
|
||||
import { detectLanguage } from "./detect-lang"
|
||||
import { getNoteSlug } from "./helpers"
|
||||
|
||||
export const expandCrossbellNote = async ({
|
||||
|
|
@ -17,6 +23,7 @@ export const expandCrossbellNote = async ({
|
|||
keyword,
|
||||
useHTML,
|
||||
disableAutofill,
|
||||
translateTo,
|
||||
}: {
|
||||
note: NoteEntity & {
|
||||
metadata?: {
|
||||
|
|
@ -31,6 +38,7 @@ export const expandCrossbellNote = async ({
|
|||
keyword?: string
|
||||
useHTML?: boolean
|
||||
disableAutofill?: boolean
|
||||
translateTo?: Language
|
||||
}) => {
|
||||
if (note.metadata?.uri && !note.metadata?.content) {
|
||||
note.metadata.content = await (
|
||||
|
|
@ -51,12 +59,45 @@ export const expandCrossbellNote = async ({
|
|||
let rendered
|
||||
if (expandedNote.metadata?.content?.content) {
|
||||
const { renderPageContent } = await import("~/markdown")
|
||||
rendered = renderPageContent(expandedNote.metadata.content.content, true)
|
||||
|
||||
if (translateTo) {
|
||||
try {
|
||||
const processedContent = removeMarkdown(
|
||||
expandedNote.metadata.content.content.replace(/```[^]+?```/g, ""),
|
||||
)
|
||||
const detectedLang = detectLanguage(processedContent)
|
||||
|
||||
const translation = await (
|
||||
await fetch(
|
||||
`${SITE_URL}/api/translate-note?` +
|
||||
new URLSearchParams({
|
||||
cid: toCid(expandedNote.metadata.uri || ""),
|
||||
fromLang: detectedLang,
|
||||
toLang: translateTo,
|
||||
} as any),
|
||||
)
|
||||
).json()
|
||||
|
||||
if (translation.data.content) {
|
||||
expandedNote.metadata.content.content = translation.data
|
||||
.content as string
|
||||
}
|
||||
|
||||
if (translation.data.title) {
|
||||
expandedNote.metadata.content.title = translation.data
|
||||
.title as string
|
||||
}
|
||||
} catch (e) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
|
||||
rendered = renderPageContent(expandedNote.metadata.content.content!, true)
|
||||
if (keyword) {
|
||||
const position = expandedNote.metadata.content.content
|
||||
.toLowerCase()
|
||||
.indexOf(keyword.toLowerCase())
|
||||
expandedNote.metadata.content.summary = `...${expandedNote.metadata.content.content.slice(
|
||||
expandedNote.metadata.content.summary = `...${expandedNote.metadata.content.content!.slice(
|
||||
position - 10,
|
||||
position + 100,
|
||||
)}`
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
import { calculateTokens } from "./calculate-tokens"
|
||||
import { Language } from "./types"
|
||||
|
||||
function getTolerantCount(textLength: "4k" | "16k"): number {
|
||||
const ratio = 0.8
|
||||
|
||||
if (textLength == "4k") {
|
||||
return 4000 * ratio
|
||||
}
|
||||
|
||||
return 16000
|
||||
}
|
||||
|
||||
// TODO: This is a temporary solution. Because we don't know the language of the original text, we can't calculate the tokens accurately. Currently, only "Chinese to other languages" is supported.
|
||||
export function llmModelSwitcherByTextLength(
|
||||
text: string,
|
||||
options: {
|
||||
// If true, include the response in the text length calculation with the specified language
|
||||
includeResponse?: { lang: Language }
|
||||
} = {},
|
||||
): {
|
||||
modelSize: "4k" | "16k" | undefined
|
||||
tokens: number
|
||||
} {
|
||||
const { includeResponse } = options
|
||||
const tokens = calculateTokens(text)
|
||||
|
||||
const langRatioMapping: { [key in Language]: number } = {
|
||||
en: 1.6,
|
||||
ja: 1.25,
|
||||
zh: 1,
|
||||
"zh-TW": 1,
|
||||
}
|
||||
|
||||
const finalCount =
|
||||
tokens / (includeResponse ? langRatioMapping[includeResponse.lang] : 1)
|
||||
|
||||
if (finalCount < getTolerantCount("4k")) {
|
||||
return {
|
||||
modelSize: "4k",
|
||||
tokens,
|
||||
}
|
||||
} else if (finalCount < getTolerantCount("16k")) {
|
||||
return {
|
||||
modelSize: "16k",
|
||||
tokens,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
modelSize: undefined,
|
||||
tokens,
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
import type { CharacterEntity, NoteEntity } from "crossbell"
|
||||
|
||||
export type Language = "en" | "zh" | "zh-TW" | "ja"
|
||||
|
||||
export type Site = {
|
||||
id: string
|
||||
name: string
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { gql } from "@urql/core"
|
|||
|
||||
import countCharacters from "~/lib/character-count"
|
||||
import { expandCrossbellNote } from "~/lib/expand-unit"
|
||||
import { ExpandedNote } from "~/lib/types"
|
||||
import { ExpandedNote, Language } from "~/lib/types"
|
||||
import { client } from "~/queries/graphql"
|
||||
|
||||
import filter from "../../data/filter.json"
|
||||
|
|
@ -37,6 +37,7 @@ export async function getFeed({
|
|||
useHTML,
|
||||
useImageDimensions,
|
||||
topic,
|
||||
translateTo,
|
||||
}: {
|
||||
type?: FeedType
|
||||
cursor?: string
|
||||
|
|
@ -49,6 +50,7 @@ export async function getFeed({
|
|||
useHTML?: boolean
|
||||
useImageDimensions?: boolean
|
||||
topic?: string
|
||||
translateTo?: Language
|
||||
}) {
|
||||
if (type === "search" && !searchKeyword) {
|
||||
type = "latest"
|
||||
|
|
@ -144,6 +146,7 @@ export async function getFeed({
|
|||
useScore: true,
|
||||
useHTML,
|
||||
useImageDimensions,
|
||||
translateTo,
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
|
@ -239,6 +242,7 @@ export async function getFeed({
|
|||
note: page,
|
||||
useHTML,
|
||||
useImageDimensions,
|
||||
translateTo,
|
||||
})
|
||||
if (expand.toNote) {
|
||||
if (expand.toNote.toNote) {
|
||||
|
|
@ -256,6 +260,7 @@ export async function getFeed({
|
|||
note: expand.toNote,
|
||||
useHTML,
|
||||
useImageDimensions,
|
||||
translateTo,
|
||||
})
|
||||
delete expand.toNote.toNote
|
||||
}
|
||||
|
|
@ -327,6 +332,7 @@ export async function getFeed({
|
|||
useStat: true,
|
||||
useHTML,
|
||||
useImageDimensions,
|
||||
translateTo,
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
|
@ -366,6 +372,7 @@ export async function getFeed({
|
|||
note: page,
|
||||
useHTML,
|
||||
useImageDimensions,
|
||||
translateTo,
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
|
@ -473,6 +480,7 @@ export async function getFeed({
|
|||
note: page,
|
||||
useHTML,
|
||||
useImageDimensions,
|
||||
translateTo,
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
|
@ -549,6 +557,7 @@ export async function getFeed({
|
|||
note: page,
|
||||
useHTML,
|
||||
useImageDimensions,
|
||||
translateTo,
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
|
@ -654,6 +663,7 @@ export async function getFeed({
|
|||
useHTML,
|
||||
useStat: true,
|
||||
useImageDimensions,
|
||||
translateTo,
|
||||
})
|
||||
return expand
|
||||
},
|
||||
|
|
@ -780,6 +790,7 @@ export async function getFeed({
|
|||
useHTML,
|
||||
useStat: true,
|
||||
useImageDimensions,
|
||||
translateTo,
|
||||
})
|
||||
return expand
|
||||
},
|
||||
|
|
@ -825,6 +836,7 @@ export async function getFeed({
|
|||
keyword: searchKeyword,
|
||||
useHTML,
|
||||
useImageDimensions,
|
||||
translateTo,
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
|
@ -863,6 +875,7 @@ export async function getFeed({
|
|||
useScore: true,
|
||||
useHTML,
|
||||
useImageDimensions,
|
||||
translateTo,
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import { checkSlugReservedWords } from "~/lib/slug-reserved-words"
|
|||
import { getKeys, getStorage } from "~/lib/storage"
|
||||
import {
|
||||
ExpandedNote,
|
||||
Language,
|
||||
NoteType,
|
||||
PagesSortTypes,
|
||||
PageVisibilityEnum,
|
||||
|
|
@ -421,6 +422,7 @@ export async function getPage<TRender extends boolean = false>(input: {
|
|||
noteId?: number
|
||||
handle?: string // In order to be compatible with old drafts
|
||||
disableAutofill?: boolean
|
||||
translateTo?: Language
|
||||
}) {
|
||||
const mustLocal = input.slug?.startsWith("!local-") && !input.noteId
|
||||
|
||||
|
|
@ -572,6 +574,7 @@ export async function getPage<TRender extends boolean = false>(input: {
|
|||
note: page,
|
||||
useStat: input.useStat,
|
||||
disableAutofill: input.disableAutofill,
|
||||
translateTo: input.translateTo,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import { defaultLocale, locales } from "~/i18n"
|
|||
import { toGateway } from "~/lib/ipfs-parser"
|
||||
import prisma from "~/lib/prisma.server"
|
||||
import { cacheGet } from "~/lib/redis.server"
|
||||
import { Language } from "~/lib/types"
|
||||
import * as pageModel from "~/models/page.model"
|
||||
|
||||
export const fetchGetPage = async (
|
||||
|
|
@ -31,6 +32,7 @@ export const fetchGetPage = async (
|
|||
useStat: input.useStat,
|
||||
noteId: input.noteId,
|
||||
handle: input.handle,
|
||||
translateTo: input.translateTo,
|
||||
}),
|
||||
}) as Promise<ReturnType<typeof pageModel.getPage>>
|
||||
})
|
||||
|
|
@ -147,7 +149,7 @@ export async function getSummary({
|
|||
noUpdate: true,
|
||||
noExpire: true,
|
||||
getValueFun: async () => {
|
||||
if (locales.includes(lang)) {
|
||||
if (locales.includes(lang as Language)) {
|
||||
let result
|
||||
await lock.acquire(cid, async () => {
|
||||
const meta = await prisma.metadata.findFirst({
|
||||
|
|
|
|||
Loading…
Reference in New Issue