From 62e9cce9d7aa9aaf5263c146575a4d554610e5e9 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Thu, 16 Nov 2023 17:57:29 +0800 Subject: [PATCH] feat: optimize scoring prompt and use gpt-4-1106-preview model --- package.json | 1 + pnpm-lock.yaml | 15 +++++--- src/app/api/score/route.ts | 76 +++++++++++++++++++++++--------------- 3 files changed, 57 insertions(+), 35 deletions(-) diff --git a/package.json b/package.json index ab60dcf1..dec8d6c4 100644 --- a/package.json +++ b/package.json @@ -101,6 +101,7 @@ "next-themes": "0.2.1", "node-id3": "0.2.6", "open-graph-scraper": "^6.3.2", + "openai": "^4.19.0", "pangu": "4.0.7", "path-to-regexp": "6.2.1", "pinyin-pro": "3.18.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1f79e93e..beaec764 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -187,7 +187,7 @@ dependencies: version: 0.16.9 langchain: specifier: 0.0.186 - version: 0.0.186(cheerio@1.0.0-rc.12)(ioredis@5.3.2)(jsdom@22.1.0)(ws@7.5.9) + version: 0.0.186(cheerio@1.0.0-rc.12)(ioredis@5.3.2)(jsdom@22.1.0)(ws@8.13.0) lottie-react: specifier: 2.4.0 version: 2.4.0(react-dom@18.2.0)(react@18.2.0) @@ -218,6 +218,9 @@ dependencies: open-graph-scraper: specifier: ^6.3.2 version: 6.3.2 + openai: + specifier: ^4.19.0 + version: 4.19.0 pangu: specifier: 4.0.7 version: 4.0.7 @@ -8447,7 +8450,7 @@ packages: engines: {node: '>=6'} dev: false - /langchain@0.0.186(cheerio@1.0.0-rc.12)(ioredis@5.3.2)(jsdom@22.1.0)(ws@7.5.9): + /langchain@0.0.186(cheerio@1.0.0-rc.12)(ioredis@5.3.2)(jsdom@22.1.0)(ws@8.13.0): resolution: {integrity: sha512-uXDipmw9aUrUmDNcFr2XH9ORmshWIlIb/qFKneS1K3X5upMUg7TSbaBxqV9WxuuenLUSYaoTcTy7P/pKkbqXPg==} engines: {node: '>=18'} peerDependencies: @@ -8760,12 +8763,12 @@ packages: langchainhub: 0.0.6 langsmith: 0.0.48 ml-distance: 4.0.1 - openai: 4.17.4 + openai: 4.19.0 openapi-types: 12.1.3 p-queue: 6.6.2 p-retry: 4.6.2 uuid: 9.0.1 - ws: 7.5.9 + ws: 8.13.0 yaml: 2.3.4 zod: 3.22.4 zod-to-json-schema: 3.21.4(zod@3.22.4) @@ -10051,8 +10054,8 @@ packages: validator: 13.11.0 dev: false - /openai@4.17.4: - resolution: {integrity: sha512-ThRFkl6snLbcAKS58St7N3CaKuI5WdYUvIjPvf4s+8SdymgNtOfzmZcZXVcCefx04oKFnvZJvIcTh3eAFUUhAQ==} + /openai@4.19.0: + resolution: {integrity: sha512-cJbl0noZyAaXVKBTMMq6X5BAvP1pm2rWYDBnZes99NL+Zh5/4NmlAwyuhTZEru5SqGGZIoiYKeMPXy4bm9DI0w==} hasBin: true dependencies: '@types/node': 18.18.9 diff --git a/src/app/api/score/route.ts b/src/app/api/score/route.ts index 736560fd..51cc7f63 100644 --- a/src/app/api/score/route.ts +++ b/src/app/api/score/route.ts @@ -1,4 +1,5 @@ import AsyncLock from "async-lock" +import OpenAI from "openai" import countCharacters from "~/lib/character-count" import { toGateway } from "~/lib/ipfs-parser" @@ -6,6 +7,13 @@ import prisma from "~/lib/prisma.server" import { cacheGet } from "~/lib/redis.server" import { getQuery, NextServerResponse } from "~/lib/server-helper" +let openai: OpenAI | undefined +if (process.env.OPENAI_API_KEY) { + openai = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, + }) +} + const lock = new AsyncLock() const getOriginalScore = async (cid: string) => { @@ -26,41 +34,51 @@ const getOriginalScore = async (cid: string) => { } if (countCharacters(content) > 300) { + if (!openai) { + return { + number: 100, + reason: "OpenAI not configured", + } + } + console.time(`fetching score ${cid}`) - const prompt = `According to rule 1 not too short content, rule 2 good originality and innovation, and rule 3 good fun or logic, give this article a score in the range of 0-100 and explain the reason: - "${content}" - Score:` - const response = await ( - await fetch("https://api.openai.com/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${process.env.OPENAI_API_KEY}`, + const completion = await openai.chat.completions.create({ + messages: [ + { + role: "user", + content: `You are an expert in scoring blog posts, you will evaluate this article based on three rules: +1. Content length should not be too short. +2. Originality and innovation should be good. +3. The article should provide either good fun or logical reasoning. + +Based on these criteria, you will assign a score to the article on a scale of 0-100 and provide an explanation for your decision. The evaluation should be presented in JSON format, following the structure: { "score": number, "reason": string }. + +Below you find the post content: +-------- +${content} +--------`, }, - body: JSON.stringify({ - model: "gpt-4", - temperature: 0, - messages: [ - { - role: "user", - content: prompt, - }, - ], - }), - }) - ).json() + ], + model: "gpt-4-1106-preview", + temperature: 0, + response_format: { type: "json_object" }, + }) + + let evaluate + try { + if (completion.choices?.[0]?.message?.content) { + evaluate = JSON.parse(completion.choices?.[0]?.message?.content) + } + } catch (error) {} console.timeEnd(`fetching score ${cid}`) - return { - number: parseInt(response.choices?.[0]?.message?.content?.trim()), - reason: response.choices?.[0]?.message?.content - ?.trim() - .replace(/^\d+([,.\s]*)/, "") - .trim() - .replace(/^Reason:/, "") - .trim(), + if (evaluate.reason) { + return { + number: evaluate.score, + reason: evaluate.reason, + } } } else { return {