feat: 支持敏感词审核

Author: Kerwin

README.en.md

@@ -266,6 +266,13 @@ services:
       SMTP_TSL: true
       SMTP_USERNAME: noreply@examile.com
       SMTP_PASSWORD: xxx
+      # Enable sensitive word review, because the response result is streaming, so there is currently no review.
+      AUDIT_ENABLED: false
+      # https://ai.baidu.com/ai-doc/ANTIPORN/Vk3h6xaga
+      AUDIT_PROVIDER: baidu
+      AUDIT_API_KEY: xxx
+      AUDIT_API_SECRET: xxx
+      AUDIT_TEXT_LABEL: xxx
     links:
       - database
 

README.md

@@ -270,6 +270,13 @@ services:
       SMTP_TSL: true
       SMTP_USERNAME: noreply@examile.com
       SMTP_PASSWORD: xxx
+      # 是否开启敏感词审核, 因为响应结果是流式 所以暂时没审核
+      AUDIT_ENABLED: false
+      # https://ai.baidu.com/ai-doc/ANTIPORN/Vk3h6xaga
+      AUDIT_PROVIDER: baidu
+      AUDIT_API_KEY: xxx
+      AUDIT_API_SECRET: xxx
+      AUDIT_TEXT_LABEL: xxx
     links:
       - database
 

docker-compose/docker-compose.yml

@@ -57,6 +57,13 @@ services:
       SMTP_TSL: true
       SMTP_USERNAME: ${SMTP_USERNAME}
       SMTP_PASSWORD: ${SMTP_PASSWORD}
+      # 是否开启敏感词审核, 因为响应结果是流式 所以暂时没审核
+      AUDIT_ENABLED: false
+      # https://ai.baidu.com/ai-doc/ANTIPORN/Vk3h6xaga
+      AUDIT_PROVIDER: baidu
+      AUDIT_API_KEY: 
+      AUDIT_API_SECRET:
+      AUDIT_TEXT_LABEL:
     links:
       - database
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "chatgpt-web",
-  "version": "2.11.7",
+  "version": "2.12.0",
   "private": false,
   "description": "ChatGPT Web",
   "author": "ChenZhaoYu <chenzhaoyu1994@gmail.com>",

service/src/chatgpt/index.ts

@@ -5,6 +5,9 @@ import { ChatGPTAPI, ChatGPTUnofficialProxyAPI } from 'chatgpt'
 import { SocksProxyAgent } from 'socks-proxy-agent'
 import httpsProxyAgent from 'https-proxy-agent'
 import fetch from 'node-fetch'
+import type { AuditConfig } from 'src/storage/model'
+import type { TextAuditService } from '../utils/textAudit'
+import { textAuditServices } from '../utils/textAudit'
 import { getCacheConfig, getOriginConfig } from '../storage/config'
 import { sendResponse } from '../utils'
 import { isNotEmptyString } from '../utils/is'
@@ -26,6 +29,7 @@ const ErrorCodeMessage: Record<string, string> = {
 
 let apiModel: ApiModel
 let api: ChatGPTAPI | ChatGPTUnofficialProxyAPI
+let auditService: TextAuditService
 
 export async function initApi() {
   // More Info: https://github.com/transitive-bullshit/chatgpt-api
@@ -85,6 +89,10 @@ async function chatReplyProcess(options: RequestOptions) {
   const config = await getCacheConfig()
   const model = isNotEmptyString(config.apiModel) ? config.apiModel : 'gpt-3.5-turbo'
   const { message, lastContext, process, systemMessage, temperature, top_p } = options
+
+  if ((config.auditConfig?.enabled ?? false) && !await auditText(config.auditConfig, message))
+    return sendResponse({ type: 'Fail', message: '含有敏感词 | Contains sensitive words' })
+
   try {
     const timeoutMs = (await getCacheConfig()).timeoutMs
     let options: SendMessageOptions = { timeoutMs }
@@ -120,9 +128,28 @@ async function chatReplyProcess(options: RequestOptions) {
   }
 }
 
+export function initAuditService(audit: AuditConfig) {
+  if (!audit || !audit.options || !audit.options.apiKey || !audit.options.apiSecret)
+    throw new Error('未配置 | Not configured.')
+  const Service = textAuditServices[audit.provider]
+  auditService = new Service(audit.options)
+}
+
+async function auditText(audit: AuditConfig, text: string): Promise<boolean> {
+  if (!auditService)
+    initAuditService(audit)
+
+  return await auditService.audit(text)
+}
+let cachedBanlance: number | undefined
+let cacheExpiration = 0
+
 async function fetchBalance() {
-  // 计算起始日期和结束日期
   const now = new Date().getTime()
+  if (cachedBanlance && cacheExpiration > now)
+    return Promise.resolve(cachedBanlance.toFixed(3))
+
+  // 计算起始日期和结束日期
   const startDate = new Date(now - 90 * 24 * 60 * 60 * 1000)
   const endDate = new Date(now + 24 * 60 * 60 * 1000)
 
@@ -165,9 +192,10 @@ async function fetchBalance() {
     const totalUsage = usageData.total_usage / 100
 
     // 计算剩余额度
-    const balance = totalAmount - totalUsage
+    cachedBanlance = totalAmount - totalUsage
+    cacheExpiration = now + 10 * 60 * 1000
 
-    return Promise.resolve(balance.toFixed(3))
+    return Promise.resolve(cachedBanlance.toFixed(3))
   }
   catch {
     return Promise.resolve('-')
@@ -226,4 +254,4 @@ initApi()
 
 export type { ChatContext, ChatMessage }
 
-export { chatReplyProcess, chatConfig, currentModel }
+export { chatReplyProcess, chatConfig, currentModel, auditText }

service/src/index.ts

@@ -3,10 +3,10 @@ import jwt from 'jsonwebtoken'
 import * as dotenv from 'dotenv'
 import type { RequestProps } from './types'
 import type { ChatContext, ChatMessage } from './chatgpt'
-import { chatConfig, chatReplyProcess, currentModel, initApi } from './chatgpt'
+import { auditText, chatConfig, chatReplyProcess, currentModel, initApi, initAuditService } from './chatgpt'
 import { auth } from './middleware/auth'
 import { clearConfigCache, getCacheConfig, getOriginConfig } from './storage/config'
-import type { ChatInfo, ChatOptions, Config, MailConfig, SiteConfig, UsageResponse, UserInfo } from './storage/model'
+import type { AuditConfig, ChatInfo, ChatOptions, Config, MailConfig, SiteConfig, UsageResponse, UserInfo } from './storage/model'
 import { Status } from './storage/model'
 import {
   clearChat,
@@ -594,6 +594,36 @@ router.post('/mail-test', rootAuth, async (req, res) => {
   }
 })
 
+router.post('/setting-audit', rootAuth, async (req, res) => {
+  try {
+    const config = req.body as AuditConfig
+
+    const thisConfig = await getOriginConfig()
+    thisConfig.auditConfig = config
+    const result = await updateConfig(thisConfig)
+    clearConfigCache()
+    initAuditService(config)
+    res.send({ status: 'Success', message: '操作成功 | Successfully', data: result.auditConfig })
+  }
+  catch (error) {
+    res.send({ status: 'Fail', message: error.message, data: null })
+  }
+})
+
+router.post('/audit-test', rootAuth, async (req, res) => {
+  try {
+    const { audit, text } = req.body as { audit: AuditConfig; text: string }
+    const config = await getCacheConfig()
+    initAuditService(audit)
+    const result = await auditText(audit, text)
+    initAuditService(config.auditConfig)
+    res.send({ status: 'Success', message: !result ? '含敏感词 | Contains sensitive words' : '不含敏感词 | Does not contain sensitive words.', data: null })
+  }
+  catch (error) {
+    res.send({ status: 'Fail', message: error.message, data: null })
+  }
+})
+
 app.use('', router)
 app.use('/api', router)
 app.set('trust proxy', 1)

service/src/storage/config.ts

@@ -1,7 +1,8 @@
 import { ObjectId } from 'mongodb'
 import * as dotenv from 'dotenv'
-import { isNotEmptyString } from '../utils/is'
-import { Config, MailConfig, SiteConfig } from './model'
+import type { TextAuditServiceProvider } from 'src/utils/textAudit'
+import { isNotEmptyString, isTextAuditServiceProvider } from '../utils/is'
+import { AuditConfig, Config, MailConfig, SiteConfig, TextAudioType } from './model'
 import { getConfig } from './mongo'
 
 dotenv.config()
@@ -69,9 +70,40 @@ export async function getOriginConfig() {
     if (config.siteConfig.registerReview === undefined)
       config.siteConfig.registerReview = process.env.REGISTER_REVIEW === 'true'
   }
+
+  if (config.auditConfig === undefined) {
+    config.auditConfig = new AuditConfig(
+      process.env.AUDIT_ENABLED === 'true',
+      isTextAuditServiceProvider(process.env.AUDIT_PROVIDER)
+        ? process.env.AUDIT_PROVIDER as TextAuditServiceProvider
+        : 'baidu',
+      {
+        apiKey: process.env.AUDIT_API_KEY,
+        apiSecret: process.env.AUDIT_API_SECRET,
+        label: process.env.AUDIT_TEXT_LABEL,
+      },
+      getTextAuditServiceOptionFromString(process.env.AUDIT_TEXT_TYPE),
+    )
+  }
   return config
 }
 
+function getTextAuditServiceOptionFromString(value: string): TextAudioType {
+  if (value === undefined)
+    return TextAudioType.None
+
+  switch (value.toLowerCase()) {
+    case 'request':
+      return TextAudioType.Request
+    case 'response':
+      return TextAudioType.Response
+    case 'all':
+      return TextAudioType.All
+    default:
+      return TextAudioType.None
+  }
+}
+
 export function clearConfigCache() {
   cacheExpiration = 0
   cachedConfig = null

service/src/storage/model.ts

@@ -1,4 +1,5 @@
 import type { ObjectId } from 'mongodb'
+import type { TextAuditServiceOptions, TextAuditServiceProvider } from 'src/utils/textAudit'
 
 export enum Status {
   Normal = 0,
@@ -129,6 +130,7 @@ export class Config {
     public httpsProxy?: string,
     public siteConfig?: SiteConfig,
     public mailConfig?: MailConfig,
+    public auditConfig?: AuditConfig,
   ) { }
 }
 
@@ -153,3 +155,19 @@ export class MailConfig {
     public smtpPassword: string,
   ) { }
 }
+
+export class AuditConfig {
+  constructor(
+    public enabled: boolean,
+    public provider: TextAuditServiceProvider,
+    public options: TextAuditServiceOptions,
+    public textType: TextAudioType,
+  ) { }
+}
+
+export enum TextAudioType {
+  None = 0,
+  Request = 1 << 0, // 二进制 01
+  Response = 1 << 1, // 二进制 10
+  All = Request | Response, // 二进制 11
+}

service/src/utils/is.ts

@@ -1,3 +1,6 @@
+import { TextAudioType } from '../storage/model'
+import type { TextAuditServiceProvider } from './textAudit'
+
 export function isNumber<T extends number>(value: T | unknown): value is number {
   return Object.prototype.toString.call(value) === '[object Number]'
 }
@@ -21,3 +24,16 @@ export function isFunction<T extends (...args: any[]) => any | void | never>(val
 export function isEmail(value: any): boolean {
   return isNotEmptyString(value) && /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/.test(value)
 }
+
+export function isTextAuditServiceProvider(value: any): value is TextAuditServiceProvider {
+  return value === 'baidu' // || value === 'ali'
+}
+
+export function isTextAudioType(value: any): value is TextAudioType {
+  return (
+    value === TextAudioType.None
+    || value === TextAudioType.Request
+    || value === TextAudioType.Response
+    || value === TextAudioType.All
+  )
+}

service/src/utils/textAudit.ts

@@ -0,0 +1,91 @@
+import fetch from 'node-fetch'
+
+export interface TextAuditServiceOptions {
+  apiKey: string
+  apiSecret: string
+  label: string
+}
+
+export interface TextAuditService {
+  audit(text: string): Promise<boolean>
+}
+
+/**
+ * https://ai.baidu.com/ai-doc/ANTIPORN/Vk3h6xaga
+ */
+export class BaiduTextAuditService implements TextAuditService {
+  private accessToken: string
+  private expiredTime: number
+
+  constructor(private options: TextAuditServiceOptions) { }
+
+  async audit(text: string): Promise<boolean> {
+    if (!await this.refreshAccessToken())
+      return
+    const url = `https://aip.baidubce.com/rest/2.0/solution/v1/text_censor/v2/user_defined?access_token=${this.accessToken}`
+    let headers: {
+      'Content-Type': 'application/x-www-form-urlencoded'
+      'Accept': 'application/json'
+    }
+    const response = await fetch(url, { headers, method: 'POST', body: `text=${encodeURIComponent(text)}` })
+    const data = await response.json() as { conclusionType: number; data: any }
+
+    if (data.error_msg)
+      throw new Error(data.error_msg)
+
+    // 审核结果类型，可取值1、2、3、4，分别代表1：合规，2：不合规，3：疑似，4：审核失败
+    if (data.conclusionType === 1)
+      return true
+
+    // https://ai.baidu.com/ai-doc/ANTIPORN/Nk3h6xbb2#%E7%BB%86%E5%88%86%E6%A0%87%E7%AD%BE%E5%AF%B9%E7%85%A7%E8%A1%A8
+
+    // 3 仅政治
+    const safe = data.data.filter(d => d.subType === 3).length <= 0
+    if (!safe || !this.options.label)
+      return safe
+    const str = JSON.stringify(data)
+    for (const l of this.options.label.split(',')) {
+      if (str.indexOf(l))
+        return false
+    }
+    return true
+  }
+
+  async refreshAccessToken() {
+    if (!this.options.apiKey || !this.options.apiSecret)
+      throw new Error('未配置 | Not configured.')
+
+    try {
+      if (this.accessToken && Math.floor(new Date().getTime() / 1000) <= this.expiredTime)
+        return true
+
+      const url = `https://aip.baidubce.com/oauth/2.0/token?client_id=${this.options.apiKey}&client_secret=${this.options.apiSecret}&grant_type=client_credentials`
+      let headers: {
+        'Content-Type': 'application/json'
+        'Accept': 'application/json'
+      }
+      const response = await fetch(url, { headers })
+      const data = (await response.json()) as { access_token: string; expires_in: number }
+
+      this.accessToken = data.access_token
+      this.expiredTime = Math.floor(new Date().getTime() / 1000) + (+data.expires_in)
+      return true
+    }
+    catch (error) {
+      global.console.error(`百度审核${error}`)
+    }
+    return false
+  }
+}
+
+export type TextAuditServiceProvider = 'baidu' //  | 'ali'
+
+export type TextAuditServices = {
+  [key in TextAuditServiceProvider]: new (
+    options: TextAuditServiceOptions,
+  ) => TextAuditService;
+}
+
+export const textAuditServices: TextAuditServices = {
+  baidu: BaiduTextAuditService,
+}