fix: remove imageKeys

Author: xqvvu

packages/global/core/dataset/api.d.ts

@@ -139,7 +139,6 @@ export type PushDatasetDataChunkProps = {
   q?: string;
   a?: string;
   imageId?: string;
-  imageKeys?: string[];
   chunkIndex?: number;
   indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];
 };

packages/global/core/dataset/apiDataset/type.d.ts

@@ -40,7 +40,6 @@ export type ApiDatasetServerType = {
 export type ApiFileReadContentResponse = {
   title?: string;
   rawText: string;
-  imageKeys?: string[];
 };
 
 export type APIFileReadResponse = {

packages/global/core/dataset/controller.d.ts

@@ -9,7 +9,6 @@ export type CreateDatasetDataProps = {
   q: string;
   a?: string;
   imageId?: string;
-  imageKeys?: string[];
   indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];
   indexPrefix?: string;
 };

packages/global/core/dataset/type.d.ts

@@ -148,7 +148,6 @@ export type DatasetDataFieldType = {
   q: string; // large chunks or question
   a?: string; // answer or custom content
   imageId?: string;
-  imageKeys?: string[];
 };
 export type DatasetDataSchemaType = DatasetDataFieldType & {
   _id: string;
@@ -193,7 +192,6 @@ export type DatasetTrainingSchemaType = {
   q: string;
   a: string;
   imageId?: string;
-  imageKeys?: string[];
   imageDescMap?: Record<string, string>;
   chunkIndex: number;
   indexSize?: number;

packages/service/common/buffer/rawText/controller.ts

@@ -18,21 +18,18 @@ export const addRawTextBuffer = async ({
   sourceId,
   sourceName,
   text,
-  expiredTime,
-  imageKeys = []
+  expiredTime
 }: {
   sourceId: string;
   sourceName: string;
   text: string;
   expiredTime: Date;
-  imageKeys?: string[];
 }) => {
   const gridBucket = getGridBucket();
   const metadata = {
     sourceId,
     sourceName,
-    expiredTime,
-    imageKeys
+    expiredTime
   };
 
   const buffer = Buffer.from(text);
@@ -109,8 +106,7 @@ export const getRawTextBuffer = async (sourceId: string) => {
 
     return {
       text: rawText,
-      sourceName: bufferData.metadata?.sourceName || '',
-      imageKeys: bufferData.metadata?.imageKeys || []
+      sourceName: bufferData.metadata?.sourceName || ''
     };
   });
 };

packages/service/common/buffer/rawText/schema.ts

@@ -6,8 +6,7 @@ const RawTextBufferSchema = new Schema({
   metadata: {
     sourceId: { type: String, required: true },
     sourceName: { type: String, required: true },
-    expiredTime: { type: Date, required: true },
-    imageKeys: { type: [String], required: true }
+    expiredTime: { type: Date, required: true }
   }
 });
 RawTextBufferSchema.index({ 'metadata.sourceId': 'hashed' });
@@ -19,6 +18,5 @@ export const MongoRawTextBufferSchema = getMongoModel<{
     sourceId: string;
     sourceName: string;
     expiredTime: Date;
-    imageKeys: string[];
   };
 }>(`${bucketName}.files`, RawTextBufferSchema);

packages/service/common/file/read/utils.ts

@@ -1,4 +1,3 @@
-import { uploadMongoImg } from '../image/controller';
 import FormData from 'form-data';
 import fs from 'fs';
 import type { ReadFileResponse } from '../../../worker/readFile/type';
@@ -9,12 +8,9 @@ import { matchMdImg } from '@fastgpt/global/common/string/markdown';
 import { createPdfParseUsage } from '../../../support/wallet/usage/controller';
 import { useDoc2xServer } from '../../../thirdProvider/doc2x';
 import { readRawContentFromBuffer } from '../../../worker/function';
-import { getS3DatasetSource } from '../../s3/sources/dataset';
-import type { ParsedFileContentS3KeyParams } from '../../s3/sources/dataset/type';
-import { getNanoid } from '@fastgpt/global/common/string/tools';
-import path from 'path';
-import { S3Sources } from '../../s3/type';
-import { randomUUID } from 'crypto';
+import { uploadImage2S3Bucket } from '../../s3/utils';
+import { Mimes } from '../../s3/constants';
+import { addDays } from 'date-fns';
 
 export type readRawTextByLocalFileParams = {
   teamId: string;
@@ -42,7 +38,8 @@ export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParam
     encoding: params.encoding,
     buffer,
     imageKeyOptions: {
-      prefix: params.uploadKey
+      prefix: params.uploadKey,
+      expiredTime: addDays(new Date(), 1)
     }
   });
 };
@@ -71,11 +68,10 @@ export const readS3FileContentByBuffer = async ({
   getFormatText?: boolean;
   imageKeyOptions: {
     prefix: string;
-    hasTTL?: boolean;
+    expiredTime?: Date;
   };
 }): Promise<{
   rawText: string;
-  imageKeys?: string[];
 }> => {
   const systemParse = () =>
     readRawContentFromBuffer({
@@ -171,26 +167,23 @@ export const readS3FileContentByBuffer = async ({
   addLog.debug(`Parse file success, time: ${Date.now() - start}ms. `);
 
   // markdown data format
-  const uploadedImageKeys: string[] = [];
   if (imageList && imageList.length > 0) {
     addLog.debug(`Processing ${imageList.length} images from parsed document`);
 
     await batchRun(imageList, async (item) => {
       const src = await (async () => {
         try {
-          const { prefix, hasTTL } = imageKeyOptions;
-          const ext = item.mime.split('/')[1].replace('x-', '');
-          const imageKey = await getS3DatasetSource().uploadDatasetImage({
+          const { prefix, expiredTime } = imageKeyOptions;
+          const ext = `.${item.mime.split('/')[1].replace('x-', '')}`;
+
+          return await uploadImage2S3Bucket('private', {
             base64Img: `data:${item.mime};base64,${item.base64}`,
-            mimetype: `${ext}`,
-            filename: `${item.uuid}.${ext}`,
             uploadKey: `${prefix}/${item.uuid}.${ext}`,
-            hasTTL
+            mimetype: Mimes[ext as keyof typeof Mimes],
+            filename: `${item.uuid}.${ext}`,
+            expiredTime
           });
-          uploadedImageKeys.push(imageKey);
-          return imageKey;
         } catch (error) {
-          // Don't add to uploadedImageKeys if upload failed, but still continue processing
           return `[Image Upload Failed: ${item.uuid}]`;
         }
       })();
@@ -199,63 +192,9 @@ export const readS3FileContentByBuffer = async ({
         formatText = formatText.replace(item.uuid, src);
       }
     });
-
-    // Log summary of image processing
-    addLog.info(`Image processing completed`, {
-      total: imageList.length,
-      successful: uploadedImageKeys.length,
-      failed: imageList.length - uploadedImageKeys.length
-    });
   }
 
-  addLog.debug(`Upload file to S3 success, time: ${Date.now() - start}ms`, {
-    uploadedImageKeysCount: uploadedImageKeys.length,
-    uploadedImageKeys
-  });
-
   return {
-    rawText: getFormatText ? formatText || rawText : rawText,
-    imageKeys: uploadedImageKeys
+    rawText: getFormatText ? formatText || rawText : rawText
   };
 };
-
-export const parsedFileContentS3Key = {
-  // 临时的文件路径（比如 evaluation)
-  temp: (appId: string) => `chat/${appId}/temp/parsed/${randomUUID()}`,
-
-  // 对话中上传的文件的解析结果的图片的 Key
-  chat: ({ appId, chatId, uId }: { chatId: string; uId: string; appId: string }) =>
-    `chat/${appId}/${uId}/${chatId}/parsed`,
-
-  // 上传数据集的文件的解析结果的图片的 Key
-  dataset: (params: ParsedFileContentS3KeyParams) => {
-    const { datasetId, mimetype, filename, parentFileKey } = params;
-
-    const extension = mimetype;
-    const image = (() => {
-      if (filename) {
-        return Boolean(path.extname(filename))
-          ? `${getNanoid(6)}-${filename}`
-          : `${getNanoid(6)}-${filename}.${extension}`;
-      }
-      return `${getNanoid(6)}.${extension}`;
-    })();
-
-    const parentFilename = parentFileKey?.slice().split('/').at(-1);
-    const parsedParentFilename = parentFilename
-      ? `parsed-${path.basename(parentFilename, path.extname(parentFilename))}`
-      : '';
-    const parsedParentFileKey = parentFileKey
-      ?.split('/')
-      .slice(0, -1)
-      .concat(parsedParentFilename)
-      .join('/');
-
-    return {
-      key: parsedParentFileKey
-        ? `${parsedParentFileKey}/${image}`
-        : [S3Sources.dataset, datasetId, image].join('/'),
-      filename: image
-    };
-  }
-};

packages/service/common/s3/sources/dataset/index.ts

@@ -11,9 +11,7 @@ import {
   type GetDatasetFileContentParams,
   GetDatasetFileContentParamsSchema,
   type UploadDatasetFileByBufferParams,
-  UploadDatasetFileByBufferParamsSchema,
-  type UploadDatasetImageParams,
-  UploadDatasetImageParamsSchema
+  UploadDatasetFileByBufferParamsSchema
 } from './type';
 import { MongoS3TTL } from '../../schema';
 import {
@@ -37,7 +35,7 @@ import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
 type DatasetObjectKey = `${typeof S3Sources.dataset}/${string}`;
 
 class S3DatasetSource {
-  private bucket: S3PrivateBucket;
+  public bucket: S3PrivateBucket;
   private static instance: S3DatasetSource;
 
   constructor() {
@@ -65,7 +63,11 @@ class S3DatasetSource {
     return await this.bucket.createPostPresignedUrl({ rawKey, filename }, { expiredHours: 3 });
   }
 
-  // 前缀删除
+  /**
+   * 可以根据 datasetId 或者 prefix 删除文件
+   * 如果存在 rawPrefix 则优先使用 rawPrefix 去删除文件，否则使用 datasetId 拼接前缀去删除文件
+   * 比如根据被解析的文档前缀去删除解析出来的图片
+   **/
   deleteDatasetFilesByPrefix(params: DeleteDatasetFilesByPrefixParams) {
     const { datasetId, rawPrefix } = DeleteDatasetFilesByPrefixParamsSchema.parse(params);
     const prefix = rawPrefix || [S3Sources.dataset, datasetId].filter(Boolean).join('/');
@@ -132,8 +134,7 @@ class S3DatasetSource {
     if (fileBuffer) {
       return {
         rawText: fileBuffer.text,
-        filename: fileBuffer.sourceName,
-        imageKeys: fileBuffer.imageKeys
+        filename: fileBuffer.sourceName
       };
     }
 
@@ -151,7 +152,7 @@ class S3DatasetSource {
 
     const encoding = detectFileEncoding(buffer);
     const prefix = `${path.dirname(fileId)}/${path.basename(fileId, path.extname(fileId))}-parsed`;
-    const { rawText, imageKeys } = await readS3FileContentByBuffer({
+    const { rawText } = await readS3FileContentByBuffer({
       teamId,
       tmbId,
       extension,
@@ -169,47 +170,15 @@ class S3DatasetSource {
       sourceId: bufferId,
       sourceName: filename,
       text: rawText,
-      expiredTime: addMinutes(new Date(), 20),
-      imageKeys
+      expiredTime: addMinutes(new Date(), 20)
     });
 
     return {
       rawText,
-      filename,
-      imageKeys
+      filename
     };
   }
 
-  // 上传图片
-  async uploadDatasetImage(params: UploadDatasetImageParams): Promise<string> {
-    const {
-      uploadKey,
-      base64Img,
-      mimetype,
-      filename,
-      hasTTL = true
-    } = UploadDatasetImageParamsSchema.parse(params);
-
-    const base64Data = base64Img.split(',')[1] || base64Img;
-    const buffer = Buffer.from(base64Data, 'base64');
-
-    await this.bucket.putObject(uploadKey, buffer, buffer.length, {
-      'content-type': mimetype,
-      'upload-time': new Date().toISOString(),
-      'origin-filename': encodeURIComponent(filename)
-    });
-
-    if (hasTTL) {
-      await MongoS3TTL.create({
-        minioKey: uploadKey,
-        bucketName: this.bucket.name,
-        expiredTime: addDays(new Date(), 7)
-      });
-    }
-
-    return uploadKey;
-  }
-
   // 根据文件 Buffer 上传文件
   async uploadDatasetFileByBuffer(params: UploadDatasetFileByBufferParams): Promise<string> {
     const { datasetId, buffer, filename } = UploadDatasetFileByBufferParamsSchema.parse(params);
@@ -227,37 +196,6 @@ class S3DatasetSource {
     });
     return key;
   }
-
-  // 移除单个文件的 TTL 记录
-  async removeDatasetFileTTL(fileKey: string, session?: ClientSession): Promise<void> {
-    await MongoS3TTL.deleteOne(
-      {
-        minioKey: fileKey,
-        bucketName: this.bucket.name
-      },
-      { session }
-    );
-
-    addLog.debug('Removed TTL for dataset file', { fileKey });
-  }
-
-  // 移除多个图片的 TTL 记录
-  async removeDatasetImagesTTL(imageKeys: string[], session?: ClientSession): Promise<void> {
-    if (imageKeys.length === 0) return;
-
-    const result = await MongoS3TTL.deleteMany(
-      {
-        minioKey: { $in: imageKeys },
-        bucketName: this.bucket.name
-      },
-      { session }
-    );
-
-    addLog.debug('Removed TTL for dataset images', {
-      imageKeysCount: imageKeys.length,
-      deletedCount: result.deletedCount
-    });
-  }
 }
 
 export function getS3DatasetSource() {

packages/service/common/s3/sources/dataset/type.ts

@@ -38,20 +38,11 @@ export const UploadParsedDatasetImagesParamsSchema = z.object({
 });
 export type UploadParsedDatasetImagesParams = z.infer<typeof UploadParsedDatasetImagesParamsSchema>;
 
-export const UploadDatasetImageParamsSchema = z.object({
-  base64Img: z.string().nonempty(),
-  uploadKey: z.string().nonempty(),
-  mimetype: z.string().nonempty(),
-  filename: z.string().nonempty(),
-  hasTTL: z.boolean().optional()
-});
-export type UploadDatasetImageParams = z.infer<typeof UploadDatasetImageParamsSchema>;
-
 export const ParsedFileContentS3KeyParamsSchema = z.object({
   datasetId: ObjectIdSchema,
   mimetype: z.string().nonempty(),
   filename: z.string().optional(),
-  parentFileKey: z.string().optional() // 被解析的文件的完整key，作为图片的父目录
+  parsedFileKey: z.string().optional() // 被解析的文件的完整 key，作为图片的父目录
 });
 export type ParsedFileContentS3KeyParams = z.infer<typeof ParsedFileContentS3KeyParamsSchema>;