Skip to content

Commit 2a60b78

Browse files
xqvvuc121914yu
xqvvu
authored and
c121914yu
committed
fix: delay s3 files delete timing
1 parent 10414ee commit 2a60b78

File tree

20 files changed

+156
-249
lines changed

20 files changed

+156
-249
lines changed

‎packages/service/common/file/image/controller.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,9 +135,13 @@ export const removeImageByPath = (path?: string, session?: ClientSession) => {
135135
if (!name) return;
136136

137137
const id = name.split('.')[0];
138-
if (!id || !Types.ObjectId.isValid(id)) return;
138+
if (!id) return;
139139

140-
return MongoImage.deleteOne({ _id: id }, { session });
140+
if (Types.ObjectId.isValid(id)) {
141+
return MongoImage.deleteOne({ _id: id }, { session });
142+
} else if (getS3AvatarSource().isAvatarKey(path)) {
143+
return getS3AvatarSource().deleteAvatar(path, session);
144+
}
141145
};
142146

143147
export async function readMongoImg({ id }: { id: string }) {

‎packages/service/common/file/read/utils.ts‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParam
4141
tmbId: params.tmbId,
4242
encoding: params.encoding,
4343
buffer,
44-
uploadKeyPrefix: params.uploadKey
44+
imageKeyOptions: {
45+
prefix: params.uploadKey
46+
}
4547
});
4648
};
4749

@@ -55,7 +57,7 @@ export const readS3FileContentByBuffer = async ({
5557
customPdfParse = false,
5658
usageId,
5759
getFormatText = true,
58-
uploadKeyPrefix
60+
imageKeyOptions
5961
}: {
6062
teamId: string;
6163
tmbId: string;
@@ -67,7 +69,10 @@ export const readS3FileContentByBuffer = async ({
6769
customPdfParse?: boolean;
6870
usageId?: string;
6971
getFormatText?: boolean;
70-
uploadKeyPrefix: string;
72+
imageKeyOptions: {
73+
prefix: string;
74+
hasTTL?: boolean;
75+
};
7176
}): Promise<{
7277
rawText: string;
7378
imageKeys?: string[];
@@ -173,12 +178,14 @@ export const readS3FileContentByBuffer = async ({
173178
await batchRun(imageList, async (item) => {
174179
const src = await (async () => {
175180
try {
181+
const { prefix, hasTTL } = imageKeyOptions;
176182
const ext = item.mime.split('/')[1].replace('x-', '');
177183
const imageKey = await getS3DatasetSource().uploadDatasetImage({
178184
base64Img: `data:${item.mime};base64,${item.base64}`,
179185
mimetype: `${ext}`,
180186
filename: `${item.uuid}.${ext}`,
181-
uploadKey: `${uploadKeyPrefix}/${item.uuid}.${ext}`
187+
uploadKey: `${prefix}/${item.uuid}.${ext}`,
188+
hasTTL
182189
});
183190
uploadedImageKeys.push(imageKey);
184191
return imageKey;
@@ -213,11 +220,14 @@ export const readS3FileContentByBuffer = async ({
213220
};
214221

215222
export const parsedFileContentS3Key = {
223+
// 临时的文件路径（比如 evaluation)
216224
temp: (appId: string) => `chat/${appId}/temp/parsed/${randomUUID()}`,
217225

226+
// 对话中上传的文件的解析结果的图片的 Key
218227
chat: ({ appId, chatId, uId }: { chatId: string; uId: string; appId: string }) =>
219228
`chat/${appId}/${uId}/${chatId}/parsed`,
220229

230+
// 上传数据集的文件的解析结果的图片的 Key
221231
dataset: (params: ParsedFileContentS3KeyParams) => {
222232
const { datasetId, mimetype, filename, parentFileKey } = params;
223233

‎packages/service/common/s3/sources/dataset/index.ts‎

Lines changed: 20 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ class S3DatasetSource {
6767

6868
// 前缀删除
6969
deleteDatasetFilesByPrefix(params: DeleteDatasetFilesByPrefixParams) {
70-
const { datasetId } = DeleteDatasetFilesByPrefixParamsSchema.parse(params);
71-
const prefix = [S3Sources.dataset, datasetId].filter(Boolean).join('/');
70+
const { datasetId, rawPrefix } = DeleteDatasetFilesByPrefixParamsSchema.parse(params);
71+
const prefix = rawPrefix || [S3Sources.dataset, datasetId].filter(Boolean).join('/');
7272
return this.bucket.addDeleteJob({ prefix });
7373
}
7474

@@ -154,13 +154,15 @@ class S3DatasetSource {
154154
const { rawText, imageKeys } = await readS3FileContentByBuffer({
155155
teamId,
156156
tmbId,
157-
uploadKeyPrefix: prefix,
158157
extension,
159158
buffer,
160159
encoding,
161160
customPdfParse,
162161
usageId,
163-
getFormatText
162+
getFormatText,
163+
imageKeyOptions: {
164+
prefix: prefix
165+
}
164166
});
165167

166168
addRawTextBuffer({
@@ -180,8 +182,13 @@ class S3DatasetSource {
180182

181183
// 上传图片
182184
async uploadDatasetImage(params: UploadDatasetImageParams): Promise<string> {
183-
const { uploadKey, base64Img, mimetype, filename } =
184-
UploadDatasetImageParamsSchema.parse(params);
185+
const {
186+
uploadKey,
187+
base64Img,
188+
mimetype,
189+
filename,
190+
hasTTL = true
191+
} = UploadDatasetImageParamsSchema.parse(params);
185192

186193
const base64Data = base64Img.split(',')[1] || base64Img;
187194
const buffer = Buffer.from(base64Data, 'base64');
@@ -192,11 +199,13 @@ class S3DatasetSource {
192199
'origin-filename': encodeURIComponent(filename)
193200
});
194201

195-
await MongoS3TTL.create({
196-
minioKey: uploadKey,
197-
bucketName: this.bucket.name,
198-
expiredTime: addDays(new Date(), 7)
199-
});
202+
if (hasTTL) {
203+
await MongoS3TTL.create({
204+
minioKey: uploadKey,
205+
bucketName: this.bucket.name,
206+
expiredTime: addDays(new Date(), 7)
207+
});
208+
}
200209

201210
return uploadKey;
202211
}
@@ -249,19 +258,6 @@ class S3DatasetSource {
249258
deletedCount: result.deletedCount
250259
});
251260
}
252-
253-
async getFileDatasetInfo(key: string): Promise<{
254-
_id: string;
255-
datasetId: string;
256-
collectionId: string;
257-
} | null> {
258-
return await MongoDatasetData.findOne(
259-
{ $or: [{ imageKeys: { $in: [key] } }, { imageId: key }] },
260-
'datasetId collectionId'
261-
)
262-
.lean()
263-
.exec();
264-
}
265261
}
266262

267263
export function getS3DatasetSource() {

‎packages/service/common/s3/sources/dataset/type.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ export const CreateGetDatasetFileURLParamsSchema = z.object({
1515
export type CreateGetDatasetFileURLParams = z.infer<typeof CreateGetDatasetFileURLParamsSchema>;
1616

1717
export const DeleteDatasetFilesByPrefixParamsSchema = z.object({
18-
datasetId: ObjectIdSchema
18+
datasetId: ObjectIdSchema.optional(),
19+
rawPrefix: z.string().nonempty().optional()
1920
});
2021
export type DeleteDatasetFilesByPrefixParams = z.infer<
2122
typeof DeleteDatasetFilesByPrefixParamsSchema
@@ -41,7 +42,8 @@ export const UploadDatasetImageParamsSchema = z.object({
4142
base64Img: z.string().nonempty(),
4243
uploadKey: z.string().nonempty(),
4344
mimetype: z.string().nonempty(),
44-
filename: z.string().nonempty()
45+
filename: z.string().nonempty(),
46+
hasTTL: z.boolean().optional()
4547
});
4648
export type UploadDatasetImageParams = z.infer<typeof UploadDatasetImageParamsSchema>;
4749

‎packages/service/common/s3/utils.ts‎

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
11
import jwt from 'jsonwebtoken';
22
import { differenceInMilliseconds, addDays } from 'date-fns';
33
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
4-
import { S3Sources } from './type';
5-
import { getS3ChatSource } from './sources/chat';
6-
import { getS3DatasetSource } from './sources/dataset';
7-
import { EndpointUrl } from '@fastgpt/global/common/file/constants';
84

95
export function jwtSignS3ObjectKey(objectKey: string) {
106
const secret = process.env.FILE_TOKEN_KEY as string;
@@ -27,30 +23,3 @@ export function jwtVerifyS3ObjectKey(token: string) {
2723
});
2824
});
2925
}
30-
31-
export async function replaceDatasetQuoteTextWithJWT(datasetQuoteText: string) {
32-
if (!datasetQuoteText || typeof datasetQuoteText !== 'string') return datasetQuoteText as string;
33-
34-
const prefixPattern = Object.values(S3Sources)
35-
.map((pattern) => `${pattern}\\/[^\\s)]+`)
36-
.join('|');
37-
const regex = new RegExp(String.raw`(!?)\[([^\]]+)\]\((?!https?:\/\/)(${prefixPattern})\)`, 'g');
38-
const s3DatasetSource = getS3DatasetSource();
39-
const s3ChatSource = getS3ChatSource();
40-
41-
const matches = Array.from(datasetQuoteText.matchAll(regex));
42-
let content = datasetQuoteText;
43-
44-
for (const match of matches.slice().reverse()) {
45-
const [full, bang, alt, objectKey] = match;
46-
47-
if (s3DatasetSource.isDatasetObjectKey(objectKey) || s3ChatSource.isChatFileKey(objectKey)) {
48-
const url = `${EndpointUrl}/api/system/file/${jwtSignS3ObjectKey(objectKey)}`;
49-
const replacement = `${bang}[${alt}](${url})`;
50-
content =
51-
content.slice(0, match.index) + replacement + content.slice(match.index + full.length);
52-
}
53-
}
54-
55-
return content;
56-
}

‎packages/service/core/ai/llm/request.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,6 @@ export const createLLMResponse = async <T extends CompletionsBodyType>(
8686
messages: rewriteMessages
8787
});
8888

89-
console.dir(requestBody, { depth: null });
9089
// console.log(JSON.stringify(requestBody, null, 2));
9190
const { response, isStreamResponse, getEmptyResponseTip } = await createChatCompletion({
9291
body: requestBody,

‎packages/service/core/ai/llm/utils.ts‎

Lines changed: 0 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -282,46 +282,6 @@ export const loadRequestMessages = async ({
282282
return result.map((item) => item.text).join('\n');
283283
};
284284

285-
// const redis = getGlobalRedisConnection();
286-
// const prefixPattern = Object.values(S3Sources)
287-
// .map((pattern) => `${pattern}\\/[^\\s)]+`)
288-
// .join('|');
289-
// const regex = new RegExp(String.raw`(!?)\[([^\]]+)\]\((?!https?:\/\/)(${prefixPattern})\)`, 'g');
290-
291-
// TODO: 在我迁移完到 JWT 后移除这个 transformS3PreviewKey
292-
// const transformS3PreviewKey = async (
293-
// origin: string | ChatCompletionContentPartText[] | undefined
294-
// ) => {
295-
// if (!origin || typeof origin !== 'string') return origin as string;
296-
297-
// const matches = Array.from(origin.matchAll(regex));
298-
// let content = origin;
299-
300-
// for (const match of matches.slice().reverse()) {
301-
// const [full, bang, alt, objectKey] = match;
302-
303-
// const filename = objectKey.split('/').pop()?.split('-')[1];
304-
// const name = `${randomUUID()}:${filename}`;
305-
306-
// const redisKey = `chat:temp_file:${name}`;
307-
// try {
308-
// await redis.set(redisKey, objectKey);
309-
// await redis.expire(redisKey, 3600);
310-
// } catch {
311-
// continue;
312-
// }
313-
314-
// const k = new URLSearchParams({ k: name });
315-
// const link = `${EndpointUrl}${TempFileURL}?${k}`;
316-
317-
// const replacement = `${bang}[${alt}](${link})`;
318-
// content =
319-
// content.slice(0, match.index) + replacement + content.slice(match.index + full.length);
320-
// }
321-
322-
// return content;
323-
// };
324-
325285
if (messages.length === 0) {
326286
return Promise.reject(i18nT('common:core.chat.error.Messages empty'));
327287
}

‎packages/service/core/app/controller.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,6 @@ export const onDelOneApp = async ({
220220

221221
// Delete avatar
222222
await removeImageByPath(app.avatar, session);
223-
await getS3AvatarSource().deleteAvatar(app.avatar, session);
224223
};
225224

226225
// Delete chats

‎packages/service/core/chat/saveChat.ts‎

Lines changed: 2 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,4 @@
1-
import type {
2-
AIChatItemType,
3-
AIChatItemValueItemType,
4-
UserChatItemType
5-
} from '@fastgpt/global/core/chat/type.d';
1+
import type { AIChatItemType, UserChatItemType } from '@fastgpt/global/core/chat/type.d';
62
import { MongoApp } from '../app/schema';
73
import type { ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
84
import { ChatItemValueTypeEnum, ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
@@ -23,7 +19,6 @@ import { MongoChatItemResponse } from './chatItemResponseSchema';
2319
import { chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
2420
import { MongoS3TTL } from '../../common/s3/schema';
2521
import type { ClientSession } from '../../common/mongo';
26-
import { getGlobalRedisConnection } from '../../common/redis';
2722

2823
type Props = {
2924
chatId: string;
@@ -46,44 +41,6 @@ type Props = {
4641
errorMsg?: string;
4742
};
4843

49-
// TODO: 在我迁移完到 JWT 后移除这个 transformAiResponse
50-
// const transformAiResponse = async (value: AIChatItemValueItemType[]) => {
51-
// const redis = getGlobalRedisConnection();
52-
// const regex = /(!?)\[([^\]]+)\]\((https?:\/\/[^\s)]+\/api\/file\/temp[^\s)]*)\)/g;
53-
54-
// return Promise.all(
55-
// value.map(async (item) => {
56-
// if (item.type !== ChatItemValueTypeEnum.text || !item.text) return item;
57-
// let content = item.text.content;
58-
// const matches = Array.from(content.matchAll(regex));
59-
60-
// for (const match of matches.slice().reverse()) {
61-
// const [full, bang, alt, link] = match;
62-
// if (typeof match.index !== 'number') continue;
63-
64-
// try {
65-
// const url = new URL(link); // 可能会发生解析错误
66-
// const k = url.searchParams.get('k');
67-
// if (!k) continue;
68-
69-
// const redisKey = `chat:temp_file:${decodeURIComponent(k)}`;
70-
// const objectKey = await redis.get(redisKey);
71-
// if (!objectKey) continue;
72-
73-
// const replacement = `${bang}[${alt}](${objectKey})`;
74-
// content =
75-
// content.slice(0, match.index) + replacement + content.slice(match.index + full.length);
76-
// } catch {
77-
// continue;
78-
// }
79-
// }
80-
81-
// item.text.content = content;
82-
// return item;
83-
// })
84-
// );
85-
// };
86-
8744
const beforProcess = (props: Props) => {
8845
// Remove url
8946
props.userContent.value.forEach((item) => {
@@ -113,6 +70,7 @@ const afterProcess = async ({
11370
})
11471
.flat()
11572
.filter(Boolean) as string[];
73+
11674
if (fileKeys.length > 0) {
11775
await MongoS3TTL.deleteMany({ minioKey: { $in: fileKeys } }, { session });
11876
}
@@ -152,10 +110,6 @@ const formatAiContent = ({
152110
return responseItem;
153111
});
154112

155-
// aiResponse.value = await transformAiResponse(aiResponse.value);
156-
// console.log('aiResponse ========================');
157-
// console.dir(aiResponse, { depth: null });
158-
159113
return {
160114
aiResponse: {
161115
...aiResponse,

0 commit comments

Comments
 (0)