From 0847370d9545f7d4517764a52efddcfe56ed62dc Mon Sep 17 00:00:00 2001 From: wangzhankun Date: Wed, 20 Dec 2023 17:00:41 +0800 Subject: [PATCH 1/9] add openapi 2 gemini proxy, not support stream mode --- cf-openai-gemini-proxy.js | 173 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 cf-openai-gemini-proxy.js diff --git a/cf-openai-gemini-proxy.js b/cf-openai-gemini-proxy.js new file mode 100644 index 0000000..5668b77 --- /dev/null +++ b/cf-openai-gemini-proxy.js @@ -0,0 +1,173 @@ +// The deployment name you chose when you deployed the model. +const chatmodel = 'gemini-pro'; + +addEventListener("fetch", (event) => { + event.respondWith(handleRequest(event.request)); +}); + +async function handleRequest(request) { + if (request.method === 'OPTIONS') { + return handleOPTIONS(request) + } + + + + const url = new URL(request.url); + if (url.pathname === '/v1/chat/completions') { + var path = "generateContent" + var deployName = chatmodel; + } else if (url.pathname === '/v1/completions') { + var path = "generateContent" + var deployName = chatmodel; + } else { + return new Response('404 Not Found', { status: 404 }) + } + + let body; + if (request.method === 'POST') { + body = await request.json(); + } + + // if(body?.stream === true) { + // path = "streamGenerateContent"; + // } + + const authKey = request.headers.get('Authorization'); + if (!authKey) { + return new Response("Not allowed", { status: 403 }); + } + + // Remove 'Bearer ' from the start of authKey + const apiKey = authKey.replace('Bearer ', ''); + + const fetchAPI = `https://generativelanguage.googleapis.com/v1/models/${deployName}:${path}?key=${apiKey}` + + // Transform request body from OpenAI to Gemini format + const transformedBody = { + // body?.messages 是一个数组，每个元素是一个对象，包含 role 和 content 两个属性 + // 目标是把这个数组转换成 {} + // if role is 'system', then delete the message + + contents: body?.messages?.filter((message) => message.role !== 'system').map(message => ({ + role: message.role === "assistant" ? "model" : "user", + parts: { text: message.content }, + })), + + generationConfig: { + temperature: body?.temperature, + candidateCount: body?.n, + topP: body?.top_p, + } + }; + + const payload = { + method: request.method, + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(transformedBody), + }; + + const response = await fetch(fetchAPI, payload); + + // Check if the response is valid + if (!response.ok) { + return new Response(response.statusText, { status: response.status }); + } + + const geminiData = await response.json(); + + // console.log(geminiData); + + // Transform response from Gemini to OpenAI format + + // console.log(transformedResponse); + + // if (body?.stream != true) + { + const transformedResponse = transformResponse(geminiData); + return new Response(JSON.stringify(transformedResponse), { + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': '*', + 'Access-Control-Allow-Headers': '*' + } + }); + } + +} + +// Function to transform the response +function transformResponse(GeminiData) { + // Check if the 'candidates' array exists and if it's not empty + if (!GeminiData.candidates) { + // If it doesn't exist or is empty, create a default candidate message + GeminiData.candidates = [ + { + "content": { + "parts": [ + { + "text": "Oops, Model respond nothing." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0, + "safetyRatings": [ + { + "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", + "probability": "NEGLIGIBLE" + }, + { + "category": "HARM_CATEGORY_HATE_SPEECH", + "probability": "NEGLIGIBLE" + }, + { + "category": "HARM_CATEGORY_HARASSMENT", + "probability": "NEGLIGIBLE" + }, + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "probability": "NEGLIGIBLE" + } + ] + } + ]; + } + + // console.log(GeminiData.candidates); + + var ret = { + id: "chatcmpl-QXlha2FBbmROaXhpZUFyZUF3ZXNvbWUK", + object: 'chat.completion', + created: Math.floor(Date.now() / 1000), // Current Unix timestamp + model: 'gpt-3.5-turbo', // Static model name + usage: { + prompt_tokens: GeminiData.usageMetadata?.promptTokenCount, // This is a placeholder. Replace with actual token count if available + completion_tokens: GeminiData.usageMetadata?.candidatesTokenCount, // This is a placeholder. Replace with actual token count if available + total_tokens: GeminiData.usageMetadata?.totalTokenCount, // This is a placeholder. Replace with actual token count if available + }, + choices: GeminiData.candidates.map((candidate) => ({ + message: { + role: 'assistant', + content: candidate.content.parts[0].text, + }, + finish_reason: 'stop', // Static finish reason + index: candidate.index, + })), + }; + + return ret; +} + +async function handleOPTIONS(request) { + return new Response("pong", { + headers: { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': '*', + 'Access-Control-Allow-Headers': '*' + } + }) +} From 54be3560f673074c39c9b8ed16a440cabbe0df3b Mon Sep 17 00:00:00 2001 From: wangzhankun Date: Wed, 20 Dec 2023 17:22:33 +0800 Subject: [PATCH 2/9] support stream mode --- cf-openai-gemini-proxy.js | 47 +++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/cf-openai-gemini-proxy.js b/cf-openai-gemini-proxy.js index 5668b77..b365274 100644 --- a/cf-openai-gemini-proxy.js +++ b/cf-openai-gemini-proxy.js @@ -28,10 +28,6 @@ async function handleRequest(request) { body = await request.json(); } - // if(body?.stream === true) { - // path = "streamGenerateContent"; - // } - const authKey = request.headers.get('Authorization'); if (!authKey) { return new Response("Not allowed", { status: 403 }); @@ -82,10 +78,9 @@ async function handleRequest(request) { // Transform response from Gemini to OpenAI format // console.log(transformedResponse); + const transformedResponse = transformResponse(geminiData); - // if (body?.stream != true) - { - const transformedResponse = transformResponse(geminiData); + if (body?.stream != true) { return new Response(JSON.stringify(transformedResponse), { headers: { 'Content-Type': 'application/json', @@ -94,8 +89,46 @@ async function handleRequest(request) { 'Access-Control-Allow-Headers': '*' } }); + } else { + let { readable, writable } = new TransformStream(); + streamResponse(transformedResponse, writable); + return new Response(readable, { + headers: { + 'Content-Type': 'text/event-stream', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': '*', + 'Access-Control-Allow-Headers': '*' + } + }); } +} + +function streamResponse(response, writable) { + let encoder = new TextEncoder(); + let writer = writable.getWriter(); + + let content = response.choices[0].message.content; + + let chunks = content.split("\n\n") || []; + chunks.forEach((chunk, i) => { + let chunkResponse = { + ...response, + object: "chat.completion.chunk", + choices: [{ + index: response.choices[0].index, + delta: { ...response.choices[0].message, content: chunk }, + finish_reason: i === chunks.length - 1 ? 'stop' : null // Set 'stop' for the last chunk + }], + usage: null + }; + + writer.write(encoder.encode(`data: ${JSON.stringify(chunkResponse)}\n\n`)); + }); + + // Write the done signal + writer.write(encoder.encode(`data: [DONE]\n`)); + writer.close(); } // Function to transform the response From 1456d2fed367776e7acaea29f261bc168ef9538d Mon Sep 17 00:00:00 2001 From: wangzhankun Date: Fri, 22 Dec 2023 12:45:13 +0800 Subject: [PATCH 3/9] fix bug for openai2gemini --- cf-openai-gemini-proxy.js | 65 +++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/cf-openai-gemini-proxy.js b/cf-openai-gemini-proxy.js index b365274..160a623 100644 --- a/cf-openai-gemini-proxy.js +++ b/cf-openai-gemini-proxy.js @@ -39,22 +39,7 @@ async function handleRequest(request) { const fetchAPI = `https://generativelanguage.googleapis.com/v1/models/${deployName}:${path}?key=${apiKey}` // Transform request body from OpenAI to Gemini format - const transformedBody = { - // body?.messages 是一个数组，每个元素是一个对象，包含 role 和 content 两个属性 - // 目标是把这个数组转换成 {} - // if role is 'system', then delete the message - - contents: body?.messages?.filter((message) => message.role !== 'system').map(message => ({ - role: message.role === "assistant" ? "model" : "user", - parts: { text: message.content }, - })), - - generationConfig: { - temperature: body?.temperature, - candidateCount: body?.n, - topP: body?.top_p, - } - }; + const transformedBody = transform2GeminiRequest(body); const payload = { method: request.method, @@ -103,6 +88,54 @@ async function handleRequest(request) { } } +function transform2GeminiRequest(body) { + + let messages = body?.messages || []; + if (messages.length === 0) { + messages.push({ role: 'user', content: '' }); + } else { + // 如果相邻的两个 message 的 role 相同，那么就把它们合并成一个 message + let mergedMessages = []; + let lastRole = null; + messages.forEach((message) => { + if (message.role === 'system') { + message.role = 'user'; + }else if(message.role === 'assistant'){ + message.role = 'model'; + } + + if (lastRole === message.role) { + mergedMessages[mergedMessages.length - 1].content += message.content + '\n'; + } else { + mergedMessages.push(message); + } + lastRole = message.role; + }); + + messages = mergedMessages; + } + + var ret = { + // body?.messages 是一个数组，每个元素是一个对象，包含 role 和 content 两个属性 + // 目标是把这个数组转换成 {} + // if role is 'system', then delete the message + + contents: messages.map(message => ({ + role: message.role, + parts: { text: message.content }, + })), + + generationConfig: { + temperature: body?.temperature, + candidateCount: body?.n, + topP: body?.top_p, + } + }; + + console.log(ret); + return ret; +} + function streamResponse(response, writable) { let encoder = new TextEncoder(); let writer = writable.getWriter(); From 58c66461d399b611d3351635a6416cfb81cf3a4c Mon Sep 17 00:00:00 2001 From: wangzhankun Date: Fri, 22 Dec 2023 20:21:09 +0800 Subject: [PATCH 4/9] add function call convert --- cf-openai-gemini-proxy.js | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/cf-openai-gemini-proxy.js b/cf-openai-gemini-proxy.js index 160a623..5bfe866 100644 --- a/cf-openai-gemini-proxy.js +++ b/cf-openai-gemini-proxy.js @@ -88,6 +88,38 @@ async function handleRequest(request) { } } +// 现在 gemini 还不支持 function，所以这个函数暂时没用 +function convert2GeminiFunctionDeclaration(tools, tool_choice) { + if (tools === undefined || tool_choice === undefined || tool_choice === "none") { + return []; + } + + // TODO - add support for tool_choice + + const result = []; + const functionDeclarations = []; + + for (const tool of tools) { + if (tool.type === "function") { + var functionDeclaration = { + name: tool.function.name, + description: tool.function.description, + parameters: tool.function.parameters, + }; + functionDeclarations.push(functionDeclaration); + } + } + + if (functionDeclarations.length > 0) { + const toolObject = { + functionDeclarations, + }; + result.push(toolObject); + } + + return result; +} + function transform2GeminiRequest(body) { let messages = body?.messages || []; @@ -100,7 +132,7 @@ function transform2GeminiRequest(body) { messages.forEach((message) => { if (message.role === 'system') { message.role = 'user'; - }else if(message.role === 'assistant'){ + } else if (message.role === 'assistant') { message.role = 'model'; } From a0bc32745f65c0000284a80cbdb855fea1bdf118 Mon Sep 17 00:00:00 2001 From: wangzhankun Date: Fri, 22 Dec 2023 20:41:41 +0800 Subject: [PATCH 5/9] add embedding call --- cf-openai-gemini-proxy.js | 120 ++++++++++++++++++++++++++------------ 1 file changed, 82 insertions(+), 38 deletions(-) diff --git a/cf-openai-gemini-proxy.js b/cf-openai-gemini-proxy.js index 5bfe866..cbab4af 100644 --- a/cf-openai-gemini-proxy.js +++ b/cf-openai-gemini-proxy.js @@ -1,5 +1,6 @@ // The deployment name you chose when you deployed the model. const chatmodel = 'gemini-pro'; +const embeddmodel = 'embedding-001'; addEventListener("fetch", (event) => { event.respondWith(handleRequest(event.request)); @@ -19,7 +20,11 @@ async function handleRequest(request) { } else if (url.pathname === '/v1/completions') { var path = "generateContent" var deployName = chatmodel; - } else { + } else if (url.pathname === '/v1/embeddings') { + var path = "embedContent" + var deployName = embeddmodel; + } + else { return new Response('404 Not Found', { status: 404 }) } @@ -38,34 +43,34 @@ async function handleRequest(request) { const fetchAPI = `https://generativelanguage.googleapis.com/v1/models/${deployName}:${path}?key=${apiKey}` - // Transform request body from OpenAI to Gemini format - const transformedBody = transform2GeminiRequest(body); - - const payload = { - method: request.method, - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(transformedBody), - }; - - const response = await fetch(fetchAPI, payload); - - // Check if the response is valid - if (!response.ok) { - return new Response(response.statusText, { status: response.status }); - } - - const geminiData = await response.json(); - - // console.log(geminiData); - - // Transform response from Gemini to OpenAI format - - // console.log(transformedResponse); - const transformedResponse = transformResponse(geminiData); - - if (body?.stream != true) { + if (deployName === embeddmodel) { + const transformedBody = { + "model": "models/embedding-001", + "content": { + "parts": [ + { + "text": body?.input + } + ] + } + }; + const payload = { + method: request.method, + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(transformedBody), + }; + const response = await fetch(fetchAPI, payload); + if (!response.ok) { + return new Response(response.statusText, { status: response.status }); + } + const geminiData = await response.json(); + const transformedResponse = { + "object": "embedding", + "embedding": geminiData?.embedding?.values || [], + "index": 0 + }; return new Response(JSON.stringify(transformedResponse), { headers: { 'Content-Type': 'application/json', @@ -75,16 +80,55 @@ async function handleRequest(request) { } }); } else { - let { readable, writable } = new TransformStream(); - streamResponse(transformedResponse, writable); - return new Response(readable, { + + // Transform request body from OpenAI to Gemini format + const transformedBody = transform2GeminiRequest(body); + + const payload = { + method: request.method, headers: { - 'Content-Type': 'text/event-stream', - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Methods': '*', - 'Access-Control-Allow-Headers': '*' - } - }); + "Content-Type": "application/json", + }, + body: JSON.stringify(transformedBody), + }; + + const response = await fetch(fetchAPI, payload); + + // Check if the response is valid + if (!response.ok) { + return new Response(response.statusText, { status: response.status }); + } + + const geminiData = await response.json(); + + // console.log(geminiData); + + // Transform response from Gemini to OpenAI format + + // console.log(transformedResponse); + const transformedResponse = transformResponse(geminiData); + + if (body?.stream != true) { + return new Response(JSON.stringify(transformedResponse), { + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': '*', + 'Access-Control-Allow-Headers': '*' + } + }); + } else { + let { readable, writable } = new TransformStream(); + streamResponse(transformedResponse, writable); + return new Response(readable, { + headers: { + 'Content-Type': 'text/event-stream', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': '*', + 'Access-Control-Allow-Headers': '*' + } + }); + } } } From 3e2104a8230ec3005cc3a3e35477744634b6e469 Mon Sep 17 00:00:00 2001 From: wangzhankun Date: Fri, 22 Dec 2023 21:07:53 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=E9=87=8D=E6=9E=84=20cf-openai-gemini-proxy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cf-openai-gemini-proxy.js | 221 +++++++++++++++++--------------------- 1 file changed, 100 insertions(+), 121 deletions(-) diff --git a/cf-openai-gemini-proxy.js b/cf-openai-gemini-proxy.js index cbab4af..9216374 100644 --- a/cf-openai-gemini-proxy.js +++ b/cf-openai-gemini-proxy.js @@ -10,9 +10,17 @@ async function handleRequest(request) { if (request.method === 'OPTIONS') { return handleOPTIONS(request) } + const url = new URL(request.url); + if (url.pathname === '/v1/chat/completions' || url.pathname === '/v1/completions') { + return handleRequestWithTransform(request, transformCommonRequest, transformCommonResponse); + } else if (url.pathname === '/v1/embeddings') { + return handleRequestWithTransform(request, transformEmbeddingRequest, transformEmbeddingResponse); + } else { + return new Response('404 Not Found for ' + url.pathname, { status: 404 }) + } +} - - +function transformURL(request) { const url = new URL(request.url); if (url.pathname === '/v1/chat/completions') { var path = "generateContent" @@ -23,54 +31,39 @@ async function handleRequest(request) { } else if (url.pathname === '/v1/embeddings') { var path = "embedContent" var deployName = embeddmodel; - } - else { - return new Response('404 Not Found', { status: 404 }) - } - - let body; - if (request.method === 'POST') { - body = await request.json(); + } else { + return null; } const authKey = request.headers.get('Authorization'); - if (!authKey) { - return new Response("Not allowed", { status: 403 }); + const apiKey = authKey.replace('Bearer ', ''); + return `https://generativelanguage.googleapis.com/v1/models/${deployName}:${path}?key=${apiKey}` +} + +async function handleRequestWithTransform(request, transformRequestBody, transformResponseBody) { + let body = await request.json(); + const fetchAPI = transformURL(request); + if (fetchAPI === null) { + return new Response('404 Not Found', { status: 404 }) } - // Remove 'Bearer ' from the start of authKey - const apiKey = authKey.replace('Bearer ', ''); - const fetchAPI = `https://generativelanguage.googleapis.com/v1/models/${deployName}:${path}?key=${apiKey}` + const transformedBody = transformRequestBody(body); + const payload = { + method: request.method, + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(transformedBody), + }; + const response = await fetch(fetchAPI, payload); + if (!response.ok) { + return new Response(response.statusText, { status: response.status }); + } + const geminiData = await response.json(); + const transformedResponse = transformResponseBody(geminiData); - if (deployName === embeddmodel) { - const transformedBody = { - "model": "models/embedding-001", - "content": { - "parts": [ - { - "text": body?.input - } - ] - } - }; - const payload = { - method: request.method, - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(transformedBody), - }; - const response = await fetch(fetchAPI, payload); - if (!response.ok) { - return new Response(response.statusText, { status: response.status }); - } - const geminiData = await response.json(); - const transformedResponse = { - "object": "embedding", - "embedding": geminiData?.embedding?.values || [], - "index": 0 - }; + if (body?.stream != true) { return new Response(JSON.stringify(transformedResponse), { headers: { 'Content-Type': 'application/json', @@ -80,58 +73,20 @@ async function handleRequest(request) { } }); } else { - - // Transform request body from OpenAI to Gemini format - const transformedBody = transform2GeminiRequest(body); - - const payload = { - method: request.method, + let { readable, writable } = new TransformStream(); + streamResponse(transformedResponse, writable); + return new Response(readable, { headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(transformedBody), - }; - - const response = await fetch(fetchAPI, payload); - - // Check if the response is valid - if (!response.ok) { - return new Response(response.statusText, { status: response.status }); - } - - const geminiData = await response.json(); - - // console.log(geminiData); - - // Transform response from Gemini to OpenAI format - - // console.log(transformedResponse); - const transformedResponse = transformResponse(geminiData); - - if (body?.stream != true) { - return new Response(JSON.stringify(transformedResponse), { - headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Methods': '*', - 'Access-Control-Allow-Headers': '*' - } - }); - } else { - let { readable, writable } = new TransformStream(); - streamResponse(transformedResponse, writable); - return new Response(readable, { - headers: { - 'Content-Type': 'text/event-stream', - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Methods': '*', - 'Access-Control-Allow-Headers': '*' - } - }); - } + 'Content-Type': 'text/event-stream', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': '*', + 'Access-Control-Allow-Headers': '*' + } + }); } } + // 现在 gemini 还不支持 function，所以这个函数暂时没用 function convert2GeminiFunctionDeclaration(tools, tool_choice) { if (tools === undefined || tool_choice === undefined || tool_choice === "none") { @@ -164,7 +119,28 @@ function convert2GeminiFunctionDeclaration(tools, tool_choice) { return result; } -function transform2GeminiRequest(body) { +function transformEmbeddingRequest(body) { + return { + "model": "models/embedding-001", + "content": { + "parts": [ + { + "text": body?.input + } + ] + } + }; +} + +function transformEmbeddingResponse(geminiData) { + return { + "object": "embedding", + "embedding": geminiData?.embedding?.values || [], + "index": 0 + }; +} + +function transformCommonRequest(body) { let messages = body?.messages || []; if (messages.length === 0) { @@ -212,36 +188,8 @@ function transform2GeminiRequest(body) { return ret; } -function streamResponse(response, writable) { - let encoder = new TextEncoder(); - let writer = writable.getWriter(); - - let content = response.choices[0].message.content; - - let chunks = content.split("\n\n") || []; - chunks.forEach((chunk, i) => { - let chunkResponse = { - ...response, - object: "chat.completion.chunk", - choices: [{ - index: response.choices[0].index, - delta: { ...response.choices[0].message, content: chunk }, - finish_reason: i === chunks.length - 1 ? 'stop' : null // Set 'stop' for the last chunk - }], - usage: null - }; - - writer.write(encoder.encode(`data: ${JSON.stringify(chunkResponse)}\n\n`)); - }); - - // Write the done signal - writer.write(encoder.encode(`data: [DONE]\n`)); - - writer.close(); -} - // Function to transform the response -function transformResponse(GeminiData) { +function transformCommonResponse(GeminiData) { // Check if the 'candidates' array exists and if it's not empty if (!GeminiData.candidates) { // If it doesn't exist or is empty, create a default candidate message @@ -304,6 +252,37 @@ function transformResponse(GeminiData) { return ret; } + +function streamResponse(response, writable) { + let encoder = new TextEncoder(); + let writer = writable.getWriter(); + + let content = response.choices[0].message.content; + + let chunks = content.split("\n\n") || []; + chunks.forEach((chunk, i) => { + let chunkResponse = { + ...response, + object: "chat.completion.chunk", + choices: [{ + index: response.choices[0].index, + delta: { ...response.choices[0].message, content: chunk }, + finish_reason: i === chunks.length - 1 ? 'stop' : null // Set 'stop' for the last chunk + }], + usage: null + }; + + writer.write(encoder.encode(`data: ${JSON.stringify(chunkResponse)}\n\n`)); + }); + + // Write the done signal + writer.write(encoder.encode(`data: [DONE]\n`)); + + writer.close(); +} + + + async function handleOPTIONS(request) { return new Response("pong", { headers: { From 4203c98476ba80d364457a135ad5cc1195ff4b9a Mon Sep 17 00:00:00 2001 From: wangzhankun Date: Sun, 7 Jan 2024 13:23:48 +0800 Subject: [PATCH 7/9] add proxyintro.md --- ProxyIntro.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 ProxyIntro.md diff --git a/ProxyIntro.md b/ProxyIntro.md new file mode 100644 index 0000000..e960c42 --- /dev/null +++ b/ProxyIntro.md @@ -0,0 +1,10 @@ + +# cf-openai-gemini-proxy.js +|chatgpt3.5 feature|support?| +|---|---| +|single conversion| yes| +|multi conversion| yes| +|stream content|yes| +|embedded| yes| +|function call|no| +|多模态|no| From 3f5219d758fe43e28188cfabfc98ecc6b9477f7a Mon Sep 17 00:00:00 2001 From: wangzhankun Date: Sun, 7 Jan 2024 17:05:53 +0800 Subject: [PATCH 8/9] add cf-openai-qwen-proxy.js --- ProxyIntro.md | 11 ++ cf-openai-qwen-proxy.js | 246 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 cf-openai-qwen-proxy.js diff --git a/ProxyIntro.md b/ProxyIntro.md index e960c42..98335e4 100644 --- a/ProxyIntro.md +++ b/ProxyIntro.md @@ -8,3 +8,14 @@ |embedded| yes| |function call|no| |多模态|no| + +# cf-openai-qwen-proxy.js + +|chatgpt3.5 feature|support?| +|---|---| +|single conversion| yes| +|multi conversion| yes| +|stream content|yes| +|embedded| no| +|function call|no| +|多模态|no| \ No newline at end of file diff --git a/cf-openai-qwen-proxy.js b/cf-openai-qwen-proxy.js new file mode 100644 index 0000000..29d8f87 --- /dev/null +++ b/cf-openai-qwen-proxy.js @@ -0,0 +1,246 @@ +// The deployment name you chose when you deployed the model. +const chatmodel = 'text-generation'; +const embeddmodel = 'embedding-001'; + +addEventListener("fetch", (event) => { + event.respondWith(handleRequest(event.request)); +}); + +async function handleRequest(request) { + if (request.method === 'OPTIONS') { + return handleOPTIONS(request) + } + const url = new URL(request.url); + if (url.pathname === '/v1/chat/completions' || url.pathname === '/v1/completions') { + return handleRequestWithTransform(request, transformCommonRequest, transformCommonResponse); + } else if (url.pathname === '/v1/embeddings') { + return handleRequestWithTransform(request, transformEmbeddingRequest, transformEmbeddingResponse); + } else { + return new Response('404 Not Found for ' + url.pathname, { status: 404 }) + } +} + +function transformURL(request) { + const url = new URL(request.url); + if (url.pathname === '/v1/chat/completions') { + var path = "generation" + var deployName = chatmodel; + } else if (url.pathname === '/v1/completions') { + var path = "generation" + var deployName = chatmodel; + } else if (url.pathname === '/v1/embeddings') { + return null; + } else { + return null; + } + + return `https://dashscope.aliyuncs.com/api/v1/services/aigc/${deployName}/${path}` +} + +async function handleRequestWithTransform(request, transformRequestBody, transformResponseBody) { + let body = await request.json(); + const fetchAPI = transformURL(request); + // console.log(fetchAPI); + + if (fetchAPI === null) { + return new Response('404 Not Found', { status: 404 }) + } + + const transformedBody = transformRequestBody(body); + const payload = { + method: request.method, + headers: { + 'Content-Type': 'application/json', + 'Authorization': request.headers.get('Authorization') + }, + body: JSON.stringify(transformedBody), + }; + // console.log(payload); + + const response = await fetch(fetchAPI, payload); + if (!response.ok) { + return new Response(response.statusText, { status: response.status }); + } + const response_data = await response.json(); + + if(response_data?.code) + { + // 出现了错误，返回400 + return new Response(JSON.stringify(transformedBody) + '\n' + JSON.stringify(response_data) + '\n', { + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': '*', + 'Access-Control-Allow-Headers': '*' + }, + status: 400 + }); + } + + // console.log(response_data); + + const transformedResponse = transformResponseBody(response_data); + // console.log(transformedResponse); + + if (body?.stream != true) { + return new Response(JSON.stringify(transformedResponse), { + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': '*', + 'Access-Control-Allow-Headers': '*' + } + }); + } else { + let { readable, writable } = new TransformStream(); + streamResponse(transformedResponse, writable); + return new Response(readable, { + headers: { + 'Content-Type': 'text/event-stream', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': '*', + 'Access-Control-Allow-Headers': '*' + } + }); + } +} + + +// 现在 gemini 还不支持 function，所以这个函数暂时没用 +function convert2GeminiFunctionDeclaration(tools, tool_choice) { + if (tools === undefined || tool_choice === undefined || tool_choice === "none") { + return []; + } + + // TODO - add support for tool_choice + + const result = []; + const functionDeclarations = []; + + for (const tool of tools) { + if (tool.type === "function") { + var functionDeclaration = { + name: tool.function.name, + description: tool.function.description, + parameters: tool.function.parameters, + }; + functionDeclarations.push(functionDeclaration); + } + } + + if (functionDeclarations.length > 0) { + const toolObject = { + functionDeclarations, + }; + result.push(toolObject); + } + + return result; +} + +function transformEmbeddingRequest(body) { + return { + "model": "models/embedding-001", + "content": { + "parts": [ + { + "text": body?.input + } + ] + } + }; +} + +function transformEmbeddingResponse(geminiData) { + return { + "object": "embedding", + "embedding": geminiData?.embedding?.values || [], + "index": 0 + }; +} + +function transformCommonRequest(openaiRequest) { + + var qwenRequest = { + "model": openaiRequest.model.replace("gpt-3.5-turbo", "qwen-turbo"), + "input": { + "messages": openaiRequest.messages + }, + "parameters": { + "top_p": openaiRequest?.top_p, + "top_k": openaiRequest?.candidateCount, + "seed": openaiRequest?.seed, + } + }; + + return qwenRequest; +} + +// Function to transform the response +function transformCommonResponse(qwenResponse) { + + var openaiResponse = { + "id": qwenResponse.request_id, + "object": "chat.completion", + "created": Math.floor(Date.now() / 1000), + "model": "gpt-3.5-turbo-0613", + "system_fingerprint": "fp_44709d6fcb", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": qwenResponse.output.text, + }, + "logprobs": null, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": qwenResponse?.usage.input_tokens, + "completion_tokens": qwenResponse?.usage.output_tokens, + "total_tokens": qwenResponse?.usage.input_tokens + qwenResponse?.usage.output_tokens + } + }; + + return openaiResponse; +} + + +function streamResponse(response, writable) { + let encoder = new TextEncoder(); + let writer = writable.getWriter(); + + let content = response.choices[0].message.content; + + let chunks = content.split("\n\n") || []; + chunks.forEach((chunk, i) => { + let chunkResponse = { + ...response, + object: "chat.completion.chunk", + choices: [{ + index: response.choices[0].index, + delta: { ...response.choices[0].message, content: chunk }, + finish_reason: i === chunks.length - 1 ? 'stop' : null // Set 'stop' for the last chunk + }], + usage: null + }; + + writer.write(encoder.encode(`data: ${JSON.stringify(chunkResponse)}\n\n`)); + }); + + // Write the done signal + writer.write(encoder.encode(`data: [DONE]\n`)); + + writer.close(); +} + + + +async function handleOPTIONS(request) { + return new Response("pong", { + headers: { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': '*', + 'Access-Control-Allow-Headers': '*' + } + }) +} From 79cf5e2d464212ad891d68bbcaa6e311341929f8 Mon Sep 17 00:00:00 2001 From: wangzhankun Date: Wed, 10 Jan 2024 13:33:55 +0800 Subject: [PATCH 9/9] add qwen embedding --- cf-openai-qwen-proxy.js | 76 ++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/cf-openai-qwen-proxy.js b/cf-openai-qwen-proxy.js index 29d8f87..92a066a 100644 --- a/cf-openai-qwen-proxy.js +++ b/cf-openai-qwen-proxy.js @@ -1,6 +1,7 @@ // The deployment name you chose when you deployed the model. -const chatmodel = 'text-generation'; -const embeddmodel = 'embedding-001'; +const base = 'https://dashscope.aliyuncs.com/api/v1/services'; +const chatmodel = 'aigc/text-generation'; +const embeddmodel = 'embeddings/text-embedding'; addEventListener("fetch", (event) => { event.respondWith(handleRequest(event.request)); @@ -22,19 +23,14 @@ async function handleRequest(request) { function transformURL(request) { const url = new URL(request.url); - if (url.pathname === '/v1/chat/completions') { - var path = "generation" - var deployName = chatmodel; - } else if (url.pathname === '/v1/completions') { - var path = "generation" - var deployName = chatmodel; + if (url.pathname === '/v1/chat/completions' || url.pathname === '/v1/completions') { + return `${base}/${chatmodel}/generation` } else if (url.pathname === '/v1/embeddings') { - return null; + return `${base}/${embeddmodel}/text-embedding` } else { return null; } - return `https://dashscope.aliyuncs.com/api/v1/services/aigc/${deployName}/${path}` } async function handleRequestWithTransform(request, transformRequestBody, transformResponseBody) { @@ -63,8 +59,7 @@ async function handleRequestWithTransform(request, transformRequestBody, transfo } const response_data = await response.json(); - if(response_data?.code) - { + if (response_data?.code) { // 出现了错误，返回400 return new Response(JSON.stringify(transformedBody) + '\n' + JSON.stringify(response_data) + '\n', { headers: { @@ -138,24 +133,51 @@ function convert2GeminiFunctionDeclaration(tools, tool_choice) { return result; } -function transformEmbeddingRequest(body) { - return { - "model": "models/embedding-001", - "content": { - "parts": [ - { - "text": body?.input - } - ] - } - }; +function transformEmbeddingRequest(openaiRequest) { + // 判断是否是array + if (Array.isArray(openaiRequest?.input)) { + return { + "model": "text-embedding-v1", + "input": { "texts": openaiRequest?.input }, + "parameters": { + "text_type": "query" + } + }; + } else if (typeof openaiRequest?.input === 'string') { + return { + "model": "text-embedding-v1", + "input": { "texts": [openaiRequest?.input] }, + "parameters": { + "text_type": "query" + } + }; + } else + return { + "model": "text-embedding-v1", + "input": { "texts": [] }, + "parameters": { + "text_type": "query" + } + }; } -function transformEmbeddingResponse(geminiData) { +function transformEmbeddingResponse(qwenResponse) { + const { output: { embeddings }, usage } = qwenResponse; + + const data = embeddings.map((item) => ({ + object: "embedding", + embedding: item.embedding, + index: item.text_index + })); + return { - "object": "embedding", - "embedding": geminiData?.embedding?.values || [], - "index": 0 + object: "list", + data, + model: "text-embedding-ada-002", + usage: { + prompt_tokens: usage.total_tokens, + total_tokens: usage.total_tokens + } }; }