Skip to content

feat: add support for custom LLM endpoints and unify model selection across providers. #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
amitb0ra wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add support for custom LLM endpoints and unify model selection across providers. #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 19 additions & 21 deletions settings/settings.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,41 +5,42 @@ import {

export enum SettingEnum {
ID_LLM_PROVIDER = `llm-provider`,
ID_GROQ_API_KEY = `groq-model`,
ID_GROQ_MODEL = `groq-api-key`,
ID_MODEL = `model`,
ID_GROQ_API_KEY = `groq-api-key`,
ID_GEMINI_API_KEY = `gemini-api-key`,
ID_RC_MODEL = `rc-model`,
ID_CUSTOM_URL = `custom-url`,
ID_RC_AUTH_TOKEN = `rc-auth-token`,
ID_RC_USER_ID = `rc-user-id`,
KEY_GROQ = `groq`,
KEY_GEMINI = `gemini`,
KEY_RC = `rocketchat`,
KEY_CUSTOM = `custom`,
}


export const settings: ISetting[] = [
{
id: SettingEnum.ID_LLM_PROVIDER,
i18nLabel: "Service Provider",
i18nDescription: "Who's service are you using?",
i18nDescription: "Which service are you using?",
type: SettingType.SELECT,
values: [
{ key: SettingEnum.KEY_GROQ, i18nLabel: "1. Groqcloud (groq.com)" },
{ key: SettingEnum.KEY_GEMINI, i18nLabel: "2. Google AI Studio" },
{ key: SettingEnum.KEY_RC, i18nLabel: "3. In House (Rocket Chat)" },
{ key: SettingEnum.KEY_CUSTOM, i18nLabel: "3. Custom Model" },
],
required: true,
public: true,
packageValue: SettingEnum.KEY_GROQ,
},
{
id: SettingEnum.ID_GROQ_MODEL,
id: SettingEnum.ID_MODEL,
i18nLabel: "Model Name (required for all providers)",
i18nDescription: "Model to use (e.g., llama-3.1-8b-instant for Groq/Self-Hosted, gemini-2.5-flash for Gemini)",
type: SettingType.STRING,
packageValue: "",
packageValue: "llama-3.1-8b-instant",
required: false,
public: false,
i18nLabel: "Groq Model (required for Groqcloud)",
i18nPlaceholder: "Groq Model",
i18nPlaceholder: "llama-3.1-8b-instant",
},
{
id: SettingEnum.ID_GROQ_API_KEY,
Expand All @@ -60,22 +61,19 @@ export const settings: ISetting[] = [
i18nPlaceholder: "Gemini API Key",
},
{
id: SettingEnum.ID_RC_MODEL,
i18nLabel: "Model selection (required for In House (Rocket Chat))",
i18nDescription: "AI model to use for summarization.",
type: SettingType.SELECT,
values: [
{ key: "llama3-8b:1234", i18nLabel: "Llama3 8B" },
{ key: "mistral-7b", i18nLabel: "Mistral 7B" },
],
id: SettingEnum.ID_CUSTOM_URL,
i18nLabel: "Custom Model Base URL (required for Custom Model)",
i18nDescription: "Base API URL for your custom model (e.g., http://localhost:11434/v1/chat/completions for Ollama)",
type: SettingType.STRING,
required: false,
public: false,
packageValue: "llama3-8b:1234",
packageValue: "",
i18nPlaceholder: "http://localhost:11434/v1/chat/completions",
},
{
id: SettingEnum.ID_RC_AUTH_TOKEN,
i18nLabel:
"Personal Access Token (required for In House (Rocket Chat))",
"Personal Access Token (required for file summary)",
i18nDescription: "Must be filled to enable file summary add-on",
type: SettingType.PASSWORD,
required: false,
Expand All @@ -84,7 +82,7 @@ export const settings: ISetting[] = [
},
{
id: SettingEnum.ID_RC_USER_ID,
i18nLabel: "User ID (required for In House (Rocket Chat))",
i18nLabel: "User ID (required for file summary)",
i18nDescription: "Must be filled to enable file summary add-on",
type: SettingType.STRING,
required: false,
Expand Down
6 changes: 3 additions & 3 deletions utils/AIProvider.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { IHttp, IRead } from "@rocket.chat/apps-engine/definition/accessors";
import { createTextCompletionGroq } from "./GroqModels";
import { createTextCompletionGemini } from "./GeminiModel";
import { createTextCompletionRocketChat } from "./RCInHouseHostedModels";
import { createTextCompletionCustom } from "./CustomModels";
import { SettingEnum } from "../settings/settings";

export async function createTextCompletion(
Expand All @@ -25,8 +25,8 @@ export async function createTextCompletion(
case SettingEnum.KEY_GEMINI: {
return await createTextCompletionGemini(read, http, prompt);
}
case SettingEnum.KEY_RC: {
return await createTextCompletionRocketChat(read, http, prompt);
case SettingEnum.KEY_CUSTOM: {
return await createTextCompletionCustom(read, http, prompt);
}
default:
throw new Error(`Unsupported AI provider: ${aiProvider}`);
Expand Down
25 changes: 16 additions & 9 deletions utils/RCInHouseHostedModels.ts → utils/CustomModels.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,31 @@ import { IHttp, IRead } from "@rocket.chat/apps-engine/definition/accessors";
import { SettingEnum } from "../settings/settings";


export async function createTextCompletionRocketChat(
export async function createTextCompletionCustom(
read: IRead,
http: IHttp,
prompt: string
): Promise<string> {
const rcModel = await read
const baseUrl = await read
.getEnvironmentReader()
.getSettings()
.getValueById(SettingEnum.ID_RC_MODEL);
.getValueById(SettingEnum.ID_CUSTOM_URL);

if (!rcModel) {
throw new Error("rcModel not configured.");
if (!baseUrl) {
throw new Error("Custom model base URL not configured.");
}

const url = `http://${rcModel}/v1`;
const customModel = await read
.getEnvironmentReader()
.getSettings()
.getValueById(SettingEnum.ID_MODEL);

if (!customModel) {
throw new Error("Model not configured.");
}

const body = {
rcModel,
model: customModel,
messages: [
{
role: "system",
Expand All @@ -29,15 +36,15 @@ export async function createTextCompletionRocketChat(
temperature: 0,
};

const response = await http.post(url + "/chat/completions", {
const response = await http.post(baseUrl, {
headers: {
"Content-Type": "application/json",
},
content: JSON.stringify(body),
});

if (!response.content) {
console.log("Something is wrong with AI. Please try again later");
console.log(response);
throw new Error("Something is wrong with AI. Please try again later");
}

Expand Down
11 changes: 10 additions & 1 deletion utils/GeminiModel.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,16 @@ export async function createTextCompletionGemini(
throw new Error("geminiApiKey not configured.");
}

const url = `https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=${geminiApiKey}`;
const geminiModel = await read
.getEnvironmentReader()
.getSettings()
.getValueById(SettingEnum.ID_MODEL);

if (!geminiModel) {
throw new Error("Model not configured.");
}

const url = `https://generativelanguage.googleapis.com/v1beta/models/${geminiModel}:generateContent?key=${geminiApiKey}`;

const body = {
contents: [{ parts: [{ text: prompt }] }],
Expand Down
6 changes: 3 additions & 3 deletions utils/GroqModels.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,14 @@ export async function createTextCompletionGroq(
const groqModel = await read
.getEnvironmentReader()
.getSettings()
.getValueById(SettingEnum.ID_GROQ_MODEL);
.getValueById(SettingEnum.ID_MODEL);

if (!groqModel) {
throw new Error("groqModel not configured.");
throw new Error("Model not configured.");
}

const body = {
groqModel,
model: groqModel,
messages: [
{
role: "system",
Expand Down