feat: add IBM COS and AWS S3

.gitignore

@@ -46,3 +46,4 @@ documents/warmup_ocr.pdf
 documents/openrag-documentation.pdf
 documents/ibm_anthropic.pdf
 documents/docling.pdf
+/opensearch-data-new-lf

docker-compose.yml

@@ -81,6 +81,14 @@ services:
       - WEBHOOK_BASE_URL=${WEBHOOK_BASE_URL}
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT}
+      - AWS_REGION=${AWS_REGION}
+      - IBM_COS_API_KEY=${IBM_COS_API_KEY}
+      - IBM_COS_SERVICE_INSTANCE_ID=${IBM_COS_SERVICE_INSTANCE_ID}
+      - IBM_COS_ENDPOINT=${IBM_COS_ENDPOINT}
+      - IBM_COS_HMAC_ACCESS_KEY_ID=${IBM_COS_HMAC_ACCESS_KEY_ID}
+      - IBM_COS_HMAC_SECRET_ACCESS_KEY=${IBM_COS_HMAC_SECRET_ACCESS_KEY}
+      - IBM_COS_AUTH_ENDPOINT=${IBM_COS_AUTH_ENDPOINT}
       - OPENSEARCH_INDEX_NAME=${OPENSEARCH_INDEX_NAME:-documents}
       - LANGFLOW_KEY=${LANGFLOW_KEY}
       - LANGFLOW_KEY_RETRIES=${LANGFLOW_KEY_RETRIES:-15}

frontend/app/api/mutations/useConnectConnectorMutation.ts

@@ -81,6 +81,11 @@ export const useConnectConnectorMutation = () => {
           `state=${result.connection_id}`;
 
         window.location.href = authUrl;
+      } else {
+        // Direct-auth connector (e.g. IBM COS) — credentials already verified,
+        // no OAuth redirect needed. Refresh connector status.
+        queryClient.invalidateQueries({ queryKey: ["connectors"] });
+        toast.success(`${connector.name} connected successfully`);
       }
     },
   });

frontend/app/api/mutations/useIBMCOSConfigureMutation.ts

@@ -0,0 +1,40 @@
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+
+export interface IBMCOSConfigurePayload {
+    auth_mode: "iam" | "hmac";
+    endpoint: string;
+    // IAM
+    api_key?: string;
+    service_instance_id?: string;
+    auth_endpoint?: string;
+    // HMAC
+    hmac_access_key?: string;
+    hmac_secret_key?: string;
+    // Bucket selection
+    bucket_names?: string[];
+    // Updating an existing connection
+    connection_id?: string;
+}
+
+async function configureIBMCOS(payload: IBMCOSConfigurePayload) {
+    const res = await fetch("/api/connectors/ibm_cos/configure", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(payload),
+    });
+    const data = await res.json();
+    if (!res.ok) throw new Error(data.error || "Failed to configure IBM COS");
+    return data as { connection_id: string; status: string };
+}
+
+export function useIBMCOSConfigureMutation() {
+    const queryClient = useQueryClient();
+
+    return useMutation({
+        mutationFn: configureIBMCOS,
+        onSuccess: () => {
+            queryClient.invalidateQueries({ queryKey: ["connectors"] });
+            queryClient.invalidateQueries({ queryKey: ["ibm-cos-defaults"] });
+        },
+    });
+}

frontend/app/api/mutations/useS3ConfigureMutation.ts

@@ -0,0 +1,33 @@
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+
+export interface S3ConfigurePayload {
+    access_key?: string;
+    secret_key?: string;
+    endpoint_url?: string;
+    region?: string;
+    bucket_names?: string[];
+    connection_id?: string;
+}
+
+async function configureS3(payload: S3ConfigurePayload) {
+    const res = await fetch("/api/connectors/aws_s3/configure", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(payload),
+    });
+    const data = await res.json();
+    if (!res.ok) throw new Error(data.error || "Failed to configure S3");
+    return data as { connection_id: string; status: string };
+}
+
+export function useS3ConfigureMutation() {
+    const queryClient = useQueryClient();
+
+    return useMutation({
+        mutationFn: configureS3,
+        onSuccess: () => {
+            queryClient.invalidateQueries({ queryKey: ["connectors"] });
+            queryClient.invalidateQueries({ queryKey: ["s3-defaults"] });
+        },
+    });
+}

frontend/app/api/mutations/useSyncConnector.ts

@@ -47,6 +47,10 @@ const syncConnector = async ({
       size?: number;
     }>;
     settings?: any;
+    /** When true, ingest all files from the connector (bypasses the re-sync gate). */
+    sync_all?: boolean;
+    /** Restrict ingest to these bucket names (IBM COS). */
+    bucket_filter?: string[];
   };
 }): Promise<SyncResponse> => {
   const response = await fetch(`/api/connectors/${connectorType}/sync`, {

frontend/app/api/queries/useIBMCOSBucketStatusQuery.ts

@@ -0,0 +1,34 @@
+import { useQuery } from "@tanstack/react-query";
+
+export interface IBMCOSBucketStatus {
+  name: string;
+  ingested_count: number;
+  is_synced: boolean;
+}
+
+async function fetchIBMCOSBucketStatus(
+  connectionId: string,
+): Promise<IBMCOSBucketStatus[]> {
+  const res = await fetch(
+    `/api/connectors/ibm_cos/${connectionId}/bucket-status`,
+  );
+  if (!res.ok) {
+    const err = await res.json().catch(() => ({}));
+    throw new Error(err.error || "Failed to fetch bucket status");
+  }
+  const data = await res.json();
+  return data.buckets as IBMCOSBucketStatus[];
+}
+
+export function useIBMCOSBucketStatusQuery(
+  connectionId: string | null | undefined,
+  options?: { enabled?: boolean },
+) {
+  return useQuery<IBMCOSBucketStatus[]>({
+    queryKey: ["ibm-cos-bucket-status", connectionId],
+    queryFn: () => fetchIBMCOSBucketStatus(connectionId!),
+    enabled: (options?.enabled ?? true) && !!connectionId,
+    staleTime: 0,
+    refetchOnMount: "always",
+  });
+}

frontend/app/api/queries/useIBMCOSBucketsQuery.ts

@@ -0,0 +1,23 @@
+import { useQuery } from "@tanstack/react-query";
+
+async function fetchIBMCOSBuckets(connectionId: string): Promise<string[]> {
+    const res = await fetch(`/api/connectors/ibm_cos/${connectionId}/buckets`);
+    if (!res.ok) {
+        const err = await res.json().catch(() => ({}));
+        throw new Error(err.error || "Failed to list buckets");
+    }
+    const data = await res.json();
+    return data.buckets as string[];
+}
+
+export function useIBMCOSBucketsQuery(
+    connectionId: string | null | undefined,
+    options?: { enabled?: boolean },
+) {
+    return useQuery<string[]>({
+        queryKey: ["ibm-cos-buckets", connectionId],
+        queryFn: () => fetchIBMCOSBuckets(connectionId!),
+        enabled: (options?.enabled ?? true) && !!connectionId,
+        staleTime: 30_000,
+    });
+}

frontend/app/api/queries/useIBMCOSDefaultsQuery.ts

@@ -0,0 +1,28 @@
+import { useQuery } from "@tanstack/react-query";
+
+export interface IBMCOSDefaults {
+    api_key_set: boolean;
+    service_instance_id: string;
+    endpoint: string;
+    hmac_access_key_set: boolean;
+    hmac_secret_key_set: boolean;
+    auth_mode: "iam" | "hmac";
+    bucket_names: string[];
+    connection_id: string | null;
+    disable_iam: boolean;
+}
+
+async function fetchIBMCOSDefaults(): Promise<IBMCOSDefaults> {
+    const res = await fetch("/api/connectors/ibm_cos/defaults");
+    if (!res.ok) throw new Error("Failed to fetch IBM COS defaults");
+    return res.json();
+}
+
+export function useIBMCOSDefaultsQuery(options?: { enabled?: boolean }) {
+    return useQuery<IBMCOSDefaults>({
+        queryKey: ["ibm-cos-defaults"],
+        queryFn: fetchIBMCOSDefaults,
+        enabled: options?.enabled ?? true,
+        staleTime: 0,
+    });
+}

frontend/app/api/queries/useS3BucketStatusQuery.ts

@@ -0,0 +1,30 @@
+import { useQuery } from "@tanstack/react-query";
+
+export interface S3BucketStatus {
+    name: string;
+    ingested_count: number;
+    is_synced: boolean;
+}
+
+async function fetchS3BucketStatus(connectionId: string): Promise<S3BucketStatus[]> {
+    const res = await fetch(`/api/connectors/aws_s3/${connectionId}/bucket-status`);
+    if (!res.ok) {
+        const err = await res.json().catch(() => ({}));
+        throw new Error(err.error || "Failed to fetch bucket status");
+    }
+    const data = await res.json();
+    return data.buckets as S3BucketStatus[];
+}
+
+export function useS3BucketStatusQuery(
+    connectionId: string | null | undefined,
+    options?: { enabled?: boolean },
+) {
+    return useQuery<S3BucketStatus[]>({
+        queryKey: ["s3-bucket-status", connectionId],
+        queryFn: () => fetchS3BucketStatus(connectionId!),
+        enabled: (options?.enabled ?? true) && !!connectionId,
+        staleTime: 0,
+        refetchOnMount: "always",
+    });
+}

frontend/app/api/queries/useS3DefaultsQuery.ts

@@ -0,0 +1,25 @@
+import { useQuery } from "@tanstack/react-query";
+
+export interface S3Defaults {
+    access_key_set: boolean;
+    secret_key_set: boolean;
+    endpoint: string;
+    region: string;
+    bucket_names: string[];
+    connection_id: string | null;
+}
+
+async function fetchS3Defaults(): Promise<S3Defaults> {
+    const res = await fetch("/api/connectors/aws_s3/defaults");
+    if (!res.ok) throw new Error("Failed to fetch S3 defaults");
+    return res.json();
+}
+
+export function useS3DefaultsQuery(options?: { enabled?: boolean }) {
+    return useQuery<S3Defaults>({
+        queryKey: ["s3-defaults"],
+        queryFn: fetchS3Defaults,
+        enabled: options?.enabled ?? true,
+        staleTime: 0,
+    });
+}

frontend/app/knowledge/page.tsx

@@ -34,7 +34,10 @@ import {
   DeleteConfirmationDialog,
   formatFilesToDelete,
 } from "../../components/delete-confirmation-dialog";
+import AwsLogo from "../../components/icons/aws-logo";
 import GoogleDriveIcon from "../../components/icons/google-drive-logo";
+import IBMCOSIcon from "../../components/icons/ibm-cos-icon";
+import IBMLogo from "../../components/icons/ibm-logo";
 import OneDriveIcon from "../../components/icons/one-drive-logo";
 import SharePointIcon from "../../components/icons/share-point-logo";
 import { useDeleteDocument } from "../api/mutations/useDeleteDocument";
@@ -59,6 +62,10 @@ function getSourceIcon(connectorType?: string) {
       return <Globe className="h-4 w-4 text-muted-foreground flex-shrink-0" />;
     case "s3":
       return <Cloud className="h-4 w-4 text-foreground flex-shrink-0" />;
+    case "ibm_cos":
+      return <IBMCOSIcon className="h-4 w-4 flex-shrink-0" />;
+    case "aws_s3":
+      return <AwsLogo className="h-4 w-4 flex-shrink-0" />;
     default:
       return (
         <FileIcon className="h-4 w-4 text-muted-foreground flex-shrink-0" />

frontend/app/settings/_components/anthropic-settings-dialog.tsx

@@ -138,7 +138,13 @@ const AnthropicSettingsDialog = ({
   };
 
   return (
-    <Dialog open={open} onOpenChange={(o) => { setShowRemoveConfirm(false); setOpen(o); }}>
+    <Dialog
+      open={open}
+      onOpenChange={(o) => {
+        setShowRemoveConfirm(false);
+        setOpen(o);
+      }}
+    >
       <DialogContent className="max-w-2xl">
         <FormProvider {...methods}>
           <form onSubmit={handleSubmit(onSubmit)} className="grid gap-4">

frontend/app/settings/_components/card-icon.tsx

@@ -16,7 +16,7 @@ export default function CardIcon({
       className={cn(
         "w-8 h-8 rounded flex items-center justify-center border",
         isActive
-          ? activeBgColor
+          ? `${activeBgColor} text-black`
           : "bg-muted grayscale group-hover:bg-background",
       )}
     >

frontend/app/settings/_components/connector-card.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { Loader2, Plus, RefreshCcw, Trash2 } from "lucide-react";
+import { Loader2, Plus, RefreshCcw, Settings2, Trash2 } from "lucide-react";
 import Link from "next/link";
 import { Button } from "@/components/ui/button";
 import {
@@ -29,6 +29,8 @@ interface ConnectorCardProps {
   onConnect: (connector: Connector) => void;
   onDisconnect: (connector: Connector) => void;
   onNavigateToKnowledge: (connector: Connector) => void;
+  /** Optional: open a connector-specific settings/edit dialog */
+  onConfigure?: (connector: Connector) => void;
 }
 
 export default function ConnectorCard({
@@ -38,22 +40,19 @@ export default function ConnectorCard({
   onConnect,
   onDisconnect,
   onNavigateToKnowledge,
+  onConfigure,
 }: ConnectorCardProps) {
   console.log(connector);
   const isConnected =
     connector.status === "connected" && connector.connectionId;
-  connector?.status === "connected" && connector?.connectionId;
 
   return (
     <Card className="group relative flex flex-col hover:bg-secondary-hover hover:border-muted-foreground transition-colors">
       <CardHeader className="pb-2">
         <div className="flex flex-col items-start justify-between">
           <div className="flex flex-col gap-4 mb-2 w-full">
             <div className="flex items-center justify-between mb-1">
-              <CardIcon
-                isActive={!!(connector?.available && isConnected)}
-                activeBgColor="bg-white"
-              >
+              <CardIcon isActive={!!isConnected} activeBgColor="bg-white">
                 {connector.icon}
               </CardIcon>
               {isConnected ? (
@@ -68,7 +67,7 @@ export default function ConnectorCard({
                 {connector.name}
               </CardTitle>
               <CardDescription className="text-sm">
-                {connector?.available
+                {isConnected || connector?.available
                   ? `${connector.name} is configured.`
                   : "Not configured."}
               </CardDescription>
@@ -93,15 +92,17 @@ export default function ConnectorCard({
                 </Button>
                 <Button
                   variant="outline"
-                  onClick={() => onConnect(connector)}
+                  onClick={() =>
+                    onConfigure ? onConfigure(connector) : onConnect(connector)
+                  }
                   disabled={isConnecting || isDisconnecting}
                   className="cursor-pointer"
                   size="iconMd"
                 >
                   {isConnecting ? (
                     <RefreshCcw className="h-4 w-4 animate-spin" />
                   ) : (
-                    <RefreshCcw className="h-4 w-4" />
+                    <Settings2 className="h-4 w-4" />
                   )}
                 </Button>
                 <Button
@@ -120,7 +121,9 @@ export default function ConnectorCard({
               </div>
             ) : (
               <Button
-                onClick={() => onConnect(connector)}
+                onClick={() =>
+                  onConfigure ? onConfigure(connector) : onConnect(connector)
+                }
                 disabled={isConnecting}
                 className="w-full cursor-pointer group-hover:bg-background group-hover:border-zinc-700 group-hover:text-primary"
                 size="sm"