chore: Move and run Python SDK integration tests

.gitignore

@@ -46,3 +46,4 @@ documents/warmup_ocr.pdf
 documents/openrag-documentation.pdf
 documents/ibm_anthropic.pdf
 documents/docling.pdf
+/opensearch-data-new-lf

Makefile

@@ -665,9 +665,11 @@ test-unit: ## Run unit tests only
 	@echo "$(PURPLE)Unit tests complete.$(NC)"
 
 test-integration: ## Run integration tests (requires infrastructure)
-	@echo "$(YELLOW)Running integration tests (requires infrastructure)...$(NC)"
-	@echo "$(CYAN)Make sure to run 'make dev-local' first!$(NC)"
-	uv run pytest tests/integration/ -v
+	@echo "$(CYAN)════════════════════════════════════════$(NC)"
+	@echo "$(PURPLE) Core Integration Tests$(NC)"
+	@echo "$(CYAN)════════════════════════════════════════$(NC)"
+	@echo "$(YELLOW)Make sure to run 'make dev-local' first!$(NC)"
+	uv run pytest tests/integration/core/ -v
 
 test-ci: ## Start infra, run integration + SDK tests, tear down (uses DockerHub images)
 	@set -e; \
@@ -681,12 +683,15 @@ test-ci: ## Start infra, run integration + SDK tests, tear down (uses DockerHub
 		chmod 600 keys/private_key.pem 2>/dev/null || true; \
 		chmod 644 keys/public_key.pem 2>/dev/null || true; \
 	fi; \
+	echo "::group::Cleanup, Pull & Build Images"; \
 	echo "$(YELLOW)Cleaning up old containers and volumes...$(NC)"; \
 	$(COMPOSE_CMD) down -v 2>/dev/null || true; \
 	echo "$(YELLOW)Pulling latest images...$(NC)"; \
 	$(COMPOSE_CMD) pull; \
 	echo "$(YELLOW)Building OpenSearch image override...$(NC)"; \
 	$(CONTAINER_RUNTIME) build --no-cache -t langflowai/openrag-opensearch:latest -f Dockerfile .; \
+	echo "::endgroup::"; \
+	echo "::group::Start Infrastructure"; \
 	echo "$(YELLOW)Starting infra (OpenSearch + Dashboards + Langflow + Backend + Frontend) with CPU containers$(NC)"; \
 	OPENSEARCH_HOST=opensearch $(COMPOSE_CMD) up -d opensearch dashboards langflow openrag-backend openrag-frontend; \
 	echo "$(CYAN)Architecture: $$(uname -m), Platform: $$(uname -s)$(NC)"; \
@@ -752,31 +757,42 @@ test-ci: ## Start infra, run integration + SDK tests, tear down (uses DockerHub
 		$(COMPOSE_CMD) down -v 2>/dev/null || true; \
 		exit 1; \
 	fi; \
-	echo "$(PURPLE)Running integration tests$(NC)"; \
+	echo "::endgroup::"; \
+	echo "::group::Core Integration Tests"; \
+	echo "$(CYAN)════════════════════════════════════════$(NC)"; \
+	echo "$(PURPLE) Core Integration Tests$(NC)"; \
+	echo "$(CYAN)════════════════════════════════════════$(NC)"; \
 	LOG_LEVEL=$${LOG_LEVEL:-DEBUG} \
 	GOOGLE_OAUTH_CLIENT_ID="" \
 	GOOGLE_OAUTH_CLIENT_SECRET="" \
 	OPENSEARCH_HOST=localhost OPENSEARCH_PORT=9200 \
 	OPENSEARCH_USERNAME=admin OPENSEARCH_PASSWORD=$${OPENSEARCH_PASSWORD} \
 	DISABLE_STARTUP_INGEST=$${DISABLE_STARTUP_INGEST:-true} \
-	uv run pytest tests/integration -vv -s -o log_cli=true --log-cli-level=DEBUG; \
+	uv run pytest tests/integration/core -vv -s -o log_cli=true --log-cli-level=DEBUG; \
 	TEST_RESULT=$$?; \
+	echo "::endgroup::"; \
 	echo ""; \
 	echo "$(YELLOW)Waiting for frontend at http://localhost:3000...$(NC)"; \
 	for i in $$(seq 1 60); do \
 		curl -s http://localhost:3000/ >/dev/null 2>&1 && break || sleep 2; \
 	done; \
-	echo "$(PURPLE)Running Python SDK integration tests$(NC)"; \
-	cd sdks/python && \
-	uv sync --extra dev && \
-	OPENRAG_URL=http://localhost:3000 uv run pytest tests/test_integration.py -vv -s || TEST_RESULT=1; \
-	cd ../..; \
-	echo "$(PURPLE)Running TypeScript SDK integration tests$(NC)"; \
+	echo "::group::SDK Integration Tests (Python)"; \
+	echo "$(CYAN)════════════════════════════════════════$(NC)"; \
+	echo "$(PURPLE) SDK Integration Tests (Python)$(NC)"; \
+	echo "$(CYAN)════════════════════════════════════════$(NC)"; \
+	uv pip install -e sdks/python; \
+	SDK_TESTS_ONLY=true OPENRAG_URL=http://localhost:3000 uv run pytest tests/integration/sdk/ -vv -s || TEST_RESULT=1; \
+	echo "::endgroup::"; \
+	echo "::group::SDK Integration Tests (TypeScript)"; \
+	echo "$(CYAN)════════════════════════════════════════$(NC)"; \
+	echo "$(PURPLE) SDK Integration Tests (TypeScript)$(NC)"; \
+	echo "$(CYAN)════════════════════════════════════════$(NC)"; \
 	cd sdks/typescript && \
 	npm install && npm run build && \
 	OPENRAG_URL=http://localhost:3000 npm test || TEST_RESULT=1; \
 	cd ../..; \
-	echo "$(CYAN)=================================$(NC)"; \
+	echo "::endgroup::"; \
+	echo "$(CYAN)════════════════════════════════════════$(NC)"; \
 	echo ""; \
 	($(call test_jwt_opensearch)) || TEST_RESULT=1; \
 	echo "$(YELLOW)Tearing down infra$(NC)"; \
@@ -796,13 +812,16 @@ test-ci-local: ## Same as test-ci but builds all images locally
 		chmod 600 keys/private_key.pem 2>/dev/null || true; \
 		chmod 644 keys/public_key.pem 2>/dev/null || true; \
 	fi; \
+	echo "::group::Cleanup & Build Images"; \
 	echo "$(YELLOW)Cleaning up old containers and volumes...$(NC)"; \
 	$(COMPOSE_CMD) down -v 2>/dev/null || true; \
 	echo "$(YELLOW)Building all images locally...$(NC)"; \
 	$(CONTAINER_RUNTIME) build -t langflowai/openrag-opensearch:latest -f Dockerfile .; \
 	$(CONTAINER_RUNTIME) build -t langflowai/openrag-backend:latest -f Dockerfile.backend .; \
 	$(CONTAINER_RUNTIME) build -t langflowai/openrag-frontend:latest -f Dockerfile.frontend .; \
 	$(CONTAINER_RUNTIME) build -t langflowai/openrag-langflow:latest -f Dockerfile.langflow .; \
+	echo "::endgroup::"; \
+	echo "::group::Start Infrastructure"; \
 	echo "$(YELLOW)Starting infra (OpenSearch + Dashboards + Langflow + Backend + Frontend) with CPU containers$(NC)"; \
 	echo "$(CYAN)Architecture: $$(uname -m), Platform: $$(uname -s)$(NC)"; \
 	OPENSEARCH_HOST=opensearch $(COMPOSE_CMD) up -d opensearch dashboards langflow openrag-backend openrag-frontend; \
@@ -868,31 +887,42 @@ test-ci-local: ## Same as test-ci but builds all images locally
 		$(COMPOSE_CMD) down -v 2>/dev/null || true; \
 		exit 1; \
 	fi; \
-	echo "$(PURPLE)Running integration tests$(NC)"; \
+	echo "::endgroup::"; \
+	echo "::group::Core Integration Tests"; \
+	echo "$(CYAN)════════════════════════════════════════$(NC)"; \
+	echo "$(PURPLE) Core Integration Tests$(NC)"; \
+	echo "$(CYAN)════════════════════════════════════════$(NC)"; \
 	LOG_LEVEL=$${LOG_LEVEL:-DEBUG} \
 	GOOGLE_OAUTH_CLIENT_ID="" \
 	GOOGLE_OAUTH_CLIENT_SECRET="" \
 	OPENSEARCH_HOST=localhost OPENSEARCH_PORT=9200 \
 	OPENSEARCH_USERNAME=admin OPENSEARCH_PASSWORD=$${OPENSEARCH_PASSWORD} \
 	DISABLE_STARTUP_INGEST=$${DISABLE_STARTUP_INGEST:-true} \
-	uv run pytest tests/integration -vv -s -o log_cli=true --log-cli-level=DEBUG; \
+	uv run pytest tests/integration/core -vv -s -o log_cli=true --log-cli-level=DEBUG; \
 	TEST_RESULT=$$?; \
+	echo "::endgroup::"; \
 	echo ""; \
 	echo "$(YELLOW)Waiting for frontend at http://localhost:3000...$(NC)"; \
 	for i in $$(seq 1 60); do \
 		curl -s http://localhost:3000/ >/dev/null 2>&1 && break || sleep 2; \
 	done; \
-	echo "$(PURPLE)Running Python SDK integration tests$(NC)"; \
-	cd sdks/python && \
-	uv sync --extra dev && \
-	OPENRAG_URL=http://localhost:3000 uv run pytest tests/test_integration.py -vv -s || TEST_RESULT=1; \
-	cd ../..; \
-	echo "$(PURPLE)Running TypeScript SDK integration tests$(NC)"; \
+	echo "::group::SDK Integration Tests (Python)"; \
+	echo "$(CYAN)════════════════════════════════════════$(NC)"; \
+	echo "$(PURPLE) SDK Integration Tests (Python)$(NC)"; \
+	echo "$(CYAN)════════════════════════════════════════$(NC)"; \
+	uv pip install -e sdks/python; \
+	SDK_TESTS_ONLY=true OPENRAG_URL=http://localhost:3000 uv run pytest tests/integration/sdk/ -vv -s || TEST_RESULT=1; \
+	echo "::endgroup::"; \
+	echo "::group::SDK Integration Tests (TypeScript)"; \
+	echo "$(CYAN)════════════════════════════════════════$(NC)"; \
+	echo "$(PURPLE) SDK Integration Tests (TypeScript)$(NC)"; \
+	echo "$(CYAN)════════════════════════════════════════$(NC)"; \
 	cd sdks/typescript && \
 	npm install && npm run build && \
 	OPENRAG_URL=http://localhost:3000 npm test || TEST_RESULT=1; \
 	cd ../..; \
-	echo "$(CYAN)=================================$(NC)"; \
+	echo "::endgroup::"; \
+	echo "$(CYAN)════════════════════════════════════════$(NC)"; \
 	echo ""; \
 	if [ $$TEST_RESULT -ne 0 ]; then \
 		echo "$(RED)=== Tests failed, dumping container logs ===$(NC)"; \
@@ -914,11 +944,12 @@ test-os-jwt: ## Test JWT authentication against OpenSearch
 	@$(call test_jwt_opensearch)
 
 test-sdk: ## Run SDK integration tests (requires running OpenRAG at localhost:3000)
-	@echo "$(YELLOW)Running SDK integration tests...$(NC)"
-	@echo "$(CYAN)Make sure OpenRAG is running at localhost:3000 (make dev)$(NC)"
-	@echo ""
-	@echo "$(PURPLE)Running Python SDK tests...$(NC)"
-	cd sdks/python && uv sync --extra dev && OPENRAG_URL=http://localhost:3000 uv run pytest tests/test_integration.py -vv -s
+	@echo "$(CYAN)════════════════════════════════════════$(NC)"
+	@echo "$(PURPLE) SDK Integration Tests (Python)$(NC)"
+	@echo "$(CYAN)════════════════════════════════════════$(NC)"
+	@echo "$(YELLOW)Make sure OpenRAG is running at localhost:3000 (make dev)$(NC)"
+	uv pip install -e sdks/python
+	SDK_TESTS_ONLY=true OPENRAG_URL=http://localhost:3000 uv run pytest tests/integration/sdk/ -vv -s
 	@echo ""
 	@echo "$(PURPLE)Running TypeScript SDK tests...$(NC)"
 	cd sdks/typescript && npm install && npm run build && OPENRAG_URL=http://localhost:3000 npm test

docker-compose.yml

@@ -80,6 +80,14 @@ services:
       - WEBHOOK_BASE_URL=${WEBHOOK_BASE_URL}
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT}
+      - AWS_REGION=${AWS_REGION}
+      - IBM_COS_API_KEY=${IBM_COS_API_KEY}
+      - IBM_COS_SERVICE_INSTANCE_ID=${IBM_COS_SERVICE_INSTANCE_ID}
+      - IBM_COS_ENDPOINT=${IBM_COS_ENDPOINT}
+      - IBM_COS_HMAC_ACCESS_KEY_ID=${IBM_COS_HMAC_ACCESS_KEY_ID}
+      - IBM_COS_HMAC_SECRET_ACCESS_KEY=${IBM_COS_HMAC_SECRET_ACCESS_KEY}
+      - IBM_COS_AUTH_ENDPOINT=${IBM_COS_AUTH_ENDPOINT}
       - OPENSEARCH_INDEX_NAME=${OPENSEARCH_INDEX_NAME:-documents}
       - LANGFLOW_KEY=${LANGFLOW_KEY}
       - LANGFLOW_KEY_RETRIES=${LANGFLOW_KEY_RETRIES:-15}

frontend/app/api/mutations/useConnectConnectorMutation.ts

@@ -81,6 +81,11 @@ export const useConnectConnectorMutation = () => {
           `state=${result.connection_id}`;
 
         window.location.href = authUrl;
+      } else {
+        // Direct-auth connector (e.g. IBM COS) — credentials already verified,
+        // no OAuth redirect needed. Refresh connector status.
+        queryClient.invalidateQueries({ queryKey: ["connectors"] });
+        toast.success(`${connector.name} connected successfully`);
       }
     },
   });

frontend/app/api/mutations/useIBMCOSConfigureMutation.ts

@@ -0,0 +1,40 @@
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+
+export interface IBMCOSConfigurePayload {
+    auth_mode: "iam" | "hmac";
+    endpoint: string;
+    // IAM
+    api_key?: string;
+    service_instance_id?: string;
+    auth_endpoint?: string;
+    // HMAC
+    hmac_access_key?: string;
+    hmac_secret_key?: string;
+    // Bucket selection
+    bucket_names?: string[];
+    // Updating an existing connection
+    connection_id?: string;
+}
+
+async function configureIBMCOS(payload: IBMCOSConfigurePayload) {
+    const res = await fetch("/api/connectors/ibm_cos/configure", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(payload),
+    });
+    const data = await res.json();
+    if (!res.ok) throw new Error(data.error || "Failed to configure IBM COS");
+    return data as { connection_id: string; status: string };
+}
+
+export function useIBMCOSConfigureMutation() {
+    const queryClient = useQueryClient();
+
+    return useMutation({
+        mutationFn: configureIBMCOS,
+        onSuccess: () => {
+            queryClient.invalidateQueries({ queryKey: ["connectors"] });
+            queryClient.invalidateQueries({ queryKey: ["ibm-cos-defaults"] });
+        },
+    });
+}

frontend/app/api/mutations/useS3ConfigureMutation.ts

@@ -0,0 +1,33 @@
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+
+export interface S3ConfigurePayload {
+    access_key?: string;
+    secret_key?: string;
+    endpoint_url?: string;
+    region?: string;
+    bucket_names?: string[];
+    connection_id?: string;
+}
+
+async function configureS3(payload: S3ConfigurePayload) {
+    const res = await fetch("/api/connectors/aws_s3/configure", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(payload),
+    });
+    const data = await res.json();
+    if (!res.ok) throw new Error(data.error || "Failed to configure S3");
+    return data as { connection_id: string; status: string };
+}
+
+export function useS3ConfigureMutation() {
+    const queryClient = useQueryClient();
+
+    return useMutation({
+        mutationFn: configureS3,
+        onSuccess: () => {
+            queryClient.invalidateQueries({ queryKey: ["connectors"] });
+            queryClient.invalidateQueries({ queryKey: ["s3-defaults"] });
+        },
+    });
+}

frontend/app/api/mutations/useSyncConnector.ts

@@ -47,6 +47,10 @@ const syncConnector = async ({
       size?: number;
     }>;
     settings?: any;
+    /** When true, ingest all files from the connector (bypasses the re-sync gate). */
+    sync_all?: boolean;
+    /** Restrict ingest to these bucket names (IBM COS). */
+    bucket_filter?: string[];
   };
 }): Promise<SyncResponse> => {
   const response = await fetch(`/api/connectors/${connectorType}/sync`, {

frontend/app/api/queries/useIBMCOSBucketStatusQuery.ts

@@ -0,0 +1,34 @@
+import { useQuery } from "@tanstack/react-query";
+
+export interface IBMCOSBucketStatus {
+  name: string;
+  ingested_count: number;
+  is_synced: boolean;
+}
+
+async function fetchIBMCOSBucketStatus(
+  connectionId: string,
+): Promise<IBMCOSBucketStatus[]> {
+  const res = await fetch(
+    `/api/connectors/ibm_cos/${connectionId}/bucket-status`,
+  );
+  if (!res.ok) {
+    const err = await res.json().catch(() => ({}));
+    throw new Error(err.error || "Failed to fetch bucket status");
+  }
+  const data = await res.json();
+  return data.buckets as IBMCOSBucketStatus[];
+}
+
+export function useIBMCOSBucketStatusQuery(
+  connectionId: string | null | undefined,
+  options?: { enabled?: boolean },
+) {
+  return useQuery<IBMCOSBucketStatus[]>({
+    queryKey: ["ibm-cos-bucket-status", connectionId],
+    queryFn: () => fetchIBMCOSBucketStatus(connectionId!),
+    enabled: (options?.enabled ?? true) && !!connectionId,
+    staleTime: 0,
+    refetchOnMount: "always",
+  });
+}

frontend/app/api/queries/useIBMCOSBucketsQuery.ts

@@ -0,0 +1,23 @@
+import { useQuery } from "@tanstack/react-query";
+
+async function fetchIBMCOSBuckets(connectionId: string): Promise<string[]> {
+    const res = await fetch(`/api/connectors/ibm_cos/${connectionId}/buckets`);
+    if (!res.ok) {
+        const err = await res.json().catch(() => ({}));
+        throw new Error(err.error || "Failed to list buckets");
+    }
+    const data = await res.json();
+    return data.buckets as string[];
+}
+
+export function useIBMCOSBucketsQuery(
+    connectionId: string | null | undefined,
+    options?: { enabled?: boolean },
+) {
+    return useQuery<string[]>({
+        queryKey: ["ibm-cos-buckets", connectionId],
+        queryFn: () => fetchIBMCOSBuckets(connectionId!),
+        enabled: (options?.enabled ?? true) && !!connectionId,
+        staleTime: 30_000,
+    });
+}

frontend/app/api/queries/useIBMCOSDefaultsQuery.ts

@@ -0,0 +1,28 @@
+import { useQuery } from "@tanstack/react-query";
+
+export interface IBMCOSDefaults {
+    api_key_set: boolean;
+    service_instance_id: string;
+    endpoint: string;
+    hmac_access_key_set: boolean;
+    hmac_secret_key_set: boolean;
+    auth_mode: "iam" | "hmac";
+    bucket_names: string[];
+    connection_id: string | null;
+    disable_iam: boolean;
+}
+
+async function fetchIBMCOSDefaults(): Promise<IBMCOSDefaults> {
+    const res = await fetch("/api/connectors/ibm_cos/defaults");
+    if (!res.ok) throw new Error("Failed to fetch IBM COS defaults");
+    return res.json();
+}
+
+export function useIBMCOSDefaultsQuery(options?: { enabled?: boolean }) {
+    return useQuery<IBMCOSDefaults>({
+        queryKey: ["ibm-cos-defaults"],
+        queryFn: fetchIBMCOSDefaults,
+        enabled: options?.enabled ?? true,
+        staleTime: 0,
+    });
+}