feat: Add get all documents

Author: tjtanjin

config/env/.env.template

@@ -47,4 +47,4 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
 GITHUB_REPO_OWNER=react-chatbotify
 GITHUB_REPO_NAME=core-library-documentation
 GITHUB_DOCS_PATH='docs'
-GITHUB_TOKEN=GITHUB_TOKEN; # optional, unlikely needed
+GITHUB_TOKEN=GITHUB_TOKEN # create a personal access token on github (no perms required, just for higher auth rate limits)

src/api/controllers/__tests__/ragManagement.test.ts

@@ -1,7 +1,7 @@
 import { Request, Response } from 'express';
 
 import { initializedRagService } from '../../services/ragService';
-import { createDocument, deleteDocument, getDocument, updateDocument } from '../ragManagement'; // Adjust path as needed
+import { createDocument, deleteDocument, getDocument, getAllDocuments, updateDocument } from '../ragManagement'; // Adjust path as needed
 
 // Mock the initializedRagService
 jest.mock('../../services/ragService', () => ({
@@ -10,6 +10,7 @@ jest.mock('../../services/ragService', () => ({
     deleteDocument: jest.fn(),
     getParentDocumentContent: jest.fn(),
     updateDocument: jest.fn(),
+    getAllDocumentIds: jest.fn(), // Added mock for getAllDocumentIds
     // Add other methods if they are called and need mocking, e.g. documentExists
   }),
 }));
@@ -213,4 +214,55 @@ describe('RAG Management Controllers', () => {
       expect(res.json).toHaveBeenCalledWith({ details: 'Service error', error: 'Internal Server Error' });
     });
   });
+
+  describe('getAllDocuments', () => {
+    it('should return 200 and all document IDs', async () => {
+      const req = mockRequest() as Request;
+      const res = mockResponse() as Response;
+      const expectedIds = ['id1', 'id2'];
+      mockRagService.getAllDocumentIds.mockResolvedValue(expectedIds);
+
+      await getAllDocuments(req, res);
+
+      expect(mockRagService.getAllDocumentIds).toHaveBeenCalled();
+      expect(res.status).toHaveBeenCalledWith(200);
+      expect(res.json).toHaveBeenCalledWith({ documentIds: expectedIds });
+    });
+
+    it('should return 200 and an empty array if no documents exist', async () => {
+      const req = mockRequest() as Request;
+      const res = mockResponse() as Response;
+      mockRagService.getAllDocumentIds.mockResolvedValue([]);
+
+      await getAllDocuments(req, res);
+
+      expect(mockRagService.getAllDocumentIds).toHaveBeenCalled();
+      expect(res.status).toHaveBeenCalledWith(200);
+      expect(res.json).toHaveBeenCalledWith({ documentIds: [] });
+    });
+
+    it('should return 503 if RAG service is not ready', async () => {
+      const req = mockRequest() as Request;
+      const res = mockResponse() as Response;
+      mockRagService.getAllDocumentIds.mockRejectedValue(new Error('ChromaDB collection is not initialized'));
+
+      await getAllDocuments(req, res);
+
+      expect(mockRagService.getAllDocumentIds).toHaveBeenCalled();
+      expect(res.status).toHaveBeenCalledWith(503);
+      expect(res.json).toHaveBeenCalledWith({ error: 'Service Unavailable: RAG service is not ready.' });
+    });
+
+    it('should return 500 for other errors', async () => {
+      const req = mockRequest() as Request;
+      const res = mockResponse() as Response;
+      mockRagService.getAllDocumentIds.mockRejectedValue(new Error('Something went wrong'));
+
+      await getAllDocuments(req, res);
+
+      expect(mockRagService.getAllDocumentIds).toHaveBeenCalled();
+      expect(res.status).toHaveBeenCalledWith(500);
+      expect(res.json).toHaveBeenCalledWith({ error: 'Internal Server Error', details: 'Something went wrong' });
+    });
+  });
 });

src/api/controllers/ragManagement.ts

@@ -27,6 +27,21 @@ export const createDocument = async (req: Request, res: Response) => {
   }
 };
 
+export const getAllDocuments = async (req: Request, res: Response) => {
+  try {
+    const ragService = await initializedRagService;
+    const documentIds = await ragService.getAllDocumentIds();
+    return res.status(200).json({ documentIds });
+  } catch (error: any) {
+    Logger.error('Error in getAllDocuments:', error);
+    // Check for specific service readiness errors if applicable
+    if (error.message && error.message.includes('ChromaDB collection is not initialized')) {
+      return res.status(503).json({ error: 'Service Unavailable: RAG service is not ready.' });
+    }
+    return res.status(500).json({ error: 'Internal Server Error', details: error.message });
+  }
+};
+
 export const getDocument = async (req: Request, res: Response) => {
   try {
     const { documentId } = req.params;

src/api/routers/ragManagement.ts

@@ -1,6 +1,12 @@
 import { Router } from 'express';
 
-import { createDocument, deleteDocument, getDocument, updateDocument } from '../controllers/ragManagement';
+import {
+  createDocument,
+  deleteDocument,
+  getDocument,
+  getAllDocuments, // <-- Import added here
+  updateDocument,
+} from '../controllers/ragManagement';
 import { apiKeyAuth } from '../middleware/auth';
 
 const ragManagementRouter = Router();
@@ -10,6 +16,7 @@ ragManagementRouter.use(apiKeyAuth);
 
 // Define routes
 ragManagementRouter.post('/documents', createDocument);
+ragManagementRouter.get('/documents:all', getAllDocuments);
 ragManagementRouter.get('/documents/:documentId', getDocument);
 ragManagementRouter.put('/documents/:documentId', updateDocument);
 ragManagementRouter.delete('/documents/:documentId', deleteDocument);

src/api/services/mongoService.ts

@@ -43,6 +43,12 @@ class MongoService {
     const collection = await this.getParentDocumentCollection();
     await collection.deleteOne({ _id: documentId });
   }
+
+  async getAllDocumentIds(): Promise<string[]> {
+    const collection = await this.getParentDocumentCollection();
+    const documents = await collection.find({}, { projection: { _id: 1 } }).toArray();
+    return documents.map((doc) => doc._id);
+  }
 }
 
 export const mongoService = new MongoService();

src/api/services/ragService.ts

@@ -4,7 +4,7 @@ import { v4 as uuidv4 } from 'uuid';
 
 import { config } from '../config';
 import Logger from '../logger';
-import { generateEmbeddings } from './llmWrapper';
+import { generateEmbeddings } from '../services/llmWrapper';
 import { mongoService } from './mongoService';
 
 export class RAGService {
@@ -293,6 +293,17 @@ export class RAGService {
       throw error;
     }
   }
+
+  public async getAllDocumentIds(): Promise<string[]> {
+    try {
+      const ids = await mongoService.getAllDocumentIds();
+      Logger.info(`Retrieved ${ids.length} document IDs from mongoService.`);
+      return ids;
+    } catch (error) {
+      Logger.error('Error retrieving all document IDs via mongoService:', error);
+      throw error; // Re-throw to allow controller to handle
+    }
+  }
 }
 
 // Export a promise that resolves when the service is initialized

src/api/swagger/ragManagement.ts

@@ -5,127 +5,109 @@ const API_VERSION = process.env.API_VERSION || 'v1';
 const ragManagementPaths = {
   [`/api/${API_VERSION}/rag/manage/documents`]: {
     post: {
+      summary: 'Create a new document in the RAG system.',
+      tags: ['RAG Management'],
+      security: [{ ApiKeyAuth: [] }],
       requestBody: {
+        required: true,
         content: {
           'multipart/form-data': {
             schema: {
+              type: 'object',
               properties: {
                 documentId: {
+                  type: 'string',
                   description: 'Unique identifier for the document (e.g., filename or a unique ID).',
                   example: 'my-document-123',
-                  type: 'string',
                 },
                 markdownFile: {
-                  description: 'The Markdown file to upload.',
-                  format: 'binary',
                   type: 'string',
+                  format: 'binary',
+                  description: 'The Markdown file to upload.',
                 },
               },
               required: ['documentId', 'markdownFile'],
-              type: 'object',
             },
           },
         },
-        required: true,
       },
       responses: {
         '201': { description: 'Document added successfully.' },
         '400': { description: 'Bad Request: Invalid input.' },
         '401': { description: 'Unauthorized: API key is missing or invalid.' },
         '500': { description: 'Internal Server Error.' },
       },
-      security: [{ ApiKeyAuth: [] }],
-      summary: 'Create a new document in the RAG system.',
-      tags: ['RAG Management'],
     },
   },
   [`/api/${API_VERSION}/rag/manage/documents/{documentId}`]: {
-    delete: {
-      parameters: [
-        {
-          description: 'Identifier of the document to delete.',
-          in: 'path',
-          name: 'documentId',
-          required: true,
-          schema: { type: 'string' },
-        },
-      ],
-      responses: {
-        '200': { description: 'Document deleted successfully.' },
-        // "204": { description: "Document deleted successfully (No Content)." }, // Alternative
-        '401': { description: 'Unauthorized: API key is missing or invalid.' },
-        '404': { description: 'Not Found: Document not found (or already deleted).' },
-        '500': { description: 'Internal Server Error.' },
-      },
-      security: [{ ApiKeyAuth: [] }],
-      summary: 'Delete a document by its ID.',
-      tags: ['RAG Management'],
-    },
     get: {
+      summary: 'Retrieve a document by its ID.',
+      tags: ['RAG Management'],
+      security: [{ ApiKeyAuth: [] }],
       parameters: [
         {
-          description: 'Identifier of the document to retrieve.',
-          in: 'path',
           name: 'documentId',
+          in: 'path',
           required: true,
+          description: 'Identifier of the document to retrieve.',
           schema: { type: 'string' },
         },
       ],
       responses: {
         '200': {
+          description: 'Successful retrieval of document content.',
           content: {
             'application/json': {
               schema: {
+                type: 'object',
                 properties: {
+                  documentId: { type: 'string' },
                   content: {
-                    description: 'The full markdown content of the document.',
                     type: 'string',
+                    description: 'The full markdown content of the document.',
                   },
-                  documentId: { type: 'string' },
                 },
-                type: 'object',
               },
             },
           },
-          description: 'Successful retrieval of document content.',
         },
         '401': { description: 'Unauthorized: API key is missing or invalid.' },
         '404': { description: 'Not Found: Document not found.' },
         '500': { description: 'Internal Server Error.' },
       },
-      security: [{ ApiKeyAuth: [] }],
-      summary: 'Retrieve a document by its ID.',
-      tags: ['RAG Management'],
     },
     put: {
+      summary: 'Update an existing document.',
+      tags: ['RAG Management'],
+      security: [{ ApiKeyAuth: [] }],
       parameters: [
         {
-          description: 'Identifier of the document to update.',
-          in: 'path',
           name: 'documentId',
+          in: 'path',
           required: true,
+          description: 'Identifier of the document to update.',
           schema: { type: 'string' },
         },
       ],
       requestBody: {
+        required: true,
         content: {
           'multipart/form-data': {
             // Consistent with POST
             schema: {
+              type: 'object',
               properties: {
                 // documentId is in path, so not needed here again unless we want to allow changing it (unusual for PUT)
                 markdownFile: {
-                  description: 'The new Markdown file to replace the existing document.',
-                  format: 'binary',
                   type: 'string',
+                  format: 'binary',
+                  description: 'The new Markdown file to replace the existing document.',
                 },
               },
               required: ['markdownFile'],
-              type: 'object',
             },
           },
         },
-        required: true,
       },
       responses: {
         '200': { description: 'Document updated successfully.' },
@@ -134,10 +116,59 @@ const ragManagementPaths = {
         '404': { description: 'Not Found: Document not found.' },
         '500': { description: 'Internal Server Error.' },
       },
+    },
+    delete: {
+      summary: 'Delete a document by its ID.',
+      tags: ['RAG Management'],
       security: [{ ApiKeyAuth: [] }],
-      summary: 'Update an existing document.',
+      parameters: [
+        {
+          name: 'documentId',
+          in: 'path',
+          required: true,
+          description: 'Identifier of the document to delete.',
+          schema: { type: 'string' },
+        },
+      ],
+      responses: {
+        '200': { description: 'Document deleted successfully.' },
+        // "204": { description: "Document deleted successfully (No Content)." }, // Alternative
+        '401': { description: 'Unauthorized: API key is missing or invalid.' },
+        '404': { description: 'Not Found: Document not found (or already deleted).' },
+        '500': { description: 'Internal Server Error.' },
+      },
+    },
+  },
+  [`/api/${API_VERSION}/rag/manage/documents:all`]: {
+    get: {
+      summary: 'Get all document IDs',
       tags: ['RAG Management'],
+      security: [{ ApiKeyAuth: [] }],
+      responses: {
+        '200': {
+          description: 'Successful retrieval of all document IDs.',
+          content: {
+            'application/json': {
+              schema: {
+                type: 'object',
+                properties: {
+                  documentIds: {
+                    type: 'array',
+                    items: { type: 'string' },
+                    example: ['doc-abc-123', 'doc-def-456'],
+                  },
+                },
+              },
+            },
+          },
+        },
+        '401': { description: 'Unauthorized: API key is missing or invalid.' },
+        '503': { description: 'Service Unavailable: RAG service is not ready.' },
+        '500': { description: 'Internal Server Error.' },
+      },
     },
+    // Note: Based on the router, a POST operation to this path also exists.
+    // Its definition should be added here if it's different from POST /rag/manage/documents.
   },
 };
 

src/jobs/index.ts

@@ -5,4 +5,5 @@ void runSyncDocuments();
 
 // todo: ideally a cronjob should be used to only spin up job pod when required
 // but nvm this is good enough for now
+// todo: consider caching locally in mongodb to reduce requests to github
 setInterval(() => void runSyncDocuments(), 86400000);