feat: Improve index and embedding cleanup logic, add combined summary API

- add support for the API to obtain combined summaries
- adjust cleanup strategies based on the codebase switch status
- batch process timeout events to improve performance
- optimize logging and error handling
- adjust the default file scanning limit to 10000

Author: WayneWang00

.gitignore

@@ -15,4 +15,5 @@ test/tmp
 localhost
 tmp/codegraph/*.profile
 .costrict
-.coignore
+.coignore
+.documents

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/gin-gonic/gin v1.10.1
 	github.com/golang/mock v1.7.0-rc.1
 	github.com/google/uuid v1.6.0
+	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/mattn/go-sqlite3 v1.14.30
 	github.com/pelletier/go-toml/v2 v2.2.4
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
@@ -41,7 +42,6 @@ require (
 	github.com/go-playground/validator/v10 v10.27.0 // indirect
 	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect

internal/config/config.go

@@ -97,7 +97,7 @@ var DefaultFileIncludePatterns = []string{
 
 var DefaultConfigScan = ConfigScan{
 	MaxFileSizeKB:        10240,                       // Default maximum file size in KB
-	MaxFileCount:         100000,                      // Default maximum file count
+	MaxFileCount:         10000,                       // Default maximum file count
 	FolderIgnorePatterns: DefaultFolderIgnorePatterns, // Default folder ignore patterns
 	FileIncludePatterns:  DefaultFileIncludePatterns,  // Default file include patterns
 }

internal/dto/syncer.go

@@ -92,3 +92,29 @@ type DeleteEmbeddingResp struct {
 	Success bool        `json:"success"`
 	Data    interface{} `json:"data"`
 }
+
+// CombinedSummaryReq 获取组合摘要请求
+type CombinedSummaryReq struct {
+	ClientId     string `form:"clientId" binding:"required"`
+	CodebasePath string `form:"codebasePath" binding:"required"`
+}
+
+// CombinedSummaryResp 获取组合摘要响应
+type CombinedSummaryResp struct {
+	Code    int                 `json:"code"`
+	Message string              `json:"message"`
+	Success bool                `json:"success"`
+	Data    CombinedSummaryData `json:"data"`
+}
+
+type CombinedSummaryData struct {
+	TotalFiles int
+	Embedding  EmbeddingSummary `json:"embedding"`
+}
+
+type EmbeddingSummary struct {
+	Status      string `json:"status"`
+	UpdatedAt   string `json:"updatedAt"`
+	TotalFiles  int    `json:"totalFiles"`
+	TotalChunks int    `json:"totalChunks"`
+}

internal/handler/backend.go

@@ -127,17 +127,14 @@ func (h *BackendHandler) SearchCallGraph(c *gin.Context) {
 	}
 	h.logger.Info("search callgraph request: ClientId=%s, Workspace=%s, FilePath=%s", req.ClientId, req.CodebasePath, req.FilePath)
 	callGraph, err := h.codebaseService.QueryCallGraph(c, &req)
-	if  err != nil{
+	if err != nil {
 		h.logger.Error("search callgraph err:%v", err)
 		response.Error(c, http.StatusBadRequest, err)
 		return
 	}
-	response.OkJson(c,callGraph)
+	response.OkJson(c, callGraph)
 }
 
-
-
-
 // GetFileContent 获取源文件内容接口
 // @Summary 获取文件内容
 // @Description 获取源文件内容，以二进制流形式返回

internal/job/event_processor_job.go

@@ -52,7 +52,7 @@ func (j *EventProcessorJob) Start(ctx context.Context) {
 			j.logger.Error("recovered from panic in event processor job: %v", r)
 		}
 	}()
-	j.logger.Info("starting event processor job")
+	j.logger.Info("starting embedding event processor job with interval: %v", j.embeddingInterval)
 
 	// 立即执行一次事件处理
 	authInfo := config.GetAuthInfo()

internal/job/index_clean_job.go

@@ -138,10 +138,20 @@ func (j *IndexCleanJob) cleanupExpiredWorkspaceIndexes(ctx context.Context) {
 			j.logger.Error("recovered from panic in index clean job: %v", r)
 		}
 	}()
-	j.logger.Info("start to clean up expired workspace indexes with expiry period %.0f hours", j.expiryPeriod.Hours())
+
+	// 获取 codebase 开关状态
+	codebaseEnv := j.storageRepo.GetCodebaseEnv()
+	if codebaseEnv == nil {
+		j.logger.Warn("failed to get codebase env")
+		return
+	}
+
+	j.logger.Info("start to clean up expired workspace indexes with expiry period %.0f hours, codebase switch: %s",
+		j.expiryPeriod.Hours(), codebaseEnv.Switch)
+
 	workspaces, err := j.workspaceRepo.ListWorkspaces()
 	if err != nil {
-		j.logger.Error("list workspaces failed with %v", err)
+		j.logger.Warn("list workspaces failed with %v", err)
 		return
 	}
 
@@ -151,13 +161,23 @@ func (j *IndexCleanJob) cleanupExpiredWorkspaceIndexes(ctx context.Context) {
 	}
 
 	for _, workspace := range workspaces {
-		// 活跃中 更新时间小于过期间隔 索引数量为0 跳过
-		if workspace.Active == dto.True || time.Since(workspace.UpdatedAt) < j.expiryPeriod ||
-			workspace.CodegraphFileNum == 0 {
-			continue
+		// 如果 codebase 开关为 on，则使用原来的逻辑
+		if codebaseEnv.Switch == dto.SwitchOn {
+			// 活跃中 更新时间小于过期间隔 索引数量为0 跳过
+			if workspace.Active == dto.True || time.Since(workspace.UpdatedAt) < j.expiryPeriod ||
+				workspace.CodegraphFileNum == 0 {
+				continue
+			}
+		} else {
+			// 如果 codebase 开关为 off，则检查是否过期一天
+			if time.Since(workspace.UpdatedAt) < 24*time.Hour ||
+				workspace.CodegraphFileNum == 0 {
+				continue
+			}
 		}
-		j.logger.Info("workspace %s updated_at %s exceeds expiry period %.0f hours, start to cleanup.",
-			workspace.WorkspacePath, workspace.UpdatedAt.Format("2006-01-02 15:04:05"), j.expiryPeriod.Hours())
+
+		j.logger.Info("workspace %s updated_at %s exceeds expiry period, start to cleanup.",
+			workspace.WorkspacePath, workspace.UpdatedAt.Format("2006-01-02 15:04:05"))
 		// 清理索引 （有更新数据库为0的逻辑）
 		if err = j.indexer.RemoveAllIndexes(ctx, workspace.WorkspacePath); err != nil {
 			j.logger.Error("remove indexes failed with %v", err)
@@ -177,9 +197,16 @@ func (j *IndexCleanJob) cleanupInactiveWorkspaceEmbeddings(ctx context.Context)
 		}
 	}()
 
+	// 获取 codebase 开关状态
+	codebaseEnv := j.storageRepo.GetCodebaseEnv()
+	if codebaseEnv == nil {
+		j.logger.Warn("failed to get codebase env")
+		return
+	}
+
 	workspaces, err := j.workspaceRepo.ListWorkspaces()
 	if err != nil {
-		j.logger.Error("list workspaces failed with %v", err)
+		j.logger.Warn("list workspaces failed with %v", err)
 		return
 	}
 
@@ -192,12 +219,77 @@ func (j *IndexCleanJob) cleanupInactiveWorkspaceEmbeddings(ctx context.Context)
 	authInfo := config.GetAuthInfo()
 	clientId := authInfo.ClientId
 	for _, workspace := range workspaces {
-		// 只处理 active 为 false, embedding_file_num 不为 0, 且更新时间超过过期间隔的工作区
-		if workspace.Active == dto.True || time.Since(workspace.UpdatedAt) < j.embeddingExpiryPeriod ||
-			workspace.EmbeddingFileNum == 0 {
+		if workspace.EmbeddingFileNum == 0 {
+			continue
+		}
+		// 调用 FetchCombinedSummary 方法获取结果
+		summaryReq := dto.CombinedSummaryReq{
+			ClientId:     clientId,
+			CodebasePath: workspace.WorkspacePath,
+		}
+
+		summaryResp, err := j.syncRepo.FetchCombinedSummary(summaryReq)
+		if err != nil {
+			j.logger.Warn("failed to fetch combined summary for workspace %s: %v", workspace.WorkspacePath, err)
 			continue
 		}
 
+		// 判断状态，若为 failed 则复用下面的更新 workspace、删除 event、删除 codebaseConfig 和 embeddingConfig 逻辑
+		if summaryResp.Data.Embedding.TotalChunks == 0 {
+			j.logger.Info("workspace %s embedding total chunks is 0, start to cleanup.",
+				workspace.WorkspacePath)
+
+			// 更新 workspace
+			updateWorkspace := map[string]interface{}{
+				"file_num":                    0,
+				"embedding_file_num":          0,
+				"embedding_ts":                0,
+				"embedding_message":           "",
+				"embedding_failed_file_paths": "",
+			}
+			if err := j.workspaceRepo.UpdateWorkspaceByMap(workspace.WorkspacePath, updateWorkspace); err != nil {
+				j.logger.Error("update workspace failed with %v", err)
+				continue
+			}
+
+			// 1. 删除这个 workspace 的所有 event 表记录
+			if err := j.cleanupWorkspaceEvents(workspace.WorkspacePath); err != nil {
+				j.logger.Error("failed to cleanup events for workspace %s: %v", workspace.WorkspacePath, err)
+				continue
+			}
+
+			// 2. 删除 codebaseConfig
+			codebaseId := utils.GenerateCodebaseID(workspace.WorkspacePath)
+			if err := j.storageRepo.DeleteCodebaseConfig(codebaseId); err != nil {
+				j.logger.Error("failed to delete codebase config for workspace %s: %v", workspace.WorkspacePath, err)
+				continue
+			}
+
+			// 3. 删除 embeddingConfig
+			embeddingId := utils.GenerateEmbeddingID(workspace.WorkspacePath)
+			if err := j.embeddingRepo.DeleteEmbeddingConfig(embeddingId); err != nil {
+				j.logger.Error("failed to delete embedding config for workspace %s: %v", workspace.WorkspacePath, err)
+				continue
+			}
+
+			j.logger.Info("workspace %s embeddings cleaned up successfully.", workspace.WorkspacePath)
+			cleanedCount++
+			continue
+		}
+
+		// 如果 codebase 开关为 on，则使用原来的逻辑
+		if codebaseEnv.Switch == dto.SwitchOn {
+			// 只处理 active 为 false, 且更新时间超过过期间隔的工作区
+			if workspace.Active == dto.True || time.Since(workspace.UpdatedAt) < j.embeddingExpiryPeriod {
+				continue
+			}
+		} else {
+			// 如果 codebase 开关为 off，则检查是否过期一天
+			if time.Since(workspace.UpdatedAt) < 24*time.Hour {
+				continue
+			}
+		}
+
 		j.logger.Info("workspace %s is inactive and exceeds expiry period, start to cleanup embeddings.",
 			workspace.WorkspacePath)
 

internal/repository/syncer.go

@@ -21,11 +21,12 @@ import (
 
 // Server API paths
 const (
-	API_UPLOAD_TOKEN      = "/codebase-embedder/api/v1/files/token"
-	API_UPLOAD_FILE       = "/codebase-embedder/api/v1/files/upload"
-	API_FILE_STATUS       = "/codebase-embedder/api/v1/files/status"
-	API_GET_CODEBASE_HASH = "/codebase-embedder/api/v1/codebases/hash"
-	API_DELETE_EMBEDDING  = "/codebase-embedder/api/v1/embeddings"
+	API_UPLOAD_TOKEN         = "/codebase-embedder/api/v1/files/token"
+	API_UPLOAD_FILE          = "/codebase-embedder/api/v1/files/upload"
+	API_FILE_STATUS          = "/codebase-embedder/api/v1/files/status"
+	API_GET_CODEBASE_HASH    = "/codebase-embedder/api/v1/codebases/hash"
+	API_DELETE_EMBEDDING     = "/codebase-embedder/api/v1/embeddings"
+	API_GET_COMBINED_SUMMARY = "/codebase-embedder/api/v1/combined/summary"
 )
 
 type SyncInterface interface {
@@ -37,6 +38,7 @@ type SyncInterface interface {
 	FetchUploadToken(req dto.UploadTokenReq) (*dto.UploadTokenResp, error)
 	FetchFileStatus(req dto.FileStatusReq) (*dto.FileStatusResp, error)
 	DeleteEmbedding(req dto.DeleteEmbeddingReq) (*dto.DeleteEmbeddingResp, error)
+	FetchCombinedSummary(req dto.CombinedSummaryReq) (*dto.CombinedSummaryResp, error)
 }
 
 type HTTPSync struct {
@@ -428,3 +430,35 @@ func (hs *HTTPSync) DeleteEmbedding(req dto.DeleteEmbeddingReq) (*dto.DeleteEmbe
 	hs.logger.Info("HTTP %s %s completed in %v, status: %d", "DELETE", url, duration, responseData.Code)
 	return &responseData, nil
 }
+
+// FetchCombinedSummary fetches combined summary from server
+func (hs *HTTPSync) FetchCombinedSummary(req dto.CombinedSummaryReq) (*dto.CombinedSummaryResp, error) {
+	hs.logger.Info("fetching combined summary from server")
+
+	// 验证配置
+	authInfo := config.GetAuthInfo()
+	if err := hs.ValidateSyncConfig(authInfo); err != nil {
+		return nil, err
+	}
+
+	// 构建请求URL
+	url := fmt.Sprintf("%s%s", authInfo.ServerURL, API_GET_COMBINED_SUMMARY)
+
+	// 构建查询参数
+	queryParams := map[string]string{
+		"clientId":     req.ClientId,
+		"codebasePath": req.CodebasePath,
+	}
+
+	// 执行请求
+	var responseData dto.CombinedSummaryResp
+	hs.logger.Info("sending HTTP %s request to: %s", "GET", url)
+	startTime := time.Now()
+	if err := hs.httpClient.DoGetRequest(url, queryParams, authInfo.Token, &responseData); err != nil {
+		return nil, err
+	}
+	duration := time.Since(startTime)
+	hs.logger.Info("HTTP %s %s completed in %v, status: %d", "GET", url, duration, responseData.Code)
+
+	return &responseData, nil
+}

internal/service/embedding_status.go

@@ -166,19 +166,33 @@ func (sc *embeddingStatusService) checkWorkspaceBuildingStates(workspacePath str
 
 	// 首先处理超时的事件
 	var validEvents []*model.Event
+	var timeoutEvents []*model.Event
 	for _, event := range events {
 		if nowTime.Sub(event.UpdatedAt) > time.Minute*3 {
-			updateEvent := &model.Event{ID: event.ID, EmbeddingStatus: model.EmbeddingStatusBuildFailed}
-			err := sc.eventRepo.UpdateEvent(updateEvent)
-			if err != nil {
-				sc.logger.Error("failed to update event status: %v", err)
-			}
-			sc.buildFilePathFailed(event)
+			timeoutEvents = append(timeoutEvents, event)
 			continue
 		}
 		validEvents = append(validEvents, event)
 	}
 
+	// 批量处理超时事件
+	if len(timeoutEvents) > 0 {
+		// 构建超时，批量更新事件状态为构建失败
+		eventIDs := make([]int64, 0, len(timeoutEvents))
+		for _, event := range timeoutEvents {
+			eventIDs = append(eventIDs, event.ID)
+		}
+		updateErr := sc.eventRepo.UpdateEventsEmbeddingStatus(eventIDs, model.EmbeddingStatusBuildFailed)
+		if updateErr != nil {
+			sc.logger.Error("failed to update events status to buildFailed: %v", updateErr)
+		}
+		// 批量处理构建失败事件
+		err := sc.batchHandleBuildFailed(workspacePath, timeoutEvents)
+		if err != nil {
+			sc.logger.Error("failed to batch handle build failed: %v", err)
+		}
+	}
+
 	// 按syncId分组有效事件
 	syncIdEventsMap := make(map[string][]*model.Event)
 	for _, event := range validEvents {
@@ -227,16 +241,30 @@ func (sc *embeddingStatusService) checkWorkspaceUploadingStates(workspacePath st
 
 	// 检查每个event的上传状态
 	nowTime := time.Now()
+	var timeoutEvents []*model.Event
 	for _, event := range events {
 		if nowTime.Sub(event.UpdatedAt) < time.Minute*3 {
 			continue
 		}
-		updateEvent := &model.Event{ID: event.ID, EmbeddingStatus: model.EmbeddingStatusUploadFailed}
-		err := sc.eventRepo.UpdateEvent(updateEvent)
+		timeoutEvents = append(timeoutEvents, event)
+	}
+
+	// 批量处理超时事件
+	if len(timeoutEvents) > 0 {
+		// 构建超时，批量更新事件状态为上传失败
+		eventIDs := make([]int64, 0, len(timeoutEvents))
+		for _, event := range timeoutEvents {
+			eventIDs = append(eventIDs, event.ID)
+		}
+		updateErr := sc.eventRepo.UpdateEventsEmbeddingStatus(eventIDs, model.EmbeddingStatusUploadFailed)
+		if updateErr != nil {
+			sc.logger.Error("failed to update events status to uploadFailed: %v", updateErr)
+		}
+		// 批量处理上传失败事件
+		err := sc.batchHandleBuildFailed(workspacePath, timeoutEvents)
 		if err != nil {
-			sc.logger.Error("failed to update event status: %v", err)
+			sc.logger.Error("failed to batch handle build failed: %v", err)
 		}
-		sc.buildFilePathFailed(event)
 	}
 
 	return nil

test/mocks/mock_http_sync.go

@@ -49,3 +49,8 @@ func (m *MockHTTPSync) DeleteEmbedding(req dto.DeleteEmbeddingReq) (*dto.DeleteE
 	args := m.Called(req)
 	return args.Get(0).(*dto.DeleteEmbeddingResp), args.Error(1)
 }
+
+func (m *MockHTTPSync) FetchCombinedSummary(req dto.CombinedSummaryReq) (*dto.CombinedSummaryResp, error) {
+	args := m.Called(req)
+	return args.Get(0).(*dto.CombinedSummaryResp), args.Error(1)
+}