diff --git a/dev-share-api/Configuration/VectorDbSettings.cs b/dev-share-api/Configuration/VectorDbSettings.cs new file mode 100644 index 0000000..5729bec --- /dev/null +++ b/dev-share-api/Configuration/VectorDbSettings.cs @@ -0,0 +1,30 @@ +namespace DevShare.Api.Configuration; + +public class VectorDbSettings +{ + public const string SectionName = "VectorDb"; + + // Collections + public string ResourceCollection { get; set; } = "DevShare_Resource"; + public string InsightCollection { get; set; } = "DevShare_Insight"; + + // Vector dimensions + public uint Dimensions { get; set; } = 384; // MiniLM-L6-v2 dimension + + // Thresholds + public float DenseScoreThreshold { get; set; } = 0.65f; // Balanced semantic similarity + public float SparseScoreThreshold { get; set; } = 0.15f; // Good token overlap + public float HybridScoreThreshold { get; set; } = 0.53f; // Optimal fusion threshold + + // Vector configurations + public VectorConfig Dense { get; set; } = new(); + public VectorConfig Sparse { get; set; } = new(); +} + +public class VectorConfig +{ + public string Name { get; set; } = string.Empty; + public bool OnDisk { get; set; } + public ulong MinTokenLength { get; set; } = 2; + public ulong MaxTokenLength { get; set; } = 10; +} \ No newline at end of file diff --git a/dev-share-api/Controllers/ApiController.cs b/dev-share-api/Controllers/ApiController.cs index 51661d1..f234cd8 100644 --- a/dev-share-api/Controllers/ApiController.cs +++ b/dev-share-api/Controllers/ApiController.cs @@ -173,7 +173,14 @@ public async Task> InitVectorDB() return Ok(); } - [HttpPost("embedding/indexing")] + [HttpPut("vector/updateCollection")] + public async Task> UpdateCollection([FromBody] string collectionName) + { + await _vectorService.UpdateCollectionAsync(collectionName); + return Ok(); + } + + [HttpPost("vector/indexing")] public async Task> Indexing([FromBody] string collectionName, string field) { return Ok(await _vectorService.IndexingAsync(collectionName, field)); diff --git a/dev-share-api/Handle/SummarizeShareChainHandle.cs b/dev-share-api/Handle/SummarizeShareChainHandle.cs index d4aed5c..65b5733 100644 --- a/dev-share-api/Handle/SummarizeShareChainHandle.cs +++ b/dev-share-api/Handle/SummarizeShareChainHandle.cs @@ -26,11 +26,15 @@ public override async Task IsSkip(ResourceShareContext context) protected override async Task ProcessAsync(ResourceShareContext context) { + if (string.IsNullOrWhiteSpace(context.ExtractResult)) + { + return HandlerResult.Fail("No content provided for summarization"); + } var summary = await _summaryService.SummarizeAsync(context.ExtractResult); context.Summary = summary.Summary; context.Title = summary.Title; return HandlerResult.Success(); } - + } \ No newline at end of file diff --git a/dev-share-api/Program.cs b/dev-share-api/Program.cs index 18f818f..779c59c 100644 --- a/dev-share-api/Program.cs +++ b/dev-share-api/Program.cs @@ -1,3 +1,5 @@ +using DevShare.Api.Configuration; + var builder = WebApplication.CreateBuilder(args); // Configuration @@ -20,6 +22,9 @@ .AddInfrastructureServices(builder.Configuration) .AddApplicationServices(); +builder.Services.Configure( + builder.Configuration.GetSection(VectorDbSettings.SectionName)); + var app = builder.Build(); // Middleware Configuration diff --git a/dev-share-api/Services/DependencyInjection.cs b/dev-share-api/Services/DependencyInjection.cs index aa32bf7..21073e9 100644 --- a/dev-share-api/Services/DependencyInjection.cs +++ b/dev-share-api/Services/DependencyInjection.cs @@ -40,7 +40,9 @@ public static IServiceCollection AddInfrastructureServices( // Database services.AddDbContext(options => - options.UseSqlServer(configuration.GetConnectionString("DefaultConnection"))); + { + options.UseSqlServer(configuration.GetConnectionString("DefaultConnection")); + }); // HTTP Client diff --git a/dev-share-api/Services/SummaryService.cs b/dev-share-api/Services/SummaryService.cs index 473dd42..d53eea9 100644 --- a/dev-share-api/Services/SummaryService.cs +++ b/dev-share-api/Services/SummaryService.cs @@ -22,14 +22,21 @@ public SummaryService(AzureOpenAIClient openAIClient) public async Task SummarizeAsync(string content) { - var messages = new List { - new SystemChatMessage($@" - You are a helpful summarization assistant. + new SystemChatMessage(@" + You are a specialized AI assistant focused on technical content summarization. + Your task is to: - 1. Summarize the article in no more than 100 words. - 2. Extract or infer a clear, appropriate title, no more than 12 words. + 1. Create a semantically-rich summary that: + - Preserves key technical terms and domain-specific vocabulary + - Maintains semantic relationships between concepts + - Uses clear, factual language without metaphors + - Includes important numerical values and specific details + - Maximum length: 100 words + - Format: Single paragraph, no bullets + + 2. Extract or create a clear, appropriate title (max 12 words) Always call the `generate_summary` function with your result in JSON: {{ @@ -38,13 +45,15 @@ 1. Summarize the article in no more than 100 words. }} Guidelines: - - Avoid fabricating content. Be brief and accurate. Do not include any explanation. - - If the article lacks detail, summarize what’s available. + - Focus on technical accuracy and clarity + - Preserve key technical concepts and relationships + - Avoid explanations or additional formatting + - Be concise but informationally dense - Never return plain text. Always return structured JSON using the `generate_summary` function. - "), + "), new UserChatMessage(content) }; - + var tool = CreateGenerateSummaryTool(); return await CallToolAndDeserializeAsync( @@ -52,9 +61,9 @@ 1. Summarize the article in no more than 100 words. messages: messages, tool: tool ); - + } - + private ChatTool CreateGenerateSummaryTool() { @@ -81,7 +90,7 @@ private ChatTool CreateGenerateSummaryTool() }) ); } - + public async Task CallToolAndDeserializeAsync( string toolFunctionName, List messages, diff --git a/dev-share-api/Services/VectorService.cs b/dev-share-api/Services/VectorService.cs index f17f4d4..bef6d11 100644 --- a/dev-share-api/Services/VectorService.cs +++ b/dev-share-api/Services/VectorService.cs @@ -1,6 +1,9 @@ +using DevShare.Api.Configuration; +using Microsoft.Extensions.Options; using Models; using Qdrant.Client; using Qdrant.Client.Grpc; +using System.Text.Json; namespace Services; @@ -8,6 +11,8 @@ public interface IVectorService { Task InitializeAsync(); Task IndexingAsync(string collectionName, string fieldName); + Task UpdateCollectionAsync(string collectionName); + Task UpsertResourceAsync(string id, string url, string Content, Dictionary vectors); Task UpsertInsightAsync(string id, string url, string Content, string resourceId, Dictionary vectors); Task> SearchResourceAsync(string query, int topK); @@ -17,69 +22,27 @@ public interface IVectorService public class VectorService : IVectorService { private readonly QdrantClient _client; - private readonly string _resourceCollection = "BlotzShare_Resource"; - private readonly string _insightCollection = "BlotzShare_Insight"; - private readonly ulong _dimensions = 384; private readonly IEmbeddingService _embeddingService; - - public VectorService(QdrantClient qdrantClient, IEmbeddingService embeddingService) + private readonly VectorDbSettings _settings; + private readonly ILogger _logger; + + public VectorService( + QdrantClient qdrantClient, + IEmbeddingService embeddingService, + IOptions settings, + ILogger logger) { - _client = qdrantClient; - _embeddingService = embeddingService; + _client = qdrantClient ?? throw new ArgumentNullException(nameof(qdrantClient)); + _embeddingService = embeddingService ?? throw new ArgumentNullException(nameof(embeddingService)); + _settings = settings?.Value ?? throw new ArgumentNullException(nameof(settings)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } // init if there is no collection in vector db public async Task InitializeAsync() { - // Create resource collection - await _client.CreateCollectionAsync( - _resourceCollection, - vectorsConfig: new VectorParamsMap - { - Map = - { - ["dense_vector"] = new VectorParams { Size = _dimensions, Distance = Distance.Cosine }, - } - }, - sparseVectorsConfig: - ( - "sparse_vector", - new SparseVectorParams - { - Index = new SparseIndexConfig - { - OnDisk = false, - } - } - ) - ); - - await IndexingAsync(_resourceCollection, "content"); - - // Create insight collection - await _client.CreateCollectionAsync( - _insightCollection, - vectorsConfig: new VectorParamsMap - { - Map = - { - ["dense_vector"] = new VectorParams { Size = _dimensions, Distance = Distance.Cosine }, - } - }, - sparseVectorsConfig: - ( - "sparse_vector", - new SparseVectorParams - { - Index = new SparseIndexConfig - { - OnDisk = false, - } - } - ) - ); - - await IndexingAsync(_insightCollection, "content"); + await CreateCollectionAsync(_settings.ResourceCollection); + await CreateCollectionAsync(_settings.InsightCollection); } public async Task IndexingAsync(string collectionName, string fieldName) @@ -93,14 +56,22 @@ public async Task IndexingAsync(string collectionName, string fiel TextIndexParams = new TextIndexParams { Tokenizer = TokenizerType.Word, - MinTokenLen = 2, - MaxTokenLen = 10, + MinTokenLen = _settings.Sparse.MinTokenLength, + MaxTokenLen = _settings.Sparse.MaxTokenLength, Lowercase = true } } ); } + public async Task UpdateCollectionAsync(string collectionName) + { + await _client.UpdateCollectionAsync( + collectionName: collectionName, + sparseVectorsConfig: CreateSparseVectorConfig() + ); + } + public async Task UpsertResourceAsync(string id, string url, string content, Dictionary vectors) { var point = new PointStruct @@ -112,7 +83,7 @@ public async Task UpsertResourceAsync(string id, string url, string content, Dic ["content"] = content } }; - await _client.UpsertAsync(_resourceCollection, new List { point }); + await _client.UpsertAsync(_settings.ResourceCollection, new List { point }); } public async Task UpsertInsightAsync(string id, string url, string content, string resourceId, Dictionary vectors) @@ -127,80 +98,145 @@ public async Task UpsertInsightAsync(string id, string url, string content, stri ["resourceId"] = resourceId } }; - await _client.UpsertAsync(_insightCollection, new List { point }); + await _client.UpsertAsync(_settings.InsightCollection, new List { point }); } public async Task> SearchResourceAsync(string query, int topK) { - // Hybrid search on resource collection - var denseQueryVector = await _embeddingService.GetDenseEmbeddingAsync(query); - var (sparseIndices, sparseValues) = await _embeddingService.GetSparseEmbeddingAsync(query); + var (denseVector, sparseVector) = await GetQueryVectorsAsync(query); + var prefetchQueries = CreatePrefetchQueries(denseVector, sparseVector, topK); - var sparseTupleArray = sparseValues.Select((val, i) => (val, sparseIndices[i])).ToArray(); - var prefetch = new List - { - new() { Query = sparseTupleArray, Using = "sparse_vector", Limit = (ulong)topK }, - new() { Query = denseQueryVector, Using = "dense_vector", Limit = (ulong)topK } - }; - - - var resourceResults = await _client.QueryAsync( - collectionName: _resourceCollection, - prefetch: prefetch, + var results = await _client.QueryAsync( + collectionName: _settings.ResourceCollection, + prefetch: prefetchQueries, query: Fusion.Rrf, limit: (ulong)topK, - scoreThreshold: (float)0.7, //todo: make this dynamic payloadSelector: true, vectorsSelector: false ); - return resourceResults.Select(result => - { - var payload = result.Payload; - return new VectorResourceDto - { - Id = result.Id.Num.ToString(), - Url = payload.TryGetValue("url", out var urlVal) && urlVal.KindCase == Value.KindOneofCase.StringValue ? urlVal.StringValue : string.Empty, - Content = payload.TryGetValue("content", out var contentVal) && contentVal.KindCase == Value.KindOneofCase.StringValue ? contentVal.StringValue : string.Empty, - Score = result.Score - }; - }).ToList(); + return results.Select(MapToResourceDto).ToList(); } public async Task> SearchInsightAsync(string query, int topK) { - var denseQueryVector = await _embeddingService.GetDenseEmbeddingAsync(query); - var (sparseIndices, sparseValues) = await _embeddingService.GetSparseEmbeddingAsync(query); - - var sparseTupleArray = sparseValues.Select((val, i) => (val, sparseIndices[i])).ToArray(); - var prefetch = new List - { - new() { Query = sparseTupleArray, Using = "sparse_vector", Limit = (ulong)topK }, - new() { Query = denseQueryVector, Using = "dense_vector", Limit = (ulong)topK } - }; - + var (denseVector, sparseVector) = await GetQueryVectorsAsync(query); + var prefetchQueries = CreatePrefetchQueries(denseVector, sparseVector, topK); var insightResults = await _client.QueryAsync( - collectionName: _insightCollection, - prefetch: prefetch, + collectionName: _settings.InsightCollection, + prefetch: prefetchQueries, query: Fusion.Rrf, limit: (ulong)topK, - scoreThreshold: (float)0.9, //todo: make this dynamic payloadSelector: true, vectorsSelector: false ); - return insightResults.Select(result => + return insightResults.Select(MapToInsightDto).ToList(); + } + + private async Task CreateCollectionAsync(string collectionName) + { + try + { + await _client.CreateCollectionAsync( + collectionName, + vectorsConfig: CreateVectorConfig(), + sparseVectorsConfig: CreateSparseVectorConfig() + ); + + await IndexingAsync(collectionName, "content"); + _logger.LogInformation("Successfully created collection: {CollectionName}", collectionName); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to create collection: {CollectionName}", collectionName); + throw; + } + } + + private VectorParamsMap CreateVectorConfig() => new() + { + Map = + { + [_settings.Dense.Name] = new VectorParams + { + Size = _settings.Dimensions, + Distance = Distance.Cosine + } + } + }; + + private (string name, SparseVectorParams config) CreateSparseVectorConfig() => + ( + _settings.Sparse.Name, + new SparseVectorParams { - var payload = result.Payload; - return new VectorInsightDto + Modifier = Modifier.Idf, + Index = new SparseIndexConfig { - Id = result.Id.Num.ToString(), - Url = payload.TryGetValue("url", out var urlVal) && urlVal.KindCase == Value.KindOneofCase.StringValue ? urlVal.StringValue : string.Empty, - Content = payload.TryGetValue("content", out var contentVal) && contentVal.KindCase == Value.KindOneofCase.StringValue ? contentVal.StringValue : string.Empty, - ResourceId = payload.TryGetValue("resourceId", out var resourceIdVal) && resourceIdVal.KindCase == Value.KindOneofCase.StringValue ? resourceIdVal.StringValue : string.Empty, - Score = result.Score - }; - }).ToList(); + OnDisk = _settings.Sparse.OnDisk + } + } + ); + + private async Task<(float[] dense, (uint[] indices, float[] values) sparse)> + GetQueryVectorsAsync(string query) + { + var denseTask = _embeddingService.GetDenseEmbeddingAsync(query); + var sparseTask = _embeddingService.GetSparseEmbeddingAsync(query); + + await Task.WhenAll(denseTask, sparseTask); + return (await denseTask, await sparseTask); } + + private List CreatePrefetchQueries( + float[] denseVector, + (uint[] indices, float[] values) sparseVector, + int topK) + { + var sparseTuples = sparseVector.values + .Select((val, i) => (val, sparseVector.indices[i])) + .ToArray(); + + return new List + { + new() + { + Query = sparseTuples, + Using = _settings.Sparse.Name, + Limit = (ulong)topK, + ScoreThreshold = _settings.SparseScoreThreshold + }, + new() + { + Query = denseVector, + Using = _settings.Dense.Name, + Limit = (ulong)topK, + ScoreThreshold = _settings.DenseScoreThreshold + } + }; + } + + private static VectorResourceDto MapToResourceDto(ScoredPoint result) => new() + { + Id = result.Id.Num.ToString(), + Url = GetPayloadValue(result.Payload, "url"), + Content = GetPayloadValue(result.Payload, "content"), + Score = result.Score + }; + + private static string GetPayloadValue(IDictionary payload, string key) => + payload.TryGetValue(key, out var val) && val.KindCase == Value.KindOneofCase.StringValue + ? val.StringValue + : string.Empty; + + private static VectorInsightDto MapToInsightDto(ScoredPoint result) => new() + { + Id = result.Id.Num.ToString(), + Url = GetPayloadValue(result.Payload, "url"), + Content = GetPayloadValue(result.Payload, "content"), + ResourceId = GetPayloadValue(result.Payload, "resourceId"), + Score = result.Score + }; } \ No newline at end of file diff --git a/dev-share-api/appsettings.json b/dev-share-api/appsettings.json index 0b38196..a9fd4ec 100644 --- a/dev-share-api/appsettings.json +++ b/dev-share-api/appsettings.json @@ -1,7 +1,6 @@ { "ConnectionStrings": { - "DefaultConnection": "" - + "DefaultConnection": "" }, "Logging": { "LogLevel": { @@ -19,8 +18,32 @@ "Endpoint": "" }, "Qdrant": { - "Host": "", - "ApiKey": "", - "Collection": "" + "Host": "", + "ApiKey": "" + }, + "VectorDb": { + "ResourceCollection": "BlotzShare_Resource", + "InsightCollection": "BlotzShare_Insight", + "Dimensions": 384, + "DenseScoreThreshold": 0.5, + "SparseScoreThreshold": 0.1, + "HybridScoreThreshold": 0.51, + "Dense": { + "Name": "dense_vector", + "OnDisk": false, + "MinTokenLength": 2, + "MaxTokenLength": 10 + }, + "Sparse": { + "Name": "sparse_vector", + "OnDisk": false, + "MinTokenLength": 2, + "MaxTokenLength": 10 + }, + "_thresholdv2_comment": { + "DenseScoreThreshold": 0.5, + "SparseScoreThreshold": 0.1, + "HybridScoreThreshold": 0.51 } -} \ No newline at end of file + } +} diff --git a/python-embedding/embedding_service.py b/python-embedding/embedding_service.py index a52b08d..28ec53c 100644 --- a/python-embedding/embedding_service.py +++ b/python-embedding/embedding_service.py @@ -19,7 +19,7 @@ async def load_models(): with ThreadPoolExecutor() as pool: dense_model, sparse_model = await asyncio.gather( loop.run_in_executor(pool, TextEmbedding, "sentence-transformers/all-MiniLM-L6-v2"), - loop.run_in_executor(pool, SparseTextEmbedding, "prithivida/Splade_PP_en_v1"), + loop.run_in_executor(pool, SparseTextEmbedding, "Qdrant/bm42-all-minilm-l6-v2-attentions"), ) print("Models loaded successesful")