diff --git a/dev-share-api.Tests/TestHost.cs b/dev-share-api.Tests/TestHost.cs index 2df6d03..c3041a0 100644 --- a/dev-share-api.Tests/TestHost.cs +++ b/dev-share-api.Tests/TestHost.cs @@ -10,7 +10,8 @@ public static IServiceProvider BuildTestServiceProvider() .Build(); var services = new ServiceCollection(); - services.AddApplicationServices(config); + services.AddInfrastructureServices(config) + .AddApplicationServices(); return services.BuildServiceProvider(); } diff --git a/dev-share-api/Controllers/ApiController.cs b/dev-share-api/Controllers/ApiController.cs index 3455d46..101d5b3 100644 --- a/dev-share-api/Controllers/ApiController.cs +++ b/dev-share-api/Controllers/ApiController.cs @@ -7,6 +7,8 @@ using System.Text; using Executor; using System.Collections.Concurrent; +using System.Text.Json; +using Newtonsoft.Json.Linq; namespace Controllers; @@ -15,25 +17,26 @@ namespace Controllers; [Route("api")] public class ExtractController : ControllerBase { - private readonly ISummaryService _summaryService; + private readonly IEmbeddingService _embeddingService; private readonly IVectorService _vectorService; - private readonly ShareChainExecutor _shareChainExecutor; - private readonly OnlineResearchService _onlineResearchService; + private readonly IResourceService _resourceService; + private readonly IOnlineResearchService _onlineResearchService; + private readonly IServiceScopeFactory _scopeFactory; private static readonly ConcurrentDictionary TaskStore = new(); public ExtractController( - ISummaryService summaryService, IEmbeddingService embeddingService, IVectorService vectorService, - ShareChainExecutor shareChainExecutor, - OnlineResearchService onlineResearchService) + IOnlineResearchService onlineResearchService, + IServiceScopeFactory scopeFactory, + IResourceService resourceService) { - _summaryService = summaryService; _embeddingService = embeddingService; _vectorService = vectorService; - _shareChainExecutor = shareChainExecutor; _onlineResearchService = onlineResearchService; + _scopeFactory = scopeFactory; + _resourceService = resourceService; } [HttpPost("share")] @@ -67,26 +70,14 @@ public async Task Share([FromBody] UrlRequest request) _ = Task.Run(async () => { + using var scope = _scopeFactory.CreateScope(); + var executor = scope.ServiceProvider.GetRequiredService(); try { - Console.WriteLine($"Extracting: {url}"); - var result = TryHtmlAgilityPack(url); - if (string.IsNullOrWhiteSpace(result)) - result = await TryPlaywright(url); - if (string.IsNullOrWhiteSpace(result)) - throw new Exception("Content extraction failed."); - - var prompt = new StringBuilder() - .AppendLine("You will receive an input text and your task is to summarize the article in no more than 100 words.") - .AppendLine("Only return the summary. Do not include any explanation.") - .AppendLine("# Article content:") - .AppendLine($"{result}") - .ToString(); - - await _shareChainExecutor.ExecuteAsync(new ResourceShareContext + await executor.ExecuteAsync(new ResourceShareContext { Url = url, - Prompt = prompt + Insight = request.Insight }); task.Status = "success"; @@ -117,6 +108,62 @@ public IActionResult GetStatus(string taskId) }); } + [HttpPost("search")] + public async Task Search([FromBody] SearchRequest request) + { + if (string.IsNullOrWhiteSpace(request.Text)) + { + return BadRequest(new { message = "Search text cannot be empty." }); + } + if (request.TopRelatives <= 0 || request.TopRelatives > 100) + return BadRequest("TopRelatives must be between 1 and 100."); + + try + { + //get vectordb data results + var resourceResults = await _vectorService.SearchResourceAsync( + query: request.Text, + topK: request.TopRelatives); + + var insightResults = await _vectorService.SearchInsightAsync( + query: request.Text, + topK: request.TopRelatives); + + if (resourceResults == null + || resourceResults.Count == 0 + || insightResults == null + || insightResults.Count == 0) + { + // Fallback to online research + var onlineResult = await _onlineResearchService.PerformOnlineResearchAsync(request.Text, request.TopRelatives); + return Ok(new { source = "online", result = onlineResult.ToList() }); + } + else + { + //2. do rerank and get reranked list + var rerankResults = GetRerankedList(resourceResults, insightResults); + + //3. get　finalResults from sql server by id + var results = new List(); + foreach (var item in rerankResults) + { + var resourceId = item.ResourceId; + var resource = await _resourceService.GetResourceById(long.Parse(resourceId)); + if (resource != null) + { + results.Add(resource); + } + } + return Ok(new { source = "vector", result = results }); + } + } + catch (Exception ex) + { + return StatusCode(500, "Search failed due to an internal error."); + } + } + + [HttpPost("vector/init")] public async Task> InitVectorDB() { @@ -155,99 +202,37 @@ public async Task ShareInsight([FromBody] ShareInsightRequest req return Ok(); } - [HttpPost("search")] - public async Task Search([FromBody] SearchRequest request) + //todo make sure the return data from service is List and List + private static List GetRerankedList(List resources, List insights) { - if (string.IsNullOrWhiteSpace(request.Text)) - { - return BadRequest("Search text cannot be empty."); - } - if (request.TopRelatives <= 0 || request.TopRelatives > 100) - return BadRequest("TopRelatives must be between 1 and 100."); - - try - { - var resourceResults = await _vectorService.SearchResourceAsync( - query: request.Text, - topK: request.TopRelatives); - - var insightResults = await _vectorService.SearchInsightAsync( - query: request.Text, - topK: request.TopRelatives); - - if (resourceResults == null - || resourceResults.Count == 0 - || insightResults == null - || insightResults.Count == 0) - { - // Fallback to online research - var onlineResult = await _onlineResearchService.PerformOnlineResearchAsync(request.Text); - return Ok(new { source = "online", result = onlineResult }); - } - - return Ok(new { source = "vector", result = resourceResults }); - } - catch (Exception) - { - return StatusCode(500, "Search failed due to an internal error."); - } - } - - private string? TryHtmlAgilityPack(string url) - { - try - { - var web = new HtmlWeb + // averge comment.score + var insightGroups = insights + .GroupBy(c => c.ResourceId) + .ToDictionary( + g => g.Key, + g => g.Average(c => c.Score) + ); + + // content.score find table + var resourceScores = resources + .ToDictionary(c => c.Id, c => c.Score); + + // union all contentId + var allResourceIds = resourceScores.Keys + .Union(insightGroups.Keys) + .Distinct(); + + var result = allResourceIds + .Select(id => new Rerank { - // 设置 User-Agent，防止部分网站屏蔽爬虫 - UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + - "AppleWebKit/537.36 (KHTML, like Gecko) " + - "Chrome/120.0.0.0 Safari/537.36" - }; - var doc = web.Load(url); - - //TODO 编码问题 - // using var client = new HttpClient(); - // var bytes = client.GetByteArrayAsync(url).Result; - // var html = System.Text.Encoding.UTF8.GetString(bytes); - // var doc = new HtmlDocument(); - // doc.LoadHtml(html); - - - // 提取网页标题 - var titleNode = doc.DocumentNode.SelectSingleNode("//title"); - Console.WriteLine("Title: " + titleNode?.InnerText); - - // 提取所有段落文本 - var paragraphs = doc.DocumentNode.SelectNodes("//p"); - if (paragraphs == null) return null; - - var title = titleNode?.InnerText.Trim() ?? ""; - var paragraphText = string.Join("\n", paragraphs - .Select(p => p.InnerText.Trim()) - .Where(t => !string.IsNullOrWhiteSpace(t))); - - return title + "\n\n" + paragraphText; - } - catch - { - return null; - } - } - - // 使用 Playwright 模拟浏览器加载网页并提取段落内容（用于 CSR 页面） - private async Task TryPlaywright(string url) - { - // 启动 Playwright 浏览器（无头模式） - using var playwright = await Playwright.CreateAsync(); - await using var browser = await playwright.Chromium.LaunchAsync(new() { Headless = true }); - - // 打开新页面并导航到目标地址，等待网络空闲（页面渲染完成） - var page = await browser.NewPageAsync(); - await page.GotoAsync(url, new PageGotoOptions { WaitUntil = WaitUntilState.NetworkIdle }); - - // 提取所有

元素的 innerText，去除空行 - var text = await page.EvalOnSelectorAllAsync("p", "els => els.map(e => e.innerText).filter(t => t.trim().length > 0)"); - return string.Join("\n", text); + ResourceId = id, + Score = + (resourceScores.TryGetValue(id, out var rScore) ? rScore : 0) * 0.7 + + (insightGroups.TryGetValue(id, out var iAvg) ? iAvg : 0) * 0.3 + }) + .OrderByDescending(r => r.Score) + .ToList(); + + return result; } } \ No newline at end of file diff --git a/dev-share-api/Executor/ShareChainExecutor.cs b/dev-share-api/Executor/ShareChainExecutor.cs index fc32d11..7ad3fc2 100644 --- a/dev-share-api/Executor/ShareChainExecutor.cs +++ b/dev-share-api/Executor/ShareChainExecutor.cs @@ -17,7 +17,7 @@ public ShareChainExecutor(IEnumerable handlers, IResourceServ public async Task ExecuteAsync(ResourceShareContext context) { - preHandle(context); + await preHandle(context); foreach (var handler in _handlers) { // Check if the handler should be skipped @@ -30,7 +30,7 @@ public async Task ExecuteAsync(ResourceShareContext context) } } - private async void preHandle(ResourceShareContext context) + private async Task preHandle(ResourceShareContext context) { ResourceDto resourceDto = await _resourceService.GetResourceByUrl(UrlManageUtil.NormalizeUrl(context.Url)); if (resourceDto != null) diff --git a/dev-share-api/Handle/DatabaseShareChainHandle.cs b/dev-share-api/Handle/DatabaseShareChainHandle.cs index 7efef38..49c3fd7 100644 --- a/dev-share-api/Handle/DatabaseShareChainHandle.cs +++ b/dev-share-api/Handle/DatabaseShareChainHandle.cs @@ -5,10 +5,14 @@ namespace Services; public class DatabaseShareChainHandle : BaseShareChainHandle { private readonly IVectorService _vectorService; + private readonly IUserInsightService _userInsightService; + private readonly IResourceService _resourceService; - public DatabaseShareChainHandle(IVectorService vectorService) + public DatabaseShareChainHandle(IVectorService vectorService, IUserInsightService userInsightService, IResourceService resourceService) { _vectorService = vectorService; + _userInsightService = userInsightService; + _resourceService = resourceService; } protected override void Validate(ResourceShareContext context) @@ -19,23 +23,45 @@ protected override void Validate(ResourceShareContext context) protected override async Task ProcessAsync(ResourceShareContext context) { - var resourceId = IdGeneratorUtil.GetNextId().ToString(); + + var resourceId = 0L; if (context.ExistingResource == null) { + resourceId = IdGeneratorUtil.GetNextId(); + await _resourceService.AddResourceAsync( + new ResourceDto + { + ResourceId = resourceId, + Content = context.Summary, + Url = context.Url + }); await _vectorService.UpsertResourceAsync( + resourceId.ToString(), context.Url!, - resourceId, context.Summary!, context.ResourceVectors!); } - + else + { + resourceId = context.ExistingResource.ResourceId; + } + await _vectorService.UpsertInsightAsync( IdGeneratorUtil.GetNextId().ToString(), context.Url!, context.Insight!, - resourceId, + resourceId.ToString(), context.InsightVectors!); + + await _userInsightService.AddUserInsightAsync( + new UserInsightDto + { + ResourceId = resourceId, + Content = context.Insight + }); + return HandlerResult.Success(); + } } \ No newline at end of file diff --git a/dev-share-api/Handle/EmbeddingShareChainHandle.cs b/dev-share-api/Handle/EmbeddingShareChainHandle.cs index 90ba0cb..07e2863 100644 --- a/dev-share-api/Handle/EmbeddingShareChainHandle.cs +++ b/dev-share-api/Handle/EmbeddingShareChainHandle.cs @@ -20,7 +20,10 @@ protected override void Validate(ResourceShareContext context) protected override async Task ProcessAsync(ResourceShareContext context) { - context.ResourceVectors = await GetVectors(context.Summary); + if(context.ExistingResource == null) + { + context.ResourceVectors = await GetVectors(context.Summary); + } if (!string.IsNullOrWhiteSpace(context.Insight)) { diff --git a/dev-share-api/Handle/ExtractShareChainHandle.cs b/dev-share-api/Handle/ExtractShareChainHandle.cs new file mode 100644 index 0000000..3069c59 --- /dev/null +++ b/dev-share-api/Handle/ExtractShareChainHandle.cs @@ -0,0 +1,91 @@ +using Models; +using HtmlAgilityPack; +using Microsoft.Playwright; + +namespace Services; + +public class ExtractShareChainHandle : BaseShareChainHandle +{ + + protected override void Validate(ResourceShareContext context) + { + // if (context.ExistingResource == null && string.IsNullOrWhiteSpace(context.Prompt)) + // throw new ArgumentNullException(nameof(context.Prompt), "Prompt cannot be null or empty."); + } + + public override async Task IsSkip(ResourceShareContext context) + { + return context.ExistingResource != null; + } + + protected override async Task ProcessAsync(ResourceShareContext context) + { + var url = context.Url; + Console.WriteLine($"Extracting: {url}"); + var result = TryHtmlAgilityPack(url); + if (string.IsNullOrWhiteSpace(result)) + result = await TryPlaywright(url); + if (string.IsNullOrWhiteSpace(result)) + throw new Exception("Content extraction failed."); + context.ExtractResult = result; + return HandlerResult.Success(); + } + + private string? TryHtmlAgilityPack(string url) + { + try + { + var web = new HtmlWeb + { + // 设置 User-Agent，防止部分网站屏蔽爬虫 + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + + "AppleWebKit/537.36 (KHTML, like Gecko) " + + "Chrome/120.0.0.0 Safari/537.36" + }; + var doc = web.Load(url); + + //TODO 编码问题 + // using var client = new HttpClient(); + // var bytes = client.GetByteArrayAsync(url).Result; + // var html = System.Text.Encoding.UTF8.GetString(bytes); + // var doc = new HtmlDocument(); + // doc.LoadHtml(html); + + + // 提取网页标题 + var titleNode = doc.DocumentNode.SelectSingleNode("//title"); + Console.WriteLine("Title: " + titleNode?.InnerText); + + // 提取所有段落文本 + var paragraphs = doc.DocumentNode.SelectNodes("//p"); + if (paragraphs == null) return null; + + var title = titleNode?.InnerText.Trim() ?? ""; + var paragraphText = string.Join("\n", paragraphs + .Select(p => p.InnerText.Trim()) + .Where(t => !string.IsNullOrWhiteSpace(t))); + + return title + "\n\n" + paragraphText; + } + catch + { + return null; + } + } + + // 使用 Playwright 模拟浏览器加载网页并提取段落内容（用于 CSR 页面） + private async Task TryPlaywright(string url) + { + // 启动 Playwright 浏览器（无头模式） + using var playwright = await Playwright.CreateAsync(); + await using var browser = await playwright.Chromium.LaunchAsync(new() { Headless = true }); + + // 打开新页面并导航到目标地址，等待网络空闲（页面渲染完成） + var page = await browser.NewPageAsync(); + await page.GotoAsync(url, new PageGotoOptions { WaitUntil = WaitUntilState.NetworkIdle }); + + // 提取所有

元素的 innerText，去除空行 + var text = await page.EvalOnSelectorAllAsync("p", "els => els.map(e => e.innerText).filter(t => t.trim().length > 0)"); + return string.Join("\n", text); + } +} \ No newline at end of file diff --git a/dev-share-api/Handle/SummarizeShareChainHandle.cs b/dev-share-api/Handle/SummarizeShareChainHandle.cs index a484b16..e2b5ec3 100644 --- a/dev-share-api/Handle/SummarizeShareChainHandle.cs +++ b/dev-share-api/Handle/SummarizeShareChainHandle.cs @@ -1,3 +1,4 @@ +using System.Text; using Models; namespace Services; @@ -25,7 +26,13 @@ public override async Task IsSkip(ResourceShareContext context) protected override async Task ProcessAsync(ResourceShareContext context) { - var summary = await _summaryService.SummarizeAsync(context.Prompt); + var prompt = new StringBuilder() + .AppendLine("You will receive an input text and your task is to summarize the article in no more than 100 words.") + .AppendLine("Only return the summary. Do not include any explanation.") + .AppendLine("# Article content:") + .AppendLine($"{context.ExtractResult}") + .ToString(); + var summary = await _summaryService.SummarizeAsync(prompt); context.Summary = summary; return HandlerResult.Success(); } diff --git a/dev-share-api/Models/Rerank.cs b/dev-share-api/Models/Rerank.cs new file mode 100644 index 0000000..dd8bacb --- /dev/null +++ b/dev-share-api/Models/Rerank.cs @@ -0,0 +1,7 @@ +namespace Models; + +public class Rerank +{ + public string ResourceId { get; set; } + public double Score { get; set; } +} \ No newline at end of file diff --git a/dev-share-api/Models/ResourceDTO.cs b/dev-share-api/Models/ResourceDTO.cs deleted file mode 100644 index 7ad49a6..0000000 --- a/dev-share-api/Models/ResourceDTO.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Entities; - -namespace Models; - -public class ResourceDto -{ - public long ResourceId { get; set; } - public string Url { get; set; } - public string NormalizeUrl { get; set; } - public string Content { get; set; } - public List UserInsights { get; set; } -} \ No newline at end of file diff --git a/dev-share-api/Models/ResourceDto.cs b/dev-share-api/Models/ResourceDto.cs new file mode 100644 index 0000000..8fecada --- /dev/null +++ b/dev-share-api/Models/ResourceDto.cs @@ -0,0 +1,10 @@ +namespace Models; + +public class ResourceDto +{ + public long ResourceId { get; set; } + public required string Url { get; set; } + public string? NormalizeUrl { get; set; } + public required string Content { get; set; } + public List? UserInsights { get; set; } +} \ No newline at end of file diff --git a/dev-share-api/Models/ResourceShareContext.cs b/dev-share-api/Models/ResourceShareContext.cs index e4b4967..fc8c1de 100644 --- a/dev-share-api/Models/ResourceShareContext.cs +++ b/dev-share-api/Models/ResourceShareContext.cs @@ -9,8 +9,7 @@ public class ResourceShareContext public string? Summary { get; set; } public Dictionary? ResourceVectors { get; set; } public Dictionary? InsightVectors { get; set; } - public string? Prompt { get; set; } public ResourceDto? ExistingResource { get; set; } - - + public string? ExtractResult { get; set; } + } \ No newline at end of file diff --git a/dev-share-api/Models/UrlRequest.cs b/dev-share-api/Models/UrlRequest.cs index 3a376a5..a1a2984 100644 --- a/dev-share-api/Models/UrlRequest.cs +++ b/dev-share-api/Models/UrlRequest.cs @@ -4,5 +4,5 @@ public class UrlRequest { public string? Url { get; set; } - public string? Prompt { get; set; } + public string? Insight { get; set; } } \ No newline at end of file diff --git a/dev-share-api/Models/UserInsightDTO.cs b/dev-share-api/Models/UserInsightDto.cs similarity index 100% rename from dev-share-api/Models/UserInsightDTO.cs rename to dev-share-api/Models/UserInsightDto.cs diff --git a/dev-share-api/Services/DependencyInjection.cs b/dev-share-api/Services/DependencyInjection.cs index 43fc583..aa32bf7 100644 --- a/dev-share-api/Services/DependencyInjection.cs +++ b/dev-share-api/Services/DependencyInjection.cs @@ -41,6 +41,7 @@ public static IServiceCollection AddInfrastructureServices( // Database services.AddDbContext(options => options.UseSqlServer(configuration.GetConnectionString("DefaultConnection"))); + // HTTP Client services.AddHttpClient("FastEmbed", client => @@ -55,6 +56,7 @@ public static IServiceCollection AddApplicationServices(this IServiceCollection { //Not allowed to alter the sort of the following code. services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/dev-share-api/Services/OnlineResearchService.cs b/dev-share-api/Services/OnlineResearchService.cs index bcfe7c4..310154d 100644 --- a/dev-share-api/Services/OnlineResearchService.cs +++ b/dev-share-api/Services/OnlineResearchService.cs @@ -1,5 +1,4 @@ using Azure.AI.OpenAI; -using Microsoft.Extensions.Options; using System.Text.Json; using Models; using OpenAI.Chat; @@ -8,29 +7,25 @@ namespace Services; public interface IOnlineResearchService { - Task> PerformOnlineResearchAsync(string query, int topK); + Task> PerformOnlineResearchAsync(string query, int topK); } public class OnlineResearchService : IOnlineResearchService { private readonly AzureOpenAIClient _client; - private readonly string _deploymentName = "gpt-4o-mini"; // Set this to your deployment name - private readonly ILogger _logger; + private readonly string _deploymentName = "gpt-4o-mini"; private static readonly JsonSerializerOptions _jsonOptions = new() { PropertyNameCaseInsensitive = true, WriteIndented = true }; - public OnlineResearchService( - AzureOpenAIClient openAIClient, - ILogger logger) + public OnlineResearchService(AzureOpenAIClient openAIClient) { _client = openAIClient ?? throw new ArgumentNullException(nameof(openAIClient)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public async Task> PerformOnlineResearchAsync(string query, int topK = 3) + public async Task> PerformOnlineResearchAsync(string query, int topK = 3) { if (string.IsNullOrWhiteSpace(query)) { @@ -44,7 +39,6 @@ public async Task> PerformOnlineResearchAsync(str } catch (Exception ex) { - _logger.LogError(ex, "Error performing online research for query: {Query}", query); throw; } } @@ -58,7 +52,7 @@ private async Task GetOpenAIResponseAsync(string query, int topK) return response.Content?.FirstOrDefault()?.Text ?? string.Empty; } - private async Task> ParseResponseToVectorResourceDtos(string response) + private static async Task> ParseResponseToVectorResourceDtos(string response) { if (string.IsNullOrWhiteSpace(response)) { @@ -67,9 +61,16 @@ private async Task> ParseResponseToVectorResource try { - // Try parsing as array first + // Clean the response by removing Markdown code block and escapes + var cleanedResponse = response + .Replace("```json", "") + .Replace("```", "") + .Replace("\\n", "") + .Replace("\n", "") + .Trim(); + var results = await Task.Run(() => - JsonSerializer.Deserialize(response, _jsonOptions)); + JsonSerializer.Deserialize(cleanedResponse, _jsonOptions)); if (results?.Any() == true) { @@ -78,7 +79,7 @@ private async Task> ParseResponseToVectorResource // Try parsing as single object if array fails var singleResult = await Task.Run(() => - JsonSerializer.Deserialize(response, _jsonOptions)); + JsonSerializer.Deserialize(cleanedResponse, _jsonOptions)); return singleResult != null ? new[] { singleResult } @@ -86,7 +87,6 @@ private async Task> ParseResponseToVectorResource } catch (JsonException ex) { - _logger.LogWarning(ex, "Failed to parse OpenAI response: {Response}", response); return new[] { CreateFallbackDto(response) }; } } @@ -98,15 +98,11 @@ private static string GeneratePrompt(string query, int topK) [ {{ - ""Id"": ""unique-id-123"", ""Content"": ""First concise, factual answer here."", - ""Score"": 0.95, ""Url"": ""https://relevant-source-1.com"" }}, {{ - ""Id"": ""unique-id-456"", ""Content"": ""Second concise, factual answer here."", - ""Score"": 0.85, ""Url"": ""https://relevant-source-2.com"" }} ] @@ -116,11 +112,12 @@ private static string GeneratePrompt(string query, int topK) Return exactly {topK} JSON objects in an array. Ensure each answer is unique and relevant."; } - private static VectorResourceDto CreateFallbackDto(string content) => new() + private static ResourceDto CreateFallbackDto(string fallBackContent) { - Id = IdGeneratorUtil.GetNextId().ToString(), - Content = content, - Score = 0, - Url = string.Empty - }; + return new() + { + Content = fallBackContent, + Url = string.Empty + }; + } } diff --git a/dev-share-api/Services/ResourceService.cs b/dev-share-api/Services/ResourceService.cs index 5bafeda..2f1fb0c 100644 --- a/dev-share-api/Services/ResourceService.cs +++ b/dev-share-api/Services/ResourceService.cs @@ -29,7 +29,7 @@ public async Task AddResourceAsync(ResourceDto resourceDto) public async Task GetResourceByUrl(string normalizeUrl) { return await _dbContext.Resources - .Where(resource => resource.Url == normalizeUrl) + .Where(resource => resource.NormalizeUrl == normalizeUrl) .Select(resource => new ResourceDto { ResourceId = resource.ResourceId, diff --git a/dev-share-api/Services/VectorService.cs b/dev-share-api/Services/VectorService.cs index 2a94716..f17f4d4 100644 --- a/dev-share-api/Services/VectorService.cs +++ b/dev-share-api/Services/VectorService.cs @@ -1,7 +1,6 @@ using Models; using Qdrant.Client; using Qdrant.Client.Grpc; -using static Qdrant.Client.Grpc.Conditions; namespace Services; @@ -160,7 +159,7 @@ public async Task> SearchResourceAsync(string query, int var payload = result.Payload; return new VectorResourceDto { - Id = result.Id.ToString(), + Id = result.Id.Num.ToString(), Url = payload.TryGetValue("url", out var urlVal) && urlVal.KindCase == Value.KindOneofCase.StringValue ? urlVal.StringValue : string.Empty, Content = payload.TryGetValue("content", out var contentVal) && contentVal.KindCase == Value.KindOneofCase.StringValue ? contentVal.StringValue : string.Empty, Score = result.Score @@ -182,11 +181,11 @@ public async Task> SearchInsightAsync(string query, int t var insightResults = await _client.QueryAsync( - collectionName: _resourceCollection, + collectionName: _insightCollection, prefetch: prefetch, query: Fusion.Rrf, limit: (ulong)topK, - scoreThreshold: (float)0.7, //todo: make this dynamic + scoreThreshold: (float)0.9, //todo: make this dynamic payloadSelector: true, vectorsSelector: false ); @@ -196,7 +195,7 @@ public async Task> SearchInsightAsync(string query, int t var payload = result.Payload; return new VectorInsightDto { - Id = result.Id.ToString(), + Id = result.Id.Num.ToString(), Url = payload.TryGetValue("url", out var urlVal) && urlVal.KindCase == Value.KindOneofCase.StringValue ? urlVal.StringValue : string.Empty, Content = payload.TryGetValue("content", out var contentVal) && contentVal.KindCase == Value.KindOneofCase.StringValue ? contentVal.StringValue : string.Empty, ResourceId = payload.TryGetValue("resourceId", out var resourceIdVal) && resourceIdVal.KindCase == Value.KindOneofCase.StringValue ? resourceIdVal.StringValue : string.Empty, diff --git a/dev-share-api/appsettings.json b/dev-share-api/appsettings.json index d93d228..0b38196 100644 --- a/dev-share-api/appsettings.json +++ b/dev-share-api/appsettings.json @@ -1,6 +1,7 @@ { "ConnectionStrings": { "DefaultConnection": "" + }, "Logging": { "LogLevel": { diff --git a/dev-share-api/dev-share-api.csproj b/dev-share-api/dev-share-api.csproj index 15a2a86..2839a16 100644 --- a/dev-share-api/dev-share-api.csproj +++ b/dev-share-api/dev-share-api.csproj @@ -22,7 +22,6 @@ - diff --git a/dev-share-ui/services/search-service.tsx b/dev-share-ui/services/search-service.tsx index 4dc6b4d..28cd704 100644 --- a/dev-share-ui/services/search-service.tsx +++ b/dev-share-ui/services/search-service.tsx @@ -14,7 +14,8 @@ export async function searchResources(query: string): Promise { if (!result.ok) throw new Error(`Search failed (${result.status})`); - const dtos: VectorSearchResultDTO[] = await result.json(); + const responseJson = await result.json(); + const dtos: VectorSearchResultDTO[] = responseJson.result; return dtos.map(dto => ({ id: crypto.randomUUID(),