Skip to content

psp-10851 document content search #5179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
asanchezr merged 6 commits into from
Feb 5, 2026
Merged

psp-10851 document content search #5179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ca91759
psp-10851 document content search
devinleighsmith Jan 28, 2026
00e6f99
test corrections.
devinleighsmith Jan 28, 2026
65b2727
correct document service mappings.
devinleighsmith Jan 30, 2026
8e2f4ec
test corrections.
devinleighsmith Feb 3, 2026
e11c0cd
Merge branch 'dev' into psp-10851
asanchezr Feb 5, 2026
29768e1
Merge branch 'dev' into psp-10851
asanchezr Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions source/backend/api/Models/Configuration/MayanConfig.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,7 @@ public class MayanConfig
public int ImageRetries { get; set; }

public int PreviewPages { get; set; }

public int MaxContentResults { get; set; }
}
}
7 changes: 5 additions & 2 deletions source/backend/api/Repositories/Mayan/IEdmsDocumentRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using Pims.Api.Models;
using Pims.Api.Models.Mayan;
using Pims.Api.Models.Mayan.Document;
using Pims.Api.Models.Models.Mayan.Document;
using Pims.Api.Models.Requests.Http;
using System.Net.Http;
using System.Threading.Tasks;

namespace Pims.Api.Repositories.Mayan
{
Expand Down Expand Up @@ -56,5 +57,7 @@ public interface IEdmsDocumentRepository
Task<ExternalResponse<QueryResponse<FilePageModel>>> TryGetFilePageListAsync(long documentId, long documentFileId, int pageSize, int pageNumber);

Task<HttpResponseMessage> TryGetFilePageImage(long documentId, long documentFileId, long documentFilePageId);

Task<ExternalResponse<QueryResponse<DocumentSearchResult>>> TrySearchByDocumentContent(string contentToSearch);
}
}
21 changes: 21 additions & 0 deletions source/backend/api/Repositories/Mayan/MayanDocumentRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using Pims.Api.Models.Mayan;
using Pims.Api.Models.Mayan.Document;
using Pims.Api.Models.Mayan.Metadata;
using Pims.Api.Models.Models.Mayan.Document;
using Pims.Api.Models.Requests.Http;
using Polly.Registry;
using System;
Expand Down Expand Up @@ -429,6 +430,26 @@ public async Task<HttpResponseMessage> TryGetFilePageImage(long documentId, long
return response;
}

public async Task<ExternalResponse<QueryResponse<DocumentSearchResult>>> TrySearchByDocumentContent(string contentToSearch)
{
_logger.LogDebug("Retrieving all mayan documents that contain string {ContentToSearch}", contentToSearch);
string authenticationToken = await _authRepository.GetTokenAsync();

string endpointString = $"{_config.BaseUri}/search/advanced/documents.documentsearchresult/";
var queryParams = new System.Collections.Generic.Dictionary<string, string>
{
{ "_match_all", "false" },
{ "files__file_pages__content__content", $"\"{contentToSearch}\"" },
}; // multi-word searches need to be wrapped in quotes, as mayan will search each word separately otherwise - which has unacceptable performance.

Uri endpoint = new(QueryHelpers.AddQueryString(endpointString, queryParams));

var response = await GetAsync<QueryResponse<DocumentSearchResult>>(endpoint, authenticationToken);

_logger.LogDebug("Finished retrieving mayan search results for string {ContentToSearch}", contentToSearch);
return response;
}

private async Task<HttpResponseMessage> GetFileAsync(long documentId, long fileId)
{
string authenticationToken = await _authRepository.GetTokenAsync();
Expand Down
55 changes: 53 additions & 2 deletions source/backend/api/Services/DocumentService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
using Pims.Api.Models.Config;
using Pims.Api.Models.Mayan;
using Pims.Api.Models.Mayan.Document;
using Pims.Api.Models.Models.Mayan.Document;
using Pims.Api.Models.Requests.Document.UpdateMetadata;
using Pims.Api.Models.Requests.Document.Upload;
using Pims.Api.Models.Requests.Http;
Expand All @@ -41,6 +42,7 @@ namespace Pims.Api.Services
public class DocumentService : BaseService, IDocumentService
{
protected const string MayanGenericErrorMessage = "Error response received from Mayan Document Service";
protected const int ContentSearchLimit = 2000;
private static readonly string[] ValidExtensions =
{
"txt",
Expand Down Expand Up @@ -74,6 +76,7 @@ public class DocumentService : BaseService, IDocumentService
private readonly IAvService avService;
private readonly IMapper mapper;
private readonly IOptionsMonitor<AuthClientOptions> keycloakOptions;
private readonly IOptionsMonitor<MayanConfig> mayanOptions;

public DocumentService(
ClaimsPrincipal user,
Expand All @@ -85,6 +88,7 @@ public DocumentService(
IAvService avService,
IMapper mapper,
IOptionsMonitor<AuthClientOptions> options,
IOptionsMonitor<MayanConfig> mayanOptions,
IDocumentQueueRepository queueRepository)
: base(user, logger)
{
Expand All @@ -94,8 +98,10 @@ public DocumentService(
this.avService = avService;
this.mapper = mapper;
this.keycloakOptions = options;
_config = new MayanConfig();
configuration.Bind(MayanConfigSectionKey, _config);
this.mayanOptions = mayanOptions;
var config = new MayanConfig();
configuration?.Bind(MayanConfigSectionKey, config);
_config = config;
}

public static bool IsValidDocumentExtension(string fileName)
Expand Down Expand Up @@ -152,6 +158,41 @@ public Paged<PimsDocument> GetPage(DocumentSearchFilterModel filter)

User.ThrowIfNotAuthorized(Permissions.DocumentView);

ArgumentNullException.ThrowIfNull(filter);

if (!string.IsNullOrWhiteSpace(filter.Content))
{
var contentSearchResponse = SearchDocumentContent(filter.Content)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();

if (contentSearchResponse.Status != ExternalResponseStatus.Success)
{
throw GetMayanResponseError(contentSearchResponse.Message);
}

var mayanIds = contentSearchResponse.Payload?.Results?
.Where(r => r?.Id != null)
.Select(r => (long)r.Id.Value)
.Distinct()
.ToArray() ?? Array.Empty<long>();

var maxContentResults = mayanOptions.CurrentValue?.MaxContentResults ?? ContentSearchLimit;
if (mayanIds.Length > maxContentResults)
{
throw new BadRequestException(
$"Document content search returned more than {maxContentResults:N0} matches. Please refine your search criteria.");
}

filter.MayanDocumentIds = mayanIds;

if (mayanIds.Length == 0)
{
return new Paged<PimsDocument>(Array.Empty<PimsDocument>(), filter.Page, filter.Quantity, 0);
}
}

return documentRepository.GetPageDeep(filter);
}

Expand Down Expand Up @@ -654,6 +695,16 @@ public async Task<HttpResponseMessage> DownloadFilePageImageAsync(long mayanDocu
return result;
}

public async Task<ExternalResponse<QueryResponse<DocumentSearchResult>>> SearchDocumentContent(string contentToSearch)
{
this.Logger.LogInformation("Searching for document file content: {ContentToSearch}", contentToSearch);
this.User.ThrowIfNotAuthorized(Permissions.DocumentView);

var result = await documentStorageRepository.TrySearchByDocumentContent(contentToSearch);

return result;
}

private async Task PrecacheDocumentPreviews(long documentId, long documentFileId)
{
this.Logger.LogInformation("Precaching the first {_config.PreviewPages} pages in document {documentId}, file {documentFileId}", _config.PreviewPages, documentId, documentFileId);
Expand Down
9 changes: 6 additions & 3 deletions source/backend/api/Services/IDocumentService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;
using Pims.Api.Models;
using Pims.Api.Models.CodeTypes;
using Pims.Api.Models.Concepts.Document;
using Pims.Api.Models.Mayan;
using Pims.Api.Models.Mayan.Document;
using Pims.Api.Models.Models.Mayan.Document;
using Pims.Api.Models.Requests.Document.UpdateMetadata;
using Pims.Api.Models.Requests.Document.Upload;
using Pims.Api.Models.Requests.Http;
using Pims.Dal.Entities;
using Pims.Dal.Entities.Models;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;

namespace Pims.Api.Services
{
Expand Down Expand Up @@ -58,5 +59,7 @@ public interface IDocumentService
Task<HttpResponseMessage> DownloadFilePageImageAsync(long mayanDocumentId, long mayanFileId, long mayanFilePageId);

PimsDocument AddDocument(PimsDocument newPimsDocument);

Task<ExternalResponse<QueryResponse<DocumentSearchResult>>> SearchDocumentContent(string contentToSearch);
}
}
1 change: 1 addition & 0 deletions source/backend/api/Startup.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ public void ConfigureServices(IServiceCollection services)
services.Configure<Core.Http.Configuration.OpenIdConnectOptions>(this.Configuration.GetSection("OpenIdConnect"));
services.Configure<Keycloak.Configuration.KeycloakOptions>(this.Configuration.GetSection("Keycloak"));
services.Configure<Pims.Dal.PimsOptions>(this.Configuration.GetSection("Pims"));
services.Configure<MayanConfig>(this.Configuration.GetSection("Mayan"));
services.Configure<AllHealthCheckOptions>(this.Configuration.GetSection("HealthChecks"));
services.AddOptions();

Expand Down
3 changes: 2 additions & 1 deletion source/backend/api/appsettings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,8 @@
"ExposeErrors": false,
"UploadRetries": 4,
"ImageRetries": 2,
"PreviewPages": 10
"PreviewPages": 10,
"MaxContentResults": 2000
},
"Cdogs": {
"AuthEndpoint": "[AUTH_ENDPOINT]",
Expand Down
10 changes: 10 additions & 0 deletions source/backend/apimodels/Models/Concepts/Document/DocumentSearchFilterModel.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,16 @@ public DocumentSearchFilterModel()
/// </summary>
public string Plan { get; set; }

/// <summary>
/// get/set - The string content to search by.
/// </summary>
public string Content { get; set; }

/// <summary>
/// get/set - The Mayan document identifiers that match an external content search.
/// </summary>
public long[] MayanDocumentIds { get; set; }

/// <summary>
/// Determine if a valid filter was provided.
/// </summary>
Expand Down
50 changes: 50 additions & 0 deletions source/backend/apimodels/Models/Mayan/Document/AdvancedSearchResponseModel.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;

namespace Pims.Api.Models.Models.Mayan.Document
{
/// <summary>
/// Best-effort model for items returned by:
/// /api/v4/search/advanced/documents.documentsearchresult/...
///
/// Mayan versions/configs differ, so this includes ExtensionData to capture
/// unknown fields safely.
/// </summary>
public sealed class DocumentSearchResult
{
// Commonly present in document-ish payloads.
[JsonPropertyName("id")]
public int? Id { get; init; }

[JsonPropertyName("uuid")]
public Guid? Uuid { get; init; }

[JsonPropertyName("label")]
public string? Label { get; init; }

[JsonPropertyName("description")]
public string? Description { get; init; }

[JsonPropertyName("datetime_created")]
public DateTimeOffset? DateTimeCreated { get; init; }

[JsonPropertyName("datetime_modified")]
public DateTimeOffset? DateTimeModified { get; init; }

// Search endpoints sometimes include score / snippet-ish fields.
[JsonPropertyName("score")]
public double? Score { get; init; }

[JsonPropertyName("highlights")]
public object? Highlights { get; init; }

/// <summary>
/// Captures any additional fields returned by Mayan EDMS that are not explicitly modeled.
/// This makes your deserialization resilient to schema differences.
/// </summary>
[JsonExtensionData]
public Dictionary<string, JsonElement>? AdditionalFields { get; init; }
}
}
5 changes: 5 additions & 0 deletions source/backend/dal/Repositories/DocumentRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,11 @@ private IQueryable<PimsDocument> GetCommonQueryDeep(DocumentSearchFilterModel fi
predicate = predicate.And(pd => pd.PimsPropertyDocuments.Any(p => p != null && EF.Functions.Like(p.Property.SurveyPlanNumber.ToString(), $"%{filter.Plan}%")));
}

if (filter.MayanDocumentIds?.Any() == true)
{
predicate = predicate.And(pd => pd.MayanId.HasValue && filter.MayanDocumentIds.Contains(pd.MayanId.Value));
}

if(excludeTemplates)
{
var templateCodeType = Context.PimsDocumentTyps.FirstOrDefault(x => x.DocumentType == "CDOGTEMP");
Expand Down
Loading
Loading