From f2c135c2ced3917f2183dcde77848a8535b9b01c Mon Sep 17 00:00:00 2001 From: Viacheslav Kolesnyk <94473337+viacheslavkol@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:18:19 +0100 Subject: [PATCH 1/5] feat(reindex) Implement member tenant reindex (#855) Remove stale resources cleanup since cleanup by tenant is executed afterwards anyways executeMainMigrationPhases: migrate entities first and then relations Dedup postgres functions are not used, remove sql files with them deleteDocumentsByTenantId. Elasticsearch deletion is needed only for instance index Rework entities selection for member upload and last_updated_at logic. Now it uses last_updated_date for sub-resources; local instances(with holdings,items) are fetched by tenantId and shared instances are fetched when having holdings with member tenantId. Required work: On executeMainMigrationPhases (from staging to main) inserts set last_updated_date as constant "2000-01-01" for all entities that have it (only holdings don't have it) so it's not processed by background job. last_updated_date='2000-01-01' to be used as filter on upload stage for sub-resources. Upload stage for instances have tenantId filter which is enough to get all required entities Disable reindex status trigger (which blocks background job) on reindex start (only on member reindex) and enable on upload end. This will allow sub-resources for other tenant being processed. Notes: Since we set last_updated_date to "2000-01-01" - entities inserted during reindex will not be processed. Create alternative reindex status trigger which will only handle statuses without blocking background job. Same trigger but without usage of locks. Enable on reindex start, disable on upload end (only on member reindex). This will allow reindex status updates on "processed count" changes while preserving ability to process resources from other member tenants by the background job Remove staging table statistics Rework target tenant id cache with spring cache Add staging statuses/timings to reindex status Add already running reindex validation on full reindex Implements MSEARCH-1100 --- NEWS.md | 2 + README.md | 69 ++++ descriptors/ModuleDescriptor-template.json | 2 +- .../org/folio/search/SearchApplication.java | 2 - .../configuration/CacheConfiguration.java | 45 +++ .../configuration/SearchCacheNames.java | 2 + .../CacheConfigurationProperties.java | 36 ++ ...ndexManagementConfigurationProperties.java | 43 +++ .../ReindexConfigurationProperties.java | 36 ++ .../controller/IndexManagementController.java | 17 +- .../search/exception/ReindexException.java | 4 + .../exception/RequestValidationException.java | 11 +- .../PopulateInstanceBatchInterceptor.java | 37 +- .../model/event/ReindexRangeIndexEvent.java | 1 + .../model/event/ReindexRecordsEvent.java | 1 + .../search/model/reindex/MigrationResult.java | 13 + .../model/reindex/ReindexStatusEntity.java | 6 + .../search/model/types/ReindexStatus.java | 3 + .../AbstractResourceRepository.java | 54 +++ .../folio/search/service/IndexService.java | 26 +- .../service/reindex/ReindexCommonService.java | 78 +++- .../service/reindex/ReindexConstants.java | 15 + .../service/reindex/ReindexContext.java | 77 ++++ .../ReindexMergeRangeIndexService.java | 48 ++- .../reindex/ReindexOrchestrationService.java | 139 +++++-- .../service/reindex/ReindexService.java | 331 ++++++++++++---- .../service/reindex/ReindexStatusService.java | 74 +++- .../ReindexUploadRangeIndexService.java | 47 ++- .../reindex/StagingMigrationService.java | 364 ++++++++++++++++++ .../reindex/jdbc/CallNumberRepository.java | 126 +++++- .../jdbc/ClassificationRepository.java | 121 +++++- .../reindex/jdbc/ContributorRepository.java | 124 +++++- .../reindex/jdbc/HoldingRepository.java | 38 ++ .../service/reindex/jdbc/ItemRepository.java | 41 ++ .../reindex/jdbc/MergeInstanceRepository.java | 54 ++- .../reindex/jdbc/MergeRangeRepository.java | 9 + .../reindex/jdbc/ReindexJdbcRepository.java | 40 +- .../reindex/jdbc/ReindexStatusRepository.java | 84 +++- .../reindex/jdbc/SubjectRepository.java | 122 +++++- .../reindex/jdbc/TenantRepository.java | 2 + .../jdbc/UploadInstanceRepository.java | 122 +++++- .../reindex/jdbc/UploadRangeRepository.java | 6 + src/main/resources/application.yml | 6 + .../resources/changelog/changelog-master.xml | 6 + .../changes/v6.0/01_create_staging_tables.xml | 233 +++++++++++ .../v6.0/02_create_staging_partitions.xml | 68 ++++ .../changes/v6.0/03_add_tenant_id_indexes.xml | 105 +++++ ...add_target_tenant_id_to_reindex_status.xml | 27 ++ ...staging_time_columns_to_reindex_status.xml | 27 ++ ...sortium_member_reindex_status_function.xml | 15 + ...sortium-member-reindex-status-function.sql | 20 + ...eate_staging_child_resource_partitions.sql | 83 ++++ .../sql/create_staging_holding_partitions.sql | 20 + .../create_staging_instance_partitions.sql | 20 + .../sql/create_staging_item_partitions.sql | 20 + ...create_staging_relationship_partitions.sql | 83 ++++ .../examples/request/reindexFullRequest.yaml | 6 + .../examples/result/ReindexStatusResult.yaml | 21 +- .../reindex-instance-records-full.yaml | 6 +- .../schemas/request/reindexFullRequest.yaml | 9 + .../schemas/response/reindexStatusItem.yaml | 9 + .../repository/IndexRepositoryTest.java | 3 + .../PrimaryResourceRepositoryTest.java | 46 +++ .../reindex/ReindexCommonServiceTest.java | 246 ++++++++++++ .../service/reindex/ReindexContextTest.java | 182 +++++++++ .../ReindexMergeRangeIndexServiceTest.java | 4 +- .../ReindexOrchestrationServiceTest.java | 100 ++++- .../service/reindex/ReindexServiceTest.java | 18 +- .../reindex/ReindexStatusServiceTest.java | 186 ++++++++- .../ReindexUploadRangeIndexServiceTest.java | 38 +- .../reindex/StagingMigrationServiceTest.java | 229 +++++++++++ .../jdbc/ClassificationRepositoryIT.java | 12 +- .../reindex/jdbc/ContributorRepositoryIT.java | 3 +- .../jdbc/MergeRangeRepositoriesIT.java | 9 +- .../jdbc/ReindexJdbcRepositoriesIT.java | 5 +- .../jdbc/ReindexStatusRepositoryIT.java | 258 +++++++++++++ .../reindex/jdbc/SubjectRepositoryIT.java | 3 +- .../jdbc/UploadRangeRepositoriesIT.java | 5 +- 78 files changed, 4361 insertions(+), 242 deletions(-) create mode 100644 src/main/java/org/folio/search/configuration/CacheConfiguration.java create mode 100644 src/main/java/org/folio/search/configuration/properties/CacheConfigurationProperties.java create mode 100644 src/main/java/org/folio/search/configuration/properties/IndexManagementConfigurationProperties.java create mode 100644 src/main/java/org/folio/search/model/reindex/MigrationResult.java create mode 100644 src/main/java/org/folio/search/service/reindex/ReindexContext.java create mode 100644 src/main/java/org/folio/search/service/reindex/StagingMigrationService.java create mode 100644 src/main/resources/changelog/changes/v6.0/01_create_staging_tables.xml create mode 100644 src/main/resources/changelog/changes/v6.0/02_create_staging_partitions.xml create mode 100644 src/main/resources/changelog/changes/v6.0/03_add_tenant_id_indexes.xml create mode 100644 src/main/resources/changelog/changes/v6.0/04_add_target_tenant_id_to_reindex_status.xml create mode 100644 src/main/resources/changelog/changes/v6.0/05_add_staging_time_columns_to_reindex_status.xml create mode 100644 src/main/resources/changelog/changes/v6.0/06_create_consortium_member_reindex_status_function.xml create mode 100644 src/main/resources/changelog/changes/v6.0/sql/consortium-member-reindex-status-function.sql create mode 100644 src/main/resources/changelog/changes/v6.0/sql/create_staging_child_resource_partitions.sql create mode 100644 src/main/resources/changelog/changes/v6.0/sql/create_staging_holding_partitions.sql create mode 100644 src/main/resources/changelog/changes/v6.0/sql/create_staging_instance_partitions.sql create mode 100644 src/main/resources/changelog/changes/v6.0/sql/create_staging_item_partitions.sql create mode 100644 src/main/resources/changelog/changes/v6.0/sql/create_staging_relationship_partitions.sql create mode 100644 src/main/resources/swagger.api/examples/request/reindexFullRequest.yaml create mode 100644 src/main/resources/swagger.api/schemas/request/reindexFullRequest.yaml create mode 100644 src/test/java/org/folio/search/service/reindex/ReindexCommonServiceTest.java create mode 100644 src/test/java/org/folio/search/service/reindex/ReindexContextTest.java create mode 100644 src/test/java/org/folio/search/service/reindex/StagingMigrationServiceTest.java diff --git a/NEWS.md b/NEWS.md index 78aa2d232..41c9adaf9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -12,6 +12,7 @@ * Provides `resource-ids-streaming v1.0` * Provides `browse-inventory v1.0` * Provides `browse-authorities v1.0` +* Provides `indices v1.2` * Removed `search` * Removed `browse` @@ -30,6 +31,7 @@ * Add indexes for instance_(call_number/subject/classification/contributor) ([MSEARCH-1025](https://folio-org.atlassian.net/browse/MSEARCH-1025)) * Omit sub-resource if main value is blank ([MSEARCH-1084](https://folio-org.atlassian.net/browse/MSEARCH-1084)) * Remove excessive escaping of backslash character in sub-resources ([MSEARCH-1094](https://folio-org.atlassian.net/browse/MSEARCH-1094)) + * Implement member tenant reindex ([MSEARCH-1100](https://folio-org.atlassian.net/browse/MSEARCH-1100)) * **Instance Search** * Add support for searching by instance/holdings/item electronic access relationship ID ([MSEARCH-816](https://folio-org.atlassian.net/browse/MSEARCH-816)) * Normalize ISSN search ([MSEARCH-658](https://folio-org.atlassian.net/browse/MSEARCH-658)) diff --git a/README.md b/README.md index ccbacaaa1..46a025736 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,7 @@ and [Cross-cluster replication](https://docs.aws.amazon.com/opensearch-service/l | MAX_SEARCH_BATCH_REQUEST_IDS_COUNT | 20000 | Defines maximum batch request IDs count for searching consolidated items/holdings in consortium | | INSTANCE_CHILDREN_INDEX_ENABLED | true | Defines if module should process subjects/contributors/classifications/call-numbers in a background | | INSTANCE_CHILDREN_INDEX_DELAY_MS | 60000 | Defines the delay for scheduler that indexes subjects/contributors/classifications/call-numbers in a background | +| REINDEX_MIGRATION_WORK_MEM | 64MB | PostgreSQL work_mem value for migration operations during staging table processing. Controls memory usage before PostgreSQL writes to temporary disk files. | The module uses system user to communicate with other modules from Kafka consumers. For production deployments you MUST specify the password for this system user via env variable: @@ -424,6 +425,7 @@ x-okapi-tenant: [tenant] x-okapi-token: [JWT_TOKEN] { + "tenantId": "optional_specific_tenant", "indexSettings": { "numberOfShards": 2, "numberOfReplicas": 4, @@ -432,6 +434,7 @@ x-okapi-token: [JWT_TOKEN] } ``` +* `tenantId` parameter is optional and allows reindexing a specific consortium member. If not provided, reindexes all consortium members or single non-consortium tenant * `indexSettings` parameter is optional and defines the following Elasticsearch/Opensearch index settings: - `numberOfShards` - the number (between 1 and 100) of primary shards for the index - `numberOfReplicas` - the number of replicas (between 0 and 100) each primary shard has @@ -521,6 +524,72 @@ Only for entity type of ```instance``` we can have statuses of both Merge and Up ```status``` response field can have values of ```"MERGE_IN_PROGRESS"```, ```"MERGE_COMPLETED"``` or ```"MERGE_FAILED"``` for entity types representing Merge step and values of ```"UPLOAD_IN_PROGRESS"```, ```"UPLOAD_COMPLETED"``` or ```"UPLOAD_FAILED"``` for the entities of Upload step. +### Tenant-Specific Reindexing in Consortia + +For consortium deployments, it's often necessary to reindex data for a specific member tenant without affecting other tenants' data or shared consortium instances. +The tenant-specific reindex feature addresses this need by providing fine-grained control over which tenant's data gets reprocessed. + +#### When to Use Tenant-Specific Reindex + +- **Member tenant issues**: When a specific consortium member has data corruption or indexing problems +- **Selective updates**: When configuration changes only affect certain tenants +- **Maintenance operations**: When performing targeted maintenance without disrupting the entire consortium +- **Testing and troubleshooting**: When diagnosing issues specific to a single tenant + +#### How It Works + +1. **Data Preservation**: Shared consortium instances are preserved during tenant-specific operations +2. **Staging Process**: Data is processed through staging tables to ensure separation from other tenants +3. **Selective Cleanup**: Only documents belonging to the specified tenant are removed from OpenSearch indices +4. **Relationship Maintenance**: Instance-to-holdings/items relationships are properly maintained across the consortium + +#### Example Usage + +```http +# Reindex only tenant "university_library" in a consortium +POST /search/index/instance-records/reindex/full + +x-okapi-tenant: consortium +x-okapi-token: [JWT_TOKEN] + +{ + "tenantId": "university_library", + "indexSettings": { + "numberOfReplicas": 2, + "refreshInterval": 30 + } +} +``` + +This approach ensures that: +- Shared instances from other consortium members remain untouched +- The specified tenant's data is completely refreshed +- Index integrity is maintained throughout the process +- Other consortium members continue to have uninterrupted service + +### Staging Tables + +The reindexing process uses staging tables as a high-performance buffer to maximize throughput during large-scale data operations. +The staging tables are specifically designed for optimal write performance and minimal contention. + +#### Performance-Optimized Design + +- **Unlogged Tables**: Staging tables are unlogged for maximum write performance (no WAL overhead) +- **Partitioned Structure**: Data is partitioned to allow parallel processing across multiple workers +- **No Indexes**: Absence of indexes eliminates index maintenance overhead during bulk inserts +- **Minimal Contention**: Multiple processes can write concurrently without blocking each other + +#### Staging Process Flow + +1. **High-Speed Data Collection**: Multiple parallel processes load raw data from inventory services into staging tables without contention +2. **Batch Migration**: Data is moved to operational tables in optimized batches + +#### Performance Benefits + +- **Maximum Throughput**: Unlogged, unindexed tables allow maximum write speed during data collection +- **Reduced Contention**: Main operational tables experience minimal locking during the reindex process +- **Parallel Processing**: Partitioned staging tables enable concurrent processing across multiple workers + ## API ### CQL support diff --git a/descriptors/ModuleDescriptor-template.json b/descriptors/ModuleDescriptor-template.json index 160c37812..b2a53d35f 100644 --- a/descriptors/ModuleDescriptor-template.json +++ b/descriptors/ModuleDescriptor-template.json @@ -4,7 +4,7 @@ "provides": [ { "id": "indices", - "version": "1.1", + "version": "1.2", "handlers": [ { "methods": [ diff --git a/src/main/java/org/folio/search/SearchApplication.java b/src/main/java/org/folio/search/SearchApplication.java index ff189a559..97759c480 100644 --- a/src/main/java/org/folio/search/SearchApplication.java +++ b/src/main/java/org/folio/search/SearchApplication.java @@ -2,14 +2,12 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.cache.annotation.EnableCaching; import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.scheduling.annotation.EnableScheduling; /** * Folio search application. */ -@EnableCaching @EnableScheduling @EnableFeignClients @SpringBootApplication diff --git a/src/main/java/org/folio/search/configuration/CacheConfiguration.java b/src/main/java/org/folio/search/configuration/CacheConfiguration.java new file mode 100644 index 000000000..82fb992b9 --- /dev/null +++ b/src/main/java/org/folio/search/configuration/CacheConfiguration.java @@ -0,0 +1,45 @@ +package org.folio.search.configuration; + +import static org.folio.search.configuration.SearchCacheNames.REINDEX_TARGET_TENANT_CACHE; + +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.CaffeineSpec; +import java.util.concurrent.TimeUnit; +import lombok.RequiredArgsConstructor; +import org.folio.search.configuration.properties.CacheConfigurationProperties; +import org.springframework.cache.CacheManager; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.cache.caffeine.CaffeineCacheManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +@EnableCaching +@RequiredArgsConstructor +public class CacheConfiguration { + + private final CacheConfigurationProperties cacheProperties; + + @Bean + public CacheManager cacheManager() { + var cacheManager = new CaffeineCacheManager(); + + // Set cache names from configuration + if (cacheProperties.getCacheNames() != null && !cacheProperties.getCacheNames().isEmpty()) { + cacheManager.setCacheNames(cacheProperties.getCacheNames()); + } + + // Set default spec from configuration for all caches + var defaultSpec = cacheProperties.getCaffeine().getSpec(); + cacheManager.setCaffeineSpec(CaffeineSpec.parse(defaultSpec)); + + // Register custom cache with 10-second TTL for reindex-target-tenant + cacheManager.registerCustomCache(REINDEX_TARGET_TENANT_CACHE, + Caffeine.newBuilder() + .maximumSize(500) + .expireAfterWrite(10, TimeUnit.SECONDS) + .build()); + + return cacheManager; + } +} diff --git a/src/main/java/org/folio/search/configuration/SearchCacheNames.java b/src/main/java/org/folio/search/configuration/SearchCacheNames.java index a02dd8d9e..0f64ad259 100644 --- a/src/main/java/org/folio/search/configuration/SearchCacheNames.java +++ b/src/main/java/org/folio/search/configuration/SearchCacheNames.java @@ -13,4 +13,6 @@ public class SearchCacheNames { public static final String SEARCH_PREFERENCE_CACHE = "search-preference"; public static final String USER_TENANTS_CACHE = "user-tenants"; public static final String CONSORTIUM_TENANTS_CACHE = "consortium-tenants-cache"; + //custom cache names + public static final String REINDEX_TARGET_TENANT_CACHE = "reindex-target-tenant"; } diff --git a/src/main/java/org/folio/search/configuration/properties/CacheConfigurationProperties.java b/src/main/java/org/folio/search/configuration/properties/CacheConfigurationProperties.java new file mode 100644 index 000000000..0168861e0 --- /dev/null +++ b/src/main/java/org/folio/search/configuration/properties/CacheConfigurationProperties.java @@ -0,0 +1,36 @@ +package org.folio.search.configuration.properties; + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Data +@Component +@NoArgsConstructor +@AllArgsConstructor(staticName = "of") +@ConfigurationProperties(prefix = "spring.cache") +public class CacheConfigurationProperties { + + /** + * List of cache names to create. + */ + private List cacheNames; + + /** + * Caffeine cache specification for default caches. + */ + private Caffeine caffeine = new Caffeine(); + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static class Caffeine { + /** + * Caffeine spec string (e.g., "maximumSize=500,expireAfterWrite=3600s"). + */ + private String spec = "maximumSize=500,expireAfterWrite=3600s"; + } +} diff --git a/src/main/java/org/folio/search/configuration/properties/IndexManagementConfigurationProperties.java b/src/main/java/org/folio/search/configuration/properties/IndexManagementConfigurationProperties.java new file mode 100644 index 000000000..bc95ad31c --- /dev/null +++ b/src/main/java/org/folio/search/configuration/properties/IndexManagementConfigurationProperties.java @@ -0,0 +1,43 @@ +package org.folio.search.configuration.properties; + +import jakarta.validation.constraints.Min; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * Configuration properties for index management operations. + */ +@Data +@Component +@NoArgsConstructor +@AllArgsConstructor(staticName = "of") +@ConfigurationProperties(prefix = "folio.index-management") +public class IndexManagementConfigurationProperties { + + /** + * Batch size for delete-by-query operations to improve performance on large datasets. + */ + @Min(1) + private Integer deleteQueryBatchSize = 1_000; + + /** + * Scroll timeout in minutes for delete-by-query operations to handle large result sets. + */ + @Min(1) + private Integer deleteQueryScrollTimeoutMinutes = 5; + + /** + * Request timeout in minutes for delete-by-query operations to prevent failures on large operations. + */ + @Min(1) + private Integer deleteQueryRequestTimeoutMinutes = 30; + + /** + * Whether to refresh the index after delete-by-query operation. + * Setting to false improves performance by deferring refresh. + */ + private Boolean deleteQueryRefresh = false; +} diff --git a/src/main/java/org/folio/search/configuration/properties/ReindexConfigurationProperties.java b/src/main/java/org/folio/search/configuration/properties/ReindexConfigurationProperties.java index 738ba2bda..99cab235b 100644 --- a/src/main/java/org/folio/search/configuration/properties/ReindexConfigurationProperties.java +++ b/src/main/java/org/folio/search/configuration/properties/ReindexConfigurationProperties.java @@ -1,9 +1,12 @@ package org.folio.search.configuration.properties; +import jakarta.annotation.PostConstruct; import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.Pattern; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.extern.log4j.Log4j2; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; @@ -12,8 +15,12 @@ @NoArgsConstructor @AllArgsConstructor(staticName = "of") @ConfigurationProperties(prefix = "folio.reindex") +@Log4j2 public class ReindexConfigurationProperties { + private static final java.util.regex.Pattern WORK_MEM_VALIDATION_PATTERN = + java.util.regex.Pattern.compile("^\\d+\\s*(KB|MB|GB)$"); + /** * Defines number of locations to retrieve per inventory http request on locations reindex process. */ @@ -33,4 +40,33 @@ public class ReindexConfigurationProperties { private long mergeRangePublisherRetryIntervalMs = 1000; private int mergeRangePublisherRetryAttempts = 5; + + /** + * Defines the PostgreSQL work_mem value to set for migration operations. + * This controls the amount of memory used by query operations before PostgreSQL + * starts writing data to temporary disk files. Default is '64MB'. + * Format must be a number followed by KB, MB, or GB (e.g., "64MB", "512KB", "1GB"). + */ + @Pattern(regexp = "^\\d+\\s*(KB|MB|GB)$", + message = "work_mem must be a number followed by KB, MB, or GB (e.g., '64MB', '512KB', '1GB')") + private String migrationWorkMem = "64MB"; + + /** + * Validates the configuration properties at startup. + * This ensures that any invalid configuration fails fast during application startup. + */ + @PostConstruct + public void validateConfiguration() { + log.info("Validating reindex configuration properties..."); + + // Validate work_mem format + if (!WORK_MEM_VALIDATION_PATTERN.matcher(migrationWorkMem).matches()) { + var errorMsg = "Invalid work_mem configuration: " + migrationWorkMem + + ". Must be a number followed by KB, MB, or GB (e.g., '64MB', '512KB', '1GB')"; + log.error(errorMsg); + throw new IllegalArgumentException(errorMsg); + } + + log.info("Reindex configuration validated successfully. Migration work_mem: {}", migrationWorkMem); + } } diff --git a/src/main/java/org/folio/search/controller/IndexManagementController.java b/src/main/java/org/folio/search/controller/IndexManagementController.java index 366459522..56e224d4d 100644 --- a/src/main/java/org/folio/search/controller/IndexManagementController.java +++ b/src/main/java/org/folio/search/controller/IndexManagementController.java @@ -1,12 +1,14 @@ package org.folio.search.controller; +import static org.apache.commons.lang3.StringUtils.isNotBlank; + import java.util.List; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.folio.search.domain.dto.CreateIndexRequest; import org.folio.search.domain.dto.FolioCreateIndexResponse; import org.folio.search.domain.dto.FolioIndexOperationResponse; -import org.folio.search.domain.dto.IndexSettings; +import org.folio.search.domain.dto.ReindexFullRequest; import org.folio.search.domain.dto.ReindexJob; import org.folio.search.domain.dto.ReindexRequest; import org.folio.search.domain.dto.ReindexStatusItem; @@ -51,9 +53,16 @@ public ResponseEntity indexRecords(List reindexInstanceRecords(String tenantId, IndexSettings indexSettings) { - log.info("Attempting to run full-reindex for instance records [tenant: {}]", tenantId); - reindexService.submitFullReindex(tenantId, indexSettings); + public ResponseEntity reindexInstanceRecords(String tenantId, ReindexFullRequest reindexFullRequest) { + var targetTenantId = reindexFullRequest != null && isNotBlank(reindexFullRequest.getTenantId()) + ? reindexFullRequest.getTenantId() + : null; + var indexSettings = reindexFullRequest != null ? reindexFullRequest.getIndexSettings() : null; + + log.info("Attempting to run full-reindex for instance records [requestingTenant: {}, targetTenant: {}]", + tenantId, targetTenantId != null ? targetTenantId : "all consortium members"); + + reindexService.submitFullReindex(tenantId, indexSettings, targetTenantId); return ResponseEntity.ok().build(); } diff --git a/src/main/java/org/folio/search/exception/ReindexException.java b/src/main/java/org/folio/search/exception/ReindexException.java index c3d0bd385..0ea4c38da 100644 --- a/src/main/java/org/folio/search/exception/ReindexException.java +++ b/src/main/java/org/folio/search/exception/ReindexException.java @@ -5,4 +5,8 @@ public class ReindexException extends RuntimeException { public ReindexException(String errorMessage) { super(errorMessage); } + + public ReindexException(String errorMessage, Throwable cause) { + super(errorMessage, cause); + } } diff --git a/src/main/java/org/folio/search/exception/RequestValidationException.java b/src/main/java/org/folio/search/exception/RequestValidationException.java index 564dd0ff7..1bf3764d3 100644 --- a/src/main/java/org/folio/search/exception/RequestValidationException.java +++ b/src/main/java/org/folio/search/exception/RequestValidationException.java @@ -7,8 +7,10 @@ @Getter public class RequestValidationException extends BaseSearchException { - public static final String REQUEST_NOT_ALLOWED_MSG = + public static final String REQUEST_NOT_ALLOWED_FOR_CONSORTIUM_MEMBER_MSG = "The request not allowed for member tenant of consortium environment"; + public static final String REQUEST_NOT_ALLOWED_WITH_TARGET_TENANT_MSG = + "Target tenant ID is allowed only from consortium central tenant with a value of consortium member tenant"; private final String key; private final String value; @@ -28,6 +30,11 @@ public RequestValidationException(String message, String key, String value) { } public static RequestValidationException memberTenantNotAllowedException(String tenantId) { - return new RequestValidationException(REQUEST_NOT_ALLOWED_MSG, XOkapiHeaders.TENANT, tenantId); + return new RequestValidationException(REQUEST_NOT_ALLOWED_FOR_CONSORTIUM_MEMBER_MSG, + XOkapiHeaders.TENANT, tenantId); + } + + public static RequestValidationException targetTenantNotAllowedException(String targetTenantId) { + return new RequestValidationException(REQUEST_NOT_ALLOWED_WITH_TARGET_TENANT_MSG, "targetTenantId", targetTenantId); } } diff --git a/src/main/java/org/folio/search/integration/message/interceptor/PopulateInstanceBatchInterceptor.java b/src/main/java/org/folio/search/integration/message/interceptor/PopulateInstanceBatchInterceptor.java index 9c9903082..0d2973492 100644 --- a/src/main/java/org/folio/search/integration/message/interceptor/PopulateInstanceBatchInterceptor.java +++ b/src/main/java/org/folio/search/integration/message/interceptor/PopulateInstanceBatchInterceptor.java @@ -25,6 +25,7 @@ import org.folio.search.model.types.ReindexEntityType; import org.folio.search.model.types.ResourceType; import org.folio.search.service.consortium.ConsortiumTenantExecutor; +import org.folio.search.service.reindex.ReindexContext; import org.folio.search.service.reindex.jdbc.MergeRangeRepository; import org.folio.search.service.reindex.jdbc.ReindexJdbcRepository; import org.folio.search.utils.SearchConverterUtils; @@ -112,23 +113,31 @@ private void populate(List records) { private void process(String tenant, List batch) { var recordByResource = batch.stream().collect(Collectors.groupingBy(ResourceEvent::getResourceName)); - for (Map.Entry> recordCollection : recordByResource.entrySet()) { - var resourceType = recordCollection.getKey(); - if (ResourceType.BOUND_WITH.getName().equals(resourceType)) { - processBoundWithEvents(tenant, recordCollection); - continue; - } - var repository = repositories.get(ReindexEntityType.fromValue(resourceType)); - if (repository != null) { - var recordByOperation = getRecordByOperation(recordCollection); - saveEntities(tenant, recordByOperation.getOrDefault(true, emptyList()), repository); - deleteEntities(tenant, resourceType, recordByOperation.getOrDefault(false, emptyList()), repository); + try { + // Set reindex context for real-time event processing (not reindex mode) + ReindexContext.setReindexMode(false); + for (Map.Entry> recordCollection : recordByResource.entrySet()) { + var resourceType = recordCollection.getKey(); + if (ResourceType.BOUND_WITH.getName().equals(resourceType)) { + processBoundWithEvents(tenant, recordCollection); + continue; + } + + var repository = repositories.get(ReindexEntityType.fromValue(resourceType)); + if (repository != null) { + var recordByOperation = getRecordByOperation(recordCollection); + saveEntities(tenant, recordByOperation.getOrDefault(true, emptyList()), repository); + deleteEntities(tenant, resourceType, recordByOperation.getOrDefault(false, emptyList()), repository); - log.debug("process::Saved {} entities for resource type {} in tenant {}, " - + "sub-resource processing will be handled by background job", - recordCollection.getValue().size(), resourceType, tenant); + log.debug("process::Saved {} entities for resource type {} in tenant {}, " + + "sub-resource processing will be handled by background job", + recordCollection.getValue().size(), resourceType, tenant); + } } + } finally { + // Always clear the reindex context + ReindexContext.clear(); } } diff --git a/src/main/java/org/folio/search/model/event/ReindexRangeIndexEvent.java b/src/main/java/org/folio/search/model/event/ReindexRangeIndexEvent.java index 818ea7a09..f075f8a95 100644 --- a/src/main/java/org/folio/search/model/event/ReindexRangeIndexEvent.java +++ b/src/main/java/org/folio/search/model/event/ReindexRangeIndexEvent.java @@ -15,4 +15,5 @@ public class ReindexRangeIndexEvent implements BaseKafkaMessage { private String tenant; private String ts; + private String memberTenantId; } diff --git a/src/main/java/org/folio/search/model/event/ReindexRecordsEvent.java b/src/main/java/org/folio/search/model/event/ReindexRecordsEvent.java index c385e7cee..ca73182a9 100644 --- a/src/main/java/org/folio/search/model/event/ReindexRecordsEvent.java +++ b/src/main/java/org/folio/search/model/event/ReindexRecordsEvent.java @@ -13,6 +13,7 @@ public class ReindexRecordsEvent { private ReindexRecordType recordType; private String tenant; private String rangeId; + private String memberTenantId; @Getter public enum ReindexRecordType { diff --git a/src/main/java/org/folio/search/model/reindex/MigrationResult.java b/src/main/java/org/folio/search/model/reindex/MigrationResult.java new file mode 100644 index 000000000..7fa1b868f --- /dev/null +++ b/src/main/java/org/folio/search/model/reindex/MigrationResult.java @@ -0,0 +1,13 @@ +package org.folio.search.model.reindex; + +import lombok.Data; + +@Data +public class MigrationResult { + + private long duration; + private long totalInstances; + private long totalHoldings; + private long totalItems; + private long totalRelationships; +} diff --git a/src/main/java/org/folio/search/model/reindex/ReindexStatusEntity.java b/src/main/java/org/folio/search/model/reindex/ReindexStatusEntity.java index c7bdc0b7a..0908331a2 100644 --- a/src/main/java/org/folio/search/model/reindex/ReindexStatusEntity.java +++ b/src/main/java/org/folio/search/model/reindex/ReindexStatusEntity.java @@ -18,9 +18,13 @@ public class ReindexStatusEntity { public static final String END_TIME_MERGE_COLUMN = "end_time_merge"; public static final String START_TIME_UPLOAD_COLUMN = "start_time_upload"; public static final String END_TIME_UPLOAD_COLUMN = "end_time_upload"; + public static final String START_TIME_STAGING_COLUMN = "start_time_staging"; + public static final String END_TIME_STAGING_COLUMN = "end_time_staging"; + public static final String TARGET_TENANT_ID_COLUMN = "target_tenant_id"; private final ReindexEntityType entityType; private final ReindexStatus status; + private String targetTenantId; private int totalMergeRanges; private int processedMergeRanges; private int totalUploadRanges; @@ -29,4 +33,6 @@ public class ReindexStatusEntity { private Timestamp endTimeMerge; private Timestamp startTimeUpload; private Timestamp endTimeUpload; + private Timestamp startTimeStaging; + private Timestamp endTimeStaging; } diff --git a/src/main/java/org/folio/search/model/types/ReindexStatus.java b/src/main/java/org/folio/search/model/types/ReindexStatus.java index d4357efdc..44f7179f8 100644 --- a/src/main/java/org/folio/search/model/types/ReindexStatus.java +++ b/src/main/java/org/folio/search/model/types/ReindexStatus.java @@ -7,6 +7,9 @@ public enum ReindexStatus { MERGE_IN_PROGRESS("Merge In Progress"), MERGE_COMPLETED("Merge Completed"), MERGE_FAILED("Merge Failed"), + STAGING_IN_PROGRESS("Staging In Progress"), + STAGING_COMPLETED("Staging Completed"), + STAGING_FAILED("Staging Failed"), UPLOAD_IN_PROGRESS("Upload In Progress"), UPLOAD_COMPLETED("Upload Completed"), UPLOAD_FAILED("Upload Failed"); diff --git a/src/main/java/org/folio/search/repository/AbstractResourceRepository.java b/src/main/java/org/folio/search/repository/AbstractResourceRepository.java index 9d1d5ee00..ddb5bbad7 100644 --- a/src/main/java/org/folio/search/repository/AbstractResourceRepository.java +++ b/src/main/java/org/folio/search/repository/AbstractResourceRepository.java @@ -4,14 +4,17 @@ import static org.folio.search.model.types.IndexActionType.INDEX; import static org.folio.search.utils.SearchResponseHelper.getErrorIndexOperationResponse; import static org.folio.search.utils.SearchResponseHelper.getSuccessIndexOperationResponse; +import static org.folio.search.utils.SearchUtils.SHARED_FIELD_NAME; import static org.folio.search.utils.SearchUtils.TENANT_ID_FIELD_NAME; import static org.folio.search.utils.SearchUtils.performExceptionalOperation; import static org.opensearch.client.RequestOptions.DEFAULT; +import static org.opensearch.index.query.QueryBuilders.boolQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; import java.util.List; import lombok.extern.log4j.Log4j2; import org.apache.commons.collections4.CollectionUtils; +import org.folio.search.configuration.properties.IndexManagementConfigurationProperties; import org.folio.search.domain.dto.FolioIndexOperationResponse; import org.folio.search.model.index.SearchDocumentBody; import org.folio.search.model.types.ResourceType; @@ -22,6 +25,7 @@ import org.opensearch.action.delete.DeleteRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.client.RestHighLevelClient; +import org.opensearch.common.unit.TimeValue; import org.opensearch.index.reindex.BulkByScrollResponse; import org.opensearch.index.reindex.DeleteByQueryRequest; import org.springframework.beans.factory.annotation.Autowired; @@ -31,6 +35,7 @@ public abstract class AbstractResourceRepository implements ResourceRepository { protected RestHighLevelClient elasticsearchClient; protected IndexNameProvider indexNameProvider; + protected IndexManagementConfigurationProperties indexManagementConfig; @Override public FolioIndexOperationResponse indexResources(List documents) { @@ -59,6 +64,50 @@ public FolioIndexOperationResponse deleteResourceByTenantId(ResourceType resourc : getErrorIndexOperationResponse(getBulkByScrollResponseErrorMessage(bulkByScrollResponse)); } + /** + * Deletes documents from an index by tenant ID, preserving shared documents. + * This method uses OpenSearch delete-by-query to remove tenant-specific documents without + * dropping the entire index, enabling preservation of shared consortium data. + * + * @param resourceType resource type as {@link ResourceType} object + * @param tenantId tenant id as {@link String} object + * @return {@link FolioIndexOperationResponse} object indicating success or failure + */ + public FolioIndexOperationResponse deleteConsortiumDocumentsByTenantId(ResourceType resourceType, String tenantId) { + var indexName = indexNameProvider.getIndexName(resourceType, tenantId); + log.debug("deleteConsortiumDocumentsByTenantId:: by [indexName: {}, tenantId: {}]", indexName, tenantId); + + var deleteByQueryRequest = new DeleteByQueryRequest(indexName); + + // Configure performance settings + deleteByQueryRequest.setBatchSize(indexManagementConfig.getDeleteQueryBatchSize()); + deleteByQueryRequest.setScroll( + TimeValue.timeValueMinutes(indexManagementConfig.getDeleteQueryScrollTimeoutMinutes())); + deleteByQueryRequest.setTimeout( + TimeValue.timeValueMinutes(indexManagementConfig.getDeleteQueryRequestTimeoutMinutes())); + deleteByQueryRequest.setRefresh(indexManagementConfig.getDeleteQueryRefresh()); + + // Build query: tenantId = targetTenant AND (NOT shared = true) + var query = boolQuery() + .must(termQuery(TENANT_ID_FIELD_NAME, tenantId)) + .mustNot(termQuery(SHARED_FIELD_NAME, true)); + deleteByQueryRequest.setQuery(query); + log.info("deleteDocumentsByTenantId:: deleting tenant documents preserving shared " + + "[index: {}, tenantId: {}]", indexName, tenantId); + + var bulkByScrollResponse = performExceptionalOperation( + () -> elasticsearchClient.deleteByQuery(deleteByQueryRequest, DEFAULT), + indexName, "deleteByQueryApi"); + + var deletedCount = bulkByScrollResponse.getDeleted(); + log.info("deleteDocumentsByTenantId:: completed [index: {}, tenantId: {}, deleted: {}]", + indexName, tenantId, deletedCount); + + return bulkByScrollResponse.getBulkFailures().isEmpty() + ? getSuccessIndexOperationResponse() + : getErrorIndexOperationResponse(getBulkByScrollResponseErrorMessage(bulkByScrollResponse)); + } + @Autowired public void setIndexNameProvider(IndexNameProvider indexNameProvider) { this.indexNameProvider = indexNameProvider; @@ -69,6 +118,11 @@ public void setElasticsearchClient(RestHighLevelClient elasticsearchClient) { this.elasticsearchClient = elasticsearchClient; } + @Autowired + public void setIndexManagementConfig(IndexManagementConfigurationProperties indexManagementConfig) { + this.indexManagementConfig = indexManagementConfig; + } + protected BulkResponse executeBulkRequest(BulkRequest bulkRequest) { var indicesString = bulkRequest.requests().stream().map(DocWriteRequest::index).collect(joining(",")); return performExceptionalOperation(() -> elasticsearchClient.bulk(bulkRequest, DEFAULT), indicesString, "bulkApi"); diff --git a/src/main/java/org/folio/search/service/IndexService.java b/src/main/java/org/folio/search/service/IndexService.java index 85fa3dc91..b420bf1f2 100644 --- a/src/main/java/org/folio/search/service/IndexService.java +++ b/src/main/java/org/folio/search/service/IndexService.java @@ -101,7 +101,7 @@ public FolioIndexOperationResponse updateIndexSettings(ResourceType resourceType var index = indexNameProvider.getIndexName(resourceType, tenantId); var settings = prepareIndexDynamicSettings(indexSettings); - log.info("Attempts to update settings by [indexName: {}, settings: {}]", index, settings); + log.info("updateIndexSettings:: Attempting to update settings by [indexName: {}, settings: {}]", index, settings); return indexRepository.updateIndexSettings(index, settings.toString()); } @@ -117,7 +117,7 @@ public FolioIndexOperationResponse updateMappings(ResourceType resourceType, Str var index = indexNameProvider.getIndexName(resourceType, tenantId); var mappings = mappingHelper.getMappings(resourceType); - log.info("Attempts to update mappings by [indexName: {}, mappings: {}]", index, mappings); + log.info("updateMappings:: Attempting to update mappings by [indexName: {}, mappings: {}]", index, mappings); return indexRepository.updateMappings(index, mappings); } @@ -134,6 +134,20 @@ public void createIndexIfNotExist(ResourceType resourceType, String tenantId) { } } + /** + * Creates Elasticsearch index if it is not exist with provided settings. + * + * @param resourceType - resource name as {@link ResourceType} object. + * @param tenantId - tenant id as {@link String} object + * @param indexSettings - index settings as {@link IndexSettings} object + */ + public void createIndexIfNotExist(ResourceType resourceType, String tenantId, IndexSettings indexSettings) { + var index = indexNameProvider.getIndexName(resourceType, tenantId); + if (!indexRepository.indexExists(index)) { + createIndex(resourceType, tenantId, indexSettings); + } + } + /** * Runs reindex request for mod-inventory-storage. * @@ -167,7 +181,7 @@ && notConsortiumMemberTenant(tenantId)) { */ public ReindexJob reindexInventoryAsync(String resource) { var reindexUri = fromUriString(RESOURCE_STORAGE_REINDEX_URI).buildAndExpand(resource).toUri(); - log.info("reindexInventory:: Starting reindex for uri {}", reindexUri); + log.info("reindexInventoryAsync:: Starting reindex for uri {}", reindexUri); return resourceReindexClient.submitReindex(reindexUri); } @@ -175,13 +189,13 @@ public ReindexJob reindexInventoryAsync(String resource) { * Runs synchronous locations and location-units reindex in mod-search. */ public ReindexJob reindexInventoryLocations(String tenantId, List resources) { - log.info("reindexLocations:: Starting reindex"); + log.info("reindexInventoryLocations:: Starting reindex"); var response = new ReindexJob().id(UUID.randomUUID().toString()) .jobStatus("Completed") .submittedDate(new Date().toString()); resources.forEach(resourceType -> locationService.reindex(tenantId, resourceType)); - log.info("reindexLocations:: Reindex completed"); + log.info("reindexInventoryLocations:: Reindex completed"); return response; } @@ -207,7 +221,7 @@ private FolioCreateIndexResponse doCreateIndex(ResourceType resourceName, String var index = indexNameProvider.getIndexName(resourceName, tenantId); var mappings = mappingHelper.getMappings(resourceName); - log.info("Attempts to create index by [indexName: {}, mappings: {}, settings: {}]", + log.info("doCreateIndex:: Attempting to create index by [indexName: {}, mappings: {}, settings: {}]", index, mappings, indexSettings); return indexRepository.createIndex(index, indexSettings, mappings); } diff --git a/src/main/java/org/folio/search/service/reindex/ReindexCommonService.java b/src/main/java/org/folio/search/service/reindex/ReindexCommonService.java index ec66b8d77..0328a41c3 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexCommonService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexCommonService.java @@ -9,6 +9,8 @@ import lombok.extern.log4j.Log4j2; import org.folio.search.domain.dto.IndexSettings; import org.folio.search.model.types.ReindexEntityType; +import org.folio.search.model.types.ResourceType; +import org.folio.search.repository.PrimaryResourceRepository; import org.folio.search.service.IndexService; import org.folio.search.service.reindex.jdbc.ReindexJdbcRepository; import org.springframework.stereotype.Service; @@ -20,27 +22,93 @@ public class ReindexCommonService { private final Map repositories; private final IndexService indexService; + private final PrimaryResourceRepository resourceRepository; - public ReindexCommonService(List repositories, IndexService indexService) { + public ReindexCommonService(List repositories, IndexService indexService, + PrimaryResourceRepository resourceRepository) { this.repositories = repositories.stream() .collect(Collectors.toMap(ReindexJdbcRepository::entityType, identity(), (rep1, rep2) -> rep2)); this.indexService = indexService; + this.resourceRepository = resourceRepository; } @Transactional - public void deleteAllRecords() { - for (ReindexEntityType entityType : ReindexEntityType.values()) { - repositories.get(entityType).truncate(); + public void deleteAllRecords(String tenantId) { + for (var entityType : ReindexEntityType.values()) { + if (tenantId != null) { + // For tenant-specific refresh: only truncate staging tables + repositories.get(entityType).truncateStaging(); + } else { + // For full refresh: truncate all tables (existing behavior) + repositories.get(entityType).truncate(); + } } } + @Transactional + public void deleteRecordsByTenantId(String tenantId) { + // Delete in proper order to avoid foreign key constraint violations + // First delete relationship tables (child entities) + repositories.get(ReindexEntityType.SUBJECT).deleteByTenantId(tenantId); + repositories.get(ReindexEntityType.CONTRIBUTOR).deleteByTenantId(tenantId); + repositories.get(ReindexEntityType.CLASSIFICATION).deleteByTenantId(tenantId); + repositories.get(ReindexEntityType.CALL_NUMBER).deleteByTenantId(tenantId); + + // Then delete main entity tables (parent entities) + repositories.get(ReindexEntityType.ITEM).deleteByTenantId(tenantId); + repositories.get(ReindexEntityType.HOLDINGS).deleteByTenantId(tenantId); + repositories.get(ReindexEntityType.INSTANCE).deleteByTenantId(tenantId); + + log.info("deleteRecordsByTenantId:: Successfully deleted existing data for tenant: {}", tenantId); + } + + /** + * Deletes documents from OpenSearch indexes for a specific tenant with shared preservation. + * + * @param tenantId the tenant ID whose documents should be deleted + */ + public void deleteInstanceDocumentsByTenantId(String tenantId) { + log.info("deleteInstanceDocumentsByTenantId:: starting deletion for [tenantId: {}, ]", tenantId); + + try { + var resourceType = ResourceType.INSTANCE; + var result = resourceRepository.deleteConsortiumDocumentsByTenantId(resourceType, tenantId); + + if (result != null) { + log.debug("deleteInstanceDocumentsByTenantId:: completed for [resourceType: {}]", resourceType); + } else { + log.warn("deleteInstanceDocumentsByTenantId:: failed for [resourceType: {}]", resourceType); + } + } catch (Exception e) { + log.error("deleteInstanceDocumentsByTenantId:: error processing [tenantId: {}, error: {}]", + tenantId, e.getMessage(), e); + } + + log.info("deleteInstanceDocumentsByTenantId:: completed [tenantId: {}]", tenantId); + } + public void recreateIndex(ReindexEntityType reindexEntityType, String tenantId, IndexSettings indexSettings) { try { var resourceType = RESOURCE_NAME_MAP.get(reindexEntityType); indexService.dropIndex(resourceType, tenantId); indexService.createIndex(resourceType, tenantId, indexSettings); } catch (Exception e) { - log.warn("Index cannot be recreated for resource={}, message={}", reindexEntityType, e.getMessage()); + log.warn("recreateIndex:: Index cannot be recreated for resource={}, message={}", + reindexEntityType, e.getMessage()); + } + } + + public void ensureIndexExists(ReindexEntityType reindexEntityType, String tenantId, IndexSettings indexSettings) { + try { + var resourceType = RESOURCE_NAME_MAP.get(reindexEntityType); + if (indexSettings != null) { + indexService.createIndexIfNotExist(resourceType, tenantId, indexSettings); + } else { + indexService.createIndexIfNotExist(resourceType, tenantId); + } + } catch (Exception e) { + log.warn("ensureIndexExists:: Index existence check/creation failed for resource={}, message={}", + reindexEntityType, e.getMessage()); } } } diff --git a/src/main/java/org/folio/search/service/reindex/ReindexConstants.java b/src/main/java/org/folio/search/service/reindex/ReindexConstants.java index 6013d1ce1..6ab57a62e 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexConstants.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexConstants.java @@ -32,4 +32,19 @@ public final class ReindexConstants { public static final String SUBJECT_TABLE = "subject"; public static final String UPLOAD_RANGE_TABLE = "upload_range"; public static final String REINDEX_STATUS_TABLE = "reindex_status"; + + // Staging table names + public static final String STAGING_HOLDING_TABLE = "staging_holding"; + public static final String STAGING_INSTANCE_TABLE = "staging_instance"; + public static final String STAGING_INSTANCE_CALL_NUMBER_TABLE = "staging_instance_call_number"; + public static final String STAGING_INSTANCE_CLASSIFICATION_TABLE = "staging_instance_classification"; + public static final String STAGING_INSTANCE_CONTRIBUTOR_TABLE = "staging_instance_contributor"; + public static final String STAGING_INSTANCE_SUBJECT_TABLE = "staging_instance_subject"; + public static final String STAGING_ITEM_TABLE = "staging_item"; + + // Staging child resource table names + public static final String STAGING_SUBJECT_TABLE = "staging_subject"; + public static final String STAGING_CONTRIBUTOR_TABLE = "staging_contributor"; + public static final String STAGING_CLASSIFICATION_TABLE = "staging_classification"; + public static final String STAGING_CALL_NUMBER_TABLE = "staging_call_number"; } diff --git a/src/main/java/org/folio/search/service/reindex/ReindexContext.java b/src/main/java/org/folio/search/service/reindex/ReindexContext.java new file mode 100644 index 000000000..4fa43b853 --- /dev/null +++ b/src/main/java/org/folio/search/service/reindex/ReindexContext.java @@ -0,0 +1,77 @@ +package org.folio.search.service.reindex; + +import org.springframework.stereotype.Component; + +/** + * Context holder to track when reindex operations are active. + * Uses ThreadLocal to ensure thread-safe access during concurrent reindex operations. + * Also tracks member tenant context for consortium-specific reindex operations. + */ +@Component +public class ReindexContext { + + private static final ThreadLocal REINDEX_MODE = ThreadLocal.withInitial(() -> false); + private static final ThreadLocal MEMBER_TENANT_ID = new ThreadLocal<>(); + + /** + * Set whether the current thread is in reindex mode. + * + * @param mode true if reindex mode is active, false otherwise + */ + public static void setReindexMode(boolean mode) { + REINDEX_MODE.set(mode); + } + + /** + * Check if the current thread is in reindex mode. + * + * @return true if reindex mode is active, false otherwise + */ + public static boolean isReindexMode() { + return REINDEX_MODE.get(); + } + + /** + * Set the member tenant ID for consortium member tenant reindex operations. + * + * @param tenantId the member tenant ID + */ + public static void setMemberTenantId(String tenantId) { + MEMBER_TENANT_ID.set(tenantId); + } + + /** + * Get the current member tenant ID if set. + * + * @return the member tenant ID or null if not set + */ + public static String getMemberTenantId() { + return MEMBER_TENANT_ID.get(); + } + + /** + * Check if the current operation is a member tenant reindex. + * + * @return true if member tenant ID is set, false otherwise + */ + public static boolean isMemberTenantReindex() { + return MEMBER_TENANT_ID.get() != null; + } + + /** + * Clear the member tenant ID for the current thread. + * Should be called in finally blocks to prevent memory leaks. + */ + public static void clearMemberTenantId() { + MEMBER_TENANT_ID.remove(); + } + + /** + * Clear all reindex context for the current thread. + * Should be called in finally blocks to prevent memory leaks. + */ + public static void clear() { + REINDEX_MODE.remove(); + MEMBER_TENANT_ID.remove(); + } +} diff --git a/src/main/java/org/folio/search/service/reindex/ReindexMergeRangeIndexService.java b/src/main/java/org/folio/search/service/reindex/ReindexMergeRangeIndexService.java index 88e259832..5284ded09 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexMergeRangeIndexService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexMergeRangeIndexService.java @@ -28,20 +28,27 @@ @Service public class ReindexMergeRangeIndexService { + private static final int STATS_LOG_INTERVAL = 100; // Log stats every 100 merge ranges + private final Map repositories; private final InventoryService inventoryService; private final ReindexConfigurationProperties reindexConfig; + private final StagingMigrationService migrationService; + private int mergeRangeCounter; private InstanceChildrenResourceService instanceChildrenResourceService; public ReindexMergeRangeIndexService(List repositories, InventoryService inventoryService, - ReindexConfigurationProperties reindexConfig) { + ReindexConfigurationProperties reindexConfig, + StagingMigrationService migrationService) { this.repositories = repositories.stream() .collect(Collectors.toMap(MergeRangeRepository::entityType, Function.identity())); this.inventoryService = inventoryService; this.reindexConfig = reindexConfig; + this.migrationService = migrationService; this.instanceChildrenResourceService = null; + this.mergeRangeCounter = 0; } @Autowired(required = false) @@ -91,10 +98,27 @@ public void saveEntities(ReindexRecordsEvent event) { .map(entity -> (Map) entity) .toList(); - repositories.get(event.getRecordType().getEntityType()).saveEntities(event.getTenant(), entities); - if (instanceChildrenResourceService != null) { - instanceChildrenResourceService.persistChildrenOnReindex(event.getTenant(), - RESOURCE_NAME_MAP.get(event.getRecordType().getEntityType()), entities); + try { + // Set reindex mode for context-aware repositories + ReindexContext.setReindexMode(true); + + // Use unified repository which will route to staging based on context + var repository = repositories.get(event.getRecordType().getEntityType()); + repository.saveEntities(event.getTenant(), entities); + + if (instanceChildrenResourceService != null) { + instanceChildrenResourceService.persistChildrenOnReindex(event.getTenant(), + RESOURCE_NAME_MAP.get(event.getRecordType().getEntityType()), entities); + } + } finally { + // Only clear reindex mode, preserve member tenant context for outer scope + ReindexContext.setReindexMode(false); + } + + // Periodically log merge range counter + mergeRangeCounter++; + if (mergeRangeCounter % STATS_LOG_INTERVAL == 0) { + log.info("saveEntities:: Processed {} merge ranges", mergeRangeCounter); } } @@ -134,4 +158,18 @@ private ReindexEntityType asEntityType(InventoryRecordType recordType) { default -> ReindexEntityType.ITEM; }; } + + public void performStagingMigration(String targetTenantId) { + if (targetTenantId != null) { + log.info("performStagingMigration:: Starting tenant-specific migration of staging tables for tenant: {}", + targetTenantId); + var result = migrationService.migrateAllStagingTables(targetTenantId); + log.info("performStagingMigration:: Tenant-specific migration completed for {}: instances={}, holdings={}, " + + "items={}, relationships={}", + targetTenantId, result.getTotalInstances(), result.getTotalHoldings(), + result.getTotalItems(), result.getTotalRelationships()); + } else { + log.info("performStagingMigration:: Not member tenant refresh - staging tables not used, skipping migration"); + } + } } diff --git a/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java b/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java index 3b8d7552b..40880fad4 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java @@ -3,6 +3,7 @@ import java.util.Collection; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.message.FormattedMessage; import org.folio.search.domain.dto.FolioIndexOperationResponse; import org.folio.search.exception.ReindexException; @@ -29,40 +30,88 @@ public class ReindexOrchestrationService { private final MultiTenantSearchDocumentConverter documentConverter; private final FolioExecutionContext context; - public boolean process(ReindexRangeIndexEvent event) { - log.info("process:: ReindexRangeIndexEvent [id: {}, tenantId: {}, entityType: {}, lower: {}, upper: {}, ts: {}]", - event.getId(), event.getTenant(), event.getEntityType(), event.getLower(), event.getUpper(), event.getTs()); - var resourceEvents = uploadRangeService.fetchRecordRange(event); - var documents = documentConverter.convert(resourceEvents).values().stream().flatMap(Collection::stream).toList(); - var folioIndexOperationResponse = elasticRepository.indexResources(documents); - if (folioIndexOperationResponse.getStatus() == FolioIndexOperationResponse.StatusEnum.ERROR) { - log.warn("process:: ReindexRangeIndexEvent indexing error [id: {}, error: {}]", - event.getId(), folioIndexOperationResponse.getErrorMessage()); - uploadRangeService.updateStatus(event, ReindexRangeStatus.FAIL, folioIndexOperationResponse.getErrorMessage()); - reindexStatusService.updateReindexUploadFailed(event.getEntityType()); - throw new ReindexException(folioIndexOperationResponse.getErrorMessage()); + /** + * Determines and sets the member tenant ID context for processing. + * Uses the reindex status service to get the target tenant ID from the database. + * + * @return the member tenant ID that was set, or null if none was set + */ + private String getMemberTenantIdForProcessing() { + var memberTenantId = reindexStatusService.getTargetTenantId(); + + if (StringUtils.isNotBlank(memberTenantId)) { + log.debug("getMemberTenantIdForProcessing:: Setting member tenant context: {}", memberTenantId); + ReindexContext.setMemberTenantId(memberTenantId); + return memberTenantId; } - uploadRangeService.updateStatus(event, ReindexRangeStatus.SUCCESS, null); - log.info("process:: ReindexRangeIndexEvent processed [id: {}]", event.getId()); - reindexStatusService.addProcessedUploadRanges(event.getEntityType(), 1); - return true; + return null; + } + + /** + * Determines and sets the member tenant ID context for range processing. + * + * @param event the reindex range event containing member tenant information + * @return the member tenant ID that was set, or null if none was set + */ + private String getMemberTenantIdForRangeProcessing(ReindexRangeIndexEvent event) { + var memberTenantId = event.getMemberTenantId(); + + if (StringUtils.isNotBlank(memberTenantId)) { + log.debug("getMemberTenantIdForRangeProcessing:: Setting member tenant context: {}", memberTenantId); + ReindexContext.setMemberTenantId(memberTenantId); + return memberTenantId; + } + + return null; + } + + public boolean process(ReindexRangeIndexEvent event) { + var memberTenantId = getMemberTenantIdForRangeProcessing(event); + + try { + log.info("process:: ReindexRangeIndexEvent [id: {}, tenantId: {}, memberTenantId: {}, " + + "entityType: {}, lower: {}, upper: {}, ts: {}]", + event.getId(), event.getTenant(), memberTenantId, + event.getEntityType(), event.getLower(), event.getUpper(), event.getTs()); + + var resourceEvents = uploadRangeService.fetchRecordRange(event); + var documents = documentConverter.convert(resourceEvents).values().stream().flatMap(Collection::stream).toList(); + var folioIndexOperationResponse = elasticRepository.indexResources(documents); + if (folioIndexOperationResponse.getStatus() == FolioIndexOperationResponse.StatusEnum.ERROR) { + log.warn("process:: ReindexRangeIndexEvent indexing error [id: {}, error: {}]", + event.getId(), folioIndexOperationResponse.getErrorMessage()); + uploadRangeService.updateStatus(event, ReindexRangeStatus.FAIL, folioIndexOperationResponse.getErrorMessage()); + reindexStatusService.updateReindexUploadFailed(event.getEntityType()); + throw new ReindexException(folioIndexOperationResponse.getErrorMessage()); + } + uploadRangeService.updateStatus(event, ReindexRangeStatus.SUCCESS, null); + + log.info("process:: ReindexRangeIndexEvent processed [id: {}]", event.getId()); + reindexStatusService.addProcessedUploadRanges(event.getEntityType(), 1); + return true; + } finally { + // Clean up member tenant context + if (memberTenantId != null) { + ReindexContext.clearMemberTenantId(); + } + } } public boolean process(ReindexRecordsEvent event) { - log.info("process:: ReindexRecordsEvent [rangeId: {}, tenantId: {}, recordType: {}, recordsCount: {}]", - event.getRangeId(), event.getTenant(), event.getRecordType(), event.getRecords().size()); + var memberTenantId = getMemberTenantIdForProcessing(); var entityType = event.getRecordType().getEntityType(); try { + log.info("process:: ReindexRecordsEvent [rangeId: {}, tenantId: {}, memberTenantId: {}, " + + "recordType: {}, recordsCount: {}]", + event.getRangeId(), event.getTenant(), memberTenantId, event.getRecordType(), event.getRecords().size()); + mergeRangeService.saveEntities(event); - reindexStatusService.addProcessedMergeRanges(entityType, 1); mergeRangeService.updateStatus(entityType, event.getRangeId(), ReindexRangeStatus.SUCCESS, null); + reindexStatusService.addProcessedMergeRanges(entityType, 1); log.info("process:: ReindexRecordsEvent processed [rangeId: {}, recordType: {}]", event.getRangeId(), event.getRecordType()); - if (reindexStatusService.isMergeCompleted()) { - reindexService.submitUploadReindex(context.getTenantId(), ReindexEntityType.supportUploadTypes()); - } } catch (PessimisticLockingFailureException ex) { log.warn(new FormattedMessage("process:: ReindexRecordsEvent indexing recoverable error" + " [rangeId: {}, error: {}]", event.getRangeId(), ex.getMessage()), ex); @@ -72,8 +121,54 @@ public boolean process(ReindexRecordsEvent event) { event.getRangeId(), ex.getMessage()), ex); reindexStatusService.updateReindexMergeFailed(entityType); mergeRangeService.updateStatus(entityType, event.getRangeId(), ReindexRangeStatus.FAIL, ex.getMessage()); + return true; + } finally { + // Clean up member tenant context + if (memberTenantId != null) { + ReindexContext.clearMemberTenantId(); + } + } + + if (reindexStatusService.isMergeCompleted()) { + // Get targetTenantId before migration + var targetTenantId = reindexStatusService.getTargetTenantId(); + + // Perform migration of staging tables + log.info("process:: Merge completed. Starting migration of staging tables"); + performStagingMigration(targetTenantId); + + // Check if this is a tenant-specific reindex that requires OpenSearch document cleanup + if (targetTenantId != null) { + log.info("process:: Starting tenant-specific upload phase with document cleanup [targetTenant: {}]", + targetTenantId); + reindexService.submitUploadReindexWithTenantCleanup(context.getTenantId(), + ReindexEntityType.supportUploadTypes(), + targetTenantId); + } else { + log.info("process:: Starting standard upload phase without tenant-specific cleanup"); + reindexService.submitUploadReindex(context.getTenantId(), ReindexEntityType.supportUploadTypes()); + } + + log.info("process:: Migration and upload phase completed for {}", + targetTenantId != null ? "tenant: " + targetTenantId : "consortium"); } return true; } + + private void performStagingMigration(String targetTenantId) { + try { + log.info("performStagingMigration:: Starting staging migration"); + reindexStatusService.updateStagingStarted(); + + mergeRangeService.performStagingMigration(targetTenantId); + + reindexStatusService.updateStagingCompleted(); + log.info("performStagingMigration:: Migration completed successfully. Starting upload phase"); + } catch (Exception e) { + log.error("performStagingMigration:: Migration failed", e); + reindexStatusService.updateStagingFailed(); + throw new ReindexException("Migration failed: " + e.getMessage()); + } + } } diff --git a/src/main/java/org/folio/search/service/reindex/ReindexService.java b/src/main/java/org/folio/search/service/reindex/ReindexService.java index 420af6341..b691d8894 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexService.java @@ -1,5 +1,7 @@ package org.folio.search.service.reindex; +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.folio.search.configuration.SearchCacheNames.USER_TENANTS_CACHE; import static org.folio.search.model.types.ReindexStatus.MERGE_FAILED; import static org.folio.search.model.types.ReindexStatus.MERGE_IN_PROGRESS; @@ -15,6 +17,7 @@ import org.folio.search.converter.ReindexEntityTypeMapper; import org.folio.search.domain.dto.IndexSettings; import org.folio.search.domain.dto.ReindexUploadDto; +import org.folio.search.exception.ReindexException; import org.folio.search.exception.RequestValidationException; import org.folio.search.integration.folio.InventoryService; import org.folio.search.model.reindex.MergeRangeEntity; @@ -42,17 +45,14 @@ public class ReindexService { private final ReindexEntityTypeMapper entityTypeMapper; private final ReindexCommonService reindexCommonService; - public ReindexService(ConsortiumTenantService consortiumService, - SystemUserScopedExecutionService executionService, + public ReindexService(ConsortiumTenantService consortiumService, SystemUserScopedExecutionService executionService, ReindexMergeRangeIndexService mergeRangeService, - ReindexUploadRangeIndexService uploadRangeService, - ReindexStatusService statusService, + ReindexUploadRangeIndexService uploadRangeService, ReindexStatusService statusService, InventoryService inventoryService, @Qualifier("reindexFullExecutor") ExecutorService reindexFullExecutor, @Qualifier("reindexUploadExecutor") ExecutorService reindexUploadExecutor, @Qualifier("reindexPublisherExecutor") ExecutorService reindexPublisherExecutor, - ReindexEntityTypeMapper entityTypeMapper, - ReindexCommonService reindexCommonService) { + ReindexEntityTypeMapper entityTypeMapper, ReindexCommonService reindexCommonService) { this.consortiumService = consortiumService; this.executionService = executionService; this.mergeRangeService = mergeRangeService; @@ -68,75 +68,140 @@ public ReindexService(ConsortiumTenantService consortiumService, @CacheEvict(cacheNames = USER_TENANTS_CACHE, allEntries = true, beforeInvocation = true) public CompletableFuture submitFullReindex(String tenantId, IndexSettings indexSettings) { - log.info("submitFullReindex:: for [tenantId: {}]", tenantId); + return submitFullReindex(tenantId, indexSettings, null); + } - validateTenant("submitFullReindex", tenantId); + @CacheEvict(cacheNames = USER_TENANTS_CACHE, allEntries = true, beforeInvocation = true) + public CompletableFuture submitFullReindex(String tenantId, IndexSettings indexSettings, + String targetTenantId) { + log.info("submitFullReindex:: for [requestingTenant: {}, targetTenant: {}]", tenantId, + targetTenantId != null ? targetTenantId : "all consortium members"); + + validateTenant("submitFullReindex", tenantId, targetTenantId); + validateReindexNotInProgress(); + + reindexCommonService.deleteAllRecords(targetTenantId); + statusService.recreateMergeStatusRecords(targetTenantId); + + if (targetTenantId == null) { + // Full reindex - recreate all indexes (existing behavior) + recreateIndices(tenantId, ReindexEntityType.supportUploadTypes(), indexSettings); + log.info("submitFullReindex:: recreated indexes for full reindex [requestingTenant: {}]", tenantId); + } else { + // Tenant-specific reindex - ensure indexes exist without recreating existing ones + ensureIndicesExist(tenantId, ReindexEntityType.supportUploadTypes(), indexSettings); + log.info("submitFullReindex:: ensured indexes exist for tenant-specific reindex " + + "[requestingTenant: {}, targetTenant: {}]", tenantId, targetTenantId); + } - reindexCommonService.deleteAllRecords(); - statusService.recreateMergeStatusRecords(); - recreateIndices(tenantId, ReindexEntityType.supportUploadTypes(), indexSettings); + // Capture context before async execution + final var memberTenantIdContext = targetTenantId; var future = CompletableFuture.runAsync(() -> { - mergeRangeService.truncateMergeRanges(); - var rangesForAllTenants = Stream.of( - mergeRangeService.createMergeRanges(tenantId), - processForConsortium(tenantId) - ) - .flatMap(List::stream) - .toList(); - mergeRangeService.saveMergeRanges(rangesForAllTenants); - }, reindexFullExecutor) - .thenRun(() -> publishRecordsRange(tenantId)) - .handle((unused, throwable) -> { - if (throwable != null) { - log.error("initFullReindex:: process failed [tenantId: {}, error: {}]", tenantId, throwable); - statusService.updateReindexMergeFailed(); + try { + // Restore context in executor thread + if (memberTenantIdContext != null) { + ReindexContext.setMemberTenantId(memberTenantIdContext); + ReindexContext.setReindexMode(true); // Enable staging } - return unused; - }); - log.info("submitFullReindex:: submitted [tenantId: {}]", tenantId); + mergeRangeService.truncateMergeRanges(); + + List rangesForAllTenants; + if (memberTenantIdContext != null) { + // Only process the member tenant (no central tenant in merge) + rangesForAllTenants = processForConsortium(tenantId, memberTenantIdContext); + } else { + // Full consortium reindex: Process central + all members (no staging) + rangesForAllTenants = Stream.of(mergeRangeService.createMergeRanges(tenantId), // Central + processForConsortium(tenantId, null) // Members + ).flatMap(List::stream).toList(); + } + + mergeRangeService.saveMergeRanges(rangesForAllTenants); + } finally { + // Clean up context in executor thread + if (memberTenantIdContext != null) { + ReindexContext.clearMemberTenantId(); + ReindexContext.setReindexMode(false); + } + } + }, reindexFullExecutor).thenRun(() -> { + // Restore context before publishing + if (memberTenantIdContext != null) { + ReindexContext.setMemberTenantId(memberTenantIdContext); + } + try { + publishRecordsRange(tenantId, memberTenantIdContext); + } finally { + if (memberTenantIdContext != null) { + ReindexContext.clearMemberTenantId(); + } + } + }).handle((unused, throwable) -> { + if (throwable != null) { + log.error("initFullReindex:: process failed [tenantId: {}, targetTenant: {}, error: {}]", tenantId, + targetTenantId, throwable); + statusService.updateReindexMergeFailed(); + } + return unused; + }); + + log.info("submitFullReindex:: submitted [requestingTenant: {}, targetTenant: {}]", tenantId, + targetTenantId != null ? targetTenantId : "all consortium members"); return future; } - public CompletableFuture submitUploadReindex(String tenantId, - ReindexUploadDto reindexUploadDto) { - var entityTypes = entityTypeMapper.convert(reindexUploadDto.getEntityTypes()) - .stream().filter(ReindexEntityType::isSupportsUpload).toList(); + public CompletableFuture submitUploadReindex(String tenantId, ReindexUploadDto reindexUploadDto) { + var entityTypes = + entityTypeMapper.convert(reindexUploadDto.getEntityTypes()).stream().filter(ReindexEntityType::isSupportsUpload) + .toList(); return submitUploadReindex(tenantId, entityTypes, true, reindexUploadDto.getIndexSettings()); } - public CompletableFuture submitUploadReindex(String tenantId, - List entityTypes) { + public CompletableFuture submitUploadReindex(String tenantId, List entityTypes) { return submitUploadReindex(tenantId, entityTypes, false, null); } - private CompletableFuture submitUploadReindex(String tenantId, - List entityTypes, - boolean recreateIndex, - IndexSettings indexSettings) { + private CompletableFuture submitUploadReindex(String tenantId, List entityTypes, + boolean recreateIndex, IndexSettings indexSettings) { log.info("submitUploadReindex:: for [tenantId: {}, entities: {}]", tenantId, entityTypes); validateUploadReindex(tenantId, entityTypes); + var targetTenantId = statusService.getTargetTenantId(); for (var reindexEntityType : entityTypes) { - statusService.recreateUploadStatusRecord(reindexEntityType); + statusService.recreateUploadStatusRecord(reindexEntityType, targetTenantId); if (recreateIndex) { reindexCommonService.recreateIndex(reindexEntityType, tenantId, indexSettings); } } + // Capture context before async execution + final var memberTenantIdContext = ReindexContext.getMemberTenantId(); + var futures = new ArrayList<>(); for (var entityType : entityTypes) { - var future = CompletableFuture.runAsync(() -> - uploadRangeService.prepareAndSendIndexRanges(entityType), reindexUploadExecutor) - .handle((unused, throwable) -> { - if (throwable != null) { - log.error("reindex upload process failed: {}", throwable.getMessage()); - statusService.updateReindexUploadFailed(entityType); + var future = CompletableFuture.runAsync(() -> { + // Restore context in executor thread + if (memberTenantIdContext != null) { + ReindexContext.setMemberTenantId(memberTenantIdContext); + } + try { + uploadRangeService.prepareAndSendIndexRanges(entityType); + } finally { + // Clean up context in executor thread + if (memberTenantIdContext != null) { + ReindexContext.clearMemberTenantId(); } - return unused; - }); + } + }, reindexUploadExecutor).handle((unused, throwable) -> { + if (throwable != null) { + log.error("submitUploadReindex:: reindex upload process failed: {}", throwable.getMessage()); + statusService.updateReindexUploadFailed(entityType); + } + return unused; + }); futures.add(future); } @@ -144,6 +209,52 @@ private CompletableFuture submitUploadReindex(String tenantId, return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); } + /** + * Submits upload reindex with tenant-specific document cleanup for member tenant reindex operations. + * This method performs OpenSearch document cleanup for the specified tenant while preserving + * shared consortium instances, then proceeds with upload processing using the member tenant context. + * + * @param tenantId the requesting tenant ID + * @param entityTypes the entity types to reindex + * @param targetTenantId the specific tenant whose documents should be cleaned up (null for full cleanup) + * @return CompletableFuture representing the upload operation + */ + public CompletableFuture submitUploadReindexWithTenantCleanup(String tenantId, + List entityTypes, + String targetTenantId) { + log.info("submitUploadReindexWithTenantCleanup:: for [tenantId: {}, entities: {}, targetTenant: {}]", tenantId, + entityTypes, targetTenantId); + + // Perform tenant-specific document cleanup before upload processing + if (targetTenantId != null) { + log.info("submitUploadReindexWithTenantCleanup:: performing OpenSearch document cleanup " + + "for tenant [{}] while preserving shared instances", targetTenantId); + try { + reindexCommonService.deleteInstanceDocumentsByTenantId(targetTenantId); + log.info("submitUploadReindexWithTenantCleanup:: completed OpenSearch cleanup for tenant [{}]", targetTenantId); + } catch (Exception e) { + log.error( + "submitUploadReindexWithTenantCleanup:: failed to cleanup OpenSearch documents " + "for tenant [{}]: {}", + targetTenantId, e.getMessage(), e); + throw new ReindexException("Failed to cleanup tenant documents before upload", e); + } + } + + // Set member tenant context for upload phase + if (targetTenantId != null) { + ReindexContext.setMemberTenantId(targetTenantId); + } + + try { + // Standard upload processing - context determines data fetching + return submitUploadReindex(tenantId, entityTypes, false, null); + } finally { + if (targetTenantId != null) { + ReindexContext.clearMemberTenantId(); + } + } + } + public CompletableFuture submitFailedMergeRangesReindex(String tenantId) { log.info("submitFailedMergeRangesReindex:: for [tenantId: {}]", tenantId); @@ -156,15 +267,13 @@ public CompletableFuture submitFailedMergeRangesReindex(String tenantId) { } log.info("submitFailedMergeRangesReindex:: for [tenantId: {}, count: {}]", tenantId, failedRanges.size()); - var entityTypes = failedRanges.stream() - .map(MergeRangeEntity::getEntityType) - .collect(Collectors.toSet()); + var entityTypes = failedRanges.stream().map(MergeRangeEntity::getEntityType).collect(Collectors.toSet()); statusService.updateReindexMergeInProgress(entityTypes); var futures = new ArrayList<>(); for (var rangeEntity : failedRanges) { - var future = CompletableFuture.runAsync(() -> - executionService.executeSystemUserScoped(rangeEntity.getTenantId(), () -> { + var future = + CompletableFuture.runAsync(() -> executionService.executeSystemUserScoped(rangeEntity.getTenantId(), () -> { inventoryService.publishReindexRecordsRange(rangeEntity); return null; }), reindexPublisherExecutor); @@ -180,32 +289,72 @@ private void recreateIndices(String tenantId, List entityType } } - private List processForConsortium(String tenantId) { - List mergeRangeEntities = new ArrayList<>(); - var memberTenants = consortiumService.getConsortiumTenants(tenantId); - for (var memberTenant : memberTenants) { - mergeRangeEntities.addAll( - executionService.executeSystemUserScoped(memberTenant, () -> mergeRangeService.createMergeRanges(memberTenant)) - ); + private void ensureIndicesExist(String tenantId, List entityTypes, IndexSettings indexSettings) { + for (var reindexEntityType : entityTypes) { + reindexCommonService.ensureIndexExists(reindexEntityType, tenantId, indexSettings); + } + } + + private List processForConsortium(String tenantId, String targetTenantId) { + var mergeRangeEntities = new ArrayList(); + + if (targetTenantId != null) { + // Member tenant reindex: Process ONLY the member tenant data + // Central tenant data will be handled in upload phase + log.info("processForConsortium:: member tenant reindex - processing only [{}]", targetTenantId); + + // Set context to indicate member tenant reindex (enables staging) + ReindexContext.setMemberTenantId(targetTenantId); + + // Process ONLY the target member tenant + mergeRangeEntities.addAll(executionService.executeSystemUserScoped(targetTenantId, + () -> mergeRangeService.createMergeRanges(targetTenantId))); + + // DO NOT process central tenant here - it will be handled in upload phase + } else { + // Full consortium reindex: Process all member tenants (existing logic) + var memberTenants = consortiumService.getConsortiumTenants(tenantId); + for (var memberTenant : memberTenants) { + log.info("processForConsortium:: processing member tenant [{}]", memberTenant); + mergeRangeEntities.addAll(executionService.executeSystemUserScoped(memberTenant, + () -> mergeRangeService.createMergeRanges(memberTenant))); + } } + return mergeRangeEntities; } - private void publishRecordsRange(String tenantId) { + private void publishRecordsRange(String tenantId, String targetTenantId) { + // Capture context before async execution + final String memberTenantIdContext = ReindexContext.getMemberTenantId(); + var futures = new ArrayList<>(); for (var entityType : ReindexEntityType.supportMergeTypes()) { var rangeEntities = mergeRangeService.fetchMergeRanges(entityType); if (CollectionUtils.isNotEmpty(rangeEntities)) { - log.info("publishRecordsRange:: publishing merge ranges [tenant: {}, entityType: {}, count: {}]", - tenantId, entityType, rangeEntities.size()); + log.info("publishRecordsRange:: publishing merge ranges " + + "[requestingTenant: {}, entityType: {}, count: {}, targetTenant: {}]", tenantId, entityType, + rangeEntities.size(), targetTenantId != null ? targetTenantId : "all"); statusService.updateReindexMergeStarted(entityType, rangeEntities.size()); for (var rangeEntity : rangeEntities) { - var publishFuture = CompletableFuture.runAsync(() -> - executionService.executeSystemUserScoped(rangeEntity.getTenantId(), () -> { - inventoryService.publishReindexRecordsRange(rangeEntity); - return null; - }), reindexPublisherExecutor); + var publishFuture = CompletableFuture.runAsync(() -> { + // Restore context in executor thread + if (memberTenantIdContext != null) { + ReindexContext.setMemberTenantId(memberTenantIdContext); + } + try { + executionService.executeSystemUserScoped(rangeEntity.getTenantId(), () -> { + inventoryService.publishReindexRecordsRange(rangeEntity); + return null; + }); + } finally { + // Clean up context in executor thread + if (memberTenantIdContext != null) { + ReindexContext.clearMemberTenantId(); + } + } + }, reindexPublisherExecutor); futures.add(publishFuture); } } @@ -226,26 +375,56 @@ private void validateUploadReindex(String tenantId, List enti var mergeEntityStatus = mergeNotComplete.get(); // full reindex is either in progress or failed throw new RequestValidationException( - "Merge phase is in progress or failed for: %s".formatted(mergeEntityStatus.getKey()), - "reindexStatus", mergeEntityStatus.getValue().getValue()); + "Merge phase is in progress or failed for: %s".formatted(mergeEntityStatus.getKey()), "reindexStatus", + mergeEntityStatus.getValue().getValue()); } - var uploadInProgress = entityTypes.stream() - .filter(statusesByType::containsKey) - .filter(entityType -> statusesByType.get(entityType) == ReindexStatus.UPLOAD_IN_PROGRESS) - .findAny(); + var uploadInProgress = entityTypes.stream().filter(statusesByType::containsKey) + .filter(entityType -> statusesByType.get(entityType) == ReindexStatus.UPLOAD_IN_PROGRESS).findAny(); if (uploadInProgress.isPresent()) { - throw new RequestValidationException( - "Reindex Upload in Progress", "entityType", uploadInProgress.get().getType() - ); + throw new RequestValidationException("Reindex Upload in Progress", "entityType", + uploadInProgress.get().getType()); + } + } + + private void validateReindexNotInProgress() { + var statusesByType = statusService.getStatusesByType(); + var inProgress = statusesByType.entrySet().stream() + .filter(status -> status.getValue() == MERGE_IN_PROGRESS || status.getValue() == ReindexStatus.UPLOAD_IN_PROGRESS) + .findFirst(); + if (inProgress.isPresent()) { + var entityStatus = inProgress.get(); + throw new RequestValidationException("Reindex is already in progress for: %s".formatted(entityStatus.getKey()), + "reindexStatus", entityStatus.getValue().getValue()); } } private void validateTenant(String operation, String tenantId) { + validateTenant(operation, tenantId, null); + } + + private void validateTenant(String operation, String tenantId, String targetTenantId) { var central = consortiumService.getCentralTenant(tenantId); if (central.isPresent() && !central.get().equals(tenantId)) { log.info("{}:: could not be started for consortium member tenant [tenantId: {}]", operation, tenantId); throw RequestValidationException.memberTenantNotAllowedException(tenantId); } + + if (isBlank(targetTenantId)) { + return; + } + + if (central.isEmpty() && isNotBlank(targetTenantId)) { + log.info("{}:: could not be started for non-consortium tenant with target tenant specified [tenantId: {}, " + + "targetTenantId: {}]", operation, tenantId, targetTenantId); + throw RequestValidationException.targetTenantNotAllowedException(targetTenantId); + } + + var consortiumMembers = consortiumService.getConsortiumTenants(tenantId); + if (central.isPresent() && !consortiumMembers.contains(targetTenantId)) { + log.info("{}:: could not be started for consortium central tenant with target tenant" + + "not being a consortium member [tenantId: {}, targetTenantId: {}]", operation, tenantId, targetTenantId); + throw RequestValidationException.targetTenantNotAllowedException(targetTenantId); + } } } diff --git a/src/main/java/org/folio/search/service/reindex/ReindexStatusService.java b/src/main/java/org/folio/search/service/reindex/ReindexStatusService.java index e23cb2754..3363c130a 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexStatusService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexStatusService.java @@ -1,11 +1,14 @@ package org.folio.search.service.reindex; import static java.util.Collections.singletonList; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.folio.search.configuration.SearchCacheNames.REINDEX_TARGET_TENANT_CACHE; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.folio.search.converter.ReindexStatusMapper; import org.folio.search.domain.dto.ReindexStatusItem; @@ -15,25 +18,20 @@ import org.folio.search.model.types.ReindexStatus; import org.folio.search.service.consortium.ConsortiumTenantProvider; import org.folio.search.service.reindex.jdbc.ReindexStatusRepository; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @Log4j2 @Service +@RequiredArgsConstructor public class ReindexStatusService { private final ReindexStatusRepository statusRepository; private final ReindexStatusMapper reindexStatusMapper; private final ConsortiumTenantProvider consortiumTenantProvider; - public ReindexStatusService(ReindexStatusRepository statusRepository, - ReindexStatusMapper reindexStatusMapper, - ConsortiumTenantProvider consortiumTenantProvider) { - this.statusRepository = statusRepository; - this.reindexStatusMapper = reindexStatusMapper; - this.consortiumTenantProvider = consortiumTenantProvider; - } - public List getReindexStatuses(String tenantId) { if (consortiumTenantProvider.isMemberTenant(tenantId)) { throw RequestValidationException.memberTenantNotAllowedException(tenantId); @@ -50,19 +48,26 @@ public Map getStatusesByType() { } @Transactional - public void recreateMergeStatusRecords() { - log.info("recreateMergeStatusRecords:: recreating status records for reindex merge."); + @CacheEvict(cacheNames = REINDEX_TARGET_TENANT_CACHE, allEntries = true) + public void recreateMergeStatusRecords(String targetTenantId) { + log.info("recreateMergeStatusRecords:: recreating status records for reindex merge [targetTenant: {}].", + targetTenantId); var statusRecords = - constructNewStatusRecords(ReindexEntityType.supportMergeTypes(), ReindexStatus.MERGE_IN_PROGRESS); + constructNewStatusRecords(ReindexEntityType.supportMergeTypes(), targetTenantId); statusRepository.truncate(); + statusRepository.recreateReindexStatusTrigger(isNotBlank(targetTenantId)); statusRepository.saveReindexStatusRecords(statusRecords); } @Transactional - public void recreateUploadStatusRecord(ReindexEntityType entityType) { + public void recreateUploadStatusRecord(ReindexEntityType entityType, String targetTenantId) { var uploadStatusEntity = new ReindexStatusEntity(entityType, ReindexStatus.UPLOAD_IN_PROGRESS); + uploadStatusEntity.setTargetTenantId(targetTenantId); statusRepository.delete(entityType); statusRepository.saveReindexStatusRecords(List.of(uploadStatusEntity)); + + log.debug("recreateUploadStatusRecord:: created upload record [entityType: {}, targetTenant: {}]", + entityType, targetTenantId); } public void addProcessedMergeRanges(ReindexEntityType entityType, int processedMergeRanges) { @@ -89,6 +94,24 @@ public void updateReindexUploadFailed(ReindexEntityType entityType) { statusRepository.setReindexUploadFailed(entityType); } + public void updateStagingStarted() { + var entityTypes = ReindexEntityType.supportMergeTypes(); + log.info("updateStagingStarted:: setting staging start time for [entityTypes: {}]", entityTypes); + statusRepository.setStagingStarted(entityTypes); + } + + public void updateStagingCompleted() { + var entityTypes = ReindexEntityType.supportMergeTypes(); + log.info("updateStagingCompleted:: setting staging end time for [entityTypes: {}]", entityTypes); + statusRepository.setStagingCompleted(entityTypes); + } + + public void updateStagingFailed() { + var entityTypes = ReindexEntityType.supportMergeTypes(); + log.info("updateStagingFailed:: setting status to STAGING_FAILED and end time for [entityTypes: {}]", entityTypes); + statusRepository.setStagingFailed(entityTypes); + } + public void updateReindexMergeStarted(ReindexEntityType entityType, int totalMergeRanges) { log.info("updateReindexMergeStarted:: for [entityType: {}, totalMergeRanges: {}]", entityType, totalMergeRanges); statusRepository.setMergeReindexStarted(entityType, totalMergeRanges); @@ -100,7 +123,7 @@ public void updateReindexMergeInProgress(Set entityTypes) { } public void updateReindexUploadStarted(ReindexEntityType entityType, int totalUploadRanges) { - log.info("updateReindexUploadStarted:: for [entityType: {}, totalMergeRanges: {}]", entityType, totalUploadRanges); + log.info("updateReindexUploadStarted:: for [entityType: {}, totalUploadRanges: {}]", entityType, totalUploadRanges); statusRepository.setUploadReindexStarted(entityType, totalUploadRanges); } @@ -109,9 +132,30 @@ public boolean isMergeCompleted() { } private List constructNewStatusRecords(List entityTypes, - ReindexStatus status) { + String targetTenantId) { return entityTypes.stream() - .map(entityType -> new ReindexStatusEntity(entityType, status)) + .map(entityType -> { + var entity = new ReindexStatusEntity(entityType, ReindexStatus.MERGE_IN_PROGRESS); + entity.setTargetTenantId(targetTenantId); + return entity; + }) .toList(); } + + /** + * Gets the target tenant ID for the current reindex operation with caching. + * Cache has 10-second TTL configured in CacheConfiguration to handle high-volume Kafka events efficiently. + * Since only one reindex runs at a time per tenant, cached value remains valid for the operation duration. + * + * @return the target tenant ID if this is a tenant-specific reindex, null for full consortium reindex + */ + @Cacheable(cacheNames = REINDEX_TARGET_TENANT_CACHE, key = "@folioExecutionContext.tenantId") + public String getTargetTenantId() { + try { + return statusRepository.getTargetTenantId(); + } catch (Exception e) { + log.debug("getTargetTenantId:: error retrieving target tenant ID: {}", e.getMessage()); + return null; + } + } } diff --git a/src/main/java/org/folio/search/service/reindex/ReindexUploadRangeIndexService.java b/src/main/java/org/folio/search/service/reindex/ReindexUploadRangeIndexService.java index c82e7747f..b65608f49 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexUploadRangeIndexService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexUploadRangeIndexService.java @@ -2,6 +2,7 @@ import static java.util.function.Function.identity; import static org.apache.commons.collections4.MapUtils.getString; +import static org.folio.search.service.reindex.StagingMigrationService.RESOURCE_REINDEX_TIMESTAMP; import static org.folio.search.utils.SearchUtils.ID_FIELD; import java.sql.Timestamp; @@ -11,6 +12,7 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import lombok.extern.log4j.Log4j2; import org.folio.search.domain.dto.ResourceEvent; import org.folio.search.model.event.ReindexRangeIndexEvent; import org.folio.search.model.reindex.UploadRangeEntity; @@ -20,6 +22,7 @@ import org.folio.spring.tools.kafka.FolioMessageProducer; import org.springframework.stereotype.Service; +@Log4j2 @Service public class ReindexUploadRangeIndexService { @@ -41,14 +44,37 @@ public void prepareAndSendIndexRanges(ReindexEntityType entityType) { .orElseThrow(() -> new UnsupportedOperationException("No repository found for entity type: " + entityType)); var uploadRanges = repository.createUploadRanges(); - statusService.updateReindexUploadStarted(entityType, uploadRanges.size()); - indexRangeEventProducer.sendMessages(prepareEvents(uploadRanges)); + + // For member tenant reindex of instances, add member tenant ID to the events + if (ReindexContext.isMemberTenantReindex()) { + var memberTenantId = ReindexContext.getMemberTenantId(); + updateStatusAndSendEvents(entityType, uploadRanges.size(), memberTenantId, uploadRanges); + } else { + updateStatusAndSendEvents(entityType, uploadRanges.size(), uploadRanges); + } } public Collection fetchRecordRange(ReindexRangeIndexEvent rangeIndexEvent) { var entityType = rangeIndexEvent.getEntityType(); var repository = repositories.get(entityType); - var recordMaps = repository.fetchByIdRange(rangeIndexEvent.getLower(), rangeIndexEvent.getUpper()); + + List> recordMaps; + if (rangeIndexEvent.getMemberTenantId() != null) { + // Use timestamp-filtered range query for member tenant child resource reindex + recordMaps = repository.fetchByIdRangeWithTimestamp( + rangeIndexEvent.getLower(), + rangeIndexEvent.getUpper(), + RESOURCE_REINDEX_TIMESTAMP); + log.debug( + "fetchRecordRange:: Fetched {} records for consortium member reindex [entityType: {}, member tenant: {}]", + recordMaps.size(), entityType, rangeIndexEvent.getMemberTenantId()); + } else { + // Use regular ID range query for standard reindex + recordMaps = repository.fetchByIdRange(rangeIndexEvent.getLower(), rangeIndexEvent.getUpper()); + log.debug("fetchRecordRange:: Fetched {} records using standard ID range [entityType: {}]", + recordMaps.size(), entityType); + } + return recordMaps.stream() .map(map -> new ResourceEvent().id(getString(map, ID_FIELD)) .resourceName(ReindexConstants.RESOURCE_NAME_MAP.get(entityType).getName()) @@ -62,7 +88,19 @@ public void updateStatus(ReindexRangeIndexEvent event, ReindexRangeStatus status repository.updateRangeStatus(event.getId(), Timestamp.from(Instant.now()), status, failCause); } - private List prepareEvents(List uploadRanges) { + private void updateStatusAndSendEvents(ReindexEntityType entityType, int rangeCount, + List rangeEntities) { + updateStatusAndSendEvents(entityType, rangeCount, null, rangeEntities); + } + + private void updateStatusAndSendEvents(ReindexEntityType entityType, int rangeCount, String memberTenantId, + List rangeEntities) { + statusService.updateReindexUploadStarted(entityType, rangeCount); + var events = prepareEvents(memberTenantId, rangeEntities); + indexRangeEventProducer.sendMessages(events); + } + + private List prepareEvents(String memberTenantId, List uploadRanges) { return uploadRanges.stream() .map(range -> { var event = new ReindexRangeIndexEvent(); @@ -70,6 +108,7 @@ private List prepareEvents(List uploa event.setEntityType(range.getEntityType()); event.setLower(range.getLower()); event.setUpper(range.getUpper()); + event.setMemberTenantId(memberTenantId); return event; }) .toList(); diff --git a/src/main/java/org/folio/search/service/reindex/StagingMigrationService.java b/src/main/java/org/folio/search/service/reindex/StagingMigrationService.java new file mode 100644 index 000000000..de2a19b6f --- /dev/null +++ b/src/main/java/org/folio/search/service/reindex/StagingMigrationService.java @@ -0,0 +1,364 @@ +package org.folio.search.service.reindex; + +import static org.folio.search.utils.JdbcUtils.getSchemaName; + +import java.sql.Timestamp; +import java.util.regex.Pattern; +import lombok.extern.log4j.Log4j2; +import org.folio.search.configuration.properties.ReindexConfigurationProperties; +import org.folio.search.exception.ReindexException; +import org.folio.search.model.reindex.MigrationResult; +import org.folio.spring.FolioExecutionContext; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Log4j2 +@Service +@SuppressWarnings("java:S2077") +public class StagingMigrationService { + + protected static final Timestamp RESOURCE_REINDEX_TIMESTAMP = Timestamp.valueOf("2000-01-01 00:00:00"); + private static final Pattern WORK_MEM_PATTERN = Pattern.compile("^\\d+\\s*(KB|MB|GB)$"); + + private final JdbcTemplate jdbcTemplate; + private final FolioExecutionContext context; + private final ReindexCommonService reindexCommonService; + private final ReindexConfigurationProperties reindexConfigurationProperties; + + public StagingMigrationService(JdbcTemplate jdbcTemplate, + FolioExecutionContext context, + ReindexCommonService reindexCommonService, + ReindexConfigurationProperties reindexConfigurationProperties) { + this.jdbcTemplate = jdbcTemplate; + this.context = context; + this.reindexCommonService = reindexCommonService; + this.reindexConfigurationProperties = reindexConfigurationProperties; + } + + @Transactional + public MigrationResult migrateAllStagingTables(String targetTenantId) { + var isMemberTenantRefresh = targetTenantId != null; + var result = new MigrationResult(); + var startTime = System.currentTimeMillis(); + + try { + // Set work_mem for this transaction to optimize query performance + setWorkMem(); + + log.info("migrateAllStagingTables:: Starting migration for: [targetTenantId: {}]", targetTenantId); + + // Analyze staging tables for better query performance + analyzeStagingTables(); + + // Handle member tenant specific operations before main migration + if (isMemberTenantRefresh) { + handleMemberTenantPreMigration(targetTenantId); + } + + // Execute the main data migration phases + executeMainMigrationPhases(result); + + var duration = System.currentTimeMillis() - startTime; + result.setDuration(duration); + + log.info("migrateAllStagingTables:: Migration complete in {} ms: {} for targetTenantId: {}", + duration, result, targetTenantId); + return result; + } catch (ReindexException ex) { + log.error("migrateAllStagingTables:: Migration failed due to reindex exception for targetTenantId {}", + targetTenantId, ex); + var message = "Failed to migrate staging tables: " + ex.getMessage(); + throw new ReindexException(message, ex.getCause()); + } catch (Exception e) { + log.error("migrateAllStagingTables:: Migration failed for targetTenantId {}", targetTenantId, e); + throw new ReindexException("Failed to migrate staging tables", e); + } + } + + private void handleMemberTenantPreMigration(String targetTenantId) { + // For member tenant refresh: simply delete existing data for this tenant from main tables + log.info("handleMemberTenantPreMigration:: Clearing existing tenant data from main tables for tenant: {}", + targetTenantId); + reindexCommonService.deleteRecordsByTenantId(targetTenantId); + log.info("handleMemberTenantPreMigration:: Main table cleanup completed for tenant: {}", targetTenantId); + } + + private void executeMainMigrationPhases(MigrationResult result) { + // Phase 1: Instances + log.info("executeMainMigrationPhases:: Starting instances migration..."); + migrateInstances(result); + log.info("executeMainMigrationPhases:: Instances migration completed"); + + // Phase 2: Holdings and Items + log.info("executeMainMigrationPhases:: Starting holdings/items migration..."); + migrateHoldings(result); + log.info("executeMainMigrationPhases:: Holdings migration completed"); + + migrateItems(result); + log.info("executeMainMigrationPhases:: Items migration completed"); + + // Phase 3: Child resources (subjects, contributors, classifications, call numbers) + log.info("executeMainMigrationPhases:: Starting child resources migration..."); + migrateSubjects(result); + log.info("executeMainMigrationPhases:: Subject migration completed"); + + migrateContributors(result); + log.info("executeMainMigrationPhases:: Contributor migration completed"); + + migrateClassifications(result); + log.info("executeMainMigrationPhases:: Classification migration completed"); + + migrateCallNumbers(result); + log.info("executeMainMigrationPhases:: Call number migration completed"); + + // Phase 4: Instance/item relationships + log.info("executeMainMigrationPhases:: Starting instance relationships migration..."); + migrateInstanceSubjects(result); + log.info("executeMainMigrationPhases:: Instance-subject migration completed"); + + migrateInstanceContributors(result); + log.info("executeMainMigrationPhases:: Instance-contributor migration complete"); + + migrateInstanceClassifications(result); + log.info("executeMainMigrationPhases:: Instance-classification migration completed"); + + migrateInstanceCallNumbers(result); + log.info("executeMainMigrationPhases:: Instance-call numbers migration completed"); + } + + private void migrateInstances(MigrationResult result) { + var schema = getSchemaName(context); + var sql = String.format(""" + INSERT INTO %s.instance (id, tenant_id, shared, is_bound_with, json, last_updated_date) + SELECT id, tenant_id, shared, is_bound_with, json, ? + FROM %s.staging_instance + ORDER BY inserted_at DESC + ON CONFLICT (id) DO UPDATE SET + tenant_id = EXCLUDED.tenant_id, + shared = EXCLUDED.shared, + is_bound_with = EXCLUDED.is_bound_with, + json = EXCLUDED.json, + last_updated_date = EXCLUDED.last_updated_date + """, schema, schema); + + var recordsUpserted = jdbcTemplate.update(sql, RESOURCE_REINDEX_TIMESTAMP); + result.setTotalInstances(recordsUpserted); + + log.debug("migrateInstances:: Instance upserted: {} records", recordsUpserted); + } + + private void migrateHoldings(MigrationResult result) { + var schema = getSchemaName(context); + var sql = String.format(""" + INSERT INTO %s.holding (id, tenant_id, instance_id, json) + SELECT id, tenant_id, instance_id, json + FROM %s.staging_holding + ORDER BY inserted_at DESC + ON CONFLICT (id, tenant_id) DO UPDATE SET + instance_id = EXCLUDED.instance_id, + json = EXCLUDED.json + """, schema, schema); + + var recordsUpserted = jdbcTemplate.update(sql); + result.setTotalHoldings(recordsUpserted); + + log.debug("migrateHoldings:: Holding upserted: {} records", recordsUpserted); + } + + private void migrateItems(MigrationResult result) { + var schema = getSchemaName(context); + var sql = String.format(""" + INSERT INTO %s.item (id, tenant_id, instance_id, holding_id, json, last_updated_date) + SELECT id, tenant_id, instance_id, holding_id, json, ? + FROM %s.staging_item + ORDER BY inserted_at DESC + ON CONFLICT (id, tenant_id) DO UPDATE SET + instance_id = EXCLUDED.instance_id, + holding_id = EXCLUDED.holding_id, + json = EXCLUDED.json, + last_updated_date = EXCLUDED.last_updated_date + """, schema, schema); + + var recordsUpserted = jdbcTemplate.update(sql, RESOURCE_REINDEX_TIMESTAMP); + result.setTotalItems(recordsUpserted); + + log.debug("migrateItems:: Item upserted: {} records", recordsUpserted); + } + + private void migrateInstanceSubjects(MigrationResult result) { + var schema = getSchemaName(context); + var sql = String.format(""" + INSERT INTO %s.instance_subject (instance_id, subject_id, tenant_id, shared) + SELECT instance_id, subject_id, tenant_id, shared + FROM %s.staging_instance_subject + ORDER BY inserted_at DESC + ON CONFLICT (subject_id, instance_id, tenant_id) DO NOTHING + """, schema, schema); + + var recordsUpserted = jdbcTemplate.update(sql); + result.setTotalRelationships(result.getTotalRelationships() + recordsUpserted); + + log.debug("migrateInstanceSubjects:: Instance-subject relationship upserted: {} records", recordsUpserted); + } + + private void migrateInstanceContributors(MigrationResult result) { + var schema = getSchemaName(context); + var sql = String.format(""" + INSERT INTO %s.instance_contributor (instance_id, contributor_id, type_id, tenant_id, shared) + SELECT instance_id, contributor_id, type_id, tenant_id, shared + FROM %s.staging_instance_contributor + ORDER BY inserted_at DESC + ON CONFLICT (contributor_id, instance_id, type_id, tenant_id) DO NOTHING + """, schema, schema); + + var recordsUpserted = jdbcTemplate.update(sql); + result.setTotalRelationships(result.getTotalRelationships() + recordsUpserted); + + log.debug("migrateInstanceContributors:: Instance-contributor relationship upserted: {} records", recordsUpserted); + } + + private void migrateInstanceClassifications(MigrationResult result) { + var schema = getSchemaName(context); + var sql = String.format(""" + INSERT INTO %s.instance_classification (instance_id, classification_id, tenant_id, shared) + SELECT instance_id, classification_id, tenant_id, shared + FROM %s.staging_instance_classification + ORDER BY inserted_at DESC + ON CONFLICT (classification_id, instance_id, tenant_id) DO NOTHING + """, schema, schema); + + var recordsUpserted = jdbcTemplate.update(sql); + result.setTotalRelationships(result.getTotalRelationships() + recordsUpserted); + + log.debug("migrateInstanceClassifications:: Instance-classification relationship upserted: {} records", + recordsUpserted); + } + + private void migrateInstanceCallNumbers(MigrationResult result) { + var schema = getSchemaName(context); + var sql = String.format(""" + INSERT INTO %s.instance_call_number (call_number_id, item_id, instance_id, tenant_id, location_id) + SELECT call_number_id, item_id, instance_id, tenant_id, location_id + FROM %s.staging_instance_call_number + ORDER BY inserted_at DESC + ON CONFLICT (call_number_id, item_id, instance_id, tenant_id) DO NOTHING + """, schema, schema); + + var recordsUpserted = jdbcTemplate.update(sql); + result.setTotalRelationships(result.getTotalRelationships() + recordsUpserted); + + log.debug("migrateInstanceCallNumbers:: Instance-call number relationship upserted: {} records", recordsUpserted); + } + + private void migrateSubjects(MigrationResult result) { + var schema = getSchemaName(context); + var sql = String.format(""" + INSERT INTO %s.subject (id, value, authority_id, source_id, type_id, last_updated_date) + SELECT id, value, authority_id, source_id, type_id, ? + FROM %s.staging_subject + ON CONFLICT (id) DO UPDATE + SET last_updated_date = EXCLUDED.last_updated_date + """, schema, schema); + + var recordsUpserted = jdbcTemplate.update(sql, RESOURCE_REINDEX_TIMESTAMP); + result.setTotalRelationships(result.getTotalRelationships() + recordsUpserted); + + log.debug("migrateSubjects:: Subject upserted: {} records", recordsUpserted); + } + + private void migrateContributors(MigrationResult result) { + var schema = getSchemaName(context); + var sql = String.format(""" + INSERT INTO %s.contributor (id, name, name_type_id, authority_id, last_updated_date) + SELECT id, name, name_type_id, authority_id, ? + FROM %s.staging_contributor + ON CONFLICT (id) DO UPDATE + SET last_updated_date = EXCLUDED.last_updated_date + """, schema, schema); + + var recordsUpserted = jdbcTemplate.update(sql, RESOURCE_REINDEX_TIMESTAMP); + result.setTotalRelationships(result.getTotalRelationships() + recordsUpserted); + + log.debug("migrateContributors:: Contributor upserted: {} records", recordsUpserted); + } + + private void migrateClassifications(MigrationResult result) { + var schema = getSchemaName(context); + var sql = String.format(""" + INSERT INTO %s.classification (id, number, type_id, last_updated_date) + SELECT id, number, type_id, ? + FROM %s.staging_classification + ON CONFLICT (id) DO UPDATE + SET last_updated_date = EXCLUDED.last_updated_date + """, schema, schema); + + var recordsUpserted = jdbcTemplate.update(sql, RESOURCE_REINDEX_TIMESTAMP); + result.setTotalRelationships(result.getTotalRelationships() + recordsUpserted); + + log.debug("migrateClassifications:: Classification upserted: {} records", recordsUpserted); + } + + private void migrateCallNumbers(MigrationResult result) { + var schema = getSchemaName(context); + var sql = String.format(""" + INSERT INTO %s.call_number + (id, call_number, call_number_prefix, call_number_suffix, call_number_type_id, last_updated_date) + SELECT id, call_number, call_number_prefix, call_number_suffix, call_number_type_id, ? + FROM %s.staging_call_number + ON CONFLICT (id) DO UPDATE + SET last_updated_date = EXCLUDED.last_updated_date + """, schema, schema); + + var recordsUpserted = jdbcTemplate.update(sql, RESOURCE_REINDEX_TIMESTAMP); + result.setTotalRelationships(result.getTotalRelationships() + recordsUpserted); + + log.debug("migrateCallNumbers:: Call number upserted: {} records", recordsUpserted); + } + + private void analyzeStagingTables() { + var schema = getSchemaName(context); + + log.info("analyzeStagingTables:: Analyzing staging tables for better query performance..."); + jdbcTemplate.execute(String.format("ANALYZE %s.staging_instance", schema)); + jdbcTemplate.execute(String.format("ANALYZE %s.staging_holding", schema)); + jdbcTemplate.execute(String.format("ANALYZE %s.staging_item", schema)); + jdbcTemplate.execute(String.format("ANALYZE %s.staging_instance_subject", schema)); + jdbcTemplate.execute(String.format("ANALYZE %s.staging_instance_contributor", schema)); + jdbcTemplate.execute(String.format("ANALYZE %s.staging_instance_classification", schema)); + jdbcTemplate.execute(String.format("ANALYZE %s.staging_instance_call_number", schema)); + jdbcTemplate.execute(String.format("ANALYZE %s.staging_subject", schema)); + jdbcTemplate.execute(String.format("ANALYZE %s.staging_contributor", schema)); + jdbcTemplate.execute(String.format("ANALYZE %s.staging_classification", schema)); + jdbcTemplate.execute(String.format("ANALYZE %s.staging_call_number", schema)); + log.info("analyzeStagingTables:: Staging tables analyzed"); + } + + /** + * Sets the PostgreSQL work_mem parameter for the current transaction using SET LOCAL. + * This optimizes query performance for memory-intensive operations during migration. + * + * @throws ReindexException if the work_mem value is invalid or the SET LOCAL command fails + */ + private void setWorkMem() { + var workMemValue = reindexConfigurationProperties.getMigrationWorkMem(); + + // Validate the work_mem format for security + if (!WORK_MEM_PATTERN.matcher(workMemValue).matches()) { + throw new ReindexException("Invalid work_mem format: " + workMemValue + + ". Must be a number followed by KB, MB, or GB (e.g., '64MB', '512KB', '1GB')"); + } + + log.info("setWorkMem:: Setting work_mem to {} for migration transaction", workMemValue); + + try { + var sql = String.format("SET LOCAL work_mem = '%s'", workMemValue); + jdbcTemplate.execute(sql); + log.debug("setWorkMem:: Successfully set work_mem to {}", workMemValue); + } catch (Exception e) { + var errorMsg = "Failed to set work_mem to " + workMemValue + " for migration transaction"; + log.error("setWorkMem:: " + errorMsg, e); + throw new ReindexException(errorMsg, e); + } + } +} diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java index e65d067be..0f98a2b6b 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java @@ -3,6 +3,8 @@ import static org.apache.commons.collections4.MapUtils.getString; import static org.folio.search.service.reindex.ReindexConstants.CALL_NUMBER_TABLE; import static org.folio.search.service.reindex.ReindexConstants.INSTANCE_CALL_NUMBER_TABLE; +import static org.folio.search.service.reindex.ReindexConstants.STAGING_CALL_NUMBER_TABLE; +import static org.folio.search.service.reindex.ReindexConstants.STAGING_INSTANCE_CALL_NUMBER_TABLE; import static org.folio.search.utils.CallNumberUtils.calculateFullCallNumber; import static org.folio.search.utils.JdbcUtils.getFullTableName; import static org.folio.search.utils.JdbcUtils.getParamPlaceholderForUuid; @@ -27,6 +29,7 @@ import org.folio.search.configuration.properties.ReindexConfigurationProperties; import org.folio.search.model.entity.ChildResourceEntityBatch; import org.folio.search.model.types.ReindexEntityType; +import org.folio.search.service.reindex.ReindexContext; import org.folio.search.utils.JdbcUtils; import org.folio.search.utils.JsonConverter; import org.folio.spring.FolioExecutionContext; @@ -37,6 +40,7 @@ @Log4j2 @Repository +@SuppressWarnings("java:S2077") public class CallNumberRepository extends UploadRangeRepository implements InstanceChildResourceRepository { private static final String SELECT_QUERY = """ @@ -145,6 +149,17 @@ LEFT JOIN ( ON CONFLICT (id) DO UPDATE SET last_updated_date = CURRENT_TIMESTAMP; """; + private static final String INSERT_STAGING_ENTITIES_SQL = """ + INSERT INTO %s ( + id, + call_number, + call_number_prefix, + call_number_suffix, + call_number_type_id, + inserted_at + ) VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP); + """; + private static final String INSERT_RELATIONS_SQL = """ INSERT INTO %s ( call_number_id, @@ -156,6 +171,17 @@ LEFT JOIN ( ON CONFLICT DO NOTHING; """; + private static final String INSERT_STAGING_RELATIONS_SQL = """ + INSERT INTO %s ( + call_number_id, + item_id, + instance_id, + tenant_id, + location_id, + inserted_at + ) VALUES (?, ?::uuid, ?::uuid, ?, ?::uuid, CURRENT_TIMESTAMP); + """; + private static final String ID_RANGE_INS_WHERE_CLAUSE = "ins.call_number_id >= ? AND ins.call_number_id <= ?"; private static final String ID_RANGE_CLAS_WHERE_CLAUSE = "c.id >= ? AND c.id <= ?"; @@ -176,8 +202,14 @@ public void deleteByInstanceIds(List itemIds, String tenantId) { @Override public void saveAll(ChildResourceEntityBatch entityBatch) { - saveResourceEntities(entityBatch); - saveRelationshipEntities(entityBatch); + // Use staging tables only for member tenant specific full reindex + if (ReindexContext.isReindexMode() && ReindexContext.isMemberTenantReindex()) { + saveResourceEntitiesToStaging(entityBatch); + saveRelationshipEntitiesToStaging(entityBatch); + } else { + saveResourceEntities(entityBatch); + saveRelationshipEntities(entityBatch); + } } @Override @@ -195,6 +227,22 @@ protected Optional subEntityTable() { return Optional.of(INSTANCE_CALL_NUMBER_TABLE); } + @Override + protected Optional stagingEntityTable() { + return Optional.of(STAGING_CALL_NUMBER_TABLE); + } + + @Override + protected Optional subEntityStagingTable() { + return Optional.of(STAGING_INSTANCE_CALL_NUMBER_TABLE); + } + + @Override + protected boolean supportsTenantSpecificDeletion() { + // Call number table doesn't have tenant_id column - it's shared across tenants + return false; + } + @Override public List> fetchByIdRange(String lower, String upper) { var sql = getFetchBySql(); @@ -280,6 +328,33 @@ private void saveResourceEntities(ChildResourceEntityBatch entityBatch) { } } + private void saveResourceEntitiesToStaging(ChildResourceEntityBatch entityBatch) { + var stagingCallNumberTable = getFullTableName(context, STAGING_CALL_NUMBER_TABLE); + var stagingCallNumberSql = INSERT_STAGING_ENTITIES_SQL.formatted(stagingCallNumberTable); + + try { + jdbcTemplate.batchUpdate(stagingCallNumberSql, entityBatch.resourceEntities(), BATCH_OPERATION_SIZE, + (statement, entity) -> { + statement.setString(1, getId(entity)); + statement.setString(2, getCallNumber(entity)); + statement.setString(3, getPrefix(entity)); + statement.setString(4, getSuffix(entity)); + statement.setString(5, getTypeId(entity)); + }); + } catch (DataAccessException e) { + log.warn("saveResourceEntitiesToStaging::Failed to save entities batch. Processing one-by-one", e); + for (var entity : entityBatch.resourceEntities()) { + try { + jdbcTemplate.update(stagingCallNumberSql, getId(entity), getCallNumber(entity), getPrefix(entity), + getSuffix(entity), getTypeId(entity)); + } catch (DataAccessException ex) { + log.debug("Failed to save staging call number entity {}: {}", getId(entity), ex.getMessage()); + } + } + } + log.debug("Saved {} call number entities to staging table", entityBatch.resourceEntities().size()); + } + private void saveRelationshipEntities(ChildResourceEntityBatch entityBatch) { var instanceCallNumberTable = getFullTableName(context, INSTANCE_CALL_NUMBER_TABLE); var instanceCallNumberSql = INSERT_RELATIONS_SQL.formatted(instanceCallNumberTable); @@ -294,12 +369,45 @@ private void saveRelationshipEntities(ChildResourceEntityBatch entityBatch) { statement.setString(5, getLocationId(entity)); }); } catch (DataAccessException e) { - log.warn("saveAll::Failed to save relations batch. Starting processing one-by-one", e); + log.warn("saveRelationshipEntities::Failed to save relations batch. Processing one-by-one", e); + for (var entityRelation : entityBatch.relationshipEntities()) { + try { + jdbcTemplate.update(instanceCallNumberSql, getCallNumberId(entityRelation), getItemId(entityRelation), + getInstanceId(entityRelation), getTenantId(entityRelation), getLocationId(entityRelation)); + } catch (DataAccessException ex) { + log.debug("Failed to save call number relationship for {}: {}", + getCallNumberId(entityRelation), ex.getMessage()); + } + } + } + } + + private void saveRelationshipEntitiesToStaging(ChildResourceEntityBatch entityBatch) { + var stagingInstanceCallNumberTable = getFullTableName(context, STAGING_INSTANCE_CALL_NUMBER_TABLE); + var stagingInstanceCallNumberSql = INSERT_STAGING_RELATIONS_SQL.formatted(stagingInstanceCallNumberTable); + + try { + jdbcTemplate.batchUpdate(stagingInstanceCallNumberSql, entityBatch.relationshipEntities(), BATCH_OPERATION_SIZE, + (statement, entity) -> { + statement.setString(1, getCallNumberId(entity)); + statement.setString(2, getItemId(entity)); + statement.setString(3, getInstanceId(entity)); + statement.setString(4, getTenantId(entity)); + statement.setString(5, getLocationId(entity)); + }); + } catch (DataAccessException e) { + log.warn("saveRelationshipEntitiesToStaging::Failed to save staging relations batch. Processing one-by-one", e); for (var entityRelation : entityBatch.relationshipEntities()) { - jdbcTemplate.update(instanceCallNumberSql, getCallNumberId(entityRelation), getItemId(entityRelation), - getInstanceId(entityRelation), getTenantId(entityRelation), getLocationId(entityRelation)); + try { + jdbcTemplate.update(stagingInstanceCallNumberSql, getCallNumberId(entityRelation), getItemId(entityRelation), + getInstanceId(entityRelation), getTenantId(entityRelation), getLocationId(entityRelation)); + } catch (DataAccessException ex) { + log.debug("Failed to save staging call number relationship for {}: {}", + getCallNumberId(entityRelation), ex.getMessage()); + } } } + log.debug("Saved {} call number relationships to staging table", entityBatch.relationshipEntities().size()); } private String getCallNumberSuffix(ResultSet rs) throws SQLException { @@ -346,6 +454,14 @@ private String getItemId(Map item) { return getString(item, "itemId"); } + @Override + public List> fetchByIdRangeWithTimestamp(String lower, String upper, Timestamp timestamp) { + var sql = SELECT_QUERY.formatted(JdbcUtils.getSchemaName(context), + ID_RANGE_INS_WHERE_CLAUSE, + ID_RANGE_CLAS_WHERE_CLAUSE + " AND c.last_updated_date = ?"); + return jdbcTemplate.query(sql, rowToMapMapper(), lower, upper, lower, upper, timestamp); + } + private String getId(Map item) { return getString(item, "id"); } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java index 3cfce24b5..944d62fb5 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java @@ -19,6 +19,7 @@ import org.folio.search.model.entity.ChildResourceEntityBatch; import org.folio.search.model.types.ReindexEntityType; import org.folio.search.service.reindex.ReindexConstants; +import org.folio.search.service.reindex.ReindexContext; import org.folio.search.utils.JdbcUtils; import org.folio.search.utils.JsonConverter; import org.folio.spring.FolioExecutionContext; @@ -29,6 +30,7 @@ @Log4j2 @Repository +@SuppressWarnings("java:S2077") public class ClassificationRepository extends UploadRangeRepository implements InstanceChildResourceRepository { private static final String SELECT_QUERY = """ @@ -123,12 +125,21 @@ WHERE instance_id IN (%2$s) %3$s VALUES (?, ?, ?) ON CONFLICT (id) DO UPDATE SET last_updated_date = CURRENT_TIMESTAMP; """; + private static final String INSERT_STAGING_ENTITIES_SQL = """ + INSERT INTO %s.staging_classification (id, number, type_id, inserted_at) + VALUES (?, ?, ?, CURRENT_TIMESTAMP); + """; private static final String INSERT_RELATIONS_SQL = """ INSERT INTO %s.instance_classification (instance_id, classification_id, tenant_id, shared) VALUES (?::uuid, ?, ?, ?) ON CONFLICT DO NOTHING; """; + private static final String INSERT_STAGING_RELATIONS_SQL = """ + INSERT INTO %s.staging_instance_classification (instance_id, classification_id, tenant_id, shared, inserted_at) + VALUES (?::uuid, ?, ?, ?, CURRENT_TIMESTAMP); + """; + private static final String ID_RANGE_INS_WHERE_CLAUSE = "ins.classification_id >= ? AND ins.classification_id <= ?"; private static final String ID_RANGE_CLAS_WHERE_CLAUSE = "c.id >= ? AND c.id <= ?"; @@ -154,6 +165,22 @@ protected Optional subEntityTable() { return Optional.of(ReindexConstants.INSTANCE_CLASSIFICATION_TABLE); } + @Override + protected Optional stagingEntityTable() { + return Optional.of(ReindexConstants.STAGING_CLASSIFICATION_TABLE); + } + + @Override + protected Optional subEntityStagingTable() { + return Optional.of(ReindexConstants.STAGING_INSTANCE_CLASSIFICATION_TABLE); + } + + @Override + protected boolean supportsTenantSpecificDeletion() { + // Classification table doesn't have tenant_id column - it's shared across tenants + return false; + } + @Override public List> fetchByIdRange(String lower, String upper) { var sql = getFetchBySql(); @@ -206,9 +233,20 @@ public void deleteByInstanceIds(List instanceIds, String tenantId) { @Override public void saveAll(ChildResourceEntityBatch entityBatch) { + // Use staging tables only for member tenant specific full reindex + if (ReindexContext.isReindexMode() && ReindexContext.isMemberTenantReindex()) { + saveEntitiesToStaging(entityBatch.resourceEntities().stream().toList()); + saveRelationshipsToStaging(entityBatch.relationshipEntities().stream().toList()); + } else { + saveEntitiesToMain(entityBatch.resourceEntities().stream().toList()); + saveRelationshipsToMain(entityBatch.relationshipEntities().stream().toList()); + } + } + + private void saveEntitiesToMain(List> entities) { var entitiesSql = INSERT_ENTITIES_SQL.formatted(JdbcUtils.getSchemaName(context)); try { - jdbcTemplate.batchUpdate(entitiesSql, entityBatch.resourceEntities(), BATCH_OPERATION_SIZE, + jdbcTemplate.batchUpdate(entitiesSql, entities, BATCH_OPERATION_SIZE, (statement, entity) -> { statement.setString(1, (String) entity.get("id")); statement.setString(2, (String) entity.get(CLASSIFICATION_NUMBER_FIELD)); @@ -216,15 +254,44 @@ public void saveAll(ChildResourceEntityBatch entityBatch) { }); } catch (DataAccessException e) { logWarnDebugError(SAVE_ENTITIES_BATCH_ERROR_MESSAGE, e); - for (var entity : entityBatch.resourceEntities()) { - jdbcTemplate.update(entitiesSql, - entity.get("id"), entity.get(CLASSIFICATION_NUMBER_FIELD), entity.get(CLASSIFICATION_TYPE_FIELD)); + for (var entity : entities) { + try { + jdbcTemplate.update(entitiesSql, + entity.get("id"), entity.get(CLASSIFICATION_NUMBER_FIELD), entity.get(CLASSIFICATION_TYPE_FIELD)); + } catch (DataAccessException ex) { + log.debug("Failed to save classification entity {}: {}", entity.get("id"), ex.getMessage()); + } + } + } + } + + private void saveEntitiesToStaging(List> entities) { + var stagingEntitiesSql = INSERT_STAGING_ENTITIES_SQL.formatted(JdbcUtils.getSchemaName(context)); + try { + jdbcTemplate.batchUpdate(stagingEntitiesSql, entities, BATCH_OPERATION_SIZE, + (statement, entity) -> { + statement.setString(1, (String) entity.get("id")); + statement.setString(2, (String) entity.get(CLASSIFICATION_NUMBER_FIELD)); + statement.setString(3, (String) entity.get(CLASSIFICATION_TYPE_FIELD)); + }); + } catch (DataAccessException e) { + log.warn("saveEntitiesToStaging::Failed to save entities batch. Processing one-by-one", e); + for (var entity : entities) { + try { + jdbcTemplate.update(stagingEntitiesSql, entity.get("id"), + entity.get(CLASSIFICATION_NUMBER_FIELD), entity.get(CLASSIFICATION_TYPE_FIELD)); + } catch (DataAccessException ex) { + log.debug("Failed to save staging classification entity {}: {}", entity.get("id"), ex.getMessage()); + } } } + log.debug("Saved {} classification entities to staging table", entities.size()); + } + private void saveRelationshipsToMain(List> relationships) { var relationsSql = INSERT_RELATIONS_SQL.formatted(JdbcUtils.getSchemaName(context)); try { - jdbcTemplate.batchUpdate(relationsSql, entityBatch.relationshipEntities(), BATCH_OPERATION_SIZE, + jdbcTemplate.batchUpdate(relationsSql, relationships, BATCH_OPERATION_SIZE, (statement, entityRelation) -> { statement.setObject(1, entityRelation.get("instanceId")); statement.setString(2, (String) entityRelation.get("classificationId")); @@ -233,13 +300,43 @@ public void saveAll(ChildResourceEntityBatch entityBatch) { }); } catch (DataAccessException e) { logWarnDebugError(SAVE_RELATIONS_BATCH_ERROR_MESSAGE, e); - for (var entityRelation : entityBatch.relationshipEntities()) { - jdbcTemplate.update(relationsSql, entityRelation.get("instanceId"), entityRelation.get("classificationId"), - entityRelation.get("tenantId"), entityRelation.get("shared")); + for (var entityRelation : relationships) { + try { + jdbcTemplate.update(relationsSql, entityRelation.get("instanceId"), entityRelation.get("classificationId"), + entityRelation.get("tenantId"), entityRelation.get("shared")); + } catch (DataAccessException ex) { + log.debug("Failed to save classification relationship for {}: {}", + entityRelation.get("classificationId"), ex.getMessage()); + } } } } + private void saveRelationshipsToStaging(List> relationships) { + var stagingRelationsSql = INSERT_STAGING_RELATIONS_SQL.formatted(JdbcUtils.getSchemaName(context)); + try { + jdbcTemplate.batchUpdate(stagingRelationsSql, relationships, BATCH_OPERATION_SIZE, + (statement, entityRelation) -> { + statement.setObject(1, entityRelation.get("instanceId")); + statement.setString(2, (String) entityRelation.get("classificationId")); + statement.setString(3, (String) entityRelation.get("tenantId")); + statement.setObject(4, entityRelation.get("shared")); + }); + } catch (DataAccessException e) { + log.warn("saveRelationshipsToStaging::Failed to save relationships batch. Processing one-by-one", e); + for (var entityRelation : relationships) { + try { + jdbcTemplate.update(stagingRelationsSql, entityRelation.get("instanceId"), + entityRelation.get("classificationId"), entityRelation.get("tenantId"), entityRelation.get("shared")); + } catch (DataAccessException ex) { + log.debug("Failed to save staging classification relationship for {}: {}", + entityRelation.get("classificationId"), ex.getMessage()); + } + } + } + log.debug("Saved {} classification relationships to staging table", relationships.size()); + } + protected RowMapper> rowToMapMapper2() { return (rs, rowNum) -> { Map classification = new HashMap<>(); @@ -257,6 +354,14 @@ protected RowMapper> rowToMapMapper2() { }; } + @Override + public List> fetchByIdRangeWithTimestamp(String lower, String upper, Timestamp timestamp) { + var sql = SELECT_QUERY.formatted(JdbcUtils.getSchemaName(context), + ID_RANGE_INS_WHERE_CLAUSE, + ID_RANGE_CLAS_WHERE_CLAUSE + " AND c.last_updated_date = ?"); + return jdbcTemplate.query(sql, rowToMapMapper(), lower, upper, lower, upper, timestamp); + } + private String getId(ResultSet rs) throws SQLException { return rs.getString("id"); } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java index 64f4ce075..df5a81976 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java @@ -18,6 +18,7 @@ import org.folio.search.model.entity.ChildResourceEntityBatch; import org.folio.search.model.types.ReindexEntityType; import org.folio.search.service.reindex.ReindexConstants; +import org.folio.search.service.reindex.ReindexContext; import org.folio.search.utils.JdbcUtils; import org.folio.search.utils.JsonConverter; import org.folio.spring.FolioExecutionContext; @@ -28,6 +29,7 @@ @Log4j2 @Repository +@SuppressWarnings("java:S2077") public class ContributorRepository extends UploadRangeRepository implements InstanceChildResourceRepository { private static final String SELECT_QUERY = """ @@ -129,12 +131,21 @@ WHERE instance_id IN (%2$s) %3$s VALUES (?, ?, ?, ?) ON CONFLICT (id) DO UPDATE SET last_updated_date = CURRENT_TIMESTAMP; """; + private static final String INSERT_STAGING_ENTITIES_SQL = """ + INSERT INTO %s.staging_contributor (id, name, name_type_id, authority_id, inserted_at) + VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP); + """; private static final String INSERT_RELATIONS_SQL = """ INSERT INTO %s.instance_contributor (instance_id, contributor_id, type_id, tenant_id, shared) VALUES (?::uuid, ?, ?, ?, ?) ON CONFLICT DO NOTHING; """; + private static final String INSERT_STAGING_RELATIONS_SQL = """ + INSERT INTO %s.staging_instance_contributor (instance_id, contributor_id, type_id, tenant_id, shared, inserted_at) + VALUES (?::uuid, ?, ?, ?, ?, CURRENT_TIMESTAMP); + """; + private static final String ID_RANGE_INS_WHERE_CLAUSE = "ins.contributor_id >= ? AND ins.contributor_id <= ?"; private static final String ID_RANGE_CONTR_WHERE_CLAUSE = "c.id >= ? AND c.id <= ?"; @@ -159,6 +170,22 @@ protected Optional subEntityTable() { return Optional.of(ReindexConstants.INSTANCE_CONTRIBUTOR_TABLE); } + @Override + protected Optional stagingEntityTable() { + return Optional.of(ReindexConstants.STAGING_CONTRIBUTOR_TABLE); + } + + @Override + protected Optional subEntityStagingTable() { + return Optional.of(ReindexConstants.STAGING_INSTANCE_CONTRIBUTOR_TABLE); + } + + @Override + protected boolean supportsTenantSpecificDeletion() { + // Contributor table doesn't have tenant_id column - it's shared across tenants + return false; + } + @Override public List> fetchByIdRange(String lower, String upper) { var sql = getFetchBySql(); @@ -229,9 +256,20 @@ public void deleteByInstanceIds(List instanceIds, String tenantId) { @Override public void saveAll(ChildResourceEntityBatch entityBatch) { + // Use staging tables only for member tenant specific full reindex + if (ReindexContext.isReindexMode() && ReindexContext.isMemberTenantReindex()) { + saveEntitiesToStaging(entityBatch.resourceEntities().stream().toList()); + saveRelationshipsToStaging(entityBatch.relationshipEntities().stream().toList()); + } else { + saveEntitiesToMain(entityBatch.resourceEntities().stream().toList()); + saveRelationshipsToMain(entityBatch.relationshipEntities().stream().toList()); + } + } + + private void saveEntitiesToMain(List> entities) { var entitiesSql = INSERT_ENTITIES_SQL.formatted(JdbcUtils.getSchemaName(context)); try { - jdbcTemplate.batchUpdate(entitiesSql, entityBatch.resourceEntities(), BATCH_OPERATION_SIZE, + jdbcTemplate.batchUpdate(entitiesSql, entities, BATCH_OPERATION_SIZE, (statement, entity) -> { statement.setString(1, (String) entity.get("id")); statement.setString(2, (String) entity.get("name")); @@ -240,15 +278,45 @@ public void saveAll(ChildResourceEntityBatch entityBatch) { }); } catch (DataAccessException e) { logWarnDebugError(SAVE_ENTITIES_BATCH_ERROR_MESSAGE, e); - for (var entity : entityBatch.resourceEntities()) { - jdbcTemplate.update(entitiesSql, - entity.get("id"), entity.get("name"), entity.get("nameTypeId"), entity.get(AUTHORITY_ID_FIELD)); + for (var entity : entities) { + try { + jdbcTemplate.update(entitiesSql, + entity.get("id"), entity.get("name"), entity.get("nameTypeId"), entity.get(AUTHORITY_ID_FIELD)); + } catch (DataAccessException ex) { + log.debug("Failed to save contributor entity {}: {}", entity.get("id"), ex.getMessage()); + } + } + } + } + + private void saveEntitiesToStaging(List> entities) { + var stagingEntitiesSql = INSERT_STAGING_ENTITIES_SQL.formatted(JdbcUtils.getSchemaName(context)); + try { + jdbcTemplate.batchUpdate(stagingEntitiesSql, entities, BATCH_OPERATION_SIZE, + (statement, entity) -> { + statement.setString(1, (String) entity.get("id")); + statement.setString(2, (String) entity.get("name")); + statement.setString(3, (String) entity.get("nameTypeId")); + statement.setString(4, (String) entity.get(AUTHORITY_ID_FIELD)); + }); + } catch (DataAccessException e) { + log.warn("saveEntitiesToStaging::Failed to save entities batch. Processing one-by-one", e); + for (var entity : entities) { + try { + jdbcTemplate.update(stagingEntitiesSql, entity.get("id"), entity.get("name"), + entity.get("nameTypeId"), entity.get(AUTHORITY_ID_FIELD)); + } catch (DataAccessException ex) { + log.debug("Failed to save staging contributor entity {}: {}", entity.get("id"), ex.getMessage()); + } } } + log.debug("Saved {} contributor entities to staging table", entities.size()); + } + private void saveRelationshipsToMain(List> relationships) { var relationsSql = INSERT_RELATIONS_SQL.formatted(JdbcUtils.getSchemaName(context)); try { - jdbcTemplate.batchUpdate(relationsSql, entityBatch.relationshipEntities(), BATCH_OPERATION_SIZE, + jdbcTemplate.batchUpdate(relationsSql, relationships, BATCH_OPERATION_SIZE, (statement, entityRelation) -> { statement.setObject(1, entityRelation.get("instanceId")); statement.setString(2, (String) entityRelation.get("contributorId")); @@ -258,13 +326,53 @@ public void saveAll(ChildResourceEntityBatch entityBatch) { }); } catch (DataAccessException e) { logWarnDebugError(SAVE_RELATIONS_BATCH_ERROR_MESSAGE, e); - for (var entityRelation : entityBatch.relationshipEntities()) { - jdbcTemplate.update(relationsSql, entityRelation.get("instanceId"), entityRelation.get("contributorId"), - entityRelation.get(CONTRIBUTOR_TYPE_FIELD), entityRelation.get("tenantId"), entityRelation.get("shared")); + for (var entityRelation : relationships) { + try { + jdbcTemplate.update(relationsSql, entityRelation.get("instanceId"), entityRelation.get("contributorId"), + entityRelation.get(CONTRIBUTOR_TYPE_FIELD), entityRelation.get("tenantId"), entityRelation.get("shared")); + } catch (DataAccessException ex) { + log.debug("Failed to save contributor relationship for {}: {}", + entityRelation.get("contributorId"), ex.getMessage()); + } } } } + private void saveRelationshipsToStaging(List> relationships) { + var stagingRelationsSql = INSERT_STAGING_RELATIONS_SQL.formatted(JdbcUtils.getSchemaName(context)); + try { + jdbcTemplate.batchUpdate(stagingRelationsSql, relationships, BATCH_OPERATION_SIZE, + (statement, entityRelation) -> { + statement.setObject(1, entityRelation.get("instanceId")); + statement.setString(2, (String) entityRelation.get("contributorId")); + statement.setString(3, (String) entityRelation.get(CONTRIBUTOR_TYPE_FIELD)); + statement.setString(4, (String) entityRelation.get("tenantId")); + statement.setObject(5, entityRelation.get("shared")); + }); + } catch (DataAccessException e) { + log.warn("saveRelationshipsToStaging::Failed to save relationships batch. Processing one-by-one", e); + for (var entityRelation : relationships) { + try { + jdbcTemplate.update(stagingRelationsSql, entityRelation.get("instanceId"), + entityRelation.get("contributorId"), entityRelation.get(CONTRIBUTOR_TYPE_FIELD), + entityRelation.get("tenantId"), entityRelation.get("shared")); + } catch (DataAccessException ex) { + log.debug("Failed to save staging contributor relationship for {}: {}", + entityRelation.get("contributorId"), ex.getMessage()); + } + } + } + log.debug("Saved {} contributor relationships to staging table", relationships.size()); + } + + @Override + public List> fetchByIdRangeWithTimestamp(String lower, String upper, Timestamp timestamp) { + var sql = SELECT_QUERY.formatted(JdbcUtils.getSchemaName(context), + ID_RANGE_INS_WHERE_CLAUSE, + ID_RANGE_CONTR_WHERE_CLAUSE + " AND c.last_updated_date = ?"); + return jdbcTemplate.query(sql, rowToMapMapper(), lower, upper, lower, upper, timestamp); + } + private String getId(ResultSet rs) throws SQLException { return rs.getString("id"); } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/HoldingRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/HoldingRepository.java index 8289b24e6..ea29b8dee 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/HoldingRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/HoldingRepository.java @@ -4,14 +4,18 @@ import java.util.List; import java.util.Map; +import java.util.Optional; +import lombok.extern.log4j.Log4j2; import org.folio.search.configuration.properties.SearchConfigurationProperties; import org.folio.search.model.types.ReindexEntityType; import org.folio.search.service.reindex.ReindexConstants; +import org.folio.search.service.reindex.ReindexContext; import org.folio.search.utils.JsonConverter; import org.folio.spring.FolioExecutionContext; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Repository; +@Log4j2 @Repository public class HoldingRepository extends MergeRangeRepository { @@ -25,6 +29,11 @@ ON CONFLICT (id, tenant_id) json = EXCLUDED.json; """; + private static final String INSERT_STAGING_SQL = """ + INSERT INTO %s (id, tenant_id, instance_id, json, inserted_at) + VALUES (?::uuid, ?, ?::uuid, ?::jsonb, CURRENT_TIMESTAMP); + """; + protected HoldingRepository(JdbcTemplate jdbcTemplate, JsonConverter jsonConverter, FolioExecutionContext context, SearchConfigurationProperties searchConfigurationProperties) { super(jdbcTemplate, jsonConverter, context, searchConfigurationProperties); @@ -37,6 +46,14 @@ public ReindexEntityType entityType() { @Override public void saveEntities(String tenantId, List> entities) { + if (ReindexContext.isReindexMode() && ReindexContext.getMemberTenantId() != null) { + saveEntitiesToStaging(tenantId, entities); + } else { + saveEntitiesToMain(tenantId, entities); + } + } + + private void saveEntitiesToMain(String tenantId, List> entities) { var fullTableName = getFullTableName(context, entityTable()); var sql = INSERT_SQL.formatted(fullTableName); @@ -49,6 +66,22 @@ public void saveEntities(String tenantId, List> entities) { }); } + @SuppressWarnings("java:S2077") + private void saveEntitiesToStaging(String tenantId, List> entities) { + var fullTableName = getFullTableName(context, ReindexConstants.STAGING_HOLDING_TABLE); + var sql = INSERT_STAGING_SQL.formatted(fullTableName); + + jdbcTemplate.batchUpdate(sql, entities, BATCH_OPERATION_SIZE, + (statement, entity) -> { + statement.setObject(1, entity.get("id")); + statement.setString(2, tenantId); + statement.setObject(3, entity.get("instanceId")); + statement.setString(4, jsonConverter.toJson(entity)); + }); + + log.debug("Saved {} entities to staging table {}", entities.size(), ReindexConstants.STAGING_HOLDING_TABLE); + } + @Override public void deleteEntitiesForTenant(List ids, String tenantId) { deleteEntitiesForTenant(ids, tenantId, true); @@ -63,4 +96,9 @@ public void deleteEntities(List ids) { protected String entityTable() { return ReindexConstants.HOLDING_TABLE; } + + @Override + protected Optional stagingEntityTable() { + return Optional.of(ReindexConstants.STAGING_HOLDING_TABLE); + } } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ItemRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ItemRepository.java index b953ed751..013edbc8d 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ItemRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ItemRepository.java @@ -7,15 +7,19 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import lombok.extern.log4j.Log4j2; import org.folio.search.configuration.properties.SearchConfigurationProperties; import org.folio.search.model.types.ReindexEntityType; import org.folio.search.service.reindex.ReindexConstants; +import org.folio.search.service.reindex.ReindexContext; import org.folio.search.utils.JsonConverter; import org.folio.spring.FolioExecutionContext; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Repository; +@Log4j2 @Repository public class ItemRepository extends MergeRangeRepository { @@ -38,6 +42,11 @@ ON CONFLICT (id, tenant_id) ORDER BY last_updated_date ASC """; + private static final String INSERT_STAGING_SQL = """ + INSERT INTO %s (id, tenant_id, instance_id, holding_id, json, inserted_at) + VALUES (?::uuid, ?, ?::uuid, ?::uuid, ?::jsonb, CURRENT_TIMESTAMP); + """; + protected ItemRepository(JdbcTemplate jdbcTemplate, JsonConverter jsonConverter, FolioExecutionContext context, SearchConfigurationProperties searchConfigurationProperties) { super(jdbcTemplate, jsonConverter, context, searchConfigurationProperties); @@ -53,8 +62,22 @@ protected String entityTable() { return ReindexConstants.ITEM_TABLE; } + @Override + protected Optional stagingEntityTable() { + return Optional.of(ReindexConstants.STAGING_ITEM_TABLE); + } + @Override public void saveEntities(String tenantId, List> entities) { + if (ReindexContext.isReindexMode() && ReindexContext.getMemberTenantId() != null) { + saveEntitiesToStaging(tenantId, entities); + } else { + saveEntitiesToMain(tenantId, entities); + } + } + + @SuppressWarnings("java:S2077") + private void saveEntitiesToMain(String tenantId, List> entities) { var fullTableName = getFullTableName(context, entityTable()); var sql = INSERT_SQL.formatted(fullTableName); @@ -68,7 +91,25 @@ public void saveEntities(String tenantId, List> entities) { }); } + @SuppressWarnings("java:S2077") + private void saveEntitiesToStaging(String tenantId, List> entities) { + var fullTableName = getFullTableName(context, ReindexConstants.STAGING_ITEM_TABLE); + var sql = INSERT_STAGING_SQL.formatted(fullTableName); + + jdbcTemplate.batchUpdate(sql, entities, BATCH_OPERATION_SIZE, + (statement, entity) -> { + statement.setObject(1, entity.get("id")); + statement.setString(2, tenantId); + statement.setObject(3, entity.get("instanceId")); + statement.setObject(4, entity.get("holdingsRecordId")); + statement.setString(5, jsonConverter.toJson(entity)); + }); + + log.debug("Saved {} entities to staging table {}", entities.size(), ReindexConstants.STAGING_ITEM_TABLE); + } + @Override + @SuppressWarnings("java:S2077") public SubResourceResult fetchByTimestamp(String tenant, Timestamp timestamp) { var sql = SELECT_BY_UPDATED_QUERY.formatted(getSchemaName(tenant, context.getFolioModuleMetadata())); var records = jdbcTemplate.query(sql, itemRowMapper(), timestamp); diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/MergeInstanceRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/MergeInstanceRepository.java index 8a9ab6f27..e0ca4937b 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/MergeInstanceRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/MergeInstanceRepository.java @@ -7,11 +7,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import lombok.extern.log4j.Log4j2; import org.folio.search.configuration.properties.SearchConfigurationProperties; import org.folio.search.model.types.ReindexEntityType; import org.folio.search.service.consortium.ConsortiumTenantProvider; import org.folio.search.service.reindex.ReindexConstants; +import org.folio.search.service.reindex.ReindexContext; import org.folio.search.utils.JsonConverter; import org.folio.spring.FolioExecutionContext; import org.springframework.dao.DataAccessException; @@ -34,6 +36,11 @@ ON CONFLICT (id) last_updated_date = CURRENT_TIMESTAMP; """; + private static final String INSERT_STAGING_SQL = """ + INSERT INTO %s (id, tenant_id, shared, is_bound_with, json, inserted_at) + VALUES (?::uuid, ?, ?, ?, ?::jsonb, CURRENT_TIMESTAMP); + """; + private static final String UPDATE_BOUND_WITH_SQL = """ UPDATE %s SET is_bound_with = ? WHERE id = ?::uuid; """; @@ -64,8 +71,22 @@ protected String entityTable() { return ReindexConstants.INSTANCE_TABLE; } + @Override + protected Optional stagingEntityTable() { + return Optional.of(ReindexConstants.STAGING_INSTANCE_TABLE); + } + @Override public void saveEntities(String tenantId, List> entities) { + if (ReindexContext.isReindexMode() && ReindexContext.getMemberTenantId() != null) { + saveEntitiesToStaging(tenantId, entities); + } else { + saveEntitiesToMain(tenantId, entities); + } + } + + @SuppressWarnings("java:S2077") + private void saveEntitiesToMain(String tenantId, List> entities) { var fullTableName = getFullTableName(context, entityTable()); var sql = INSERT_SQL.formatted(fullTableName); var shared = consortiumTenantProvider.isCentralTenant(tenantId); @@ -80,7 +101,7 @@ public void saveEntities(String tenantId, List> entities) { statement.setString(5, jsonConverter.toJson(entity)); }); } catch (DataAccessException e) { - log.warn("saveEntities::Failed to save batch. Starting processing one-by-one", e); + log.warn("saveEntitiesToMain::Failed to save batch. Starting processing one-by-one", e); for (Map entity : entities) { jdbcTemplate.update(sql, entity.get("id"), tenantId, @@ -91,14 +112,39 @@ public void saveEntities(String tenantId, List> entities) { } } + @SuppressWarnings("java:S2077") + private void saveEntitiesToStaging(String tenantId, List> entities) { + var fullTableName = getFullTableName(context, ReindexConstants.STAGING_INSTANCE_TABLE); + var sql = INSERT_STAGING_SQL.formatted(fullTableName); + var shared = consortiumTenantProvider.isCentralTenant(tenantId); + + jdbcTemplate.batchUpdate(sql, entities, BATCH_OPERATION_SIZE, + (statement, entity) -> { + statement.setObject(1, entity.get("id")); + statement.setString(2, tenantId); + statement.setObject(3, shared); + statement.setObject(4, entity.getOrDefault("isBoundWith", false)); + statement.setString(5, jsonConverter.toJson(entity)); + }); + + log.debug("Saved {} entities to staging table {}", entities.size(), ReindexConstants.STAGING_INSTANCE_TABLE); + } + @Override + @SuppressWarnings("java:S2077") public void updateBoundWith(String tenantId, String id, boolean bound) { - var fullTableName = getFullTableName(context, entityTable()); - var sql = UPDATE_BOUND_WITH_SQL.formatted(fullTableName); - jdbcTemplate.update(sql, bound, id); + if (ReindexContext.isReindexMode()) { + // Staging tables don't need update operations, they only accumulate data + log.debug("Update operation not supported for staging table {}", ReindexConstants.STAGING_INSTANCE_TABLE); + } else { + var fullTableName = getFullTableName(context, entityTable()); + var sql = UPDATE_BOUND_WITH_SQL.formatted(fullTableName); + jdbcTemplate.update(sql, bound, id); + } } @Override + @SuppressWarnings("java:S2077") public SubResourceResult fetchByTimestamp(String tenant, Timestamp timestamp) { var sql = SELECT_BY_UPDATED_QUERY.formatted(getSchemaName(tenant, context.getFolioModuleMetadata())); var records = jdbcTemplate.query(sql, instanceRowMapper(), timestamp); diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/MergeRangeRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/MergeRangeRepository.java index a96e0fa3e..28c5efb43 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/MergeRangeRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/MergeRangeRepository.java @@ -58,6 +58,7 @@ protected MergeRangeRepository(JdbcTemplate jdbcTemplate, } @Transactional + @SuppressWarnings("java:S2077") public void saveMergeRanges(List mergeRanges) { var fullTableName = getFullTableName(context, MERGE_RANGE_TABLE); jdbcTemplate.batchUpdate(INSERT_MERGE_RANGE_SQL.formatted(fullTableName), mergeRanges, BATCH_OPERATION_SIZE, @@ -72,12 +73,14 @@ public void saveMergeRanges(List mergeRanges) { }); } + @SuppressWarnings("java:S2077") public List getMergeRanges() { var fullTableName = getFullTableName(context, MERGE_RANGE_TABLE); var sql = SELECT_MERGE_RANGES_BY_ENTITY_TYPE.formatted(fullTableName); return jdbcTemplate.query(sql, mergeRangeEntityRowMapper(), entityType().getType()); } + @SuppressWarnings("java:S2077") public List getFailedMergeRanges() { var fullTableName = getFullTableName(context, MERGE_RANGE_TABLE); var sql = SELECT_FAILED_MERGE_RANGES.formatted(fullTableName); @@ -95,11 +98,14 @@ protected String rangeTable() { public abstract void saveEntities(String tenantId, List> entities); + @Transactional public void deleteEntitiesForTenant(List ids, String tenantId) { var hard = !instanceChildrenIndexEnabled; deleteEntitiesForTenant(ids, tenantId, hard); } + @Transactional + @SuppressWarnings("java:S2077") public void deleteEntitiesForTenant(List ids, String tenantId, boolean hard) { var fullTableName = getFullTableName(context, entityTable()); var query = hard ? DELETE_SQL_FOR_TENANT : SOFT_DELETE_SQL_FOR_TENANT; @@ -111,11 +117,14 @@ public void deleteEntitiesForTenant(List ids, String tenantId, boolean h }); } + @Transactional public void deleteEntities(List ids) { var hard = !instanceChildrenIndexEnabled; deleteEntities(ids, hard); } + @Transactional + @SuppressWarnings("java:S2077") public void deleteEntities(List ids, boolean hard) { var fullTableName = getFullTableName(context, entityTable()); var query = hard ? DELETE_SQL : SOFT_DELETE_SQL; diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ReindexJdbcRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ReindexJdbcRepository.java index 87f4cd50f..3ee43e054 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ReindexJdbcRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ReindexJdbcRepository.java @@ -16,10 +16,11 @@ import org.folio.spring.FolioExecutionContext; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; +import org.springframework.transaction.annotation.Transactional; public abstract class ReindexJdbcRepository { - protected static final int BATCH_OPERATION_SIZE = 100; + public static final int BATCH_OPERATION_SIZE = 100; protected static final String LAST_UPDATED_DATE_FIELD = "lastUpdatedDate"; private static final String COUNT_SQL = "SELECT COUNT(*) FROM %s;"; @@ -52,6 +53,34 @@ public void truncate() { JdbcUtils.truncateTable(entityTable(), jdbcTemplate, context); } + public void truncateStaging() { + subEntityStagingTable().ifPresent(tableName -> JdbcUtils.truncateTable(tableName, jdbcTemplate, context)); + stagingEntityTable().ifPresent(tableName -> JdbcUtils.truncateTable(tableName, jdbcTemplate, context)); + } + + @Transactional + @SuppressWarnings("java:S2077") + public void deleteByTenantId(String tenantId) { + // Delete from sub-entity table if present + subEntityTable().ifPresent(tableName -> { + var fullTableName = getFullTableName(context, tableName); + var sql = "DELETE FROM %s WHERE tenant_id = ?".formatted(fullTableName); + jdbcTemplate.update(sql, tenantId); + }); + + // Delete from main entity table only if it supports tenant-specific deletion + if (supportsTenantSpecificDeletion()) { + var fullTableName = getFullTableName(context, entityTable()); + var sql = "DELETE FROM %s WHERE tenant_id = ?".formatted(fullTableName); + jdbcTemplate.update(sql, tenantId); + } + } + + // Override in subclasses that don't have tenant_id columns (like Subject, Contributor, etc.) + protected boolean supportsTenantSpecificDeletion() { + return true; + } + public void updateRangeStatus(UUID id, Timestamp timestamp, ReindexRangeStatus status, String failCause) { var sql = UPDATE_STATUS_SQL.formatted(getFullTableName(context, rangeTable())); jdbcTemplate.update(sql, timestamp, status.name(), failCause, id); @@ -116,8 +145,17 @@ protected Optional subEntityTable() { return Optional.empty(); } + protected Optional stagingEntityTable() { + return Optional.empty(); + } + + protected Optional subEntityStagingTable() { + return Optional.empty(); + } + protected abstract String rangeTable(); + @SuppressWarnings("java:S2077") protected void deleteByInstanceIds(String query, List instanceIds, String tenantId) { var sql = query.formatted( JdbcUtils.getSchemaName(context), diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepository.java index d8ce8aef3..42002fee3 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepository.java @@ -1,16 +1,20 @@ package org.folio.search.service.reindex.jdbc; import static org.folio.search.model.reindex.ReindexStatusEntity.END_TIME_MERGE_COLUMN; +import static org.folio.search.model.reindex.ReindexStatusEntity.END_TIME_STAGING_COLUMN; import static org.folio.search.model.reindex.ReindexStatusEntity.END_TIME_UPLOAD_COLUMN; import static org.folio.search.model.reindex.ReindexStatusEntity.PROCESSED_MERGE_RANGES_COLUMN; import static org.folio.search.model.reindex.ReindexStatusEntity.PROCESSED_UPLOAD_RANGES_COLUMN; import static org.folio.search.model.reindex.ReindexStatusEntity.START_TIME_MERGE_COLUMN; +import static org.folio.search.model.reindex.ReindexStatusEntity.START_TIME_STAGING_COLUMN; import static org.folio.search.model.reindex.ReindexStatusEntity.START_TIME_UPLOAD_COLUMN; import static org.folio.search.model.reindex.ReindexStatusEntity.STATUS_COLUMN; +import static org.folio.search.model.reindex.ReindexStatusEntity.TARGET_TENANT_ID_COLUMN; import static org.folio.search.model.reindex.ReindexStatusEntity.TOTAL_MERGE_RANGES_COLUMN; import static org.folio.search.model.reindex.ReindexStatusEntity.TOTAL_UPLOAD_RANGES_COLUMN; import static org.folio.search.service.reindex.ReindexConstants.REINDEX_STATUS_TABLE; import static org.folio.search.utils.JdbcUtils.getFullTableName; +import static org.folio.search.utils.JdbcUtils.getSchemaName; import java.sql.Timestamp; import java.time.Instant; @@ -32,8 +36,9 @@ public class ReindexStatusRepository { private static final String INSERT_REINDEX_STATUS_SQL = """ INSERT INTO %s (entity_type, status, total_merge_ranges, processed_merge_ranges, total_upload_ranges, - processed_upload_ranges, start_time_merge, end_time_merge, start_time_upload, end_time_upload) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + processed_upload_ranges, start_time_merge, end_time_merge, start_time_upload, end_time_upload, + start_time_staging, end_time_staging, target_tenant_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); """; private static final String UPDATE_SQL = """ @@ -61,6 +66,21 @@ public class ReindexStatusRepository { private static final String SELECT_MERGE_STATUS_SQL = "SELECT check_merge_completed_status()"; + private static final String SELECT_TARGET_TENANT_ID_SQL = "SELECT target_tenant_id FROM %s LIMIT 1"; + + private static final String RECREATE_REINDEX_STATUS_TRIGGER_SQL = """ + DROP TRIGGER IF EXISTS reindex_status_updated_trigger ON %s CASCADE; + CREATE TRIGGER reindex_status_updated_trigger + BEFORE UPDATE OF processed_merge_ranges, processed_upload_ranges + ON %s + FOR EACH ROW + EXECUTE FUNCTION %s.%s(); + """; + + private static final String UPDATE_REINDEX_STATUS_FUNCTION_NAME = "update_reindex_status_trigger"; + private static final String UPDATE_CONSORTIUM_MEMBER_REINDEX_STATUS_FUNCTION_NAME = + "update_consortium_member_reindex_status_trigger"; + private final FolioExecutionContext context; private final JdbcTemplate jdbcTemplate; @@ -135,6 +155,42 @@ public void setMergeInProgress(Set entityTypes) { jdbcTemplate.update(sql, ReindexStatus.MERGE_IN_PROGRESS.name()); } + @SuppressWarnings("java:S2077") + public void setStagingStarted(List entityTypes) { + var inTypes = entityTypes.stream() + .map(entityType -> "'%s'".formatted(entityType.name())) + .collect(Collectors.joining(",")); + var fullTableName = getFullTableName(context, REINDEX_STATUS_TABLE); + var sql = UPDATE_FOR_ENTITIES_SQL.formatted( + fullTableName, QUERY_TWO_COLUMNS_PLACEHOLDER.formatted(STATUS_COLUMN, START_TIME_STAGING_COLUMN), inTypes); + + jdbcTemplate.update(sql, ReindexStatus.STAGING_IN_PROGRESS.name(), Timestamp.from(Instant.now())); + } + + @SuppressWarnings("java:S2077") + public void setStagingCompleted(List entityTypes) { + var inTypes = entityTypes.stream() + .map(entityType -> "'%s'".formatted(entityType.name())) + .collect(Collectors.joining(",")); + var fullTableName = getFullTableName(context, REINDEX_STATUS_TABLE); + var sql = UPDATE_FOR_ENTITIES_SQL.formatted( + fullTableName, QUERY_TWO_COLUMNS_PLACEHOLDER.formatted(STATUS_COLUMN, END_TIME_STAGING_COLUMN), inTypes); + + jdbcTemplate.update(sql, ReindexStatus.STAGING_COMPLETED.name(), Timestamp.from(Instant.now())); + } + + @SuppressWarnings("java:S2077") + public void setStagingFailed(List entityTypes) { + var inTypes = entityTypes.stream() + .map(entityType -> "'%s'".formatted(entityType.name())) + .collect(Collectors.joining(",")); + var fullTableName = getFullTableName(context, REINDEX_STATUS_TABLE); + var sql = UPDATE_FOR_ENTITIES_SQL.formatted( + fullTableName, QUERY_TWO_COLUMNS_PLACEHOLDER.formatted(STATUS_COLUMN, END_TIME_STAGING_COLUMN), inTypes); + + jdbcTemplate.update(sql, ReindexStatus.STAGING_FAILED.name(), Timestamp.from(Instant.now())); + } + public void saveReindexStatusRecords(List statusRecords) { var fullTableName = getFullTableName(context, REINDEX_STATUS_TABLE); jdbcTemplate.batchUpdate(INSERT_REINDEX_STATUS_SQL.formatted(fullTableName), statusRecords, 10, @@ -149,6 +205,9 @@ public void saveReindexStatusRecords(List statusRecords) { statement.setTimestamp(8, entity.getEndTimeMerge()); statement.setTimestamp(9, entity.getStartTimeUpload()); statement.setTimestamp(10, entity.getEndTimeUpload()); + statement.setTimestamp(11, entity.getStartTimeStaging()); + statement.setTimestamp(12, entity.getEndTimeStaging()); + statement.setString(13, entity.getTargetTenantId()); }); } @@ -156,6 +215,24 @@ public boolean isMergeCompleted() { return Boolean.TRUE.equals(jdbcTemplate.queryForObject(SELECT_MERGE_STATUS_SQL, Boolean.class)); } + @SuppressWarnings("java:S2077") + public String getTargetTenantId() { + var fullTableName = getFullTableName(context, REINDEX_STATUS_TABLE); + var sql = SELECT_TARGET_TENANT_ID_SQL.formatted(fullTableName); + return jdbcTemplate.queryForObject(sql, String.class); + } + + @SuppressWarnings("java:S2077") + public void recreateReindexStatusTrigger(boolean isConsortiumMember) { + var schemaName = getSchemaName(context); + var fullTableName = getFullTableName(context, REINDEX_STATUS_TABLE); + var functionName = isConsortiumMember ? UPDATE_CONSORTIUM_MEMBER_REINDEX_STATUS_FUNCTION_NAME + : UPDATE_REINDEX_STATUS_FUNCTION_NAME; + var sql = RECREATE_REINDEX_STATUS_TRIGGER_SQL.formatted(fullTableName, fullTableName, + schemaName, functionName); + jdbcTemplate.execute(sql); + } + private RowMapper reindexStatusRowMapper() { return (rs, rowNum) -> { var reindexStatus = new ReindexStatusEntity( @@ -170,6 +247,9 @@ private RowMapper reindexStatusRowMapper() { reindexStatus.setEndTimeMerge(rs.getTimestamp(END_TIME_MERGE_COLUMN)); reindexStatus.setStartTimeUpload(rs.getTimestamp(START_TIME_UPLOAD_COLUMN)); reindexStatus.setEndTimeUpload(rs.getTimestamp(END_TIME_UPLOAD_COLUMN)); + reindexStatus.setStartTimeStaging(rs.getTimestamp(START_TIME_STAGING_COLUMN)); + reindexStatus.setEndTimeStaging(rs.getTimestamp(END_TIME_STAGING_COLUMN)); + reindexStatus.setTargetTenantId(rs.getString(TARGET_TENANT_ID_COLUMN)); return reindexStatus; }; } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java index 860a0b748..20303eab2 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java @@ -20,6 +20,7 @@ import org.folio.search.model.entity.ChildResourceEntityBatch; import org.folio.search.model.types.ReindexEntityType; import org.folio.search.service.reindex.ReindexConstants; +import org.folio.search.service.reindex.ReindexContext; import org.folio.search.utils.JdbcUtils; import org.folio.search.utils.JsonConverter; import org.folio.spring.FolioExecutionContext; @@ -30,6 +31,7 @@ @Log4j2 @Repository +@SuppressWarnings("java:S2077") public class SubjectRepository extends UploadRangeRepository implements InstanceChildResourceRepository { private static final String SELECT_QUERY = """ @@ -130,12 +132,21 @@ WHERE instance_id IN (%2$s) %3$s VALUES (?, ?, ?, ?, ?) ON CONFLICT (id) DO UPDATE SET last_updated_date = CURRENT_TIMESTAMP; """; + private static final String INSERT_STAGING_ENTITIES_SQL = """ + INSERT INTO %s.staging_subject (id, value, authority_id, source_id, type_id, inserted_at) + VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP); + """; private static final String INSERT_RELATIONS_SQL = """ INSERT INTO %s.instance_subject (instance_id, subject_id, tenant_id, shared) VALUES (?::uuid, ?, ?, ?) ON CONFLICT DO NOTHING; """; + private static final String INSERT_STAGING_RELATIONS_SQL = """ + INSERT INTO %s.staging_instance_subject (instance_id, subject_id, tenant_id, shared, inserted_at) + VALUES (?::uuid, ?, ?, ?, CURRENT_TIMESTAMP); + """; + private static final String ID_RANGE_INS_WHERE_CLAUSE = "ins.subject_id >= ? AND ins.subject_id <= ?"; private static final String ID_RANGE_SUBJ_WHERE_CLAUSE = "s.id >= ? AND s.id <= ?"; @@ -161,6 +172,22 @@ protected Optional subEntityTable() { return Optional.of(ReindexConstants.INSTANCE_SUBJECT_TABLE); } + @Override + protected Optional stagingEntityTable() { + return Optional.of(ReindexConstants.STAGING_SUBJECT_TABLE); + } + + @Override + protected Optional subEntityStagingTable() { + return Optional.of(ReindexConstants.STAGING_INSTANCE_SUBJECT_TABLE); + } + + @Override + protected boolean supportsTenantSpecificDeletion() { + // Subject table doesn't have tenant_id column - it's shared across tenants + return false; + } + @Override public List> fetchByIdRange(String lower, String upper) { var sql = getFetchBySql(); @@ -215,9 +242,20 @@ public void deleteByInstanceIds(List instanceIds, String tenantId) { @Override public void saveAll(ChildResourceEntityBatch entityBatch) { + // Use staging tables only for member tenant specific full reindex + if (ReindexContext.isReindexMode() && ReindexContext.isMemberTenantReindex()) { + saveEntitiesToStaging(entityBatch.resourceEntities().stream().toList()); + saveRelationshipsToStaging(entityBatch.relationshipEntities().stream().toList()); + } else { + saveEntitiesToMain(entityBatch.resourceEntities().stream().toList()); + saveRelationshipsToMain(entityBatch.relationshipEntities().stream().toList()); + } + } + + private void saveEntitiesToMain(List> entities) { var entitiesSql = INSERT_ENTITIES_SQL.formatted(JdbcUtils.getSchemaName(context)); try { - jdbcTemplate.batchUpdate(entitiesSql, entityBatch.resourceEntities(), BATCH_OPERATION_SIZE, + jdbcTemplate.batchUpdate(entitiesSql, entities, BATCH_OPERATION_SIZE, (statement, entity) -> { statement.setString(1, (String) entity.get("id")); statement.setString(2, (String) entity.get(SUBJECT_VALUE_FIELD)); @@ -227,15 +265,46 @@ public void saveAll(ChildResourceEntityBatch entityBatch) { }); } catch (DataAccessException e) { logWarnDebugError(SAVE_ENTITIES_BATCH_ERROR_MESSAGE, e); - for (var entity : entityBatch.resourceEntities()) { - jdbcTemplate.update(entitiesSql, entity.get("id"), entity.get(SUBJECT_VALUE_FIELD), - entity.get(AUTHORITY_ID_FIELD), entity.get(SUBJECT_SOURCE_ID_FIELD), entity.get(SUBJECT_TYPE_ID_FIELD)); + for (var entity : entities) { + try { + jdbcTemplate.update(entitiesSql, entity.get("id"), entity.get(SUBJECT_VALUE_FIELD), + entity.get(AUTHORITY_ID_FIELD), entity.get(SUBJECT_SOURCE_ID_FIELD), entity.get(SUBJECT_TYPE_ID_FIELD)); + } catch (DataAccessException ex) { + log.debug("Failed to save subject entity {}: {}", entity.get("id"), ex.getMessage()); + } } } + } + private void saveEntitiesToStaging(List> entities) { + var stagingEntitiesSql = INSERT_STAGING_ENTITIES_SQL.formatted(JdbcUtils.getSchemaName(context)); + try { + jdbcTemplate.batchUpdate(stagingEntitiesSql, entities, BATCH_OPERATION_SIZE, + (statement, entity) -> { + statement.setString(1, (String) entity.get("id")); + statement.setString(2, (String) entity.get(SUBJECT_VALUE_FIELD)); + statement.setString(3, (String) entity.get(AUTHORITY_ID_FIELD)); + statement.setString(4, (String) entity.get(SUBJECT_SOURCE_ID_FIELD)); + statement.setString(5, (String) entity.get(SUBJECT_TYPE_ID_FIELD)); + }); + } catch (DataAccessException e) { + log.warn("saveEntitiesToStaging::Failed to save entities batch. Processing one-by-one", e); + for (var entity : entities) { + try { + jdbcTemplate.update(stagingEntitiesSql, entity.get("id"), entity.get(SUBJECT_VALUE_FIELD), + entity.get(AUTHORITY_ID_FIELD), entity.get(SUBJECT_SOURCE_ID_FIELD), entity.get(SUBJECT_TYPE_ID_FIELD)); + } catch (DataAccessException ex) { + log.debug("Failed to save staging subject entity {}: {}", entity.get("id"), ex.getMessage()); + } + } + } + log.debug("Saved {} subject entities to staging table", entities.size()); + } + + private void saveRelationshipsToMain(List> relationships) { var relationsSql = INSERT_RELATIONS_SQL.formatted(JdbcUtils.getSchemaName(context)); try { - jdbcTemplate.batchUpdate(relationsSql, entityBatch.relationshipEntities(), BATCH_OPERATION_SIZE, + jdbcTemplate.batchUpdate(relationsSql, relationships, BATCH_OPERATION_SIZE, (statement, entityRelation) -> { statement.setObject(1, entityRelation.get("instanceId")); statement.setString(2, (String) entityRelation.get("subjectId")); @@ -244,11 +313,40 @@ public void saveAll(ChildResourceEntityBatch entityBatch) { }); } catch (DataAccessException e) { logWarnDebugError(SAVE_RELATIONS_BATCH_ERROR_MESSAGE, e); - for (var entityRelation : entityBatch.relationshipEntities()) { - jdbcTemplate.update(relationsSql, entityRelation.get("instanceId"), entityRelation.get("subjectId"), - entityRelation.get("tenantId"), entityRelation.get("shared")); + for (var entityRelation : relationships) { + try { + jdbcTemplate.update(relationsSql, entityRelation.get("instanceId"), entityRelation.get("subjectId"), + entityRelation.get("tenantId"), entityRelation.get("shared")); + } catch (DataAccessException ex) { + log.debug("Failed to save subject relationship for {}: {}", entityRelation.get("subjectId"), ex.getMessage()); + } + } + } + } + + private void saveRelationshipsToStaging(List> relationships) { + var stagingRelationsSql = INSERT_STAGING_RELATIONS_SQL.formatted(JdbcUtils.getSchemaName(context)); + try { + jdbcTemplate.batchUpdate(stagingRelationsSql, relationships, BATCH_OPERATION_SIZE, + (statement, entityRelation) -> { + statement.setObject(1, entityRelation.get("instanceId")); + statement.setString(2, (String) entityRelation.get("subjectId")); + statement.setString(3, (String) entityRelation.get("tenantId")); + statement.setObject(4, entityRelation.get("shared")); + }); + } catch (DataAccessException e) { + log.warn("saveRelationshipsToStaging::Failed to save relationships batch. Processing one-by-one", e); + for (var entityRelation : relationships) { + try { + jdbcTemplate.update(stagingRelationsSql, entityRelation.get("instanceId"), entityRelation.get("subjectId"), + entityRelation.get("tenantId"), entityRelation.get("shared")); + } catch (DataAccessException ex) { + log.debug("Failed to save staging subject relationship for {}: {}", + entityRelation.get("subjectId"), ex.getMessage()); + } } } + log.debug("Saved {} subject relationships to staging table", relationships.size()); } protected RowMapper> rowToMapMapper2() { @@ -270,6 +368,14 @@ protected RowMapper> rowToMapMapper2() { }; } + @Override + public List> fetchByIdRangeWithTimestamp(String lower, String upper, Timestamp timestamp) { + var sql = SELECT_QUERY.formatted(JdbcUtils.getSchemaName(context), + ID_RANGE_INS_WHERE_CLAUSE, + ID_RANGE_SUBJ_WHERE_CLAUSE + " AND s.last_updated_date = ?"); + return jdbcTemplate.query(sql, rowToMapMapper(), lower, upper, lower, upper, timestamp); + } + private String getId(ResultSet rs) throws SQLException { return rs.getString("id"); } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/TenantRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/TenantRepository.java index 40132253b..60a70ec6a 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/TenantRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/TenantRepository.java @@ -24,12 +24,14 @@ public class TenantRepository { private final JdbcTemplate jdbcTemplate; private final SystemProperties systemProperties; + @SuppressWarnings("java:S2077") public void saveTenant(TenantEntity tenantEntity) { String query = INSERT_QUERY.formatted(systemProperties.getSchemaName()); jdbcTemplate.update(query, tenantEntity.id(), tenantEntity.centralId(), tenantEntity.active(), tenantEntity.active()); } + @SuppressWarnings("java:S2077") public List fetchDataTenantIds() { String query = FETCH_QUERY.formatted(systemProperties.getSchemaName()); return jdbcTemplate.query(query, (rs, rowNum) -> rs.getString("id")); diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/UploadInstanceRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/UploadInstanceRepository.java index 36765a694..3f3727ab5 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/UploadInstanceRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/UploadInstanceRepository.java @@ -2,13 +2,17 @@ import static org.folio.search.utils.JdbcUtils.getFullTableName; +import java.sql.Timestamp; import java.util.Collections; import java.util.List; import java.util.Map; +import lombok.extern.log4j.Log4j2; import org.folio.search.configuration.properties.ReindexConfigurationProperties; import org.folio.search.model.types.ReindexEntityType; +import org.folio.search.service.consortium.ConsortiumTenantService; import org.folio.search.service.reindex.RangeGenerator; import org.folio.search.service.reindex.ReindexConstants; +import org.folio.search.service.reindex.ReindexContext; import org.folio.search.utils.JdbcUtils; import org.folio.search.utils.JsonConverter; import org.folio.spring.FolioExecutionContext; @@ -17,6 +21,7 @@ import org.springframework.stereotype.Repository; @Repository +@Log4j2 public class UploadInstanceRepository extends UploadRangeRepository { private static final String SELECT_SQL_TEMPLATE = """ @@ -56,10 +61,14 @@ aggregated_items AS ( private static final String IDS_RANGE_WHERE_CLAUSE = "%1$s >= ?::uuid AND %1$s <= ?::uuid"; private static final String INSTANCE_IDS_WHERE_CLAUSE = "%s IN (%s)"; + private final ConsortiumTenantService consortiumTenantService; + protected UploadInstanceRepository(JdbcTemplate jdbcTemplate, JsonConverter jsonConverter, FolioExecutionContext context, - ReindexConfigurationProperties reindexConfig) { + ReindexConfigurationProperties reindexConfig, + ConsortiumTenantService consortiumTenantService) { super(jdbcTemplate, jsonConverter, context, reindexConfig); + this.consortiumTenantService = consortiumTenantService; } @Override @@ -98,6 +107,101 @@ public List> fetchByIds(List ids) { }, rowToMapMapper()); } + private List> fetchForMemberTenantReindex(String lower, String upper) { + var memberTenantId = ReindexContext.getMemberTenantId(); + var centralTenantId = consortiumTenantService.getCentralTenant(context.getTenantId()) + .orElseThrow(() -> new IllegalStateException("No central tenant found")); + + return fetchConditionalInstances(centralTenantId, memberTenantId, lower, upper); + } + + /** + * Fetches instances conditionally for member tenant reindex: + * 1. Local instances (tenant_id = memberTenantId) + * 2. Shared instances that have holdings belonging to memberTenantId + * + * @param centralTenantId Central tenant where merged data resides + * @param memberTenantId Member tenant being reindexed + * @param lower Lower UUID bound for range processing + * @param upper Upper UUID bound for range processing + * @return List of instance maps with holdings and items + */ + private List> fetchConditionalInstances( + String centralTenantId, String memberTenantId, String lower, String upper) { + + log.info("fetchConditionalInstances:: Fetching instances for member tenant reindex " + + "[memberTenant: {}, centralTenant: {}, range: {}-{}]", + memberTenantId, centralTenantId, lower, upper); + + var moduleMetadata = context.getFolioModuleMetadata(); + var centralSchema = JdbcUtils.getSchemaName(centralTenantId, moduleMetadata); + + // SQL to fetch both local and relevant shared instances + var sql = buildConditionalInstanceQuery(centralSchema); + + var results = jdbcTemplate.query(sql, ps -> { + ps.setObject(1, lower); // UUID range lower bound + ps.setObject(2, upper); // UUID range upper bound + ps.setString(3, memberTenantId); // For local instances + ps.setObject(4, lower); // UUID range lower bound for shared instances + ps.setObject(5, upper); // UUID range upper bound for shared instances + ps.setString(6, memberTenantId); // For shared instances with member holdings + }, rowToMapMapper()); + + log.debug("fetchConditionalInstances:: Found {} instances for range {}-{}", + results.size(), lower, upper); + + return results; + } + + /** + * Builds SQL query to fetch instances conditionally. + * - UNION of local instances and shared instances with member holdings + * - Maintains existing JSON aggregation for holdings/items + * - Applies UUID range filtering + */ + private String buildConditionalInstanceQuery(String centralSchema) { + return """ + SELECT combined.json + || jsonb_build_object('tenantId', combined.tenant_id, + 'shared', combined.shared, + 'isBoundWith', combined.is_bound_with, + 'holdings', COALESCE(jsonb_agg(DISTINCT h.json || + jsonb_build_object('tenantId', h.tenant_id)) + FILTER (WHERE h.json IS NOT NULL), '[]'::jsonb), + 'items', COALESCE(jsonb_agg(it.json || + jsonb_build_object('tenantId', it.tenant_id)) + FILTER (WHERE it.json IS NOT NULL), '[]'::jsonb)) as json + FROM ( + -- Local instances from central tenant (after merge) + SELECT i.id, i.tenant_id, i.shared, i.is_bound_with, i.json + FROM %s.instance i + WHERE i.id >= ?::uuid AND i.id <= ?::uuid + AND i.tenant_id = ? + UNION ALL + -- Shared instances that have holdings for member tenant + SELECT i.id, i.tenant_id, i.shared, i.is_bound_with, i.json + FROM %s.instance i + WHERE i.id >= ?::uuid AND i.id <= ?::uuid + AND i.shared = true + AND EXISTS ( + SELECT 1 FROM %s.holding h + WHERE h.instance_id = i.id AND h.tenant_id = ? + ) + ) combined + LEFT JOIN %s.holding h ON h.instance_id = combined.id + LEFT JOIN %s.item it ON it.holding_id = h.id + GROUP BY combined.id, combined.tenant_id, combined.shared, + combined.is_bound_with, combined.json + """.formatted( + centralSchema, // Local instances table + centralSchema, // Shared instances table + centralSchema, // Holdings subquery table + centralSchema, // Holdings join table + centralSchema // Items join table + ); + } + @Override protected List createRanges() { var uploadRangeSize = reindexConfig.getUploadRangeSize(); @@ -107,6 +211,15 @@ protected List createRanges() { @Override public List> fetchByIdRange(String lower, String upper) { + var memberTenantId = ReindexContext.getMemberTenantId(); + + if (memberTenantId != null) { + // Member tenant reindex: Fetch from central tenant schema only + // All member tenant data is already in central tenant after merge phase + return fetchForMemberTenantReindex(lower, upper); + } + + // Full reindex: Standard fetch from main tables var sql = getFetchBySql(); return jdbcTemplate.query(sql, rowToMapMapper(), lower, upper, lower, upper, lower, upper); } @@ -128,4 +241,11 @@ protected String getFetchBySql() { protected RowMapper> rowToMapMapper() { return (rs, rowNum) -> jsonConverter.fromJsonToMap(rs.getString("json")); } + + @Override + public List> fetchByIdRangeWithTimestamp(String lower, String upper, Timestamp timestamp) { + // Instances are not child resources and don't need timestamp filtering for member tenant reindex + // This method delegates to the standard range-based fetch + return fetchByIdRange(lower, upper); + } } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/UploadRangeRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/UploadRangeRepository.java index f7ae02364..2edb2f945 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/UploadRangeRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/UploadRangeRepository.java @@ -59,6 +59,7 @@ protected UploadRangeRepository(JdbcTemplate jdbcTemplate, this.reindexConfig = reindexConfig; } + @SuppressWarnings("java:S2077") public List getUploadRanges() { var fullTableName = getFullTableName(context, UPLOAD_RANGE_TABLE); var sql = SELECT_UPLOAD_RANGE_BY_ENTITY_TYPE_SQL.formatted(fullTableName); @@ -66,6 +67,7 @@ public List getUploadRanges() { return jdbcTemplate.query(sql, uploadRangeRowMapper(), entityType().getType()); } + @SuppressWarnings("java:S2077") public List createUploadRanges() { var fullTableName = getFullTableName(context, UPLOAD_RANGE_TABLE); var deleteSql = DELETE_UPLOAD_RANGE_SQL.formatted(fullTableName); @@ -79,6 +81,9 @@ public List> fetchByIdRange(String lower, String upper) { return jdbcTemplate.query(sql, rowToMapMapper(), lower, upper); } + public abstract List> fetchByIdRangeWithTimestamp(String lower, String upper, + Timestamp timestamp); + protected String getFetchBySql() { return SELECT_RECORD_SQL.formatted(getFullTableName(context, entityTable())); } @@ -131,6 +136,7 @@ private List prepareAndSaveUploadRanges() { return ranges; } + @SuppressWarnings("java:S2077") private void upsertUploadRanges(List uploadRanges) { var fullTableName = getFullTableName(context, UPLOAD_RANGE_TABLE); jdbcTemplate.batchUpdate(UPSERT_UPLOAD_RANGE_SQL.formatted(fullTableName), uploadRanges, BATCH_OPERATION_SIZE, diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 4949597f3..7de3e09ef 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -94,6 +94,12 @@ folio: merge-range-publisher-max-pool-size: ${REINDEX_MERGE_RANGE_PUBLISHER_MAX_POOL_SIZE:6} merge-range-publisher-retry-interval-ms: ${REINDEX_MERGE_RANGE_PUBLISHER_RETRY_INTERVAL_MS:1000} merge-range-publisher-retry-attempts: ${REINDEX_MERGE_RANGE_PUBLISHER_RETRY_ATTEMPTS:5} + migration-work-mem: ${REINDEX_MIGRATION_WORK_MEM:64MB} + index-management: + delete-query-batch-size: ${DELETE_QUERY_BATCH_SIZE:1000} + delete-query-scroll-timeout-minutes: ${DELETE_QUERY_SCROLL_TIMEOUT_MINUTES:5} + delete-query-request-timeout-minutes: ${DELETE_QUERY_REQUEST_TIMEOUT_MINUTES:30} + delete-query-refresh: ${DELETE_QUERY_REFRESH:false} query: properties: request-timeout: ${SEARCH_QUERY_TIMEOUT:25s} diff --git a/src/main/resources/changelog/changelog-master.xml b/src/main/resources/changelog/changelog-master.xml index 58b004b58..4c6547beb 100644 --- a/src/main/resources/changelog/changelog-master.xml +++ b/src/main/resources/changelog/changelog-master.xml @@ -19,4 +19,10 @@ + + + + + + diff --git a/src/main/resources/changelog/changes/v6.0/01_create_staging_tables.xml b/src/main/resources/changelog/changes/v6.0/01_create_staging_tables.xml new file mode 100644 index 000000000..866db1250 --- /dev/null +++ b/src/main/resources/changelog/changes/v6.0/01_create_staging_tables.xml @@ -0,0 +1,233 @@ + + + + + + + + + + + Create staging_instance table for reindex deduplication + + + CREATE UNLOGGED TABLE staging_instance ( + id UUID NOT NULL, + tenant_id VARCHAR(100) NOT NULL, + shared BOOLEAN NOT NULL, + is_bound_with BOOLEAN NOT NULL, + json JSONB NOT NULL, + inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL + ) PARTITION BY RANGE (SUBSTRING(id::text, 1, 1)); + + + + + + + + + + + Create staging_holding table for reindex deduplication + + + CREATE UNLOGGED TABLE staging_holding ( + id UUID NOT NULL, + tenant_id VARCHAR(100) NOT NULL, + instance_id UUID NOT NULL, + json JSONB NOT NULL, + inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL + ) PARTITION BY RANGE (SUBSTRING(id::text, 1, 1)); + + + + + + + + + + + Create staging_item table for reindex deduplication + + + CREATE UNLOGGED TABLE staging_item ( + id UUID NOT NULL, + tenant_id VARCHAR(100) NOT NULL, + instance_id UUID NOT NULL, + holding_id UUID NOT NULL, + json JSONB NOT NULL, + inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL + ) PARTITION BY RANGE (SUBSTRING(id::text, 1, 1)); + + + + + + + + + + + + Create staging_instance_subject table for reindex deduplication + + + CREATE UNLOGGED TABLE staging_instance_subject ( + instance_id UUID NOT NULL, + subject_id VARCHAR(40) NOT NULL, + tenant_id VARCHAR(100) NOT NULL, + shared BOOLEAN NOT NULL, + inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL + ) PARTITION BY RANGE (SUBSTRING(instance_id::text, 1, 1)); + + + + + + + + + + + Create staging_instance_contributor table for reindex deduplication + + + CREATE UNLOGGED TABLE staging_instance_contributor ( + instance_id UUID NOT NULL, + contributor_id VARCHAR(40) NOT NULL, + type_id VARCHAR(40) NOT NULL, + tenant_id VARCHAR(100) NOT NULL, + shared BOOLEAN NOT NULL, + inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL + ) PARTITION BY RANGE (SUBSTRING(instance_id::text, 1, 1)); + + + + + + + + + + + Create staging_instance_classification table for reindex deduplication + + + CREATE UNLOGGED TABLE staging_instance_classification ( + instance_id UUID NOT NULL, + classification_id VARCHAR(40) NOT NULL, + tenant_id VARCHAR(100) NOT NULL, + shared BOOLEAN NOT NULL, + inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL + ) PARTITION BY RANGE (SUBSTRING(instance_id::text, 1, 1)); + + + + + + + + + + + Create staging_instance_call_number table for reindex deduplication + + + CREATE UNLOGGED TABLE staging_instance_call_number ( + call_number_id VARCHAR(40) NOT NULL, + item_id UUID NOT NULL, + instance_id UUID NOT NULL, + tenant_id VARCHAR(100) NOT NULL, + location_id UUID, + inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL + ) PARTITION BY RANGE (SUBSTRING(instance_id::text, 1, 1)); + + + + + + + + + + + Create staging_subject table for reindex deduplication + + + CREATE UNLOGGED TABLE staging_subject ( + id VARCHAR(40) NOT NULL, + value VARCHAR(255) NOT NULL, + authority_id VARCHAR(40), + source_id VARCHAR(40), + type_id VARCHAR(40), + inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL + ) PARTITION BY RANGE (SUBSTRING(id::text, 1, 1)); + + + + + + + + + + + Create staging_contributor table for reindex deduplication + + + CREATE UNLOGGED TABLE staging_contributor ( + id VARCHAR(40) NOT NULL, + name VARCHAR(255) NOT NULL, + name_type_id VARCHAR(40), + authority_id VARCHAR(40), + inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL + ) PARTITION BY RANGE (SUBSTRING(id::text, 1, 1)); + + + + + + + + + + + Create staging_classification table for reindex deduplication + + + CREATE UNLOGGED TABLE staging_classification ( + id VARCHAR(40) NOT NULL, + number VARCHAR(50) NOT NULL, + type_id VARCHAR(40), + inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL + ) PARTITION BY RANGE (SUBSTRING(id::text, 1, 1)); + + + + + + + + + + + Create staging_call_number table for reindex deduplication + + + CREATE UNLOGGED TABLE staging_call_number ( + id VARCHAR(40) NOT NULL, + call_number VARCHAR(50) NOT NULL, + call_number_prefix VARCHAR(20), + call_number_suffix VARCHAR(25), + call_number_type_id VARCHAR(40), + inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL + ) PARTITION BY RANGE (SUBSTRING(id::text, 1, 1)); + + + + diff --git a/src/main/resources/changelog/changes/v6.0/02_create_staging_partitions.xml b/src/main/resources/changelog/changes/v6.0/02_create_staging_partitions.xml new file mode 100644 index 000000000..2db0430f3 --- /dev/null +++ b/src/main/resources/changelog/changes/v6.0/02_create_staging_partitions.xml @@ -0,0 +1,68 @@ + + + + + + + + Create partitions for staging_instance table + + + + + + + + Create partitions for staging_holding table + + + + + + + + Create partitions for staging_item table + + + + + + + + + + + + + + Create partitions for staging relationship tables + + + + + + + + + + + + + Create partitions for staging child resource tables + + + + diff --git a/src/main/resources/changelog/changes/v6.0/03_add_tenant_id_indexes.xml b/src/main/resources/changelog/changes/v6.0/03_add_tenant_id_indexes.xml new file mode 100644 index 000000000..ee549a884 --- /dev/null +++ b/src/main/resources/changelog/changes/v6.0/03_add_tenant_id_indexes.xml @@ -0,0 +1,105 @@ + + + + + + + + + + + Create index on instance.tenant_id for tenant deletion performance + + + + + + + + + + + + + + Create index on holding.tenant_id for tenant deletion performance + + + + + + + + + + + + + + Create index on item.tenant_id for tenant deletion performance + + + + + + + + + + + + + + Create index on instance_subject.tenant_id for tenant deletion performance + + + + + + + + + + + + + + Create index on instance_contributor.tenant_id for tenant deletion performance + + + + + + + + + + + + + + Create index on instance_classification.tenant_id for tenant deletion performance + + + + + + + + + + + + + + Create index on instance_call_number.tenant_id for tenant deletion performance + + + + + + + diff --git a/src/main/resources/changelog/changes/v6.0/04_add_target_tenant_id_to_reindex_status.xml b/src/main/resources/changelog/changes/v6.0/04_add_target_tenant_id_to_reindex_status.xml new file mode 100644 index 000000000..ef456cc99 --- /dev/null +++ b/src/main/resources/changelog/changes/v6.0/04_add_target_tenant_id_to_reindex_status.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + Add target_tenant_id column to reindex_status table for tenant-specific reindex operations + + + + + + + + + + + + diff --git a/src/main/resources/changelog/changes/v6.0/05_add_staging_time_columns_to_reindex_status.xml b/src/main/resources/changelog/changes/v6.0/05_add_staging_time_columns_to_reindex_status.xml new file mode 100644 index 000000000..c56c4cfd7 --- /dev/null +++ b/src/main/resources/changelog/changes/v6.0/05_add_staging_time_columns_to_reindex_status.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + Add start_time_staging and end_time_staging columns to reindex_status table for tracking staging phase timing + + + + + + + + + + + + diff --git a/src/main/resources/changelog/changes/v6.0/06_create_consortium_member_reindex_status_function.xml b/src/main/resources/changelog/changes/v6.0/06_create_consortium_member_reindex_status_function.xml new file mode 100644 index 000000000..d36241666 --- /dev/null +++ b/src/main/resources/changelog/changes/v6.0/06_create_consortium_member_reindex_status_function.xml @@ -0,0 +1,15 @@ + + + + + Create consortium member reindex status trigger function + + + + + diff --git a/src/main/resources/changelog/changes/v6.0/sql/consortium-member-reindex-status-function.sql b/src/main/resources/changelog/changes/v6.0/sql/consortium-member-reindex-status-function.sql new file mode 100644 index 000000000..effd48bab --- /dev/null +++ b/src/main/resources/changelog/changes/v6.0/sql/consortium-member-reindex-status-function.sql @@ -0,0 +1,20 @@ +CREATE OR REPLACE FUNCTION update_consortium_member_reindex_status_trigger() + RETURNS TRIGGER AS +$$ +BEGIN + -- update status and end time for merge + IF OLD.status = 'MERGE_IN_PROGRESS' and NEW.total_merge_ranges = NEW.processed_merge_ranges + THEN + NEW.status = 'MERGE_COMPLETED'; + NEW.end_time_merge = current_timestamp; + ELSE + -- update status and end time for upload + IF OLD.status = 'UPLOAD_IN_PROGRESS' and NEW.total_upload_ranges = NEW.processed_upload_ranges + THEN + NEW.status = 'UPLOAD_COMPLETED'; + NEW.end_time_upload = current_timestamp; + END IF; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; diff --git a/src/main/resources/changelog/changes/v6.0/sql/create_staging_child_resource_partitions.sql b/src/main/resources/changelog/changes/v6.0/sql/create_staging_child_resource_partitions.sql new file mode 100644 index 000000000..ee75f8e58 --- /dev/null +++ b/src/main/resources/changelog/changes/v6.0/sql/create_staging_child_resource_partitions.sql @@ -0,0 +1,83 @@ +-- Create partitions for staging_subject table based on first character of id +-- Numeric partitions (0-9) +CREATE TABLE staging_subject_0 PARTITION OF staging_subject FOR VALUES FROM ('0') TO ('1'); +CREATE TABLE staging_subject_1 PARTITION OF staging_subject FOR VALUES FROM ('1') TO ('2'); +CREATE TABLE staging_subject_2 PARTITION OF staging_subject FOR VALUES FROM ('2') TO ('3'); +CREATE TABLE staging_subject_3 PARTITION OF staging_subject FOR VALUES FROM ('3') TO ('4'); +CREATE TABLE staging_subject_4 PARTITION OF staging_subject FOR VALUES FROM ('4') TO ('5'); +CREATE TABLE staging_subject_5 PARTITION OF staging_subject FOR VALUES FROM ('5') TO ('6'); +CREATE TABLE staging_subject_6 PARTITION OF staging_subject FOR VALUES FROM ('6') TO ('7'); +CREATE TABLE staging_subject_7 PARTITION OF staging_subject FOR VALUES FROM ('7') TO ('8'); +CREATE TABLE staging_subject_8 PARTITION OF staging_subject FOR VALUES FROM ('8') TO ('9'); +CREATE TABLE staging_subject_9 PARTITION OF staging_subject FOR VALUES FROM ('9') TO ('a'); + +-- Alphabetic partitions (a-f for hash-based IDs) +CREATE TABLE staging_subject_a PARTITION OF staging_subject FOR VALUES FROM ('a') TO ('b'); +CREATE TABLE staging_subject_b PARTITION OF staging_subject FOR VALUES FROM ('b') TO ('c'); +CREATE TABLE staging_subject_c PARTITION OF staging_subject FOR VALUES FROM ('c') TO ('d'); +CREATE TABLE staging_subject_d PARTITION OF staging_subject FOR VALUES FROM ('d') TO ('e'); +CREATE TABLE staging_subject_e PARTITION OF staging_subject FOR VALUES FROM ('e') TO ('f'); +CREATE TABLE staging_subject_f PARTITION OF staging_subject FOR VALUES FROM ('f') TO ('g'); + +-- Create partitions for staging_contributor table based on first character of id +-- Numeric partitions (0-9) +CREATE TABLE staging_contributor_0 PARTITION OF staging_contributor FOR VALUES FROM ('0') TO ('1'); +CREATE TABLE staging_contributor_1 PARTITION OF staging_contributor FOR VALUES FROM ('1') TO ('2'); +CREATE TABLE staging_contributor_2 PARTITION OF staging_contributor FOR VALUES FROM ('2') TO ('3'); +CREATE TABLE staging_contributor_3 PARTITION OF staging_contributor FOR VALUES FROM ('3') TO ('4'); +CREATE TABLE staging_contributor_4 PARTITION OF staging_contributor FOR VALUES FROM ('4') TO ('5'); +CREATE TABLE staging_contributor_5 PARTITION OF staging_contributor FOR VALUES FROM ('5') TO ('6'); +CREATE TABLE staging_contributor_6 PARTITION OF staging_contributor FOR VALUES FROM ('6') TO ('7'); +CREATE TABLE staging_contributor_7 PARTITION OF staging_contributor FOR VALUES FROM ('7') TO ('8'); +CREATE TABLE staging_contributor_8 PARTITION OF staging_contributor FOR VALUES FROM ('8') TO ('9'); +CREATE TABLE staging_contributor_9 PARTITION OF staging_contributor FOR VALUES FROM ('9') TO ('a'); + +-- Alphabetic partitions (a-f for hash-based IDs) +CREATE TABLE staging_contributor_a PARTITION OF staging_contributor FOR VALUES FROM ('a') TO ('b'); +CREATE TABLE staging_contributor_b PARTITION OF staging_contributor FOR VALUES FROM ('b') TO ('c'); +CREATE TABLE staging_contributor_c PARTITION OF staging_contributor FOR VALUES FROM ('c') TO ('d'); +CREATE TABLE staging_contributor_d PARTITION OF staging_contributor FOR VALUES FROM ('d') TO ('e'); +CREATE TABLE staging_contributor_e PARTITION OF staging_contributor FOR VALUES FROM ('e') TO ('f'); +CREATE TABLE staging_contributor_f PARTITION OF staging_contributor FOR VALUES FROM ('f') TO ('g'); + +-- Create partitions for staging_classification table based on first character of id +-- Numeric partitions (0-9) +CREATE TABLE staging_classification_0 PARTITION OF staging_classification FOR VALUES FROM ('0') TO ('1'); +CREATE TABLE staging_classification_1 PARTITION OF staging_classification FOR VALUES FROM ('1') TO ('2'); +CREATE TABLE staging_classification_2 PARTITION OF staging_classification FOR VALUES FROM ('2') TO ('3'); +CREATE TABLE staging_classification_3 PARTITION OF staging_classification FOR VALUES FROM ('3') TO ('4'); +CREATE TABLE staging_classification_4 PARTITION OF staging_classification FOR VALUES FROM ('4') TO ('5'); +CREATE TABLE staging_classification_5 PARTITION OF staging_classification FOR VALUES FROM ('5') TO ('6'); +CREATE TABLE staging_classification_6 PARTITION OF staging_classification FOR VALUES FROM ('6') TO ('7'); +CREATE TABLE staging_classification_7 PARTITION OF staging_classification FOR VALUES FROM ('7') TO ('8'); +CREATE TABLE staging_classification_8 PARTITION OF staging_classification FOR VALUES FROM ('8') TO ('9'); +CREATE TABLE staging_classification_9 PARTITION OF staging_classification FOR VALUES FROM ('9') TO ('a'); + +-- Alphabetic partitions (a-f for hash-based IDs) +CREATE TABLE staging_classification_a PARTITION OF staging_classification FOR VALUES FROM ('a') TO ('b'); +CREATE TABLE staging_classification_b PARTITION OF staging_classification FOR VALUES FROM ('b') TO ('c'); +CREATE TABLE staging_classification_c PARTITION OF staging_classification FOR VALUES FROM ('c') TO ('d'); +CREATE TABLE staging_classification_d PARTITION OF staging_classification FOR VALUES FROM ('d') TO ('e'); +CREATE TABLE staging_classification_e PARTITION OF staging_classification FOR VALUES FROM ('e') TO ('f'); +CREATE TABLE staging_classification_f PARTITION OF staging_classification FOR VALUES FROM ('f') TO ('g'); + +-- Create partitions for staging_call_number table based on first character of id +-- Numeric partitions (0-9) +CREATE TABLE staging_call_number_0 PARTITION OF staging_call_number FOR VALUES FROM ('0') TO ('1'); +CREATE TABLE staging_call_number_1 PARTITION OF staging_call_number FOR VALUES FROM ('1') TO ('2'); +CREATE TABLE staging_call_number_2 PARTITION OF staging_call_number FOR VALUES FROM ('2') TO ('3'); +CREATE TABLE staging_call_number_3 PARTITION OF staging_call_number FOR VALUES FROM ('3') TO ('4'); +CREATE TABLE staging_call_number_4 PARTITION OF staging_call_number FOR VALUES FROM ('4') TO ('5'); +CREATE TABLE staging_call_number_5 PARTITION OF staging_call_number FOR VALUES FROM ('5') TO ('6'); +CREATE TABLE staging_call_number_6 PARTITION OF staging_call_number FOR VALUES FROM ('6') TO ('7'); +CREATE TABLE staging_call_number_7 PARTITION OF staging_call_number FOR VALUES FROM ('7') TO ('8'); +CREATE TABLE staging_call_number_8 PARTITION OF staging_call_number FOR VALUES FROM ('8') TO ('9'); +CREATE TABLE staging_call_number_9 PARTITION OF staging_call_number FOR VALUES FROM ('9') TO ('a'); + +-- Alphabetic partitions (a-f for hash-based IDs) +CREATE TABLE staging_call_number_a PARTITION OF staging_call_number FOR VALUES FROM ('a') TO ('b'); +CREATE TABLE staging_call_number_b PARTITION OF staging_call_number FOR VALUES FROM ('b') TO ('c'); +CREATE TABLE staging_call_number_c PARTITION OF staging_call_number FOR VALUES FROM ('c') TO ('d'); +CREATE TABLE staging_call_number_d PARTITION OF staging_call_number FOR VALUES FROM ('d') TO ('e'); +CREATE TABLE staging_call_number_e PARTITION OF staging_call_number FOR VALUES FROM ('e') TO ('f'); +CREATE TABLE staging_call_number_f PARTITION OF staging_call_number FOR VALUES FROM ('f') TO ('g'); diff --git a/src/main/resources/changelog/changes/v6.0/sql/create_staging_holding_partitions.sql b/src/main/resources/changelog/changes/v6.0/sql/create_staging_holding_partitions.sql new file mode 100644 index 000000000..805f886f6 --- /dev/null +++ b/src/main/resources/changelog/changes/v6.0/sql/create_staging_holding_partitions.sql @@ -0,0 +1,20 @@ +-- Create partitions for staging_holding table based on first character of UUID +-- Numeric partitions (0-9) +CREATE TABLE staging_holding_0 PARTITION OF staging_holding FOR VALUES FROM ('0') TO ('1'); +CREATE TABLE staging_holding_1 PARTITION OF staging_holding FOR VALUES FROM ('1') TO ('2'); +CREATE TABLE staging_holding_2 PARTITION OF staging_holding FOR VALUES FROM ('2') TO ('3'); +CREATE TABLE staging_holding_3 PARTITION OF staging_holding FOR VALUES FROM ('3') TO ('4'); +CREATE TABLE staging_holding_4 PARTITION OF staging_holding FOR VALUES FROM ('4') TO ('5'); +CREATE TABLE staging_holding_5 PARTITION OF staging_holding FOR VALUES FROM ('5') TO ('6'); +CREATE TABLE staging_holding_6 PARTITION OF staging_holding FOR VALUES FROM ('6') TO ('7'); +CREATE TABLE staging_holding_7 PARTITION OF staging_holding FOR VALUES FROM ('7') TO ('8'); +CREATE TABLE staging_holding_8 PARTITION OF staging_holding FOR VALUES FROM ('8') TO ('9'); +CREATE TABLE staging_holding_9 PARTITION OF staging_holding FOR VALUES FROM ('9') TO ('a'); + +-- Alphabetic partitions (a-f for UUIDs) +CREATE TABLE staging_holding_a PARTITION OF staging_holding FOR VALUES FROM ('a') TO ('b'); +CREATE TABLE staging_holding_b PARTITION OF staging_holding FOR VALUES FROM ('b') TO ('c'); +CREATE TABLE staging_holding_c PARTITION OF staging_holding FOR VALUES FROM ('c') TO ('d'); +CREATE TABLE staging_holding_d PARTITION OF staging_holding FOR VALUES FROM ('d') TO ('e'); +CREATE TABLE staging_holding_e PARTITION OF staging_holding FOR VALUES FROM ('e') TO ('f'); +CREATE TABLE staging_holding_f PARTITION OF staging_holding FOR VALUES FROM ('f') TO ('g'); diff --git a/src/main/resources/changelog/changes/v6.0/sql/create_staging_instance_partitions.sql b/src/main/resources/changelog/changes/v6.0/sql/create_staging_instance_partitions.sql new file mode 100644 index 000000000..80f542a16 --- /dev/null +++ b/src/main/resources/changelog/changes/v6.0/sql/create_staging_instance_partitions.sql @@ -0,0 +1,20 @@ +-- Create partitions for staging_instance table based on first character of UUID +-- Numeric partitions (0-9) +CREATE TABLE staging_instance_0 PARTITION OF staging_instance FOR VALUES FROM ('0') TO ('1'); +CREATE TABLE staging_instance_1 PARTITION OF staging_instance FOR VALUES FROM ('1') TO ('2'); +CREATE TABLE staging_instance_2 PARTITION OF staging_instance FOR VALUES FROM ('2') TO ('3'); +CREATE TABLE staging_instance_3 PARTITION OF staging_instance FOR VALUES FROM ('3') TO ('4'); +CREATE TABLE staging_instance_4 PARTITION OF staging_instance FOR VALUES FROM ('4') TO ('5'); +CREATE TABLE staging_instance_5 PARTITION OF staging_instance FOR VALUES FROM ('5') TO ('6'); +CREATE TABLE staging_instance_6 PARTITION OF staging_instance FOR VALUES FROM ('6') TO ('7'); +CREATE TABLE staging_instance_7 PARTITION OF staging_instance FOR VALUES FROM ('7') TO ('8'); +CREATE TABLE staging_instance_8 PARTITION OF staging_instance FOR VALUES FROM ('8') TO ('9'); +CREATE TABLE staging_instance_9 PARTITION OF staging_instance FOR VALUES FROM ('9') TO ('a'); + +-- Alphabetic partitions (a-f for UUIDs) +CREATE TABLE staging_instance_a PARTITION OF staging_instance FOR VALUES FROM ('a') TO ('b'); +CREATE TABLE staging_instance_b PARTITION OF staging_instance FOR VALUES FROM ('b') TO ('c'); +CREATE TABLE staging_instance_c PARTITION OF staging_instance FOR VALUES FROM ('c') TO ('d'); +CREATE TABLE staging_instance_d PARTITION OF staging_instance FOR VALUES FROM ('d') TO ('e'); +CREATE TABLE staging_instance_e PARTITION OF staging_instance FOR VALUES FROM ('e') TO ('f'); +CREATE TABLE staging_instance_f PARTITION OF staging_instance FOR VALUES FROM ('f') TO ('g'); diff --git a/src/main/resources/changelog/changes/v6.0/sql/create_staging_item_partitions.sql b/src/main/resources/changelog/changes/v6.0/sql/create_staging_item_partitions.sql new file mode 100644 index 000000000..8d2f6f638 --- /dev/null +++ b/src/main/resources/changelog/changes/v6.0/sql/create_staging_item_partitions.sql @@ -0,0 +1,20 @@ +-- Create partitions for staging_item table based on first character of UUID +-- Numeric partitions (0-9) +CREATE TABLE staging_item_0 PARTITION OF staging_item FOR VALUES FROM ('0') TO ('1'); +CREATE TABLE staging_item_1 PARTITION OF staging_item FOR VALUES FROM ('1') TO ('2'); +CREATE TABLE staging_item_2 PARTITION OF staging_item FOR VALUES FROM ('2') TO ('3'); +CREATE TABLE staging_item_3 PARTITION OF staging_item FOR VALUES FROM ('3') TO ('4'); +CREATE TABLE staging_item_4 PARTITION OF staging_item FOR VALUES FROM ('4') TO ('5'); +CREATE TABLE staging_item_5 PARTITION OF staging_item FOR VALUES FROM ('5') TO ('6'); +CREATE TABLE staging_item_6 PARTITION OF staging_item FOR VALUES FROM ('6') TO ('7'); +CREATE TABLE staging_item_7 PARTITION OF staging_item FOR VALUES FROM ('7') TO ('8'); +CREATE TABLE staging_item_8 PARTITION OF staging_item FOR VALUES FROM ('8') TO ('9'); +CREATE TABLE staging_item_9 PARTITION OF staging_item FOR VALUES FROM ('9') TO ('a'); + +-- Alphabetic partitions (a-f for UUIDs) +CREATE TABLE staging_item_a PARTITION OF staging_item FOR VALUES FROM ('a') TO ('b'); +CREATE TABLE staging_item_b PARTITION OF staging_item FOR VALUES FROM ('b') TO ('c'); +CREATE TABLE staging_item_c PARTITION OF staging_item FOR VALUES FROM ('c') TO ('d'); +CREATE TABLE staging_item_d PARTITION OF staging_item FOR VALUES FROM ('d') TO ('e'); +CREATE TABLE staging_item_e PARTITION OF staging_item FOR VALUES FROM ('e') TO ('f'); +CREATE TABLE staging_item_f PARTITION OF staging_item FOR VALUES FROM ('f') TO ('g'); diff --git a/src/main/resources/changelog/changes/v6.0/sql/create_staging_relationship_partitions.sql b/src/main/resources/changelog/changes/v6.0/sql/create_staging_relationship_partitions.sql new file mode 100644 index 000000000..5993841a0 --- /dev/null +++ b/src/main/resources/changelog/changes/v6.0/sql/create_staging_relationship_partitions.sql @@ -0,0 +1,83 @@ +-- Create partitions for staging_instance_subject table based on first character of instance_id +-- Numeric partitions (0-9) +CREATE TABLE staging_instance_subject_0 PARTITION OF staging_instance_subject FOR VALUES FROM ('0') TO ('1'); +CREATE TABLE staging_instance_subject_1 PARTITION OF staging_instance_subject FOR VALUES FROM ('1') TO ('2'); +CREATE TABLE staging_instance_subject_2 PARTITION OF staging_instance_subject FOR VALUES FROM ('2') TO ('3'); +CREATE TABLE staging_instance_subject_3 PARTITION OF staging_instance_subject FOR VALUES FROM ('3') TO ('4'); +CREATE TABLE staging_instance_subject_4 PARTITION OF staging_instance_subject FOR VALUES FROM ('4') TO ('5'); +CREATE TABLE staging_instance_subject_5 PARTITION OF staging_instance_subject FOR VALUES FROM ('5') TO ('6'); +CREATE TABLE staging_instance_subject_6 PARTITION OF staging_instance_subject FOR VALUES FROM ('6') TO ('7'); +CREATE TABLE staging_instance_subject_7 PARTITION OF staging_instance_subject FOR VALUES FROM ('7') TO ('8'); +CREATE TABLE staging_instance_subject_8 PARTITION OF staging_instance_subject FOR VALUES FROM ('8') TO ('9'); +CREATE TABLE staging_instance_subject_9 PARTITION OF staging_instance_subject FOR VALUES FROM ('9') TO ('a'); + +-- Alphabetic partitions (a-f for UUIDs) +CREATE TABLE staging_instance_subject_a PARTITION OF staging_instance_subject FOR VALUES FROM ('a') TO ('b'); +CREATE TABLE staging_instance_subject_b PARTITION OF staging_instance_subject FOR VALUES FROM ('b') TO ('c'); +CREATE TABLE staging_instance_subject_c PARTITION OF staging_instance_subject FOR VALUES FROM ('c') TO ('d'); +CREATE TABLE staging_instance_subject_d PARTITION OF staging_instance_subject FOR VALUES FROM ('d') TO ('e'); +CREATE TABLE staging_instance_subject_e PARTITION OF staging_instance_subject FOR VALUES FROM ('e') TO ('f'); +CREATE TABLE staging_instance_subject_f PARTITION OF staging_instance_subject FOR VALUES FROM ('f') TO ('g'); + +-- Create partitions for staging_instance_contributor table based on first character of instance_id +-- Numeric partitions (0-9) +CREATE TABLE staging_instance_contributor_0 PARTITION OF staging_instance_contributor FOR VALUES FROM ('0') TO ('1'); +CREATE TABLE staging_instance_contributor_1 PARTITION OF staging_instance_contributor FOR VALUES FROM ('1') TO ('2'); +CREATE TABLE staging_instance_contributor_2 PARTITION OF staging_instance_contributor FOR VALUES FROM ('2') TO ('3'); +CREATE TABLE staging_instance_contributor_3 PARTITION OF staging_instance_contributor FOR VALUES FROM ('3') TO ('4'); +CREATE TABLE staging_instance_contributor_4 PARTITION OF staging_instance_contributor FOR VALUES FROM ('4') TO ('5'); +CREATE TABLE staging_instance_contributor_5 PARTITION OF staging_instance_contributor FOR VALUES FROM ('5') TO ('6'); +CREATE TABLE staging_instance_contributor_6 PARTITION OF staging_instance_contributor FOR VALUES FROM ('6') TO ('7'); +CREATE TABLE staging_instance_contributor_7 PARTITION OF staging_instance_contributor FOR VALUES FROM ('7') TO ('8'); +CREATE TABLE staging_instance_contributor_8 PARTITION OF staging_instance_contributor FOR VALUES FROM ('8') TO ('9'); +CREATE TABLE staging_instance_contributor_9 PARTITION OF staging_instance_contributor FOR VALUES FROM ('9') TO ('a'); + +-- Alphabetic partitions (a-f for UUIDs) +CREATE TABLE staging_instance_contributor_a PARTITION OF staging_instance_contributor FOR VALUES FROM ('a') TO ('b'); +CREATE TABLE staging_instance_contributor_b PARTITION OF staging_instance_contributor FOR VALUES FROM ('b') TO ('c'); +CREATE TABLE staging_instance_contributor_c PARTITION OF staging_instance_contributor FOR VALUES FROM ('c') TO ('d'); +CREATE TABLE staging_instance_contributor_d PARTITION OF staging_instance_contributor FOR VALUES FROM ('d') TO ('e'); +CREATE TABLE staging_instance_contributor_e PARTITION OF staging_instance_contributor FOR VALUES FROM ('e') TO ('f'); +CREATE TABLE staging_instance_contributor_f PARTITION OF staging_instance_contributor FOR VALUES FROM ('f') TO ('g'); + +-- Create partitions for staging_instance_classification table based on first character of instance_id +-- Numeric partitions (0-9) +CREATE TABLE staging_instance_classification_0 PARTITION OF staging_instance_classification FOR VALUES FROM ('0') TO ('1'); +CREATE TABLE staging_instance_classification_1 PARTITION OF staging_instance_classification FOR VALUES FROM ('1') TO ('2'); +CREATE TABLE staging_instance_classification_2 PARTITION OF staging_instance_classification FOR VALUES FROM ('2') TO ('3'); +CREATE TABLE staging_instance_classification_3 PARTITION OF staging_instance_classification FOR VALUES FROM ('3') TO ('4'); +CREATE TABLE staging_instance_classification_4 PARTITION OF staging_instance_classification FOR VALUES FROM ('4') TO ('5'); +CREATE TABLE staging_instance_classification_5 PARTITION OF staging_instance_classification FOR VALUES FROM ('5') TO ('6'); +CREATE TABLE staging_instance_classification_6 PARTITION OF staging_instance_classification FOR VALUES FROM ('6') TO ('7'); +CREATE TABLE staging_instance_classification_7 PARTITION OF staging_instance_classification FOR VALUES FROM ('7') TO ('8'); +CREATE TABLE staging_instance_classification_8 PARTITION OF staging_instance_classification FOR VALUES FROM ('8') TO ('9'); +CREATE TABLE staging_instance_classification_9 PARTITION OF staging_instance_classification FOR VALUES FROM ('9') TO ('a'); + +-- Alphabetic partitions (a-f for UUIDs) +CREATE TABLE staging_instance_classification_a PARTITION OF staging_instance_classification FOR VALUES FROM ('a') TO ('b'); +CREATE TABLE staging_instance_classification_b PARTITION OF staging_instance_classification FOR VALUES FROM ('b') TO ('c'); +CREATE TABLE staging_instance_classification_c PARTITION OF staging_instance_classification FOR VALUES FROM ('c') TO ('d'); +CREATE TABLE staging_instance_classification_d PARTITION OF staging_instance_classification FOR VALUES FROM ('d') TO ('e'); +CREATE TABLE staging_instance_classification_e PARTITION OF staging_instance_classification FOR VALUES FROM ('e') TO ('f'); +CREATE TABLE staging_instance_classification_f PARTITION OF staging_instance_classification FOR VALUES FROM ('f') TO ('g'); + +-- Create partitions for staging_instance_call_number table based on first character of instance_id +-- Numeric partitions (0-9) +CREATE TABLE staging_instance_call_number_0 PARTITION OF staging_instance_call_number FOR VALUES FROM ('0') TO ('1'); +CREATE TABLE staging_instance_call_number_1 PARTITION OF staging_instance_call_number FOR VALUES FROM ('1') TO ('2'); +CREATE TABLE staging_instance_call_number_2 PARTITION OF staging_instance_call_number FOR VALUES FROM ('2') TO ('3'); +CREATE TABLE staging_instance_call_number_3 PARTITION OF staging_instance_call_number FOR VALUES FROM ('3') TO ('4'); +CREATE TABLE staging_instance_call_number_4 PARTITION OF staging_instance_call_number FOR VALUES FROM ('4') TO ('5'); +CREATE TABLE staging_instance_call_number_5 PARTITION OF staging_instance_call_number FOR VALUES FROM ('5') TO ('6'); +CREATE TABLE staging_instance_call_number_6 PARTITION OF staging_instance_call_number FOR VALUES FROM ('6') TO ('7'); +CREATE TABLE staging_instance_call_number_7 PARTITION OF staging_instance_call_number FOR VALUES FROM ('7') TO ('8'); +CREATE TABLE staging_instance_call_number_8 PARTITION OF staging_instance_call_number FOR VALUES FROM ('8') TO ('9'); +CREATE TABLE staging_instance_call_number_9 PARTITION OF staging_instance_call_number FOR VALUES FROM ('9') TO ('a'); + +-- Alphabetic partitions (a-f for UUIDs) +CREATE TABLE staging_instance_call_number_a PARTITION OF staging_instance_call_number FOR VALUES FROM ('a') TO ('b'); +CREATE TABLE staging_instance_call_number_b PARTITION OF staging_instance_call_number FOR VALUES FROM ('b') TO ('c'); +CREATE TABLE staging_instance_call_number_c PARTITION OF staging_instance_call_number FOR VALUES FROM ('c') TO ('d'); +CREATE TABLE staging_instance_call_number_d PARTITION OF staging_instance_call_number FOR VALUES FROM ('d') TO ('e'); +CREATE TABLE staging_instance_call_number_e PARTITION OF staging_instance_call_number FOR VALUES FROM ('e') TO ('f'); +CREATE TABLE staging_instance_call_number_f PARTITION OF staging_instance_call_number FOR VALUES FROM ('f') TO ('g'); diff --git a/src/main/resources/swagger.api/examples/request/reindexFullRequest.yaml b/src/main/resources/swagger.api/examples/request/reindexFullRequest.yaml new file mode 100644 index 000000000..35d63773f --- /dev/null +++ b/src/main/resources/swagger.api/examples/request/reindexFullRequest.yaml @@ -0,0 +1,6 @@ +value: + tenantId: "college" + indexSettings: + numberOfShards: 1 + numberOfReplicas: 1 + refreshInterval: 1 diff --git a/src/main/resources/swagger.api/examples/result/ReindexStatusResult.yaml b/src/main/resources/swagger.api/examples/result/ReindexStatusResult.yaml index bc10d89f6..b47cf3af5 100644 --- a/src/main/resources/swagger.api/examples/result/ReindexStatusResult.yaml +++ b/src/main/resources/swagger.api/examples/result/ReindexStatusResult.yaml @@ -1,6 +1,7 @@ value: - entityType: 'instance' - status: 'Upload Completed' + status: 'UPLOAD_COMPLETED' + targetTenantId: 'college' totalMergeRanges: 3 processedMergeRanges: 3 totalUploadRanges: 2 @@ -10,18 +11,30 @@ value: startTimeUpload: '2024-04-01T01:37:36.15755006Z' endTimeUpload: '2024-04-01T01:37:37.15755006Z' - entityType: 'item' - status: 'Merge In Progress' + status: 'MERGE_IN_PROGRESS' + targetTenantId: 'college' totalMergeRanges: 3 processedMergeRanges: 2 startTimeMerge: '2024-04-01T01:37:34.15755006Z' + - entityType: 'holdings' + status: 'STAGING_COMPLETED' + targetTenantId: 'college' + totalMergeRanges: 3 + processedMergeRanges: 2 + startTimeMerge: '2024-04-01T01:37:34.15755006Z' + endTimeMerge: '2024-04-01T01:37:35.15755006Z' + startTimeStaging: '2024-04-01T01:37:30.15755006Z' + endTimeStaging: '2024-04-01T01:37:33.15755006Z' - entityType: 'contributor' - status: 'Upload Completed' + status: 'UPLOAD_COMPLETED' + targetTenantId: 'college' totalUploadRanges: 3 processedUploadRanges: 3 startTimeUpload: '2024-04-01T01:37:34.15755006Z' endTimeUpload: '2024-04-01T01:37:35.15755006Z' - entityType: 'classification' - status: 'Upload Failed' + status: 'UPLOAD_FAILED' + targetTenantId: 'college' totalUploadRanges: 2 processedUploadRanges: 1 startTimeUpload: '2024-04-01T01:37:36.15755006Z' diff --git a/src/main/resources/swagger.api/paths/reindex-instance-records/reindex-instance-records-full.yaml b/src/main/resources/swagger.api/paths/reindex-instance-records/reindex-instance-records-full.yaml index dff50dd52..49a5b2869 100644 --- a/src/main/resources/swagger.api/paths/reindex-instance-records/reindex-instance-records-full.yaml +++ b/src/main/resources/swagger.api/paths/reindex-instance-records/reindex-instance-records-full.yaml @@ -8,10 +8,10 @@ post: content: application/json: examples: - indexSettings: - $ref: '../../examples/request/indexSettings.yaml' + reindexFullRequest: + $ref: '../../examples/request/reindexFullRequest.yaml' schema: - $ref: '../../schemas/entity/indexSettings.yaml' + $ref: '../../schemas/request/reindexFullRequest.yaml' parameters: - $ref: '../../parameters/x-okapi-tenant-header.yaml' responses: diff --git a/src/main/resources/swagger.api/schemas/request/reindexFullRequest.yaml b/src/main/resources/swagger.api/schemas/request/reindexFullRequest.yaml new file mode 100644 index 000000000..b13e87741 --- /dev/null +++ b/src/main/resources/swagger.api/schemas/request/reindexFullRequest.yaml @@ -0,0 +1,9 @@ +description: Full reindex request with optional tenant filtering +type: object +properties: + tenantId: + type: string + description: Optional specific tenant ID to reindex. If not provided, reindexes all consortium members. + pattern: '^[a-zA-Z0-9_.-]+$' + indexSettings: + $ref: '../entity/indexSettings.yaml' diff --git a/src/main/resources/swagger.api/schemas/response/reindexStatusItem.yaml b/src/main/resources/swagger.api/schemas/response/reindexStatusItem.yaml index 8b3f1c8ae..fb31c651c 100644 --- a/src/main/resources/swagger.api/schemas/response/reindexStatusItem.yaml +++ b/src/main/resources/swagger.api/schemas/response/reindexStatusItem.yaml @@ -7,6 +7,9 @@ properties: status: type: string description: Reindex status + targetTenantId: + type: string + description: tenantId of consortium member being re-indexed totalMergeRanges: type: integer description: Total merge ranges to process for entity @@ -31,3 +34,9 @@ properties: endTimeUpload: type: string description: End time of reindex upload phase for entity + startTimeStaging: + type: string + description: Start time of reindex staging phase for entity + endTimeStaging: + type: string + description: End time of reindex staging phase for entity diff --git a/src/test/java/org/folio/search/repository/IndexRepositoryTest.java b/src/test/java/org/folio/search/repository/IndexRepositoryTest.java index 1a8198dca..847426f5b 100644 --- a/src/test/java/org/folio/search/repository/IndexRepositoryTest.java +++ b/src/test/java/org/folio/search/repository/IndexRepositoryTest.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.util.List; +import org.folio.search.configuration.properties.IndexManagementConfigurationProperties; import org.folio.search.exception.SearchOperationException; import org.folio.spring.testing.type.UnitTest; import org.junit.jupiter.api.Test; @@ -46,6 +47,8 @@ class IndexRepositoryTest { private RestHighLevelClient restHighLevelClient; @Mock private IndicesClient indices; + @Mock + private IndexManagementConfigurationProperties indexManagementConfig; @Test void createIndex_positive() throws IOException { diff --git a/src/test/java/org/folio/search/repository/PrimaryResourceRepositoryTest.java b/src/test/java/org/folio/search/repository/PrimaryResourceRepositoryTest.java index b77abd560..16b5b66c0 100644 --- a/src/test/java/org/folio/search/repository/PrimaryResourceRepositoryTest.java +++ b/src/test/java/org/folio/search/repository/PrimaryResourceRepositoryTest.java @@ -6,6 +6,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.folio.search.utils.SearchResponseHelper.getErrorIndexOperationResponse; import static org.folio.search.utils.SearchResponseHelper.getSuccessIndexOperationResponse; +import static org.folio.support.TestConstants.INDEX_NAME; +import static org.folio.support.TestConstants.TENANT_ID; import static org.folio.support.utils.TestUtils.searchDocumentBody; import static org.folio.support.utils.TestUtils.searchDocumentBodyToDelete; import static org.mockito.ArgumentMatchers.any; @@ -17,8 +19,10 @@ import java.io.IOException; import java.util.List; +import org.folio.search.configuration.properties.IndexManagementConfigurationProperties; import org.folio.search.exception.SearchOperationException; import org.folio.search.model.index.SearchDocumentBody; +import org.folio.search.model.types.ResourceType; import org.folio.spring.testing.type.UnitTest; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -32,6 +36,9 @@ import org.opensearch.action.delete.DeleteRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.client.RestHighLevelClient; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.reindex.BulkByScrollResponse; +import org.opensearch.index.reindex.DeleteByQueryRequest; @UnitTest @ExtendWith(MockitoExtension.class) @@ -43,6 +50,8 @@ class PrimaryResourceRepositoryTest { private RestHighLevelClient restHighLevelClient; @Mock private IndexNameProvider indexNameProvider; + @Mock + private IndexManagementConfigurationProperties indexManagementConfig; @BeforeEach void setUp() { @@ -97,4 +106,41 @@ void indexResources_negative_throwsException() throws IOException { .hasMessage("Failed to perform elasticsearch request " + "[index=index_name, type=bulkApi, message: err]"); } + + @Test + void deleteDocumentsByTenantId_positive() throws IOException { + var deleteByQueryRequestCaptor = ArgumentCaptor.forClass(DeleteByQueryRequest.class); + var bulkByScrollResponse = mock(BulkByScrollResponse.class); + + when(bulkByScrollResponse.getDeleted()).thenReturn(5L); + when(indexNameProvider.getIndexName(ResourceType.INSTANCE, TENANT_ID)).thenReturn(INDEX_NAME); + when(restHighLevelClient.deleteByQuery(deleteByQueryRequestCaptor.capture(), eq(DEFAULT))) + .thenReturn(bulkByScrollResponse); + + var response = resourceRepository.deleteConsortiumDocumentsByTenantId(ResourceType.INSTANCE, TENANT_ID); + + assertThat(response).isEqualTo(getSuccessIndexOperationResponse()); + var capturedRequest = deleteByQueryRequestCaptor.getValue(); + assertThat(capturedRequest.indices()).containsExactly(INDEX_NAME); + + // Verify query structure - should have bool query with must and must_not clauses + var query = capturedRequest.getSearchRequest().source().query(); + assertThat(query).isInstanceOf(BoolQueryBuilder.class); + var boolQuery = (BoolQueryBuilder) query; + assertThat(boolQuery.must()).hasSize(1); + assertThat(boolQuery.mustNot()).hasSize(1); + } + + @Test + void deleteDocumentsByTenantId_negative_throwsException() throws IOException { + when(restHighLevelClient.deleteByQuery(any(DeleteByQueryRequest.class), eq(DEFAULT))) + .thenThrow(new IOException("delete error")); + when(indexNameProvider.getIndexName(ResourceType.INSTANCE, TENANT_ID)).thenReturn(INDEX_NAME); + + assertThatThrownBy(() -> resourceRepository.deleteConsortiumDocumentsByTenantId(ResourceType.INSTANCE, TENANT_ID)) + .isInstanceOf(SearchOperationException.class) + .hasCauseExactlyInstanceOf(IOException.class) + .hasMessage("Failed to perform elasticsearch request " + + "[index=folio_instance_test_tenant, type=deleteByQueryApi, message: delete error]"); + } } diff --git a/src/test/java/org/folio/search/service/reindex/ReindexCommonServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexCommonServiceTest.java new file mode 100644 index 000000000..dd58e0812 --- /dev/null +++ b/src/test/java/org/folio/search/service/reindex/ReindexCommonServiceTest.java @@ -0,0 +1,246 @@ +package org.folio.search.service.reindex; + +import static org.folio.search.model.types.ReindexEntityType.CALL_NUMBER; +import static org.folio.search.model.types.ReindexEntityType.CLASSIFICATION; +import static org.folio.search.model.types.ReindexEntityType.CONTRIBUTOR; +import static org.folio.search.model.types.ReindexEntityType.HOLDINGS; +import static org.folio.search.model.types.ReindexEntityType.INSTANCE; +import static org.folio.search.model.types.ReindexEntityType.ITEM; +import static org.folio.search.model.types.ReindexEntityType.SUBJECT; +import static org.folio.support.TestConstants.TENANT_ID; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import org.folio.search.domain.dto.FolioCreateIndexResponse; +import org.folio.search.domain.dto.FolioIndexOperationResponse; +import org.folio.search.domain.dto.IndexSettings; +import org.folio.search.model.types.ResourceType; +import org.folio.search.repository.PrimaryResourceRepository; +import org.folio.search.service.IndexService; +import org.folio.search.service.reindex.jdbc.CallNumberRepository; +import org.folio.search.service.reindex.jdbc.ClassificationRepository; +import org.folio.search.service.reindex.jdbc.ContributorRepository; +import org.folio.search.service.reindex.jdbc.HoldingRepository; +import org.folio.search.service.reindex.jdbc.ItemRepository; +import org.folio.search.service.reindex.jdbc.ReindexJdbcRepository; +import org.folio.search.service.reindex.jdbc.SubjectRepository; +import org.folio.spring.testing.type.UnitTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@UnitTest +@ExtendWith(MockitoExtension.class) +class ReindexCommonServiceTest { + + private ReindexCommonService service; + + @Mock + private ReindexJdbcRepository instanceRepository; + @Mock + private HoldingRepository holdingRepository; + @Mock + private ItemRepository itemRepository; + @Mock + private SubjectRepository subjectRepository; + @Mock + private ContributorRepository contributorRepository; + @Mock + private ClassificationRepository classificationRepository; + @Mock + private CallNumberRepository callNumberRepository; + @Mock + private IndexService indexService; + @Mock + private PrimaryResourceRepository resourceRepository; + + @BeforeEach + void setUp() { + when(instanceRepository.entityType()).thenReturn(INSTANCE); + when(holdingRepository.entityType()).thenReturn(HOLDINGS); + when(itemRepository.entityType()).thenReturn(ITEM); + when(subjectRepository.entityType()).thenReturn(SUBJECT); + when(contributorRepository.entityType()).thenReturn(CONTRIBUTOR); + when(classificationRepository.entityType()).thenReturn(CLASSIFICATION); + when(callNumberRepository.entityType()).thenReturn(CALL_NUMBER); + + List repositories = List.of( + instanceRepository, holdingRepository, itemRepository, + subjectRepository, contributorRepository, classificationRepository, callNumberRepository + ); + + service = new ReindexCommonService(repositories, indexService, resourceRepository); + } + + @Test + void deleteAllRecords_withNullTenantId_shouldTruncateAllTables() { + // Act + service.deleteAllRecords(null); + + // Assert - verify truncate is called for all entity types + verify(instanceRepository).truncate(); + verify(holdingRepository).truncate(); + verify(itemRepository).truncate(); + verify(subjectRepository).truncate(); + verify(contributorRepository).truncate(); + verify(classificationRepository).truncate(); + verify(callNumberRepository).truncate(); + + // Verify truncateStaging is NOT called + verify(instanceRepository, times(0)).truncateStaging(); + } + + @Test + void deleteAllRecords_withTenantId_shouldTruncateOnlyStagingTables() { + // Act + service.deleteAllRecords(TENANT_ID); + + // Assert - verify truncateStaging is called for all entity types + verify(instanceRepository).truncateStaging(); + verify(holdingRepository).truncateStaging(); + verify(itemRepository).truncateStaging(); + verify(subjectRepository).truncateStaging(); + verify(contributorRepository).truncateStaging(); + verify(classificationRepository).truncateStaging(); + verify(callNumberRepository).truncateStaging(); + + // Verify regular truncate is NOT called + verify(instanceRepository, times(0)).truncate(); + } + + @Test + void deleteRecordsByTenantId_shouldDeleteInCorrectOrder() { + // Act + service.deleteRecordsByTenantId(TENANT_ID); + + // Assert - verify order: child entities first, then parent entities + var inOrder = inOrder( + subjectRepository, contributorRepository, classificationRepository, callNumberRepository, + itemRepository, holdingRepository, instanceRepository + ); + + // Child entities (relationships) first + inOrder.verify(subjectRepository).deleteByTenantId(TENANT_ID); + inOrder.verify(contributorRepository).deleteByTenantId(TENANT_ID); + inOrder.verify(classificationRepository).deleteByTenantId(TENANT_ID); + inOrder.verify(callNumberRepository).deleteByTenantId(TENANT_ID); + + // Then parent entities + inOrder.verify(itemRepository).deleteByTenantId(TENANT_ID); + inOrder.verify(holdingRepository).deleteByTenantId(TENANT_ID); + inOrder.verify(instanceRepository).deleteByTenantId(TENANT_ID); + } + + @Test + void deleteInstanceDocumentsByTenantId_shouldCallDeleteOnRepository() { + // Arrange + when(resourceRepository.deleteConsortiumDocumentsByTenantId(ResourceType.INSTANCE, TENANT_ID)) + .thenReturn(new FolioIndexOperationResponse()); + + // Act + service.deleteInstanceDocumentsByTenantId(TENANT_ID); + + // Assert + verify(resourceRepository).deleteConsortiumDocumentsByTenantId(ResourceType.INSTANCE, TENANT_ID); + } + + @Test + void deleteInstanceDocumentsByTenantId_whenExceptionOccurs_shouldLogAndContinue() { + // Arrange + when(resourceRepository.deleteConsortiumDocumentsByTenantId(any(), any())) + .thenThrow(new RuntimeException("Test exception")); + + // Act - should not throw exception + assertDoesNotThrow(() -> service.deleteInstanceDocumentsByTenantId(TENANT_ID)); + } + + @Test + void recreateIndex_shouldCallIndexService() { + // Arrange + var indexSettings = new IndexSettings(); + doNothing().when(indexService).dropIndex(any(), any()); + when(indexService.createIndex(any(), any(), any())).thenReturn(new FolioCreateIndexResponse()); + + // Act + service.recreateIndex(INSTANCE, TENANT_ID, indexSettings); + + // Assert + verify(indexService).dropIndex(ResourceType.INSTANCE, TENANT_ID); + verify(indexService).createIndex(ResourceType.INSTANCE, TENANT_ID, indexSettings); + } + + @Test + void recreateIndex_withNullSettings_shouldCallIndexServiceWithoutSettings() { + // Arrange + doNothing().when(indexService).dropIndex(any(), any()); + when(indexService.createIndex(any(), any(), any())).thenReturn(new FolioCreateIndexResponse()); + + // Act + service.recreateIndex(INSTANCE, TENANT_ID, null); + + // Assert + verify(indexService).dropIndex(ResourceType.INSTANCE, TENANT_ID); + verify(indexService).createIndex(ResourceType.INSTANCE, TENANT_ID, null); + } + + @Test + void ensureIndexExists_shouldCallCreateIndexIfNotExist() { + // Arrange + var indexSettings = new IndexSettings(); + doNothing().when(indexService).createIndexIfNotExist(any(), any(), any()); + + // Act + service.ensureIndexExists(INSTANCE, TENANT_ID, indexSettings); + + // Assert + verify(indexService).createIndexIfNotExist(ResourceType.INSTANCE, TENANT_ID, indexSettings); + } + + @Test + void ensureIndexExists_withNullSettings_shouldCallCreateIndexIfNotExistWithoutSettings() { + // Arrange + doNothing().when(indexService).createIndexIfNotExist(any(), any()); + + // Act + service.ensureIndexExists(INSTANCE, TENANT_ID, null); + + // Assert + verify(indexService).createIndexIfNotExist(ResourceType.INSTANCE, TENANT_ID); + } + + @Test + void ensureIndexExists_whenExceptionOccurs_shouldNotPropagateException() { + // Arrange + doThrow(new RuntimeException("Test exception")) + .when(indexService).createIndexIfNotExist(any(), any()); + + // Act - should not throw exception + service.ensureIndexExists(INSTANCE, TENANT_ID, null); + + // Assert + verify(indexService).createIndexIfNotExist(ResourceType.INSTANCE, TENANT_ID); + } + + @Test + void recreateIndex_whenExceptionOccurs_shouldNotPropagateException() { + // Arrange + doThrow(new RuntimeException("Test exception")) + .when(indexService).dropIndex(any(ResourceType.class), any()); + + // Act - should not throw exception + service.recreateIndex(INSTANCE, TENANT_ID, null); + + // Assert + verify(indexService).dropIndex(ResourceType.INSTANCE, TENANT_ID); + } +} + diff --git a/src/test/java/org/folio/search/service/reindex/ReindexContextTest.java b/src/test/java/org/folio/search/service/reindex/ReindexContextTest.java new file mode 100644 index 000000000..4318edc83 --- /dev/null +++ b/src/test/java/org/folio/search/service/reindex/ReindexContextTest.java @@ -0,0 +1,182 @@ +package org.folio.search.service.reindex; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.folio.spring.testing.type.UnitTest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +@UnitTest +class ReindexContextTest { + + @AfterEach + void tearDown() { + // Clean up after each test to prevent side effects + ReindexContext.clear(); + } + + @Test + void reindexMode_defaultValue_shouldBeFalse() { + assertThat(ReindexContext.isReindexMode()).isFalse(); + } + + @Test + void setReindexMode_shouldUpdateValue() { + ReindexContext.setReindexMode(true); + assertThat(ReindexContext.isReindexMode()).isTrue(); + + ReindexContext.setReindexMode(false); + assertThat(ReindexContext.isReindexMode()).isFalse(); + } + + @Test + void memberTenantId_defaultValue_shouldBeNull() { + assertThat(ReindexContext.getMemberTenantId()).isNull(); + assertThat(ReindexContext.isMemberTenantReindex()).isFalse(); + } + + @Test + void setMemberTenantId_shouldUpdateValue() { + var tenantId = "test_tenant"; + ReindexContext.setMemberTenantId(tenantId); + + assertThat(ReindexContext.getMemberTenantId()).isEqualTo(tenantId); + assertThat(ReindexContext.isMemberTenantReindex()).isTrue(); + } + + @Test + void clearMemberTenantId_shouldResetToNull() { + ReindexContext.setMemberTenantId("test_tenant"); + ReindexContext.clearMemberTenantId(); + + assertThat(ReindexContext.getMemberTenantId()).isNull(); + assertThat(ReindexContext.isMemberTenantReindex()).isFalse(); + } + + @Test + void clear_shouldResetAllValues() { + ReindexContext.setReindexMode(true); + ReindexContext.setMemberTenantId("test_tenant"); + + ReindexContext.clear(); + + assertThat(ReindexContext.isReindexMode()).isFalse(); + assertThat(ReindexContext.getMemberTenantId()).isNull(); + assertThat(ReindexContext.isMemberTenantReindex()).isFalse(); + } + + @Test + void threadLocal_shouldIsolateValuesBetweenThreads() throws InterruptedException { + var mainThreadTenantId = "main_tenant"; + var workerThreadTenantId = "worker_tenant"; + + ReindexContext.setMemberTenantId(mainThreadTenantId); + ReindexContext.setReindexMode(true); + + var latch = new CountDownLatch(1); + var workerResults = new ArrayList(); + var workerModes = new ArrayList(); + + var thread = new Thread(() -> { + try { + // Worker thread should have default values initially + workerResults.add(ReindexContext.getMemberTenantId()); + workerModes.add(ReindexContext.isReindexMode()); + + // Set different values in worker thread + ReindexContext.setMemberTenantId(workerThreadTenantId); + ReindexContext.setReindexMode(false); + + workerResults.add(ReindexContext.getMemberTenantId()); + workerModes.add(ReindexContext.isReindexMode()); + } finally { + ReindexContext.clear(); + latch.countDown(); + } + }); + + thread.start(); + var completed = latch.await(5, TimeUnit.SECONDS); + assertThat(completed).as("Thread should complete within timeout").isTrue(); + + // Main thread should still have its original values + assertThat(ReindexContext.getMemberTenantId()).isEqualTo(mainThreadTenantId); + assertThat(ReindexContext.isReindexMode()).isTrue(); + + // Worker thread should have had isolated values + assertThat(workerResults.get(0)).isNull(); // Initial value + assertThat(workerModes.get(0)).isFalse(); // Initial value + assertThat(workerResults.get(1)).isEqualTo(workerThreadTenantId); // After set + assertThat(workerModes.get(1)).isFalse(); // After set + } + + @Test + void multipleThreads_shouldMaintainIsolation() throws InterruptedException { + var threadCount = 5; + var latch = new CountDownLatch(threadCount); + var results = new ArrayList(); + + try (var executor = Executors.newFixedThreadPool(threadCount)) { + for (int i = 0; i < threadCount; i++) { + var tenantId = "tenant_" + i; + executor.submit(() -> { + try { + ReindexContext.setMemberTenantId(tenantId); + ReindexContext.setReindexMode(true); + + synchronized (results) { + results.add(ReindexContext.getMemberTenantId()); + } + } finally { + ReindexContext.clear(); + latch.countDown(); + } + }); + } + + var completed = latch.await(5, TimeUnit.SECONDS); + assertThat(completed).as("All threads should complete within timeout").isTrue(); + } + + // Each thread should have read its own tenant ID + assertThat(results).hasSize(threadCount); + for (int i = 0; i < threadCount; i++) { + assertThat(results).contains("tenant_" + i); + } + } + + @Test + void isMemberTenantReindex_shouldReturnFalseWhenTenantIdIsNull() { + ReindexContext.setMemberTenantId(null); + assertThat(ReindexContext.isMemberTenantReindex()).isFalse(); + } + + @Test + void isMemberTenantReindex_shouldReturnTrueWhenTenantIdIsSet() { + ReindexContext.setMemberTenantId("test_tenant"); + assertThat(ReindexContext.isMemberTenantReindex()).isTrue(); + } + + @Test + void clearMemberTenantId_shouldNotAffectReindexMode() { + ReindexContext.setReindexMode(true); + ReindexContext.setMemberTenantId("test_tenant"); + + ReindexContext.clearMemberTenantId(); + + assertThat(ReindexContext.isReindexMode()).isTrue(); + assertThat(ReindexContext.getMemberTenantId()).isNull(); + } + + @Test + void settingEmptyString_shouldBeConsideredAsSet() { + ReindexContext.setMemberTenantId(""); + assertThat(ReindexContext.isMemberTenantReindex()).isTrue(); + assertThat(ReindexContext.getMemberTenantId()).isEmpty(); + } +} + diff --git a/src/test/java/org/folio/search/service/reindex/ReindexMergeRangeIndexServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexMergeRangeIndexServiceTest.java index f1edf40eb..8595b118a 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexMergeRangeIndexServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexMergeRangeIndexServiceTest.java @@ -55,6 +55,7 @@ class ReindexMergeRangeIndexServiceTest { private @Mock InventoryService inventoryService; private @Mock ReindexConfigurationProperties config; private @Mock InstanceChildrenResourceService instanceChildrenResourceService; + private @Mock StagingMigrationService stagingMigrationService; private ReindexMergeRangeIndexService service; private Map repositoryMap; @@ -63,7 +64,8 @@ class ReindexMergeRangeIndexServiceTest { void setUp() { var repositories = List.of(instanceRepository, itemRepository, holdingRepository); repositories.forEach(repository -> when(repository.entityType()).thenCallRealMethod()); - service = new ReindexMergeRangeIndexService(repositories, inventoryService, config); + // Create service with unified repositories + service = new ReindexMergeRangeIndexService(repositories, inventoryService, config, stagingMigrationService); service.setInstanceChildrenResourceService(instanceChildrenResourceService); repositoryMap = repositories.stream() .collect(Collectors.toMap(ReindexJdbcRepository::entityType, Function.identity())); diff --git a/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java index 6b1227b98..fd6c28670 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java @@ -1,6 +1,7 @@ package org.folio.search.service.reindex; import static java.util.Collections.emptyList; +import static org.folio.support.TestConstants.MEMBER_TENANT_ID; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -37,15 +38,19 @@ class ReindexOrchestrationServiceTest { @Mock - private ReindexUploadRangeIndexService uploadRangeIndexService; + private ReindexUploadRangeIndexService uploadRangeService; @Mock - private ReindexMergeRangeIndexService mergeRangeIndexService; + private ReindexMergeRangeIndexService mergeRangeService; @Mock private ReindexStatusService reindexStatusService; @Mock private PrimaryResourceRepository elasticRepository; @Mock private MultiTenantSearchDocumentConverter documentConverter; + @Mock + private ReindexService reindexService; + @Mock + private org.folio.spring.FolioExecutionContext context; @InjectMocks private ReindexOrchestrationService service; @@ -58,7 +63,7 @@ void process_shouldProcessSuccessfully() { var folioIndexOperationResponse = new FolioIndexOperationResponse() .status(FolioIndexOperationResponse.StatusEnum.SUCCESS); - when(uploadRangeIndexService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); + when(uploadRangeService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); when(documentConverter.convert(List.of(resourceEvent))).thenReturn(Map.of("key", List.of(SearchDocumentBody.of(null, IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX)))); when(elasticRepository.indexResources(any())).thenReturn(folioIndexOperationResponse); @@ -68,7 +73,7 @@ void process_shouldProcessSuccessfully() { // Assert assertTrue(result); - verify(uploadRangeIndexService).fetchRecordRange(event); + verify(uploadRangeService).fetchRecordRange(event); verify(documentConverter).convert(List.of(resourceEvent)); verify(elasticRepository).indexResources(any()); verify(reindexStatusService).addProcessedUploadRanges(event.getEntityType(), 1); @@ -83,7 +88,7 @@ void process_shouldThrowReindexException_whenElasticSearchReportsError() { .status(FolioIndexOperationResponse.StatusEnum.ERROR) .errorMessage("Error occurred during indexing."); - when(uploadRangeIndexService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); + when(uploadRangeService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); when(documentConverter.convert(List.of(resourceEvent))).thenReturn(Map.of("key", List.of(SearchDocumentBody.of(null, IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX)))); when(elasticRepository.indexResources(any())).thenReturn(folioIndexOperationResponse); @@ -91,7 +96,7 @@ void process_shouldThrowReindexException_whenElasticSearchReportsError() { // Act & Assert assertThrows(ReindexException.class, () -> service.process(event)); - verify(uploadRangeIndexService).fetchRecordRange(event); + verify(uploadRangeService).fetchRecordRange(event); verify(documentConverter).convert(List.of(resourceEvent)); verify(elasticRepository).indexResources(any()); verify(reindexStatusService).updateReindexUploadFailed(event.getEntityType()); @@ -104,11 +109,15 @@ void process_positive_reindexRecordsEvent() { event.setRecordType(ReindexRecordsEvent.ReindexRecordType.INSTANCE); event.setRecords(emptyList()); + when(reindexStatusService.getTargetTenantId()).thenReturn(MEMBER_TENANT_ID); + when(reindexStatusService.isMergeCompleted()).thenReturn(false); + service.process(event); - verify(mergeRangeIndexService).saveEntities(event); + verify(reindexStatusService).getTargetTenantId(); + verify(mergeRangeService).saveEntities(event); verify(reindexStatusService).addProcessedMergeRanges(ReindexEntityType.INSTANCE, 1); - verify(mergeRangeIndexService) + verify(mergeRangeService) .updateStatus(ReindexEntityType.INSTANCE, event.getRangeId(), ReindexRangeStatus.SUCCESS, null); } @@ -119,12 +128,13 @@ void process_negative_reindexRecordsEvent_shouldFailMergeOnException() { event.setRecordType(ReindexRecordsEvent.ReindexRecordType.INSTANCE); event.setRecords(emptyList()); var failCause = "exception occurred"; - doThrow(new RuntimeException(failCause)).when(mergeRangeIndexService).saveEntities(event); + doThrow(new RuntimeException(failCause)).when(mergeRangeService).saveEntities(event); service.process(event); + verify(reindexStatusService).getTargetTenantId(); verify(reindexStatusService).updateReindexMergeFailed(ReindexEntityType.INSTANCE); - verify(mergeRangeIndexService) + verify(mergeRangeService) .updateStatus(ReindexEntityType.INSTANCE, event.getRangeId(), ReindexRangeStatus.FAIL, failCause); verifyNoMoreInteractions(reindexStatusService); } @@ -135,14 +145,80 @@ void process_negative_reindexRecordsEvent_shouldNotFailMergeOnPessimisticLocking event.setRangeId(UUID.randomUUID().toString()); event.setRecordType(ReindexRecordsEvent.ReindexRecordType.INSTANCE); event.setRecords(emptyList()); - doThrow(new PessimisticLockingFailureException("Deadlock")).when(mergeRangeIndexService).saveEntities(event); + doThrow(new PessimisticLockingFailureException("Deadlock")).when(mergeRangeService).saveEntities(event); assertThrows(ReindexException.class, () -> service.process(event)); - verifyNoMoreInteractions(mergeRangeIndexService); + verifyNoMoreInteractions(mergeRangeService); + verify(reindexStatusService).getTargetTenantId(); verifyNoMoreInteractions(reindexStatusService); } + @Test + void process_reindexRecordsEvent_shouldTriggerStagingMigrationAndUploadWhenMergeCompleted() { + // given + var event = new ReindexRecordsEvent(); + event.setRangeId(UUID.randomUUID().toString()); + event.setRecordType(ReindexRecordsEvent.ReindexRecordType.INSTANCE); + event.setRecords(emptyList()); + + when(reindexStatusService.isMergeCompleted()).thenReturn(true); + when(reindexStatusService.getTargetTenantId()).thenReturn(null); + when(context.getTenantId()).thenReturn("test-tenant"); + + // act + service.process(event); + + // assert + verify(mergeRangeService).saveEntities(event); + verify(reindexStatusService).isMergeCompleted(); + verify(mergeRangeService).performStagingMigration(null); + } + + @Test + void process_reindexRecordsEvent_shouldNotTriggerUploadWhenMergeNotCompleted() { + // given + var event = new ReindexRecordsEvent(); + event.setRangeId(UUID.randomUUID().toString()); + event.setRecordType(ReindexRecordsEvent.ReindexRecordType.INSTANCE); + event.setRecords(emptyList()); + + when(reindexStatusService.isMergeCompleted()).thenReturn(false); + + // act + service.process(event); + + // assert + verify(mergeRangeService).saveEntities(event); + verify(reindexStatusService).isMergeCompleted(); + verify(mergeRangeService, org.mockito.Mockito.never()).performStagingMigration(any()); + } + + @Test + void process_reindexRangeIndexEvent_shouldHandleMemberTenantContext() { + // given + var event = reindexEvent(); + event.setMemberTenantId(MEMBER_TENANT_ID); + var resourceEvent = new ResourceEvent(); + var folioIndexOperationResponse = new FolioIndexOperationResponse() + .status(FolioIndexOperationResponse.StatusEnum.SUCCESS); + + when(uploadRangeService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); + when(documentConverter.convert(List.of(resourceEvent))) + .thenReturn(Map.of("key", List.of(SearchDocumentBody.of(null, + IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX)))); + when(elasticRepository.indexResources(any())).thenReturn(folioIndexOperationResponse); + + // act + service.process(event); + + // assert + verify(uploadRangeService).fetchRecordRange(event); + verify(documentConverter).convert(List.of(resourceEvent)); + verify(elasticRepository).indexResources(any()); + verify(reindexStatusService).addProcessedUploadRanges(event.getEntityType(), 1); + } + private ReindexRangeIndexEvent reindexEvent() { var event = new ReindexRangeIndexEvent(); event.setId(UUID.randomUUID()); diff --git a/src/test/java/org/folio/search/service/reindex/ReindexServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexServiceTest.java index 7a8828a43..90bc683ea 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexServiceTest.java @@ -1,7 +1,7 @@ package org.folio.search.service.reindex; import static java.util.Collections.emptyList; -import static org.folio.search.exception.RequestValidationException.REQUEST_NOT_ALLOWED_MSG; +import static org.folio.search.exception.RequestValidationException.REQUEST_NOT_ALLOWED_FOR_CONSORTIUM_MEMBER_MSG; import static org.folio.search.model.types.ReindexEntityType.HOLDINGS; import static org.folio.search.model.types.ReindexEntityType.INSTANCE; import static org.folio.search.model.types.ReindexEntityType.ITEM; @@ -80,7 +80,7 @@ void submitFullReindex_negative_shouldFailForEcsMemberTenant() { when(consortiumService.getCentralTenant(TENANT_ID)).thenReturn(Optional.of("central")); assertThrows(RequestValidationException.class, () -> reindexService.submitFullReindex(TENANT_ID, null), - REQUEST_NOT_ALLOWED_MSG); + REQUEST_NOT_ALLOWED_FOR_CONSORTIUM_MEMBER_MSG); } @Test @@ -104,8 +104,8 @@ void submitFullReindex_positive() throws InterruptedException { reindexService.submitFullReindex(tenant, indexSettings); ThreadUtils.sleep(Duration.ofSeconds(1)); - verify(reindexCommonService).deleteAllRecords(); - verify(statusService).recreateMergeStatusRecords(); + verify(reindexCommonService).deleteAllRecords(null); + verify(statusService).recreateMergeStatusRecords(null); verify(reindexCommonService, times(ReindexEntityType.supportUploadTypes().size())) .recreateIndex(any(), eq(tenant), eq(indexSettings)); verify(mergeRangeService).createMergeRanges(tenant); @@ -159,7 +159,7 @@ void submitUploadReindex_negative_notAllowedToRunUploadForEcsMemberTenant() { // act & assert assertThrows(RequestValidationException.class, - () -> reindexService.submitUploadReindex(member, entityTypes), REQUEST_NOT_ALLOWED_MSG); + () -> reindexService.submitUploadReindex(member, entityTypes), REQUEST_NOT_ALLOWED_FOR_CONSORTIUM_MEMBER_MSG); } @Test @@ -200,7 +200,8 @@ void submitUploadReindex_positive() { reindexService.submitUploadReindex(TENANT_ID, List.of(ReindexEntityType.INSTANCE)); - verify(statusService).recreateUploadStatusRecord(INSTANCE); + verify(statusService).getTargetTenantId(); + verify(statusService).recreateUploadStatusRecord(eq(INSTANCE), any()); verify(uploadRangeService).prepareAndSendIndexRanges(INSTANCE); } @@ -213,7 +214,8 @@ void submitUploadReindex_positive_recreateIndex() { reindexService.submitUploadReindex(TENANT_ID, uploadDto); - verify(statusService).recreateUploadStatusRecord(INSTANCE); + verify(statusService).getTargetTenantId(); + verify(statusService).recreateUploadStatusRecord(eq(INSTANCE), any()); verify(uploadRangeService).prepareAndSendIndexRanges(INSTANCE); } @@ -234,7 +236,7 @@ void submitFailedMergeRangesReindex_negative_shouldFailForEcsMemberTenant() { when(consortiumService.getCentralTenant(TENANT_ID)).thenReturn(Optional.of("central")); assertThrows(RequestValidationException.class, () -> reindexService.submitFailedMergeRangesReindex(TENANT_ID), - REQUEST_NOT_ALLOWED_MSG); + REQUEST_NOT_ALLOWED_FOR_CONSORTIUM_MEMBER_MSG); } @Test diff --git a/src/test/java/org/folio/search/service/reindex/ReindexStatusServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexStatusServiceTest.java index b87ddadb8..58773e6cc 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexStatusServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexStatusServiceTest.java @@ -2,10 +2,12 @@ import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; -import static org.folio.search.exception.RequestValidationException.REQUEST_NOT_ALLOWED_MSG; +import static org.folio.search.exception.RequestValidationException.REQUEST_NOT_ALLOWED_FOR_CONSORTIUM_MEMBER_MSG; import static org.folio.search.model.types.ReindexEntityType.HOLDINGS; import static org.folio.search.model.types.ReindexEntityType.INSTANCE; +import static org.folio.support.TestConstants.MEMBER_TENANT_ID; import static org.folio.support.TestConstants.TENANT_ID; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @@ -82,7 +84,7 @@ void getReindexStatuses_negative_consortiumMemberTenant() { var ex = Assertions.assertThrows(RequestValidationException.class, () -> service.getReindexStatuses(TENANT_ID)); - assertThat(ex.getMessage()).isEqualTo(REQUEST_NOT_ALLOWED_MSG); + assertThat(ex.getMessage()).isEqualTo(REQUEST_NOT_ALLOWED_FOR_CONSORTIUM_MEMBER_MSG); assertThat(ex.getKey()).isEqualTo(XOkapiHeaders.TENANT); assertThat(ex.getValue()).isEqualTo(TENANT_ID); verifyNoInteractions(statusRepository); @@ -154,10 +156,11 @@ void addProcessedUploadRanges() { @Test void shouldRecreateMergeReindexStatusEntities() { // act - service.recreateMergeStatusRecords(); + service.recreateMergeStatusRecords(null); // assert verify(statusRepository).truncate(); + verify(statusRepository).recreateReindexStatusTrigger(false); verify(statusRepository).saveReindexStatusRecords(reindexStatusEntitiesCaptor.capture()); var savedEntities = reindexStatusEntitiesCaptor.getValue(); assertThat(savedEntities) @@ -179,4 +182,181 @@ void updateReindexMergeInProgress() { // assert verify(statusRepository).setMergeInProgress(entityTypes); } + + @Test + void recreateMergeStatusRecords_withTargetTenantId() { + // given + var targetTenantId = MEMBER_TENANT_ID; + + // act + service.recreateMergeStatusRecords(targetTenantId); + + // assert + verify(statusRepository).truncate(); + verify(statusRepository).recreateReindexStatusTrigger(true); + verify(statusRepository).saveReindexStatusRecords(reindexStatusEntitiesCaptor.capture()); + + var savedEntities = reindexStatusEntitiesCaptor.getValue(); + assertThat(savedEntities) + .hasSize(ReindexEntityType.supportMergeTypes().size()) + .allMatch(entity -> targetTenantId.equals(entity.getTargetTenantId()), + "All entities should have targetTenantId set"); + } + + @Test + void recreateUploadStatusRecord_shouldPreserveTargetTenantId() { + // given + var targetTenantId = MEMBER_TENANT_ID; + + // act + service.recreateUploadStatusRecord(INSTANCE, targetTenantId); + + // assert + verify(statusRepository).delete(INSTANCE); + verify(statusRepository).saveReindexStatusRecords(reindexStatusEntitiesCaptor.capture()); + + var savedEntities = reindexStatusEntitiesCaptor.getValue(); + assertThat(savedEntities) + .hasSize(1) + .first() + .satisfies(entity -> assertThat(entity.getEntityType()).isEqualTo(INSTANCE)) + .satisfies(entity -> assertThat(entity.getStatus()).isEqualTo(ReindexStatus.UPLOAD_IN_PROGRESS)) + .satisfies(entity -> assertThat(entity.getTargetTenantId()).isEqualTo(targetTenantId)); + } + + @Test + void recreateUploadStatusRecord_whenNoExistingTargetTenantId_shouldSetNull() { + // act + service.recreateUploadStatusRecord(INSTANCE, null); + + // assert + verify(statusRepository).delete(INSTANCE); + verify(statusRepository).saveReindexStatusRecords(reindexStatusEntitiesCaptor.capture()); + + var savedEntities = reindexStatusEntitiesCaptor.getValue(); + assertThat(savedEntities) + .hasSize(1) + .first() + .satisfies(entity -> assertThat(entity.getTargetTenantId()).isNull()); + } + + @Test + void updateStagingStarted_shouldSetStagingStartTimeForAllMergeTypes() { + // act + service.updateStagingStarted(); + + // assert + verify(statusRepository).setStagingStarted(ReindexEntityType.supportMergeTypes()); + } + + @Test + void updateStagingCompleted_shouldSetStagingEndTimeForAllMergeTypes() { + // act + service.updateStagingCompleted(); + + // assert + verify(statusRepository).setStagingCompleted(ReindexEntityType.supportMergeTypes()); + } + + @Test + void updateStagingFailed_shouldSetStatusAndEndTimeForAllMergeTypes() { + // act + service.updateStagingFailed(); + + // assert + verify(statusRepository).setStagingFailed(ReindexEntityType.supportMergeTypes()); + } + + @Test + void isMergeCompleted_shouldReturnRepositoryResult() { + // given + when(statusRepository.isMergeCompleted()).thenReturn(true); + + // act + var result = service.isMergeCompleted(); + + // assert + assertThat(result).isTrue(); + verify(statusRepository).isMergeCompleted(); + } + + @Test + void isMergeCompleted_shouldReturnFalseWhenNotCompleted() { + // given + when(statusRepository.isMergeCompleted()).thenReturn(false); + + // act + var result = service.isMergeCompleted(); + + // assert + assertThat(result).isFalse(); + verify(statusRepository).isMergeCompleted(); + } + + @Test + void getStatusesByType_shouldReturnMapOfStatusesByEntityType() { + // given + var statusEntities = List.of( + new ReindexStatusEntity(INSTANCE, ReindexStatus.MERGE_COMPLETED), + new ReindexStatusEntity(HOLDINGS, ReindexStatus.UPLOAD_IN_PROGRESS) + ); + when(statusRepository.getReindexStatuses()).thenReturn(statusEntities); + + // act + var result = service.getStatusesByType(); + + // assert + assertThat(result) + .hasSize(2) + .containsEntry(INSTANCE, ReindexStatus.MERGE_COMPLETED) + .containsEntry(HOLDINGS, ReindexStatus.UPLOAD_IN_PROGRESS); + } + + @Test + void getTargetTenantId_shouldReturnValueFromRepository() { + // given + var expectedTenantId = MEMBER_TENANT_ID; + when(statusRepository.getTargetTenantId()).thenReturn(expectedTenantId); + + // act + var result = service.getTargetTenantId(); + + // assert + assertThat(result).isEqualTo(expectedTenantId); + verify(statusRepository).getTargetTenantId(); + } + + @Test + void getTargetTenantId_shouldReturnNullWhenNoTargetTenant() { + // given + when(statusRepository.getTargetTenantId()).thenReturn(null); + + // act + var result = service.getTargetTenantId(); + + // assert + assertThat(result).isNull(); + verify(statusRepository).getTargetTenantId(); + } + + @Test + void recreateMergeStatusRecords_shouldClearCache() { + // given + when(statusRepository.getTargetTenantId()).thenReturn("tenant1"); + + // Cache a value + service.getTargetTenantId(); + verify(statusRepository, times(1)).getTargetTenantId(); + + // act - recreate status records should clear cache + service.recreateMergeStatusRecords("new_tenant"); + + // Cache should be cleared, so next call should hit repository again + when(statusRepository.getTargetTenantId()).thenReturn("new_tenant"); + var result = service.getTargetTenantId(); + + // assert + assertThat(result).isEqualTo("new_tenant"); + verify(statusRepository, times(2)).getTargetTenantId(); + } } diff --git a/src/test/java/org/folio/search/service/reindex/ReindexUploadRangeIndexServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexUploadRangeIndexServiceTest.java index 74527bcd5..d3d11e0fb 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexUploadRangeIndexServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexUploadRangeIndexServiceTest.java @@ -1,6 +1,7 @@ package org.folio.search.service.reindex; import static org.assertj.core.api.Assertions.assertThat; +import static org.folio.support.TestConstants.MEMBER_TENANT_ID; import static org.folio.support.TestConstants.TENANT_ID; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.verify; @@ -39,8 +40,7 @@ class ReindexUploadRangeIndexServiceTest { @BeforeEach void setUp() { when(repository.entityType()).thenReturn(ReindexEntityType.INSTANCE); - service = new ReindexUploadRangeIndexService(List.of(repository), - indexRangeEventProducer, statusService); + service = new ReindexUploadRangeIndexService(List.of(repository), indexRangeEventProducer, statusService); } @Test @@ -53,15 +53,41 @@ void prepareAndSendIndexRanges_positive(@Random UploadRangeEntity uploadRange) { // assert verify(statusService).updateReindexUploadStarted(ReindexEntityType.INSTANCE, 1); - ArgumentCaptor> captor = ArgumentCaptor.captor(); + var captor = ArgumentCaptor.>captor(); verify(indexRangeEventProducer).sendMessages(captor.capture()); - List events = captor.getValue(); + var events = captor.getValue(); assertThat(events) .hasSize(1) .extracting(ReindexRangeIndexEvent::getEntityType, ReindexRangeIndexEvent::getLower, - ReindexRangeIndexEvent::getUpper) - .containsExactly(Tuple.tuple(uploadRange.getEntityType(), uploadRange.getLower(), uploadRange.getUpper())); + ReindexRangeIndexEvent::getUpper, + ReindexRangeIndexEvent::getMemberTenantId) + .containsExactly(Tuple.tuple(uploadRange.getEntityType(), uploadRange.getLower(), uploadRange.getUpper(), null)); + } + + @Test + void prepareAndSendIndexRanges_positive_consortiumMember(@Random UploadRangeEntity uploadRange) { + // arrange + when(repository.createUploadRanges()).thenReturn(List.of(uploadRange)); + + // act + ReindexContext.setMemberTenantId(MEMBER_TENANT_ID); + service.prepareAndSendIndexRanges(ReindexEntityType.INSTANCE); + ReindexContext.clearMemberTenantId(); + + // assert + verify(statusService).updateReindexUploadStarted(ReindexEntityType.INSTANCE, 1); + var captor = ArgumentCaptor.>captor(); + verify(indexRangeEventProducer).sendMessages(captor.capture()); + var events = captor.getValue(); + assertThat(events) + .hasSize(1) + .extracting(ReindexRangeIndexEvent::getEntityType, + ReindexRangeIndexEvent::getLower, + ReindexRangeIndexEvent::getUpper, + ReindexRangeIndexEvent::getMemberTenantId) + .containsExactly(Tuple.tuple(uploadRange.getEntityType(), uploadRange.getLower(), uploadRange.getUpper(), + MEMBER_TENANT_ID)); } @Test diff --git a/src/test/java/org/folio/search/service/reindex/StagingMigrationServiceTest.java b/src/test/java/org/folio/search/service/reindex/StagingMigrationServiceTest.java new file mode 100644 index 000000000..e96851a7b --- /dev/null +++ b/src/test/java/org/folio/search/service/reindex/StagingMigrationServiceTest.java @@ -0,0 +1,229 @@ +package org.folio.search.service.reindex; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.folio.support.TestConstants.MEMBER_TENANT_ID; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.sql.Timestamp; +import java.util.List; +import org.folio.search.configuration.properties.ReindexConfigurationProperties; +import org.folio.search.exception.ReindexException; +import org.folio.spring.FolioExecutionContext; +import org.folio.spring.testing.type.UnitTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.dao.DataAccessException; +import org.springframework.jdbc.core.JdbcTemplate; + +@UnitTest +@ExtendWith(MockitoExtension.class) +class StagingMigrationServiceTest { + + private static final String TENANT_ID = "test_tenant"; + + @InjectMocks + private StagingMigrationService stagingMigrationService; + + @Mock + private JdbcTemplate jdbcTemplate; + + @Mock + private FolioExecutionContext context; + + @Mock + private ReindexCommonService reindexCommonService; + + @Mock + private ReindexConfigurationProperties reindexConfigurationProperties; + @Mock + private org.folio.spring.FolioModuleMetadata folioModuleMetadata; + + @BeforeEach + void setUp() { + lenient().when(context.getTenantId()).thenReturn(TENANT_ID); + lenient().when(context.getFolioModuleMetadata()).thenReturn(folioModuleMetadata); + lenient().when(folioModuleMetadata.getDBSchemaName(TENANT_ID)).thenReturn("test_tenant_mod_search"); + lenient().when(reindexConfigurationProperties.getMigrationWorkMem()).thenReturn("64MB"); + } + + @Test + void migrateAllStagingTables_fullReindex() { + // Arrange + when(jdbcTemplate.update(contains("staging_instance"), any(Timestamp.class))).thenReturn(10); + when(jdbcTemplate.update(contains("staging_holding"))).thenReturn(15); + when(jdbcTemplate.update(contains("staging_item"), any(Timestamp.class))).thenReturn(20); + when(jdbcTemplate.update(contains("staging_subject"), any(Timestamp.class))).thenReturn(5); + when(jdbcTemplate.update(contains("staging_contributor"), any(Timestamp.class))).thenReturn(8); + when(jdbcTemplate.update(contains("staging_classification"), any(Timestamp.class))).thenReturn(3); + when(jdbcTemplate.update(contains("staging_call_number"), any(Timestamp.class))).thenReturn(2); + when(jdbcTemplate.update(contains("staging_instance_subject"))).thenReturn(12); + when(jdbcTemplate.update(contains("staging_instance_contributor"))).thenReturn(14); + when(jdbcTemplate.update(contains("staging_instance_classification"))).thenReturn(6); + when(jdbcTemplate.update(contains("staging_instance_call_number"))).thenReturn(4); + doNothing().when(jdbcTemplate).execute(anyString()); + + // Act + var result = stagingMigrationService.migrateAllStagingTables(null); + + // Assert - verify result metrics + assertThat(result).isNotNull(); + assertThat(result.getDuration()).isGreaterThan(0); + assertThat(result.getTotalInstances()).isEqualTo(10); + assertThat(result.getTotalHoldings()).isEqualTo(15); + assertThat(result.getTotalItems()).isEqualTo(20); + // Total relationships: 5 + 8 + 3 + 2 + 12 + 14 + 6 + 4 = 54 + assertThat(result.getTotalRelationships()).isEqualTo(54); + + // Verify work_mem was set + verify(jdbcTemplate).execute(contains("SET LOCAL work_mem")); + + // Verify all analyze statements were called + verify(jdbcTemplate, times(11)).execute(contains("ANALYZE")); + + // Verify no deleteRecordsByTenantId was called for full reindex + verifyNoInteractions(reindexCommonService); + + // Verify correct timestamp is used + var expectedTimestamp = Timestamp.valueOf("2000-01-01 00:00:00"); + verify(jdbcTemplate, times(6)).update(anyString(), eq(expectedTimestamp)); + + // Verify the order of operations + var inOrder = inOrder(jdbcTemplate); + + // Phase 1: Setup + inOrder.verify(jdbcTemplate).execute(contains("SET LOCAL work_mem")); + inOrder.verify(jdbcTemplate, times(11)).execute(contains("ANALYZE")); + + // Phase 2: Instances (first main table) + inOrder.verify(jdbcTemplate).update(contains("staging_instance"), any(Timestamp.class)); + + // Phase 3: Holdings and Items + inOrder.verify(jdbcTemplate).update(contains("staging_holding")); + inOrder.verify(jdbcTemplate).update(contains("staging_item"), any(Timestamp.class)); + + // Phase 4: Child resources + inOrder.verify(jdbcTemplate).update(contains("staging_subject"), any(Timestamp.class)); + inOrder.verify(jdbcTemplate).update(contains("staging_contributor"), any(Timestamp.class)); + inOrder.verify(jdbcTemplate).update(contains("staging_classification"), any(Timestamp.class)); + inOrder.verify(jdbcTemplate).update(contains("staging_call_number"), any(Timestamp.class)); + + // Phase 5: Relationships + inOrder.verify(jdbcTemplate).update(contains("staging_instance_subject")); + inOrder.verify(jdbcTemplate).update(contains("staging_instance_contributor")); + inOrder.verify(jdbcTemplate).update(contains("staging_instance_classification")); + inOrder.verify(jdbcTemplate).update(contains("staging_instance_call_number")); + } + + @Test + void migrateAllStagingTables_memberTenantReindex_shouldDeleteExistingTenantData() { + // Arrange + var targetTenantId = MEMBER_TENANT_ID; + when(jdbcTemplate.update(anyString())).thenReturn(5); + when(jdbcTemplate.update(anyString(), any(Timestamp.class))).thenReturn(5); + doNothing().when(jdbcTemplate).execute(anyString()); + doNothing().when(reindexCommonService).deleteRecordsByTenantId(targetTenantId); + + // Act + var result = stagingMigrationService.migrateAllStagingTables(targetTenantId); + + // Assert + assertThat(result).isNotNull(); + assertThat(result.getTotalInstances()).isEqualTo(5); + assertThat(result.getTotalHoldings()).isEqualTo(5); + assertThat(result.getTotalItems()).isEqualTo(5); + + // Verify pre-migration cleanup was called + verify(reindexCommonService).deleteRecordsByTenantId(targetTenantId); + + // Verify order of operations + var inOrder = inOrder(jdbcTemplate, reindexCommonService); + inOrder.verify(jdbcTemplate).execute(contains("SET LOCAL work_mem")); + inOrder.verify(jdbcTemplate, times(11)).execute(contains("ANALYZE")); + inOrder.verify(reindexCommonService).deleteRecordsByTenantId(targetTenantId); + inOrder.verify(jdbcTemplate).update(contains("staging_instance"), any(Timestamp.class)); + } + + @Test + void migrateAllStagingTables_whenDatabaseError_shouldThrowReindexException() { + // Arrange + doThrow(new DataAccessException("Database error") {}).when(jdbcTemplate).execute(anyString()); + + // Act & Assert + assertThatThrownBy(() -> stagingMigrationService.migrateAllStagingTables(null)) + .isInstanceOf(ReindexException.class) + .hasMessageContaining("Failed to migrate staging tables") + .hasCauseInstanceOf(DataAccessException.class); + } + + @Test + void migrateAllStagingTables_whenMigrationFails_shouldThrowReindexException() { + // Arrange + doNothing().when(jdbcTemplate).execute(anyString()); + when(jdbcTemplate.update(anyString(), any(Timestamp.class))) + .thenThrow(new DataAccessException("SQL error") {}); + + // Act & Assert + assertThatThrownBy(() -> stagingMigrationService.migrateAllStagingTables(null)) + .isInstanceOf(ReindexException.class) + .hasMessageContaining("Failed to migrate staging tables"); + } + + @Test + void setWorkMem_withInvalidFormat_shouldThrowReindexException() { + // Arrange + when(reindexConfigurationProperties.getMigrationWorkMem()).thenReturn("invalid_value"); + + // Act & Assert + assertThatThrownBy(() -> stagingMigrationService.migrateAllStagingTables(null)) + .isInstanceOf(ReindexException.class) + .hasMessageContaining("Invalid work_mem format"); + } + + @Test + void setWorkMem_withValidFormats_shouldSucceed() { + // Test various valid formats + var validFormats = List.of("64MB", "512KB", "1GB", "2048MB", "100KB"); + + for (var format : validFormats) { + when(reindexConfigurationProperties.getMigrationWorkMem()).thenReturn(format); + when(jdbcTemplate.update(anyString())).thenReturn(0); + when(jdbcTemplate.update(anyString(), any(Timestamp.class))).thenReturn(0); + doNothing().when(jdbcTemplate).execute(anyString()); + + // Should not throw exception + stagingMigrationService.migrateAllStagingTables(null); + + verify(jdbcTemplate).execute("SET LOCAL work_mem = '" + format + "'"); + } + } + + @Test + void setWorkMem_whenSetFails_shouldThrowReindexException() { + // Arrange + when(reindexConfigurationProperties.getMigrationWorkMem()).thenReturn("64MB"); + doThrow(new DataAccessException("Cannot set work_mem") {}) + .when(jdbcTemplate).execute(contains("SET LOCAL work_mem")); + + // Act & Assert + assertThatThrownBy(() -> stagingMigrationService.migrateAllStagingTables(null)) + .isInstanceOf(ReindexException.class) + .hasMessageContaining("Failed to set work_mem"); + } +} + diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/ClassificationRepositoryIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/ClassificationRepositoryIT.java index a1810871a..4aac0683f 100644 --- a/src/test/java/org/folio/search/service/reindex/jdbc/ClassificationRepositoryIT.java +++ b/src/test/java/org/folio/search/service/reindex/jdbc/ClassificationRepositoryIT.java @@ -4,6 +4,8 @@ import static org.assertj.core.api.Assertions.tuple; import static org.folio.search.utils.SearchUtils.CLASSIFICATION_NUMBER_FIELD; import static org.folio.search.utils.SearchUtils.CLASSIFICATION_TYPE_FIELD; +import static org.folio.support.TestConstants.CENTRAL_TENANT_ID; +import static org.folio.support.TestConstants.MEMBER_TENANT_ID; import static org.folio.support.TestConstants.TENANT_ID; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyCollection; @@ -85,8 +87,8 @@ void deleteByInstanceIds_oneClassificationRemovedAndOneInstanceCounterDecremente .extracting("number", "instances") .contains( tuple("Sci-Fi", List.of( - Map.of("count", 1, "shared", true, "tenantId", "consortium"), - Map.of("count", 1, "shared", false, "tenantId", "member_tenant")) + Map.of("count", 1, "shared", true, "tenantId", CENTRAL_TENANT_ID), + Map.of("count", 1, "shared", false, "tenantId", MEMBER_TENANT_ID)) ) ); } @@ -97,7 +99,7 @@ void deleteByInstanceIds_OneInstanceCounterDecrementedForSharedInstance() { var instanceIds = List.of("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); // act - repository.deleteByInstanceIds(instanceIds, "member_tenant"); + repository.deleteByInstanceIds(instanceIds, MEMBER_TENANT_ID); // assert var ranges = repository.fetchByIdRange("0", "50"); @@ -106,8 +108,8 @@ void deleteByInstanceIds_OneInstanceCounterDecrementedForSharedInstance() { .extracting("number", "instances") .contains( tuple("Sci-Fi", List.of( - Map.of("count", 2, "shared", true, "tenantId", "consortium"), - Map.of("count", 1, "shared", false, "tenantId", "member_tenant")) + Map.of("count", 2, "shared", true, "tenantId", CENTRAL_TENANT_ID), + Map.of("count", 1, "shared", false, "tenantId", MEMBER_TENANT_ID)) ), tuple("Genre", List.of( Map.of("count", 1, "shared", true, "tenantId", "consortium")) diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/ContributorRepositoryIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/ContributorRepositoryIT.java index ae229a2b0..695d38a90 100644 --- a/src/test/java/org/folio/search/service/reindex/jdbc/ContributorRepositoryIT.java +++ b/src/test/java/org/folio/search/service/reindex/jdbc/ContributorRepositoryIT.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.tuple; import static org.folio.search.utils.SearchUtils.AUTHORITY_ID_FIELD; import static org.folio.search.utils.SearchUtils.CONTRIBUTOR_TYPE_FIELD; +import static org.folio.support.TestConstants.MEMBER_TENANT_ID; import static org.folio.support.TestConstants.TENANT_ID; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyCollection; @@ -87,7 +88,7 @@ void deleteByInstanceIds_oneContributorRemovedAndOneInstanceCounterDecremented() tuple("Sci-Fi", List.of( Map.of("count", 1, "shared", true, "tenantId", "consortium", "typeId", List.of("aab8fff4-49c6-4578-979e-439b6ba3600b")), - Map.of("count", 1, "shared", false, "tenantId", "member_tenant", + Map.of("count", 1, "shared", false, "tenantId", MEMBER_TENANT_ID, "typeId", List.of("9ec55e4f-6a76-427c-b47b-197046f44a53"))))); } diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/MergeRangeRepositoriesIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/MergeRangeRepositoriesIT.java index 5c8f66ce9..6cba64f46 100644 --- a/src/test/java/org/folio/search/service/reindex/jdbc/MergeRangeRepositoriesIT.java +++ b/src/test/java/org/folio/search/service/reindex/jdbc/MergeRangeRepositoriesIT.java @@ -20,6 +20,7 @@ import org.folio.search.model.types.ReindexEntityType; import org.folio.search.model.types.ReindexRangeStatus; import org.folio.search.service.consortium.ConsortiumTenantProvider; +import org.folio.search.service.consortium.ConsortiumTenantService; import org.folio.search.utils.JsonConverter; import org.folio.spring.FolioExecutionContext; import org.folio.spring.FolioModuleMetadata; @@ -46,6 +47,7 @@ class MergeRangeRepositoriesIT { private @Autowired JdbcTemplate jdbcTemplate; private @MockitoBean FolioExecutionContext context; private @MockitoBean ConsortiumTenantProvider tenantProvider; + private @MockitoBean ConsortiumTenantService consortiumTenantService; private @MockitoBean ReindexConfigurationProperties reindexConfig; private HoldingRepository holdingRepository; private ItemRepository itemRepository; @@ -61,7 +63,8 @@ void setUp() { itemRepository = new ItemRepository(jdbcTemplate, jsonConverter, context, searchConfig); instanceRepository = new MergeInstanceRepository(jdbcTemplate, jsonConverter, context, tenantProvider, searchConfig); - uploadInstanceRepository = new UploadInstanceRepository(jdbcTemplate, jsonConverter, context, reindexConfig); + uploadInstanceRepository = new UploadInstanceRepository(jdbcTemplate, jsonConverter, context, + reindexConfig, consortiumTenantService); when(context.getFolioModuleMetadata()).thenReturn(new FolioModuleMetadata() { @Override public String getModuleName() { @@ -102,13 +105,13 @@ void getMergeRanges_returnRangesList_whenMergeRangesExist() { .are(new Condition<>(range -> range.getEntityType() == ReindexEntityType.HOLDINGS, "holding range")) .extracting(MergeRangeEntity::getId, MergeRangeEntity::getTenantId) .containsExactly(tuple(UUID.fromString("b7df83a1-8b15-46c1-9a4c-9d2dbb3cf4d6"), "consortium"), - tuple(UUID.fromString("dfb20d52-7f1f-4b5b-a492-2e47d2c0ac59"), "member_tenant")); + tuple(UUID.fromString("dfb20d52-7f1f-4b5b-a492-2e47d2c0ac59"), MEMBER_TENANT_ID)); assertThat(rangesItem) .hasSize(1) .are(new Condition<>(range -> range.getEntityType() == ReindexEntityType.ITEM, "item range")) .extracting(MergeRangeEntity::getId, MergeRangeEntity::getTenantId) - .containsExactly(tuple(UUID.fromString("2f23b9fa-9e1a-44ff-a30f-61ec5f3adcc8"), "member_tenant")); + .containsExactly(tuple(UUID.fromString("2f23b9fa-9e1a-44ff-a30f-61ec5f3adcc8"), MEMBER_TENANT_ID)); assertThat(rangesInstance) .hasSize(1) diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/ReindexJdbcRepositoriesIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/ReindexJdbcRepositoriesIT.java index e41542a40..44382cbb8 100644 --- a/src/test/java/org/folio/search/service/reindex/jdbc/ReindexJdbcRepositoriesIT.java +++ b/src/test/java/org/folio/search/service/reindex/jdbc/ReindexJdbcRepositoriesIT.java @@ -13,6 +13,7 @@ import org.folio.search.model.types.ReindexEntityType; import org.folio.search.model.types.ReindexRangeStatus; import org.folio.search.service.consortium.ConsortiumTenantProvider; +import org.folio.search.service.consortium.ConsortiumTenantService; import org.folio.search.utils.JsonConverter; import org.folio.spring.FolioExecutionContext; import org.folio.spring.FolioModuleMetadata; @@ -39,6 +40,7 @@ class ReindexJdbcRepositoriesIT { private @MockitoBean FolioExecutionContext context; private @MockitoBean ReindexConfigurationProperties reindexConfig; private @MockitoBean ConsortiumTenantProvider tenantProvider; + private @MockitoBean ConsortiumTenantService consortiumTenantService; private SearchConfigurationProperties searchConfig; private MergeInstanceRepository mergeRepository; private UploadInstanceRepository uploadRepository; @@ -50,7 +52,8 @@ void setUp() { searchConfig.setIndexing(new SearchConfigurationProperties.IndexingSettings()); mergeRepository = new MergeInstanceRepository(jdbcTemplate, jsonConverter, context, tenantProvider, searchConfig); - uploadRepository = new UploadInstanceRepository(jdbcTemplate, jsonConverter, context, reindexConfig); + uploadRepository = new UploadInstanceRepository(jdbcTemplate, jsonConverter, context, + reindexConfig, consortiumTenantService); when(context.getFolioModuleMetadata()).thenReturn(new FolioModuleMetadata() { @Override public String getModuleName() { diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepositoryIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepositoryIT.java index 35d706d53..e270d9db7 100644 --- a/src/test/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepositoryIT.java +++ b/src/test/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepositoryIT.java @@ -4,13 +4,16 @@ import static org.assertj.core.api.Assertions.tuple; import static org.folio.search.model.types.ReindexEntityType.CLASSIFICATION; import static org.folio.search.model.types.ReindexEntityType.CONTRIBUTOR; +import static org.folio.search.model.types.ReindexEntityType.HOLDINGS; import static org.folio.search.model.types.ReindexEntityType.INSTANCE; +import static org.folio.search.model.types.ReindexEntityType.ITEM; import static org.folio.search.model.types.ReindexEntityType.SUBJECT; import static org.folio.search.model.types.ReindexStatus.MERGE_COMPLETED; import static org.folio.search.model.types.ReindexStatus.MERGE_IN_PROGRESS; import static org.folio.search.model.types.ReindexStatus.UPLOAD_COMPLETED; import static org.folio.search.model.types.ReindexStatus.UPLOAD_FAILED; import static org.folio.search.model.types.ReindexStatus.UPLOAD_IN_PROGRESS; +import static org.folio.support.TestConstants.MEMBER_TENANT_ID; import static org.folio.support.TestConstants.TENANT_ID; import static org.mockito.Mockito.when; @@ -138,4 +141,259 @@ void setMergeInProgress() { .hasSize(1) .allMatch(reindexStatus -> reindexStatus.getStatus() == MERGE_IN_PROGRESS); } + + @Test + @Sql("/sql/populate-reindex-status.sql") + void truncate_shouldRemoveAllRecords() { + // act + repository.truncate(); + + // assert + var statuses = repository.getReindexStatuses(); + assertThat(statuses).isEmpty(); + } + + @Test + @Sql("/sql/populate-reindex-status.sql") + void delete_shouldRemoveSpecificEntityType() { + // act + repository.delete(INSTANCE); + + // assert + var statuses = repository.getReindexStatuses(); + assertThat(statuses) + .hasSize(3) + .noneMatch(status -> status.getEntityType() == INSTANCE); + } + + @Test + @Sql("/sql/populate-reindex-status.sql") + void setMergeReindexStarted_shouldSetTotalRangesAndStartTime() { + // given + int totalRanges = 100; + + // act + repository.setMergeReindexStarted(INSTANCE, totalRanges); + + // assert + var statuses = repository.getReindexStatuses(); + assertThat(statuses) + .filteredOn(status -> status.getEntityType() == INSTANCE) + .hasSize(1) + .first() + .satisfies(status -> { + assertThat(status.getTotalMergeRanges()).isEqualTo(totalRanges); + assertThat(status.getStartTimeMerge()).isNotNull(); + }); + } + + @Test + @Sql("/sql/populate-reindex-status.sql") + void setUploadReindexStarted_shouldSetTotalRangesAndStartTime() { + // given + int totalRanges = 50; + + // act + repository.setUploadReindexStarted(CONTRIBUTOR, totalRanges); + + // assert + var statuses = repository.getReindexStatuses(); + assertThat(statuses) + .filteredOn(status -> status.getEntityType() == CONTRIBUTOR) + .hasSize(1) + .first() + .satisfies(status -> { + assertThat(status.getTotalUploadRanges()).isEqualTo(totalRanges); + assertThat(status.getStartTimeUpload()).isNotNull(); + }); + } + + @Test + @Sql("/sql/populate-reindex-status.sql") + void setMergeReindexFailed_shouldUpdateStatusAndEndTimeForMultipleEntities() { + // given + var entityTypes = java.util.List.of(INSTANCE, CONTRIBUTOR); + + // act + repository.setMergeReindexFailed(entityTypes); + + // assert + var statuses = repository.getReindexStatuses(); + assertThat(statuses) + .filteredOn(status -> entityTypes.contains(status.getEntityType())) + .hasSize(2) + .allMatch(status -> status.getStatus() == org.folio.search.model.types.ReindexStatus.MERGE_FAILED) + .allMatch(status -> status.getEndTimeMerge() != null); + } + + @Test + @Sql("/sql/populate-reindex-status.sql") + void setStagingStarted_shouldSetStartTimeForMultipleEntities() { + // given + var entityTypes = java.util.List.of(INSTANCE, CONTRIBUTOR); + + // act + repository.setStagingStarted(entityTypes); + + // assert + var statuses = repository.getReindexStatuses(); + assertThat(statuses) + .filteredOn(s -> entityTypes.contains(s.getEntityType())) + .hasSize(2) + .allMatch(s -> s.getStartTimeStaging() != null); + } + + @Test + @Sql("/sql/populate-reindex-status.sql") + void setStagingCompleted_shouldSetEndTimeForMultipleEntities() { + // given + var entityTypes = java.util.List.of(INSTANCE, SUBJECT); + + // act + repository.setStagingCompleted(entityTypes); + + // assert + var statuses = repository.getReindexStatuses(); + assertThat(statuses) + .filteredOn(s -> entityTypes.contains(s.getEntityType())) + .hasSize(2) + .allMatch(s -> s.getEndTimeStaging() != null); + } + + @Test + @Sql("/sql/populate-reindex-status.sql") + void setStagingFailed_shouldSetStatusAndEndTimeForMultipleEntities() { + // given + var entityTypes = java.util.List.of(INSTANCE, CONTRIBUTOR, CLASSIFICATION); + + // act + repository.setStagingFailed(entityTypes); + + // assert + var statuses = repository.getReindexStatuses(); + assertThat(statuses) + .filteredOn(s -> entityTypes.contains(s.getEntityType())) + .hasSize(3) + .allMatch(s -> s.getStatus() == org.folio.search.model.types.ReindexStatus.STAGING_FAILED) + .allMatch(s -> s.getEndTimeStaging() != null); + } + + @Test + void saveReindexStatusRecords_shouldInsertNewRecords() { + // given + var entity1 = new ReindexStatusEntity(INSTANCE, MERGE_IN_PROGRESS); + entity1.setTargetTenantId("target_tenant"); + entity1.setTotalMergeRanges(100); + + var entity2 = new ReindexStatusEntity(CONTRIBUTOR, MERGE_COMPLETED); + entity2.setTotalMergeRanges(50); + entity2.setProcessedMergeRanges(50); + + var statusRecords = java.util.List.of(entity1, entity2); + + // act + repository.saveReindexStatusRecords(statusRecords); + + // assert + var statuses = repository.getReindexStatuses(); + assertThat(statuses) + .hasSize(2) + .extracting(ReindexStatusEntity::getEntityType, ReindexStatusEntity::getStatus) + .containsExactlyInAnyOrder( + tuple(INSTANCE, MERGE_IN_PROGRESS), + tuple(CONTRIBUTOR, MERGE_COMPLETED) + ); + + assertThat(statuses) + .filteredOn(s -> s.getEntityType() == INSTANCE) + .first() + .satisfies(s -> { + assertThat(s.getTargetTenantId()).isEqualTo("target_tenant"); + assertThat(s.getTotalMergeRanges()).isEqualTo(100); + }); + } + + @Test + @Sql("/sql/populate-reindex-status.sql") + void isMergeCompleted_shouldReturnFalseWhenNotAllCompleted() { + // act + var result = repository.isMergeCompleted(); + + // assert - based on test data, not all are completed + assertThat(result).isFalse(); + } + + @Test + void isMergeCompleted_shouldReturnTrueWhenAllCompleted() { + var item = new ReindexStatusEntity(ITEM, MERGE_COMPLETED); + item.setTotalMergeRanges(10); + item.setProcessedMergeRanges(10); + + var instance = new ReindexStatusEntity(INSTANCE, MERGE_COMPLETED); + instance.setTotalMergeRanges(10); + instance.setProcessedMergeRanges(10); + + var holdings = new ReindexStatusEntity(HOLDINGS, MERGE_COMPLETED); + holdings.setTotalMergeRanges(10); + holdings.setProcessedMergeRanges(10); + + repository.saveReindexStatusRecords(java.util.List.of(item, instance, holdings)); + + // act + var result = repository.isMergeCompleted(); + + // assert + assertThat(result).isTrue(); + } + + @Test + void getTargetTenantId_shouldReturnNullWhenNotSet() { + // given - insert record without target tenant + var entity = new ReindexStatusEntity(INSTANCE, MERGE_IN_PROGRESS); + repository.saveReindexStatusRecords(java.util.List.of(entity)); + + // act + var result = repository.getTargetTenantId(); + + // assert + assertThat(result).isNull(); + } + + @Test + void getTargetTenantId_shouldReturnTenantIdWhenSet() { + // given + var targetTenantId = MEMBER_TENANT_ID; + var entity = new ReindexStatusEntity(INSTANCE, MERGE_IN_PROGRESS); + entity.setTargetTenantId(targetTenantId); + repository.saveReindexStatusRecords(java.util.List.of(entity)); + + // act + var result = repository.getTargetTenantId(); + + // assert + assertThat(result).isEqualTo(targetTenantId); + } + + @Test + void recreateReindexStatusTrigger_withStandardMode_shouldSucceed() { + // act - should not throw exception + repository.recreateReindexStatusTrigger(false); + + // assert - verify trigger exists by checking if we can use the table normally + var entity = new ReindexStatusEntity(INSTANCE, MERGE_IN_PROGRESS); + repository.saveReindexStatusRecords(java.util.List.of(entity)); + assertThat(repository.getReindexStatuses()).hasSize(1); + } + + @Test + void recreateReindexStatusTrigger_withConsortiumMode_shouldSucceed() { + // act - should not throw exception + repository.recreateReindexStatusTrigger(true); + + // assert - verify trigger exists by checking if we can use the table normally + var entity = new ReindexStatusEntity(CONTRIBUTOR, MERGE_IN_PROGRESS); + entity.setTargetTenantId("consortium_member"); + repository.saveReindexStatusRecords(java.util.List.of(entity)); + assertThat(repository.getReindexStatuses()).hasSize(1); + } } diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/SubjectRepositoryIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/SubjectRepositoryIT.java index 9fe28952d..d13c3dfa8 100644 --- a/src/test/java/org/folio/search/service/reindex/jdbc/SubjectRepositoryIT.java +++ b/src/test/java/org/folio/search/service/reindex/jdbc/SubjectRepositoryIT.java @@ -6,6 +6,7 @@ import static org.folio.search.utils.SearchUtils.SUBJECT_SOURCE_ID_FIELD; import static org.folio.search.utils.SearchUtils.SUBJECT_TYPE_ID_FIELD; import static org.folio.search.utils.SearchUtils.SUBJECT_VALUE_FIELD; +import static org.folio.support.TestConstants.MEMBER_TENANT_ID; import static org.folio.support.TestConstants.TENANT_ID; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyCollection; @@ -135,7 +136,7 @@ void deleteByInstanceIds_oneSubjectRemovedAndOneInstanceCounterDecremented() { .contains( tuple("Sci-Fi", List.of( Map.of("count", 1, "shared", true, "tenantId", "consortium"), - Map.of("count", 1, "shared", false, "tenantId", "member_tenant")))); + Map.of("count", 1, "shared", false, "tenantId", MEMBER_TENANT_ID)))); } @Test diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/UploadRangeRepositoriesIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/UploadRangeRepositoriesIT.java index 331260cdc..83c748532 100644 --- a/src/test/java/org/folio/search/service/reindex/jdbc/UploadRangeRepositoriesIT.java +++ b/src/test/java/org/folio/search/service/reindex/jdbc/UploadRangeRepositoriesIT.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.folio.search.configuration.properties.ReindexConfigurationProperties; +import org.folio.search.service.consortium.ConsortiumTenantService; import org.folio.search.utils.JsonConverter; import org.folio.spring.FolioExecutionContext; import org.folio.spring.FolioModuleMetadata; @@ -31,12 +32,14 @@ class UploadRangeRepositoriesIT { private @Autowired JdbcTemplate jdbcTemplate; private @MockitoBean FolioExecutionContext context; private @MockitoBean ReindexConfigurationProperties reindexConfig; + private @MockitoBean ConsortiumTenantService consortiumTenantService; private UploadInstanceRepository uploadRepository; @BeforeEach void setUp() { var jsonConverter = new JsonConverter(new ObjectMapper()); - uploadRepository = new UploadInstanceRepository(jdbcTemplate, jsonConverter, context, reindexConfig); + uploadRepository = new UploadInstanceRepository(jdbcTemplate, jsonConverter, context, + reindexConfig, consortiumTenantService); when(context.getFolioModuleMetadata()).thenReturn(new FolioModuleMetadata() { @Override public String getModuleName() { From 5961c1276446ca56af12e82418824be6f7be8fef Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Fri, 31 Oct 2025 18:15:47 +0100 Subject: [PATCH 2/5] Override staging entity table method in UploadInstanceRepository since it's used for truncation --- .../service/reindex/jdbc/UploadInstanceRepository.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/UploadInstanceRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/UploadInstanceRepository.java index 3f3727ab5..3dc1456c8 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/UploadInstanceRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/UploadInstanceRepository.java @@ -6,6 +6,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import lombok.extern.log4j.Log4j2; import org.folio.search.configuration.properties.ReindexConfigurationProperties; import org.folio.search.model.types.ReindexEntityType; @@ -81,6 +82,11 @@ protected String entityTable() { return ReindexConstants.INSTANCE_TABLE; } + @Override + protected Optional stagingEntityTable() { + return Optional.of(ReindexConstants.STAGING_INSTANCE_TABLE); + } + public List> fetchByIds(List ids) { if (ids == null || ids.isEmpty()) { return Collections.emptyList(); From 03fff4e46ffe573f02473f4c44764db21f8f0eeb Mon Sep 17 00:00:00 2001 From: Viacheslav Kolesnyk <94473337+viacheslavkol@users.noreply.github.com> Date: Mon, 24 Nov 2025 11:50:23 +0100 Subject: [PATCH 3/5] fix(reindex): Add deduplication of staging resources (#863) - Add deduplication for staging subjects/contributors/call-numbers/classifications - Remove redundant partitioning of staging tables Refs: MSEARCH-1107 --- .../reindex/jdbc/CallNumberRepository.java | 3 +- .../jdbc/ClassificationRepository.java | 3 +- .../reindex/jdbc/ContributorRepository.java | 3 +- .../reindex/jdbc/SubjectRepository.java | 3 +- .../resources/changelog/changelog-master.xml | 9 +- .../changes/v6.0/01_create_staging_tables.xml | 30 +++---- ...dexes.xml => 02_add_tenant_id_indexes.xml} | 0 .../v6.0/02_create_staging_partitions.xml | 68 --------------- ...dd_target_tenant_id_to_reindex_status.xml} | 0 ...taging_time_columns_to_reindex_status.xml} | 0 ...ortium_member_reindex_status_function.xml} | 0 ...eate_staging_child_resource_partitions.sql | 83 ------------------- .../sql/create_staging_holding_partitions.sql | 20 ----- .../create_staging_instance_partitions.sql | 20 ----- .../sql/create_staging_item_partitions.sql | 20 ----- ...create_staging_relationship_partitions.sql | 83 ------------------- .../integration/KafkaMessageListenerTest.java | 2 +- 17 files changed, 28 insertions(+), 319 deletions(-) rename src/main/resources/changelog/changes/v6.0/{03_add_tenant_id_indexes.xml => 02_add_tenant_id_indexes.xml} (100%) delete mode 100644 src/main/resources/changelog/changes/v6.0/02_create_staging_partitions.xml rename src/main/resources/changelog/changes/v6.0/{04_add_target_tenant_id_to_reindex_status.xml => 03_add_target_tenant_id_to_reindex_status.xml} (100%) rename src/main/resources/changelog/changes/v6.0/{05_add_staging_time_columns_to_reindex_status.xml => 04_add_staging_time_columns_to_reindex_status.xml} (100%) rename src/main/resources/changelog/changes/v6.0/{06_create_consortium_member_reindex_status_function.xml => 05_create_consortium_member_reindex_status_function.xml} (100%) delete mode 100644 src/main/resources/changelog/changes/v6.0/sql/create_staging_child_resource_partitions.sql delete mode 100644 src/main/resources/changelog/changes/v6.0/sql/create_staging_holding_partitions.sql delete mode 100644 src/main/resources/changelog/changes/v6.0/sql/create_staging_instance_partitions.sql delete mode 100644 src/main/resources/changelog/changes/v6.0/sql/create_staging_item_partitions.sql delete mode 100644 src/main/resources/changelog/changes/v6.0/sql/create_staging_relationship_partitions.sql diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java index 0f98a2b6b..b1bb6c856 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java @@ -157,7 +157,8 @@ LEFT JOIN ( call_number_suffix, call_number_type_id, inserted_at - ) VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP); + ) VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + ON CONFLICT (id) DO NOTHING; """; private static final String INSERT_RELATIONS_SQL = """ diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java index 944d62fb5..a66cd43d7 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java @@ -127,7 +127,8 @@ WHERE instance_id IN (%2$s) %3$s """; private static final String INSERT_STAGING_ENTITIES_SQL = """ INSERT INTO %s.staging_classification (id, number, type_id, inserted_at) - VALUES (?, ?, ?, CURRENT_TIMESTAMP); + VALUES (?, ?, ?, CURRENT_TIMESTAMP) + ON CONFLICT (id) DO NOTHING; """; private static final String INSERT_RELATIONS_SQL = """ INSERT INTO %s.instance_classification (instance_id, classification_id, tenant_id, shared) diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java index df5a81976..5e14dc217 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java @@ -133,7 +133,8 @@ WHERE instance_id IN (%2$s) %3$s """; private static final String INSERT_STAGING_ENTITIES_SQL = """ INSERT INTO %s.staging_contributor (id, name, name_type_id, authority_id, inserted_at) - VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP); + VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP) + ON CONFLICT (id) DO NOTHING; """; private static final String INSERT_RELATIONS_SQL = """ INSERT INTO %s.instance_contributor (instance_id, contributor_id, type_id, tenant_id, shared) diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java index 20303eab2..d01e79619 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java @@ -134,7 +134,8 @@ WHERE instance_id IN (%2$s) %3$s """; private static final String INSERT_STAGING_ENTITIES_SQL = """ INSERT INTO %s.staging_subject (id, value, authority_id, source_id, type_id, inserted_at) - VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP); + VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + ON CONFLICT (id) DO NOTHING; """; private static final String INSERT_RELATIONS_SQL = """ INSERT INTO %s.instance_subject (instance_id, subject_id, tenant_id, shared) diff --git a/src/main/resources/changelog/changelog-master.xml b/src/main/resources/changelog/changelog-master.xml index 4c6547beb..7a006605f 100644 --- a/src/main/resources/changelog/changelog-master.xml +++ b/src/main/resources/changelog/changelog-master.xml @@ -20,9 +20,8 @@ - - - - - + + + + diff --git a/src/main/resources/changelog/changes/v6.0/01_create_staging_tables.xml b/src/main/resources/changelog/changes/v6.0/01_create_staging_tables.xml index 866db1250..85498d977 100644 --- a/src/main/resources/changelog/changes/v6.0/01_create_staging_tables.xml +++ b/src/main/resources/changelog/changes/v6.0/01_create_staging_tables.xml @@ -21,7 +21,7 @@ is_bound_with BOOLEAN NOT NULL, json JSONB NOT NULL, inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL - ) PARTITION BY RANGE (SUBSTRING(id::text, 1, 1)); + ); @@ -41,7 +41,7 @@ instance_id UUID NOT NULL, json JSONB NOT NULL, inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL - ) PARTITION BY RANGE (SUBSTRING(id::text, 1, 1)); + ); @@ -62,7 +62,7 @@ holding_id UUID NOT NULL, json JSONB NOT NULL, inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL - ) PARTITION BY RANGE (SUBSTRING(id::text, 1, 1)); + ); @@ -83,7 +83,7 @@ tenant_id VARCHAR(100) NOT NULL, shared BOOLEAN NOT NULL, inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL - ) PARTITION BY RANGE (SUBSTRING(instance_id::text, 1, 1)); + ); @@ -104,7 +104,7 @@ tenant_id VARCHAR(100) NOT NULL, shared BOOLEAN NOT NULL, inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL - ) PARTITION BY RANGE (SUBSTRING(instance_id::text, 1, 1)); + ); @@ -124,7 +124,7 @@ tenant_id VARCHAR(100) NOT NULL, shared BOOLEAN NOT NULL, inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL - ) PARTITION BY RANGE (SUBSTRING(instance_id::text, 1, 1)); + ); @@ -145,7 +145,7 @@ tenant_id VARCHAR(100) NOT NULL, location_id UUID, inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL - ) PARTITION BY RANGE (SUBSTRING(instance_id::text, 1, 1)); + ); @@ -160,13 +160,13 @@ CREATE UNLOGGED TABLE staging_subject ( - id VARCHAR(40) NOT NULL, + id VARCHAR(40) NOT NULL UNIQUE, value VARCHAR(255) NOT NULL, authority_id VARCHAR(40), source_id VARCHAR(40), type_id VARCHAR(40), inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL - ) PARTITION BY RANGE (SUBSTRING(id::text, 1, 1)); + ); @@ -181,12 +181,12 @@ CREATE UNLOGGED TABLE staging_contributor ( - id VARCHAR(40) NOT NULL, + id VARCHAR(40) NOT NULL UNIQUE, name VARCHAR(255) NOT NULL, name_type_id VARCHAR(40), authority_id VARCHAR(40), inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL - ) PARTITION BY RANGE (SUBSTRING(id::text, 1, 1)); + ); @@ -201,11 +201,11 @@ CREATE UNLOGGED TABLE staging_classification ( - id VARCHAR(40) NOT NULL, + id VARCHAR(40) NOT NULL UNIQUE, number VARCHAR(50) NOT NULL, type_id VARCHAR(40), inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL - ) PARTITION BY RANGE (SUBSTRING(id::text, 1, 1)); + ); @@ -220,13 +220,13 @@ CREATE UNLOGGED TABLE staging_call_number ( - id VARCHAR(40) NOT NULL, + id VARCHAR(40) NOT NULL UNIQUE, call_number VARCHAR(50) NOT NULL, call_number_prefix VARCHAR(20), call_number_suffix VARCHAR(25), call_number_type_id VARCHAR(40), inserted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL - ) PARTITION BY RANGE (SUBSTRING(id::text, 1, 1)); + ); diff --git a/src/main/resources/changelog/changes/v6.0/03_add_tenant_id_indexes.xml b/src/main/resources/changelog/changes/v6.0/02_add_tenant_id_indexes.xml similarity index 100% rename from src/main/resources/changelog/changes/v6.0/03_add_tenant_id_indexes.xml rename to src/main/resources/changelog/changes/v6.0/02_add_tenant_id_indexes.xml diff --git a/src/main/resources/changelog/changes/v6.0/02_create_staging_partitions.xml b/src/main/resources/changelog/changes/v6.0/02_create_staging_partitions.xml deleted file mode 100644 index 2db0430f3..000000000 --- a/src/main/resources/changelog/changes/v6.0/02_create_staging_partitions.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - - - Create partitions for staging_instance table - - - - - - - - Create partitions for staging_holding table - - - - - - - - Create partitions for staging_item table - - - - - - - - - - - - - - Create partitions for staging relationship tables - - - - - - - - - - - - - Create partitions for staging child resource tables - - - - diff --git a/src/main/resources/changelog/changes/v6.0/04_add_target_tenant_id_to_reindex_status.xml b/src/main/resources/changelog/changes/v6.0/03_add_target_tenant_id_to_reindex_status.xml similarity index 100% rename from src/main/resources/changelog/changes/v6.0/04_add_target_tenant_id_to_reindex_status.xml rename to src/main/resources/changelog/changes/v6.0/03_add_target_tenant_id_to_reindex_status.xml diff --git a/src/main/resources/changelog/changes/v6.0/05_add_staging_time_columns_to_reindex_status.xml b/src/main/resources/changelog/changes/v6.0/04_add_staging_time_columns_to_reindex_status.xml similarity index 100% rename from src/main/resources/changelog/changes/v6.0/05_add_staging_time_columns_to_reindex_status.xml rename to src/main/resources/changelog/changes/v6.0/04_add_staging_time_columns_to_reindex_status.xml diff --git a/src/main/resources/changelog/changes/v6.0/06_create_consortium_member_reindex_status_function.xml b/src/main/resources/changelog/changes/v6.0/05_create_consortium_member_reindex_status_function.xml similarity index 100% rename from src/main/resources/changelog/changes/v6.0/06_create_consortium_member_reindex_status_function.xml rename to src/main/resources/changelog/changes/v6.0/05_create_consortium_member_reindex_status_function.xml diff --git a/src/main/resources/changelog/changes/v6.0/sql/create_staging_child_resource_partitions.sql b/src/main/resources/changelog/changes/v6.0/sql/create_staging_child_resource_partitions.sql deleted file mode 100644 index ee75f8e58..000000000 --- a/src/main/resources/changelog/changes/v6.0/sql/create_staging_child_resource_partitions.sql +++ /dev/null @@ -1,83 +0,0 @@ --- Create partitions for staging_subject table based on first character of id --- Numeric partitions (0-9) -CREATE TABLE staging_subject_0 PARTITION OF staging_subject FOR VALUES FROM ('0') TO ('1'); -CREATE TABLE staging_subject_1 PARTITION OF staging_subject FOR VALUES FROM ('1') TO ('2'); -CREATE TABLE staging_subject_2 PARTITION OF staging_subject FOR VALUES FROM ('2') TO ('3'); -CREATE TABLE staging_subject_3 PARTITION OF staging_subject FOR VALUES FROM ('3') TO ('4'); -CREATE TABLE staging_subject_4 PARTITION OF staging_subject FOR VALUES FROM ('4') TO ('5'); -CREATE TABLE staging_subject_5 PARTITION OF staging_subject FOR VALUES FROM ('5') TO ('6'); -CREATE TABLE staging_subject_6 PARTITION OF staging_subject FOR VALUES FROM ('6') TO ('7'); -CREATE TABLE staging_subject_7 PARTITION OF staging_subject FOR VALUES FROM ('7') TO ('8'); -CREATE TABLE staging_subject_8 PARTITION OF staging_subject FOR VALUES FROM ('8') TO ('9'); -CREATE TABLE staging_subject_9 PARTITION OF staging_subject FOR VALUES FROM ('9') TO ('a'); - --- Alphabetic partitions (a-f for hash-based IDs) -CREATE TABLE staging_subject_a PARTITION OF staging_subject FOR VALUES FROM ('a') TO ('b'); -CREATE TABLE staging_subject_b PARTITION OF staging_subject FOR VALUES FROM ('b') TO ('c'); -CREATE TABLE staging_subject_c PARTITION OF staging_subject FOR VALUES FROM ('c') TO ('d'); -CREATE TABLE staging_subject_d PARTITION OF staging_subject FOR VALUES FROM ('d') TO ('e'); -CREATE TABLE staging_subject_e PARTITION OF staging_subject FOR VALUES FROM ('e') TO ('f'); -CREATE TABLE staging_subject_f PARTITION OF staging_subject FOR VALUES FROM ('f') TO ('g'); - --- Create partitions for staging_contributor table based on first character of id --- Numeric partitions (0-9) -CREATE TABLE staging_contributor_0 PARTITION OF staging_contributor FOR VALUES FROM ('0') TO ('1'); -CREATE TABLE staging_contributor_1 PARTITION OF staging_contributor FOR VALUES FROM ('1') TO ('2'); -CREATE TABLE staging_contributor_2 PARTITION OF staging_contributor FOR VALUES FROM ('2') TO ('3'); -CREATE TABLE staging_contributor_3 PARTITION OF staging_contributor FOR VALUES FROM ('3') TO ('4'); -CREATE TABLE staging_contributor_4 PARTITION OF staging_contributor FOR VALUES FROM ('4') TO ('5'); -CREATE TABLE staging_contributor_5 PARTITION OF staging_contributor FOR VALUES FROM ('5') TO ('6'); -CREATE TABLE staging_contributor_6 PARTITION OF staging_contributor FOR VALUES FROM ('6') TO ('7'); -CREATE TABLE staging_contributor_7 PARTITION OF staging_contributor FOR VALUES FROM ('7') TO ('8'); -CREATE TABLE staging_contributor_8 PARTITION OF staging_contributor FOR VALUES FROM ('8') TO ('9'); -CREATE TABLE staging_contributor_9 PARTITION OF staging_contributor FOR VALUES FROM ('9') TO ('a'); - --- Alphabetic partitions (a-f for hash-based IDs) -CREATE TABLE staging_contributor_a PARTITION OF staging_contributor FOR VALUES FROM ('a') TO ('b'); -CREATE TABLE staging_contributor_b PARTITION OF staging_contributor FOR VALUES FROM ('b') TO ('c'); -CREATE TABLE staging_contributor_c PARTITION OF staging_contributor FOR VALUES FROM ('c') TO ('d'); -CREATE TABLE staging_contributor_d PARTITION OF staging_contributor FOR VALUES FROM ('d') TO ('e'); -CREATE TABLE staging_contributor_e PARTITION OF staging_contributor FOR VALUES FROM ('e') TO ('f'); -CREATE TABLE staging_contributor_f PARTITION OF staging_contributor FOR VALUES FROM ('f') TO ('g'); - --- Create partitions for staging_classification table based on first character of id --- Numeric partitions (0-9) -CREATE TABLE staging_classification_0 PARTITION OF staging_classification FOR VALUES FROM ('0') TO ('1'); -CREATE TABLE staging_classification_1 PARTITION OF staging_classification FOR VALUES FROM ('1') TO ('2'); -CREATE TABLE staging_classification_2 PARTITION OF staging_classification FOR VALUES FROM ('2') TO ('3'); -CREATE TABLE staging_classification_3 PARTITION OF staging_classification FOR VALUES FROM ('3') TO ('4'); -CREATE TABLE staging_classification_4 PARTITION OF staging_classification FOR VALUES FROM ('4') TO ('5'); -CREATE TABLE staging_classification_5 PARTITION OF staging_classification FOR VALUES FROM ('5') TO ('6'); -CREATE TABLE staging_classification_6 PARTITION OF staging_classification FOR VALUES FROM ('6') TO ('7'); -CREATE TABLE staging_classification_7 PARTITION OF staging_classification FOR VALUES FROM ('7') TO ('8'); -CREATE TABLE staging_classification_8 PARTITION OF staging_classification FOR VALUES FROM ('8') TO ('9'); -CREATE TABLE staging_classification_9 PARTITION OF staging_classification FOR VALUES FROM ('9') TO ('a'); - --- Alphabetic partitions (a-f for hash-based IDs) -CREATE TABLE staging_classification_a PARTITION OF staging_classification FOR VALUES FROM ('a') TO ('b'); -CREATE TABLE staging_classification_b PARTITION OF staging_classification FOR VALUES FROM ('b') TO ('c'); -CREATE TABLE staging_classification_c PARTITION OF staging_classification FOR VALUES FROM ('c') TO ('d'); -CREATE TABLE staging_classification_d PARTITION OF staging_classification FOR VALUES FROM ('d') TO ('e'); -CREATE TABLE staging_classification_e PARTITION OF staging_classification FOR VALUES FROM ('e') TO ('f'); -CREATE TABLE staging_classification_f PARTITION OF staging_classification FOR VALUES FROM ('f') TO ('g'); - --- Create partitions for staging_call_number table based on first character of id --- Numeric partitions (0-9) -CREATE TABLE staging_call_number_0 PARTITION OF staging_call_number FOR VALUES FROM ('0') TO ('1'); -CREATE TABLE staging_call_number_1 PARTITION OF staging_call_number FOR VALUES FROM ('1') TO ('2'); -CREATE TABLE staging_call_number_2 PARTITION OF staging_call_number FOR VALUES FROM ('2') TO ('3'); -CREATE TABLE staging_call_number_3 PARTITION OF staging_call_number FOR VALUES FROM ('3') TO ('4'); -CREATE TABLE staging_call_number_4 PARTITION OF staging_call_number FOR VALUES FROM ('4') TO ('5'); -CREATE TABLE staging_call_number_5 PARTITION OF staging_call_number FOR VALUES FROM ('5') TO ('6'); -CREATE TABLE staging_call_number_6 PARTITION OF staging_call_number FOR VALUES FROM ('6') TO ('7'); -CREATE TABLE staging_call_number_7 PARTITION OF staging_call_number FOR VALUES FROM ('7') TO ('8'); -CREATE TABLE staging_call_number_8 PARTITION OF staging_call_number FOR VALUES FROM ('8') TO ('9'); -CREATE TABLE staging_call_number_9 PARTITION OF staging_call_number FOR VALUES FROM ('9') TO ('a'); - --- Alphabetic partitions (a-f for hash-based IDs) -CREATE TABLE staging_call_number_a PARTITION OF staging_call_number FOR VALUES FROM ('a') TO ('b'); -CREATE TABLE staging_call_number_b PARTITION OF staging_call_number FOR VALUES FROM ('b') TO ('c'); -CREATE TABLE staging_call_number_c PARTITION OF staging_call_number FOR VALUES FROM ('c') TO ('d'); -CREATE TABLE staging_call_number_d PARTITION OF staging_call_number FOR VALUES FROM ('d') TO ('e'); -CREATE TABLE staging_call_number_e PARTITION OF staging_call_number FOR VALUES FROM ('e') TO ('f'); -CREATE TABLE staging_call_number_f PARTITION OF staging_call_number FOR VALUES FROM ('f') TO ('g'); diff --git a/src/main/resources/changelog/changes/v6.0/sql/create_staging_holding_partitions.sql b/src/main/resources/changelog/changes/v6.0/sql/create_staging_holding_partitions.sql deleted file mode 100644 index 805f886f6..000000000 --- a/src/main/resources/changelog/changes/v6.0/sql/create_staging_holding_partitions.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Create partitions for staging_holding table based on first character of UUID --- Numeric partitions (0-9) -CREATE TABLE staging_holding_0 PARTITION OF staging_holding FOR VALUES FROM ('0') TO ('1'); -CREATE TABLE staging_holding_1 PARTITION OF staging_holding FOR VALUES FROM ('1') TO ('2'); -CREATE TABLE staging_holding_2 PARTITION OF staging_holding FOR VALUES FROM ('2') TO ('3'); -CREATE TABLE staging_holding_3 PARTITION OF staging_holding FOR VALUES FROM ('3') TO ('4'); -CREATE TABLE staging_holding_4 PARTITION OF staging_holding FOR VALUES FROM ('4') TO ('5'); -CREATE TABLE staging_holding_5 PARTITION OF staging_holding FOR VALUES FROM ('5') TO ('6'); -CREATE TABLE staging_holding_6 PARTITION OF staging_holding FOR VALUES FROM ('6') TO ('7'); -CREATE TABLE staging_holding_7 PARTITION OF staging_holding FOR VALUES FROM ('7') TO ('8'); -CREATE TABLE staging_holding_8 PARTITION OF staging_holding FOR VALUES FROM ('8') TO ('9'); -CREATE TABLE staging_holding_9 PARTITION OF staging_holding FOR VALUES FROM ('9') TO ('a'); - --- Alphabetic partitions (a-f for UUIDs) -CREATE TABLE staging_holding_a PARTITION OF staging_holding FOR VALUES FROM ('a') TO ('b'); -CREATE TABLE staging_holding_b PARTITION OF staging_holding FOR VALUES FROM ('b') TO ('c'); -CREATE TABLE staging_holding_c PARTITION OF staging_holding FOR VALUES FROM ('c') TO ('d'); -CREATE TABLE staging_holding_d PARTITION OF staging_holding FOR VALUES FROM ('d') TO ('e'); -CREATE TABLE staging_holding_e PARTITION OF staging_holding FOR VALUES FROM ('e') TO ('f'); -CREATE TABLE staging_holding_f PARTITION OF staging_holding FOR VALUES FROM ('f') TO ('g'); diff --git a/src/main/resources/changelog/changes/v6.0/sql/create_staging_instance_partitions.sql b/src/main/resources/changelog/changes/v6.0/sql/create_staging_instance_partitions.sql deleted file mode 100644 index 80f542a16..000000000 --- a/src/main/resources/changelog/changes/v6.0/sql/create_staging_instance_partitions.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Create partitions for staging_instance table based on first character of UUID --- Numeric partitions (0-9) -CREATE TABLE staging_instance_0 PARTITION OF staging_instance FOR VALUES FROM ('0') TO ('1'); -CREATE TABLE staging_instance_1 PARTITION OF staging_instance FOR VALUES FROM ('1') TO ('2'); -CREATE TABLE staging_instance_2 PARTITION OF staging_instance FOR VALUES FROM ('2') TO ('3'); -CREATE TABLE staging_instance_3 PARTITION OF staging_instance FOR VALUES FROM ('3') TO ('4'); -CREATE TABLE staging_instance_4 PARTITION OF staging_instance FOR VALUES FROM ('4') TO ('5'); -CREATE TABLE staging_instance_5 PARTITION OF staging_instance FOR VALUES FROM ('5') TO ('6'); -CREATE TABLE staging_instance_6 PARTITION OF staging_instance FOR VALUES FROM ('6') TO ('7'); -CREATE TABLE staging_instance_7 PARTITION OF staging_instance FOR VALUES FROM ('7') TO ('8'); -CREATE TABLE staging_instance_8 PARTITION OF staging_instance FOR VALUES FROM ('8') TO ('9'); -CREATE TABLE staging_instance_9 PARTITION OF staging_instance FOR VALUES FROM ('9') TO ('a'); - --- Alphabetic partitions (a-f for UUIDs) -CREATE TABLE staging_instance_a PARTITION OF staging_instance FOR VALUES FROM ('a') TO ('b'); -CREATE TABLE staging_instance_b PARTITION OF staging_instance FOR VALUES FROM ('b') TO ('c'); -CREATE TABLE staging_instance_c PARTITION OF staging_instance FOR VALUES FROM ('c') TO ('d'); -CREATE TABLE staging_instance_d PARTITION OF staging_instance FOR VALUES FROM ('d') TO ('e'); -CREATE TABLE staging_instance_e PARTITION OF staging_instance FOR VALUES FROM ('e') TO ('f'); -CREATE TABLE staging_instance_f PARTITION OF staging_instance FOR VALUES FROM ('f') TO ('g'); diff --git a/src/main/resources/changelog/changes/v6.0/sql/create_staging_item_partitions.sql b/src/main/resources/changelog/changes/v6.0/sql/create_staging_item_partitions.sql deleted file mode 100644 index 8d2f6f638..000000000 --- a/src/main/resources/changelog/changes/v6.0/sql/create_staging_item_partitions.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Create partitions for staging_item table based on first character of UUID --- Numeric partitions (0-9) -CREATE TABLE staging_item_0 PARTITION OF staging_item FOR VALUES FROM ('0') TO ('1'); -CREATE TABLE staging_item_1 PARTITION OF staging_item FOR VALUES FROM ('1') TO ('2'); -CREATE TABLE staging_item_2 PARTITION OF staging_item FOR VALUES FROM ('2') TO ('3'); -CREATE TABLE staging_item_3 PARTITION OF staging_item FOR VALUES FROM ('3') TO ('4'); -CREATE TABLE staging_item_4 PARTITION OF staging_item FOR VALUES FROM ('4') TO ('5'); -CREATE TABLE staging_item_5 PARTITION OF staging_item FOR VALUES FROM ('5') TO ('6'); -CREATE TABLE staging_item_6 PARTITION OF staging_item FOR VALUES FROM ('6') TO ('7'); -CREATE TABLE staging_item_7 PARTITION OF staging_item FOR VALUES FROM ('7') TO ('8'); -CREATE TABLE staging_item_8 PARTITION OF staging_item FOR VALUES FROM ('8') TO ('9'); -CREATE TABLE staging_item_9 PARTITION OF staging_item FOR VALUES FROM ('9') TO ('a'); - --- Alphabetic partitions (a-f for UUIDs) -CREATE TABLE staging_item_a PARTITION OF staging_item FOR VALUES FROM ('a') TO ('b'); -CREATE TABLE staging_item_b PARTITION OF staging_item FOR VALUES FROM ('b') TO ('c'); -CREATE TABLE staging_item_c PARTITION OF staging_item FOR VALUES FROM ('c') TO ('d'); -CREATE TABLE staging_item_d PARTITION OF staging_item FOR VALUES FROM ('d') TO ('e'); -CREATE TABLE staging_item_e PARTITION OF staging_item FOR VALUES FROM ('e') TO ('f'); -CREATE TABLE staging_item_f PARTITION OF staging_item FOR VALUES FROM ('f') TO ('g'); diff --git a/src/main/resources/changelog/changes/v6.0/sql/create_staging_relationship_partitions.sql b/src/main/resources/changelog/changes/v6.0/sql/create_staging_relationship_partitions.sql deleted file mode 100644 index 5993841a0..000000000 --- a/src/main/resources/changelog/changes/v6.0/sql/create_staging_relationship_partitions.sql +++ /dev/null @@ -1,83 +0,0 @@ --- Create partitions for staging_instance_subject table based on first character of instance_id --- Numeric partitions (0-9) -CREATE TABLE staging_instance_subject_0 PARTITION OF staging_instance_subject FOR VALUES FROM ('0') TO ('1'); -CREATE TABLE staging_instance_subject_1 PARTITION OF staging_instance_subject FOR VALUES FROM ('1') TO ('2'); -CREATE TABLE staging_instance_subject_2 PARTITION OF staging_instance_subject FOR VALUES FROM ('2') TO ('3'); -CREATE TABLE staging_instance_subject_3 PARTITION OF staging_instance_subject FOR VALUES FROM ('3') TO ('4'); -CREATE TABLE staging_instance_subject_4 PARTITION OF staging_instance_subject FOR VALUES FROM ('4') TO ('5'); -CREATE TABLE staging_instance_subject_5 PARTITION OF staging_instance_subject FOR VALUES FROM ('5') TO ('6'); -CREATE TABLE staging_instance_subject_6 PARTITION OF staging_instance_subject FOR VALUES FROM ('6') TO ('7'); -CREATE TABLE staging_instance_subject_7 PARTITION OF staging_instance_subject FOR VALUES FROM ('7') TO ('8'); -CREATE TABLE staging_instance_subject_8 PARTITION OF staging_instance_subject FOR VALUES FROM ('8') TO ('9'); -CREATE TABLE staging_instance_subject_9 PARTITION OF staging_instance_subject FOR VALUES FROM ('9') TO ('a'); - --- Alphabetic partitions (a-f for UUIDs) -CREATE TABLE staging_instance_subject_a PARTITION OF staging_instance_subject FOR VALUES FROM ('a') TO ('b'); -CREATE TABLE staging_instance_subject_b PARTITION OF staging_instance_subject FOR VALUES FROM ('b') TO ('c'); -CREATE TABLE staging_instance_subject_c PARTITION OF staging_instance_subject FOR VALUES FROM ('c') TO ('d'); -CREATE TABLE staging_instance_subject_d PARTITION OF staging_instance_subject FOR VALUES FROM ('d') TO ('e'); -CREATE TABLE staging_instance_subject_e PARTITION OF staging_instance_subject FOR VALUES FROM ('e') TO ('f'); -CREATE TABLE staging_instance_subject_f PARTITION OF staging_instance_subject FOR VALUES FROM ('f') TO ('g'); - --- Create partitions for staging_instance_contributor table based on first character of instance_id --- Numeric partitions (0-9) -CREATE TABLE staging_instance_contributor_0 PARTITION OF staging_instance_contributor FOR VALUES FROM ('0') TO ('1'); -CREATE TABLE staging_instance_contributor_1 PARTITION OF staging_instance_contributor FOR VALUES FROM ('1') TO ('2'); -CREATE TABLE staging_instance_contributor_2 PARTITION OF staging_instance_contributor FOR VALUES FROM ('2') TO ('3'); -CREATE TABLE staging_instance_contributor_3 PARTITION OF staging_instance_contributor FOR VALUES FROM ('3') TO ('4'); -CREATE TABLE staging_instance_contributor_4 PARTITION OF staging_instance_contributor FOR VALUES FROM ('4') TO ('5'); -CREATE TABLE staging_instance_contributor_5 PARTITION OF staging_instance_contributor FOR VALUES FROM ('5') TO ('6'); -CREATE TABLE staging_instance_contributor_6 PARTITION OF staging_instance_contributor FOR VALUES FROM ('6') TO ('7'); -CREATE TABLE staging_instance_contributor_7 PARTITION OF staging_instance_contributor FOR VALUES FROM ('7') TO ('8'); -CREATE TABLE staging_instance_contributor_8 PARTITION OF staging_instance_contributor FOR VALUES FROM ('8') TO ('9'); -CREATE TABLE staging_instance_contributor_9 PARTITION OF staging_instance_contributor FOR VALUES FROM ('9') TO ('a'); - --- Alphabetic partitions (a-f for UUIDs) -CREATE TABLE staging_instance_contributor_a PARTITION OF staging_instance_contributor FOR VALUES FROM ('a') TO ('b'); -CREATE TABLE staging_instance_contributor_b PARTITION OF staging_instance_contributor FOR VALUES FROM ('b') TO ('c'); -CREATE TABLE staging_instance_contributor_c PARTITION OF staging_instance_contributor FOR VALUES FROM ('c') TO ('d'); -CREATE TABLE staging_instance_contributor_d PARTITION OF staging_instance_contributor FOR VALUES FROM ('d') TO ('e'); -CREATE TABLE staging_instance_contributor_e PARTITION OF staging_instance_contributor FOR VALUES FROM ('e') TO ('f'); -CREATE TABLE staging_instance_contributor_f PARTITION OF staging_instance_contributor FOR VALUES FROM ('f') TO ('g'); - --- Create partitions for staging_instance_classification table based on first character of instance_id --- Numeric partitions (0-9) -CREATE TABLE staging_instance_classification_0 PARTITION OF staging_instance_classification FOR VALUES FROM ('0') TO ('1'); -CREATE TABLE staging_instance_classification_1 PARTITION OF staging_instance_classification FOR VALUES FROM ('1') TO ('2'); -CREATE TABLE staging_instance_classification_2 PARTITION OF staging_instance_classification FOR VALUES FROM ('2') TO ('3'); -CREATE TABLE staging_instance_classification_3 PARTITION OF staging_instance_classification FOR VALUES FROM ('3') TO ('4'); -CREATE TABLE staging_instance_classification_4 PARTITION OF staging_instance_classification FOR VALUES FROM ('4') TO ('5'); -CREATE TABLE staging_instance_classification_5 PARTITION OF staging_instance_classification FOR VALUES FROM ('5') TO ('6'); -CREATE TABLE staging_instance_classification_6 PARTITION OF staging_instance_classification FOR VALUES FROM ('6') TO ('7'); -CREATE TABLE staging_instance_classification_7 PARTITION OF staging_instance_classification FOR VALUES FROM ('7') TO ('8'); -CREATE TABLE staging_instance_classification_8 PARTITION OF staging_instance_classification FOR VALUES FROM ('8') TO ('9'); -CREATE TABLE staging_instance_classification_9 PARTITION OF staging_instance_classification FOR VALUES FROM ('9') TO ('a'); - --- Alphabetic partitions (a-f for UUIDs) -CREATE TABLE staging_instance_classification_a PARTITION OF staging_instance_classification FOR VALUES FROM ('a') TO ('b'); -CREATE TABLE staging_instance_classification_b PARTITION OF staging_instance_classification FOR VALUES FROM ('b') TO ('c'); -CREATE TABLE staging_instance_classification_c PARTITION OF staging_instance_classification FOR VALUES FROM ('c') TO ('d'); -CREATE TABLE staging_instance_classification_d PARTITION OF staging_instance_classification FOR VALUES FROM ('d') TO ('e'); -CREATE TABLE staging_instance_classification_e PARTITION OF staging_instance_classification FOR VALUES FROM ('e') TO ('f'); -CREATE TABLE staging_instance_classification_f PARTITION OF staging_instance_classification FOR VALUES FROM ('f') TO ('g'); - --- Create partitions for staging_instance_call_number table based on first character of instance_id --- Numeric partitions (0-9) -CREATE TABLE staging_instance_call_number_0 PARTITION OF staging_instance_call_number FOR VALUES FROM ('0') TO ('1'); -CREATE TABLE staging_instance_call_number_1 PARTITION OF staging_instance_call_number FOR VALUES FROM ('1') TO ('2'); -CREATE TABLE staging_instance_call_number_2 PARTITION OF staging_instance_call_number FOR VALUES FROM ('2') TO ('3'); -CREATE TABLE staging_instance_call_number_3 PARTITION OF staging_instance_call_number FOR VALUES FROM ('3') TO ('4'); -CREATE TABLE staging_instance_call_number_4 PARTITION OF staging_instance_call_number FOR VALUES FROM ('4') TO ('5'); -CREATE TABLE staging_instance_call_number_5 PARTITION OF staging_instance_call_number FOR VALUES FROM ('5') TO ('6'); -CREATE TABLE staging_instance_call_number_6 PARTITION OF staging_instance_call_number FOR VALUES FROM ('6') TO ('7'); -CREATE TABLE staging_instance_call_number_7 PARTITION OF staging_instance_call_number FOR VALUES FROM ('7') TO ('8'); -CREATE TABLE staging_instance_call_number_8 PARTITION OF staging_instance_call_number FOR VALUES FROM ('8') TO ('9'); -CREATE TABLE staging_instance_call_number_9 PARTITION OF staging_instance_call_number FOR VALUES FROM ('9') TO ('a'); - --- Alphabetic partitions (a-f for UUIDs) -CREATE TABLE staging_instance_call_number_a PARTITION OF staging_instance_call_number FOR VALUES FROM ('a') TO ('b'); -CREATE TABLE staging_instance_call_number_b PARTITION OF staging_instance_call_number FOR VALUES FROM ('b') TO ('c'); -CREATE TABLE staging_instance_call_number_c PARTITION OF staging_instance_call_number FOR VALUES FROM ('c') TO ('d'); -CREATE TABLE staging_instance_call_number_d PARTITION OF staging_instance_call_number FOR VALUES FROM ('d') TO ('e'); -CREATE TABLE staging_instance_call_number_e PARTITION OF staging_instance_call_number FOR VALUES FROM ('e') TO ('f'); -CREATE TABLE staging_instance_call_number_f PARTITION OF staging_instance_call_number FOR VALUES FROM ('f') TO ('g'); diff --git a/src/test/java/org/folio/search/integration/KafkaMessageListenerTest.java b/src/test/java/org/folio/search/integration/KafkaMessageListenerTest.java index 06b77def2..1a2ecf734 100644 --- a/src/test/java/org/folio/search/integration/KafkaMessageListenerTest.java +++ b/src/test/java/org/folio/search/integration/KafkaMessageListenerTest.java @@ -2,6 +2,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; +import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.folio.search.configuration.RetryTemplateConfiguration.KAFKA_RETRY_TEMPLATE_NAME; import static org.folio.search.domain.dto.ResourceEventType.CREATE; import static org.folio.search.domain.dto.ResourceEventType.DELETE; @@ -40,7 +41,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.springframework.retry.support.RetryTemplate.defaultInstance; -import static org.testcontainers.shaded.org.apache.commons.lang3.StringUtils.EMPTY; import java.util.List; import java.util.concurrent.Callable; From 276c98b743d48974c105e2badfae80979c3a0dab Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Fri, 13 Feb 2026 17:04:37 +0100 Subject: [PATCH 4/5] Fix merge problems, refactor to comply with checkstyle --- .../AbstractResourceRepository.java | 24 +- .../reindex/ReindexOrchestrationService.java | 35 ++- .../service/reindex/ReindexService.java | 275 +++++++++--------- .../reindex/StagingMigrationService.java | 5 +- .../reindex/jdbc/CallNumberRepository.java | 24 +- .../jdbc/ClassificationRepository.java | 2 - .../reindex/jdbc/ContributorRepository.java | 25 +- .../reindex/jdbc/ReindexJdbcRepository.java | 2 +- .../reindex/jdbc/SubjectRepository.java | 2 - .../jdbc/UploadInstanceRepository.java | 75 +++-- .../service/reindex/ReindexContextTest.java | 2 + .../ReindexOrchestrationServiceTest.java | 15 +- .../reindex/StagingMigrationServiceTest.java | 1 + 13 files changed, 261 insertions(+), 226 deletions(-) diff --git a/src/main/java/org/folio/search/repository/AbstractResourceRepository.java b/src/main/java/org/folio/search/repository/AbstractResourceRepository.java index ddb5bbad7..578fd9e52 100644 --- a/src/main/java/org/folio/search/repository/AbstractResourceRepository.java +++ b/src/main/java/org/folio/search/repository/AbstractResourceRepository.java @@ -77,15 +77,7 @@ public FolioIndexOperationResponse deleteConsortiumDocumentsByTenantId(ResourceT var indexName = indexNameProvider.getIndexName(resourceType, tenantId); log.debug("deleteConsortiumDocumentsByTenantId:: by [indexName: {}, tenantId: {}]", indexName, tenantId); - var deleteByQueryRequest = new DeleteByQueryRequest(indexName); - - // Configure performance settings - deleteByQueryRequest.setBatchSize(indexManagementConfig.getDeleteQueryBatchSize()); - deleteByQueryRequest.setScroll( - TimeValue.timeValueMinutes(indexManagementConfig.getDeleteQueryScrollTimeoutMinutes())); - deleteByQueryRequest.setTimeout( - TimeValue.timeValueMinutes(indexManagementConfig.getDeleteQueryRequestTimeoutMinutes())); - deleteByQueryRequest.setRefresh(indexManagementConfig.getDeleteQueryRefresh()); + var deleteByQueryRequest = buildDeleteByQueryRequest(indexName); // Build query: tenantId = targetTenant AND (NOT shared = true) var query = boolQuery() @@ -108,6 +100,20 @@ public FolioIndexOperationResponse deleteConsortiumDocumentsByTenantId(ResourceT : getErrorIndexOperationResponse(getBulkByScrollResponseErrorMessage(bulkByScrollResponse)); } + private DeleteByQueryRequest buildDeleteByQueryRequest(String indexName) { + var deleteByQueryRequest = new DeleteByQueryRequest(indexName); + + // Configure performance settings + deleteByQueryRequest.setBatchSize(indexManagementConfig.getDeleteQueryBatchSize()); + deleteByQueryRequest.setScroll( + TimeValue.timeValueMinutes(indexManagementConfig.getDeleteQueryScrollTimeoutMinutes())); + deleteByQueryRequest.setTimeout( + TimeValue.timeValueMinutes(indexManagementConfig.getDeleteQueryRequestTimeoutMinutes())); + deleteByQueryRequest.setRefresh(indexManagementConfig.getDeleteQueryRefresh()); + + return deleteByQueryRequest; + } + @Autowired public void setIndexNameProvider(IndexNameProvider indexNameProvider) { this.indexNameProvider = indexNameProvider; diff --git a/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java b/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java index 33aa82834..6180b0459 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java @@ -75,10 +75,9 @@ public boolean process(ReindexRangeIndexEvent event) { event.getId(), event.getTenant(), memberTenantId, event.getEntityType(), event.getLower(), event.getUpper(), event.getTs()); - var folioIndexOperationResponse = fetchRecordsAndIndexForUploadRange(event); if (folioIndexOperationResponse.getStatus() == FolioIndexOperationResponse.StatusEnum.ERROR) { - throw handleReindexUploadFailure(event,folioIndexOperationResponse.getErrorMessage()); + throw handleReindexUploadFailure(event, folioIndexOperationResponse.getErrorMessage()); } uploadRangeService.updateStatus(event, ReindexRangeStatus.SUCCESS, null); @@ -95,16 +94,13 @@ public boolean process(ReindexRangeIndexEvent event) { public boolean process(ReindexRecordsEvent event) { var memberTenantId = getMemberTenantIdForProcessing(); - var entityType = event.getRecordType().getEntityType(); try { log.info("process:: ReindexRecordsEvent [rangeId: {}, tenantId: {}, memberTenantId: {}, " + "recordType: {}, recordsCount: {}]", event.getRangeId(), event.getTenant(), memberTenantId, event.getRecordType(), event.getRecords().size()); - mergeRangeService.saveEntities(event); - mergeRangeService.updateStatus(entityType, event.getRangeId(), ReindexRangeStatus.SUCCESS, null); - reindexStatusService.addProcessedMergeRanges(entityType, 1); + persistEntities(event); log.info("process:: ReindexRecordsEvent processed [rangeId: {}, recordType: {}]", event.getRangeId(), event.getRecordType()); } catch (PessimisticLockingFailureException ex) { @@ -112,10 +108,7 @@ public boolean process(ReindexRecordsEvent event) { + " [rangeId: {}, error: {}]", event.getRangeId(), ex.getMessage()), ex); throw new ReindexException(ex.getMessage()); } catch (Exception ex) { - log.error(new FormattedMessage("process:: ReindexRecordsEvent indexing error [rangeId: {}, error: {}]", - event.getRangeId(), ex.getMessage()), ex); - reindexStatusService.updateReindexMergeFailed(entityType); - mergeRangeService.updateStatus(entityType, event.getRangeId(), ReindexRangeStatus.FAIL, ex.getMessage()); + handleReindexMergeFailure(event, ex.getMessage()); return true; } finally { // Clean up member tenant context @@ -124,6 +117,18 @@ public boolean process(ReindexRecordsEvent event) { } } + startUploadOnMergeCompletion(); + return true; + } + + private void persistEntities(ReindexRecordsEvent event) { + var entityType = event.getRecordType().getEntityType(); + mergeRangeService.saveEntities(event); + mergeRangeService.updateStatus(entityType, event.getRangeId(), ReindexRangeStatus.SUCCESS, null); + reindexStatusService.addProcessedMergeRanges(entityType, 1); + } + + private void startUploadOnMergeCompletion() { if (reindexStatusService.isMergeCompleted()) { // Get targetTenantId before migration var targetTenantId = reindexStatusService.getTargetTenantId(); @@ -147,8 +152,6 @@ public boolean process(ReindexRecordsEvent event) { log.info("process:: Migration and upload phase completed for {}", targetTenantId != null ? "tenant: " + targetTenantId : "consortium"); } - - return true; } private void performStagingMigration(String targetTenantId) { @@ -184,4 +187,12 @@ private ReindexException handleReindexUploadFailure(ReindexRangeIndexEvent event reindexStatusService.updateReindexUploadFailed(event.getEntityType()); return new ReindexException(errorMessage); } + + private void handleReindexMergeFailure(ReindexRecordsEvent event, String errorMessage) { + log.warn("handleReindexMergeFailure:: ReindexRecordsEvent indexing error [rangeId: {}, error: {}]", + event.getRangeId(), errorMessage); + var entityType = event.getRecordType().getEntityType(); + reindexStatusService.updateReindexMergeFailed(entityType); + mergeRangeService.updateStatus(entityType, event.getRangeId(), ReindexRangeStatus.FAIL, errorMessage); + } } diff --git a/src/main/java/org/folio/search/service/reindex/ReindexService.java b/src/main/java/org/folio/search/service/reindex/ReindexService.java index 871ec462a..c285a83f5 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexService.java @@ -84,66 +84,49 @@ public CompletableFuture submitFullReindex(String tenantId, IndexSettings reindexCommonService.deleteAllRecords(targetTenantId); statusService.recreateMergeStatusRecords(targetTenantId); - if (targetTenantId == null) { - // Full reindex - recreate all indexes (existing behavior) - recreateIndices(tenantId, ReindexEntityType.supportUploadTypes(), indexSettings); - log.info("submitFullReindex:: recreated indexes for full reindex [requestingTenant: {}]", tenantId); - } else { - // Tenant-specific reindex - ensure indexes exist without recreating existing ones - ensureIndicesExist(tenantId, ReindexEntityType.supportUploadTypes(), indexSettings); - log.info("submitFullReindex:: ensured indexes exist for tenant-specific reindex " - + "[requestingTenant: {}, targetTenant: {}]", tenantId, targetTenantId); - } + recreateIndices(tenantId, targetTenantId, indexSettings); // Capture context before async execution final var memberTenantIdContext = targetTenantId; - var future = CompletableFuture.runAsync(() -> { - try { - // Restore context in executor thread - if (memberTenantIdContext != null) { - ReindexContext.setMemberTenantId(memberTenantIdContext); - ReindexContext.setReindexMode(true); // Enable staging - } - - mergeRangeService.truncateMergeRanges(); + var future = CompletableFuture.runAsync(() -> recreateMergeRanges(tenantId, targetTenantId), reindexFullExecutor) + .thenRun(() -> publishRecordsRange(tenantId, memberTenantIdContext)) + .handle(handleReindexingFailure(tenantId, targetTenantId)); - List rangesForAllTenants; - if (memberTenantIdContext != null) { - // Only process the member tenant (no central tenant in merge) - rangesForAllTenants = processForConsortium(tenantId, memberTenantIdContext); - } else { - // Full consortium reindex: Process central + all members (no staging) - rangesForAllTenants = Stream.of(mergeRangeService.createMergeRanges(tenantId), // Central - processForConsortium(tenantId, null) // Members - ).flatMap(List::stream).toList(); - } + log.info("submitFullReindex:: submitted [requestingTenant: {}, targetTenant: {}]", tenantId, + targetTenantId != null ? targetTenantId : "all consortium members"); + return future; + } - mergeRangeService.saveMergeRanges(rangesForAllTenants); - } finally { - // Clean up context in executor thread - if (memberTenantIdContext != null) { - ReindexContext.clearMemberTenantId(); - ReindexContext.setReindexMode(false); - } - } - }, reindexFullExecutor).thenRun(() -> { - // Restore context before publishing + private void recreateMergeRanges(String tenantId, String memberTenantIdContext) { + try { + // Restore context in executor thread if (memberTenantIdContext != null) { ReindexContext.setMemberTenantId(memberTenantIdContext); + ReindexContext.setReindexMode(true); // Enable staging } - try { - publishRecordsRange(tenantId, memberTenantIdContext); - } finally { - if (memberTenantIdContext != null) { - ReindexContext.clearMemberTenantId(); - } + + mergeRangeService.truncateMergeRanges(); + + List rangesForAllTenants; + if (memberTenantIdContext != null) { + // Only process the member tenant (no central tenant in merge) + rangesForAllTenants = processForConsortium(tenantId, memberTenantIdContext); + } else { + // Full consortium reindex: Process central + all members (no staging) + rangesForAllTenants = Stream.of(mergeRangeService.createMergeRanges(tenantId), // Central + processForConsortium(tenantId, null) // Members + ).flatMap(List::stream).toList(); } - }).handle(handleReindexingFailure(tenantId, targetTenantId)); - log.info("submitFullReindex:: submitted [requestingTenant: {}, targetTenant: {}]", tenantId, - targetTenantId != null ? targetTenantId : "all consortium members"); - return future; + mergeRangeService.saveMergeRanges(rangesForAllTenants); + } finally { + // Clean up context in executor thread + if (memberTenantIdContext != null) { + ReindexContext.clearMemberTenantId(); + ReindexContext.setReindexMode(false); + } + } } public CompletableFuture submitUploadReindex(String tenantId, ReindexUploadDto reindexUploadDto) { @@ -162,40 +145,22 @@ private CompletableFuture submitUploadReindex(String tenantId, List(); for (var entityType : entityTypes) { - var future = CompletableFuture.runAsync(() -> { - // Restore context in executor thread - if (memberTenantIdContext != null) { - ReindexContext.setMemberTenantId(memberTenantIdContext); - } - try { - uploadRangeService.prepareAndSendIndexRanges(entityType); - } finally { - // Clean up context in executor thread - if (memberTenantIdContext != null) { - ReindexContext.clearMemberTenantId(); + var future = CompletableFuture.runAsync(() -> + sendIndexRanges(memberTenantIdContext, entityType), reindexUploadExecutor) + .handle((unused, throwable) -> { + if (throwable != null) { + log.error("submitUploadReindex:: reindex upload process failed: {}", throwable.getMessage()); + statusService.updateReindexUploadFailed(entityType); } - } - }, reindexUploadExecutor).handle((unused, throwable) -> { - if (throwable != null) { - log.error("submitUploadReindex:: reindex upload process failed: {}", throwable.getMessage()); - statusService.updateReindexUploadFailed(entityType); - } - return unused; - }); + return unused; + }); futures.add(future); } @@ -203,6 +168,32 @@ private CompletableFuture submitUploadReindex(String tenantId, List entityTypes, + boolean recreateIndex, IndexSettings indexSettings) { + var targetTenantId = statusService.getTargetTenantId(); + for (var reindexEntityType : entityTypes) { + statusService.recreateUploadStatusRecord(reindexEntityType, targetTenantId); + if (recreateIndex) { + reindexCommonService.recreateIndex(reindexEntityType, tenantId, indexSettings); + } + } + } + + private void sendIndexRanges(String memberTenantIdContext, ReindexEntityType entityType) { + // Restore context in executor thread + if (memberTenantIdContext != null) { + ReindexContext.setMemberTenantId(memberTenantIdContext); + } + try { + uploadRangeService.prepareAndSendIndexRanges(entityType); + } finally { + // Clean up context in executor thread + if (memberTenantIdContext != null) { + ReindexContext.clearMemberTenantId(); + } + } + } + /** * Submits upload reindex with tenant-specific document cleanup for member tenant reindex operations. * This method performs OpenSearch document cleanup for the specified tenant while preserving @@ -220,19 +211,7 @@ public CompletableFuture submitUploadReindexWithTenantCleanup(String tenan entityTypes, targetTenantId); // Perform tenant-specific document cleanup before upload processing - if (targetTenantId != null) { - log.info("submitUploadReindexWithTenantCleanup:: performing OpenSearch document cleanup " - + "for tenant [{}] while preserving shared instances", targetTenantId); - try { - reindexCommonService.deleteInstanceDocumentsByTenantId(targetTenantId); - log.info("submitUploadReindexWithTenantCleanup:: completed OpenSearch cleanup for tenant [{}]", targetTenantId); - } catch (Exception e) { - log.error( - "submitUploadReindexWithTenantCleanup:: failed to cleanup OpenSearch documents " + "for tenant [{}]: {}", - targetTenantId, e.getMessage(), e); - throw new ReindexException("Failed to cleanup tenant documents before upload", e); - } - } + performTenantCleanup(targetTenantId); // Set member tenant context for upload phase if (targetTenantId != null) { @@ -249,6 +228,22 @@ public CompletableFuture submitUploadReindexWithTenantCleanup(String tenan } } + private void performTenantCleanup(String targetTenantId) { + if (targetTenantId != null) { + log.info("submitUploadReindexWithTenantCleanup:: performing OpenSearch document cleanup " + + "for tenant [{}] while preserving shared instances", targetTenantId); + try { + reindexCommonService.deleteInstanceDocumentsByTenantId(targetTenantId); + log.info("submitUploadReindexWithTenantCleanup:: completed OpenSearch cleanup for tenant [{}]", targetTenantId); + } catch (Exception e) { + log.error( + "submitUploadReindexWithTenantCleanup:: failed to cleanup OpenSearch documents " + "for tenant [{}]: {}", + targetTenantId, e.getMessage(), e); + throw new ReindexException("Failed to cleanup tenant documents before upload", e); + } + } + } + public CompletableFuture submitFailedMergeRangesReindex(String tenantId) { log.info("submitFailedMergeRangesReindex:: for [tenantId: {}]", tenantId); @@ -288,15 +283,20 @@ private BiFunction handleReindexingFailure(String tenantI }; } - private void recreateIndices(String tenantId, List entityTypes, IndexSettings indexSettings) { - for (var reindexEntityType : entityTypes) { - reindexCommonService.recreateIndex(reindexEntityType, tenantId, indexSettings); - } - } - - private void ensureIndicesExist(String tenantId, List entityTypes, IndexSettings indexSettings) { - for (var reindexEntityType : entityTypes) { - reindexCommonService.ensureIndexExists(reindexEntityType, tenantId, indexSettings); + private void recreateIndices(String tenantId, String targetTenantId, IndexSettings indexSettings) { + if (targetTenantId == null) { + // Full reindex - recreate all indexes (existing behavior) + for (var reindexEntityType : ReindexEntityType.supportUploadTypes()) { + reindexCommonService.recreateIndex(reindexEntityType, tenantId, indexSettings); + } + log.info("submitFullReindex:: recreated indexes for full reindex [requestingTenant: {}]", tenantId); + } else { + // Tenant-specific reindex - ensure indexes exist without recreating existing ones + for (var reindexEntityType : ReindexEntityType.supportUploadTypes()) { + reindexCommonService.ensureIndexExists(reindexEntityType, tenantId, indexSettings); + } + log.info("submitFullReindex:: ensured indexes exist for tenant-specific reindex " + + "[requestingTenant: {}, targetTenant: {}]", tenantId, targetTenantId); } } @@ -330,41 +330,56 @@ private List processForConsortium(String tenantId, String targ } private void publishRecordsRange(String tenantId, String targetTenantId) { - // Capture context before async execution - final String memberTenantIdContext = ReindexContext.getMemberTenantId(); - - var futures = new ArrayList<>(); - for (var entityType : ReindexEntityType.supportMergeTypes()) { - var rangeEntities = mergeRangeService.fetchMergeRanges(entityType); - if (CollectionUtils.isNotEmpty(rangeEntities)) { - log.info("publishRecordsRange:: publishing merge ranges " - + "[requestingTenant: {}, entityType: {}, count: {}, targetTenant: {}]", tenantId, entityType, - rangeEntities.size(), targetTenantId != null ? targetTenantId : "all"); - - statusService.updateReindexMergeStarted(entityType, rangeEntities.size()); - for (var rangeEntity : rangeEntities) { - var publishFuture = CompletableFuture.runAsync(() -> { - // Restore context in executor thread - if (memberTenantIdContext != null) { - ReindexContext.setMemberTenantId(memberTenantIdContext); - } - try { - executionService.executeSystemUserScoped(rangeEntity.getTenantId(), () -> { - inventoryService.publishReindexRecordsRange(rangeEntity); - return null; - }); - } finally { - // Clean up context in executor thread - if (memberTenantIdContext != null) { - ReindexContext.clearMemberTenantId(); - } - } - }, reindexPublisherExecutor); - futures.add(publishFuture); + // Restore context before publishing + if (targetTenantId != null) { + ReindexContext.setMemberTenantId(targetTenantId); + } + try { + // Capture context before async execution + final String memberTenantIdContext = ReindexContext.getMemberTenantId(); + + var futures = new ArrayList>(); + for (var entityType : ReindexEntityType.supportMergeTypes()) { + var rangeEntities = mergeRangeService.fetchMergeRanges(entityType); + if (CollectionUtils.isNotEmpty(rangeEntities)) { + log.info("publishRecordsRange:: publishing merge ranges " + + "[requestingTenant: {}, entityType: {}, count: {}, targetTenant: {}]", tenantId, entityType, + rangeEntities.size(), targetTenantId != null ? targetTenantId : "all"); + + statusService.updateReindexMergeStarted(entityType, rangeEntities.size()); + submitRecordsRangePublishing(memberTenantIdContext, rangeEntities, futures); } } + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + } finally { + if (targetTenantId != null) { + ReindexContext.clearMemberTenantId(); + } + } + } + + private void submitRecordsRangePublishing(String memberTenantIdContext, List rangeEntities, + List> futures) { + for (var rangeEntity : rangeEntities) { + var publishFuture = CompletableFuture.runAsync(() -> { + // Restore context in executor thread + if (memberTenantIdContext != null) { + ReindexContext.setMemberTenantId(memberTenantIdContext); + } + try { + executionService.executeSystemUserScoped(rangeEntity.getTenantId(), () -> { + inventoryService.publishReindexRecordsRange(rangeEntity); + return null; + }); + } finally { + // Clean up context in executor thread + if (memberTenantIdContext != null) { + ReindexContext.clearMemberTenantId(); + } + } + }, reindexPublisherExecutor); + futures.add(publishFuture); } - CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); } private void validateUploadReindex(String tenantId, List entityTypes) { diff --git a/src/main/java/org/folio/search/service/reindex/StagingMigrationService.java b/src/main/java/org/folio/search/service/reindex/StagingMigrationService.java index de2a19b6f..2b83273aa 100644 --- a/src/main/java/org/folio/search/service/reindex/StagingMigrationService.java +++ b/src/main/java/org/folio/search/service/reindex/StagingMigrationService.java @@ -37,6 +37,7 @@ public StagingMigrationService(JdbcTemplate jdbcTemplate, } @Transactional + @SuppressWarnings("checkstyle:MethodLength") public MigrationResult migrateAllStagingTables(String targetTenantId) { var isMemberTenantRefresh = targetTenantId != null; var result = new MigrationResult(); @@ -45,9 +46,8 @@ public MigrationResult migrateAllStagingTables(String targetTenantId) { try { // Set work_mem for this transaction to optimize query performance setWorkMem(); - log.info("migrateAllStagingTables:: Starting migration for: [targetTenantId: {}]", targetTenantId); - + // Analyze staging tables for better query performance analyzeStagingTables(); @@ -84,6 +84,7 @@ private void handleMemberTenantPreMigration(String targetTenantId) { log.info("handleMemberTenantPreMigration:: Main table cleanup completed for tenant: {}", targetTenantId); } + @SuppressWarnings("checkstyle:MethodLength") private void executeMainMigrationPhases(MigrationResult result) { // Phase 1: Instances log.info("executeMainMigrationPhases:: Starting instances migration..."); diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java index 6df982061..1cc6fce72 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java @@ -19,6 +19,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,7 +41,6 @@ @Log4j2 @Repository -@SuppressWarnings("java:S2077") public class CallNumberRepository extends UploadRangeRepository implements InstanceChildResourceRepository { private static final String SELECT_QUERY = """ @@ -393,19 +393,23 @@ private void saveRelationshipEntitiesToStaging(ChildResourceEntityBatch entityBa }); } catch (DataAccessException e) { log.warn("saveRelationshipEntitiesToStaging::Failed to save staging relations batch. Processing one-by-one", e); - for (var entityRelation : entityBatch.relationshipEntities()) { - try { - jdbcTemplate.update(stagingInstanceCallNumberSql, getCallNumberId(entityRelation), getItemId(entityRelation), - getInstanceId(entityRelation), getTenantId(entityRelation), getLocationId(entityRelation)); - } catch (DataAccessException ex) { - log.debug("Failed to save staging call number relationship for {}: {}", - getCallNumberId(entityRelation), ex.getMessage()); - } - } + retrySaveRelationshipsToStagingOneByOne(stagingInstanceCallNumberSql, entityBatch.relationshipEntities()); } log.debug("Saved {} call number relationships to staging table", entityBatch.relationshipEntities().size()); } + private void retrySaveRelationshipsToStagingOneByOne(String sql, Collection> relationships) { + for (var entityRelation : relationships) { + try { + jdbcTemplate.update(sql, getCallNumberId(entityRelation), getItemId(entityRelation), + getInstanceId(entityRelation), getTenantId(entityRelation), getLocationId(entityRelation)); + } catch (DataAccessException ex) { + log.debug("Failed to save staging call number relationship for {}: {}", + getCallNumberId(entityRelation), ex.getMessage()); + } + } + } + private String getCallNumberSuffix(ResultSet rs) throws SQLException { return rs.getString("call_number_suffix"); } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java index 6c7bcd149..085129b7b 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java @@ -30,7 +30,6 @@ @Log4j2 @Repository -@SuppressWarnings("java:S2077") public class ClassificationRepository extends UploadRangeRepository implements InstanceChildResourceRepository { private static final String SELECT_QUERY = """ @@ -227,7 +226,6 @@ public void deleteByInstanceIds(List instanceIds, String tenantId) { deleteByInstanceIds(DELETE_QUERY, instanceIds, tenantId); } - @SuppressWarnings("checkstyle:MethodLength") @Override public void saveAll(ChildResourceEntityBatch entityBatch) { // Use staging tables only for member tenant specific full reindex diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java index a21699c6e..fab580fca 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java @@ -232,7 +232,6 @@ public void deleteByInstanceIds(List instanceIds, String tenantId) { } @Override - @SuppressWarnings("checkstyle:MethodLength") public void saveAll(ChildResourceEntityBatch entityBatch) { // Use staging tables only for member tenant specific full reindex if (ReindexContext.isReindexMode() && ReindexContext.isMemberTenantReindex()) { @@ -329,20 +328,24 @@ private void saveRelationshipsToStaging(List> relationships) }); } catch (DataAccessException e) { log.warn("saveRelationshipsToStaging::Failed to save relationships batch. Processing one-by-one", e); - for (var entityRelation : relationships) { - try { - jdbcTemplate.update(stagingRelationsSql, entityRelation.get("instanceId"), - entityRelation.get("contributorId"), entityRelation.get(CONTRIBUTOR_TYPE_FIELD), - entityRelation.get("tenantId"), entityRelation.get("shared")); - } catch (DataAccessException ex) { - log.debug("Failed to save staging contributor relationship for {}: {}", - entityRelation.get("contributorId"), ex.getMessage()); - } - } + retrySaveRelationshipsToStagingOneByOne(stagingRelationsSql, relationships); } log.debug("Saved {} contributor relationships to staging table", relationships.size()); } + private void retrySaveRelationshipsToStagingOneByOne(String sql, List> relationships) { + for (var entityRelation : relationships) { + try { + jdbcTemplate.update(sql, entityRelation.get("instanceId"), + entityRelation.get("contributorId"), entityRelation.get(CONTRIBUTOR_TYPE_FIELD), + entityRelation.get("tenantId"), entityRelation.get("shared")); + } catch (DataAccessException ex) { + log.debug("Failed to save staging contributor relationship for {}: {}", + entityRelation.get("contributorId"), ex.getMessage()); + } + } + } + @Override public List> fetchByIdRangeWithTimestamp(String lower, String upper, Timestamp timestamp) { var sql = SELECT_QUERY.formatted(JdbcUtils.getSchemaName(context), diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ReindexJdbcRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ReindexJdbcRepository.java index 35827fb32..a542a758a 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ReindexJdbcRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ReindexJdbcRepository.java @@ -20,7 +20,7 @@ public abstract class ReindexJdbcRepository { - public static final int BATCH_OPERATION_SIZE = 100; + protected static final int BATCH_OPERATION_SIZE = 100; protected static final String LAST_UPDATED_DATE_FIELD = "lastUpdatedDate"; private static final String COUNT_SQL = "SELECT COUNT(*) FROM %s;"; diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java index 9b99371a4..98c2ce477 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java @@ -31,7 +31,6 @@ @Log4j2 @Repository -@SuppressWarnings("java:S2077") public class SubjectRepository extends UploadRangeRepository implements InstanceChildResourceRepository { private static final String SELECT_QUERY = """ @@ -237,7 +236,6 @@ public void deleteByInstanceIds(List instanceIds, String tenantId) { } @Override - @SuppressWarnings("checkstyle:MethodLength") public void saveAll(ChildResourceEntityBatch entityBatch) { // Use staging tables only for member tenant specific full reindex if (ReindexContext.isReindexMode() && ReindexContext.isMemberTenantReindex()) { diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/UploadInstanceRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/UploadInstanceRepository.java index fb41f7797..b9abb250a 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/UploadInstanceRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/UploadInstanceRepository.java @@ -64,6 +64,40 @@ aggregated_items AS ( private static final String INSTANCE_IDS_WHERE_CLAUSE = "%s IN (%s)"; private static final String ITEM_NOT_DELETED_FILTER = " AND it.is_deleted = false"; + private static final String CONDITIONAL_INSTANCE_QUERY = """ + SELECT combined.json + || jsonb_build_object('tenantId', combined.tenant_id, + 'shared', combined.shared, + 'isBoundWith', combined.is_bound_with, + 'holdings', COALESCE(jsonb_agg(DISTINCT h.json || + jsonb_build_object('tenantId', h.tenant_id)) + FILTER (WHERE h.json IS NOT NULL), '[]'::jsonb), + 'items', COALESCE(jsonb_agg(it.json || + jsonb_build_object('tenantId', it.tenant_id)) + FILTER (WHERE it.json IS NOT NULL), '[]'::jsonb)) as json + FROM ( + -- Local instances from central tenant (after merge) + SELECT i.id, i.tenant_id, i.shared, i.is_bound_with, i.json + FROM %s.instance i + WHERE i.id >= ?::uuid AND i.id <= ?::uuid + AND i.tenant_id = ? + UNION ALL + -- Shared instances that have holdings for member tenant + SELECT i.id, i.tenant_id, i.shared, i.is_bound_with, i.json + FROM %s.instance i + WHERE i.id >= ?::uuid AND i.id <= ?::uuid + AND i.shared = true + AND EXISTS ( + SELECT 1 FROM %s.holding h + WHERE h.instance_id = i.id AND h.tenant_id = ? + ) + ) combined + LEFT JOIN %s.holding h ON h.instance_id = combined.id + LEFT JOIN %s.item it ON it.holding_id = h.id + GROUP BY combined.id, combined.tenant_id, combined.shared, + combined.is_bound_with, combined.json + """; + private final ConsortiumTenantService consortiumTenantService; protected UploadInstanceRepository(JdbcTemplate jdbcTemplate, JsonConverter jsonConverter, @@ -168,39 +202,7 @@ private List> fetchConditionalInstances( * - Applies UUID range filtering */ private String buildConditionalInstanceQuery(String centralSchema) { - return """ - SELECT combined.json - || jsonb_build_object('tenantId', combined.tenant_id, - 'shared', combined.shared, - 'isBoundWith', combined.is_bound_with, - 'holdings', COALESCE(jsonb_agg(DISTINCT h.json || - jsonb_build_object('tenantId', h.tenant_id)) - FILTER (WHERE h.json IS NOT NULL), '[]'::jsonb), - 'items', COALESCE(jsonb_agg(it.json || - jsonb_build_object('tenantId', it.tenant_id)) - FILTER (WHERE it.json IS NOT NULL), '[]'::jsonb)) as json - FROM ( - -- Local instances from central tenant (after merge) - SELECT i.id, i.tenant_id, i.shared, i.is_bound_with, i.json - FROM %s.instance i - WHERE i.id >= ?::uuid AND i.id <= ?::uuid - AND i.tenant_id = ? - UNION ALL - -- Shared instances that have holdings for member tenant - SELECT i.id, i.tenant_id, i.shared, i.is_bound_with, i.json - FROM %s.instance i - WHERE i.id >= ?::uuid AND i.id <= ?::uuid - AND i.shared = true - AND EXISTS ( - SELECT 1 FROM %s.holding h - WHERE h.instance_id = i.id AND h.tenant_id = ? - ) - ) combined - LEFT JOIN %s.holding h ON h.instance_id = combined.id - LEFT JOIN %s.item it ON it.holding_id = h.id - GROUP BY combined.id, combined.tenant_id, combined.shared, - combined.is_bound_with, combined.json - """.formatted( + return CONDITIONAL_INSTANCE_QUERY.formatted( centralSchema, // Local instances table centralSchema, // Shared instances table centralSchema, // Holdings subquery table @@ -209,13 +211,6 @@ AND EXISTS ( ); } - @Override - protected List createRanges() { - var uploadRangeSize = reindexConfig.getUploadRangeSize(); - var rangesCount = (int) Math.ceil((double) countEntities() / uploadRangeSize); - return RangeGenerator.createUuidRanges(rangesCount); - } - @Override public List> fetchByIdRange(String lower, String upper) { var memberTenantId = ReindexContext.getMemberTenantId(); diff --git a/src/test/java/org/folio/search/service/reindex/ReindexContextTest.java b/src/test/java/org/folio/search/service/reindex/ReindexContextTest.java index 4318edc83..f929e33c0 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexContextTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexContextTest.java @@ -70,6 +70,7 @@ void clear_shouldResetAllValues() { } @Test + @SuppressWarnings("checkstyle:MethodLength") void threadLocal_shouldIsolateValuesBetweenThreads() throws InterruptedException { var mainThreadTenantId = "main_tenant"; var workerThreadTenantId = "worker_tenant"; @@ -115,6 +116,7 @@ void threadLocal_shouldIsolateValuesBetweenThreads() throws InterruptedException } @Test + @SuppressWarnings("checkstyle:MethodLength") void multipleThreads_shouldMaintainIsolation() throws InterruptedException { var threadCount = 5; var latch = new CountDownLatch(threadCount); diff --git a/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java index 6fda15072..b063c6b13 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java @@ -25,6 +25,7 @@ import org.folio.search.model.types.ReindexRangeStatus; import org.folio.search.repository.PrimaryResourceRepository; import org.folio.search.service.converter.MultiTenantSearchDocumentConverter; +import org.folio.spring.FolioExecutionContext; import org.folio.spring.testing.type.UnitTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -50,7 +51,7 @@ class ReindexOrchestrationServiceTest { @Mock private ReindexService reindexService; @Mock - private org.folio.spring.FolioExecutionContext context; + private FolioExecutionContext context; @InjectMocks private ReindexOrchestrationService service; @@ -108,12 +109,12 @@ void process_shouldThrowReindexException_whenExceptionOccursDuringFetchRecords() var event = reindexEvent(); var exceptionMessage = "Failed to fetch records from database"; - when(uploadRangeIndexService.fetchRecordRange(event)).thenThrow(new RuntimeException(exceptionMessage)); + when(uploadRangeService.fetchRecordRange(event)).thenThrow(new RuntimeException(exceptionMessage)); // Act & Assert assertThrows(ReindexException.class, () -> service.process(event)); - verify(uploadRangeIndexService).fetchRecordRange(event); + verify(uploadRangeService).fetchRecordRange(event); verify(reindexStatusService).updateReindexUploadFailed(event.getEntityType()); } @@ -124,13 +125,13 @@ void process_shouldThrowReindexException_whenExceptionOccursDuringDocumentConver var resourceEvent = new ResourceEvent(); var exceptionMessage = "Failed to convert documents"; - when(uploadRangeIndexService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); + when(uploadRangeService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); when(documentConverter.convert(List.of(resourceEvent))).thenThrow(new RuntimeException(exceptionMessage)); // Act & Assert assertThrows(ReindexException.class, () -> service.process(event)); - verify(uploadRangeIndexService).fetchRecordRange(event); + verify(uploadRangeService).fetchRecordRange(event); verify(documentConverter).convert(List.of(resourceEvent)); verify(reindexStatusService).updateReindexUploadFailed(event.getEntityType()); } @@ -142,7 +143,7 @@ void process_shouldThrowReindexException_whenExceptionOccursDuringIndexing() { var resourceEvent = new ResourceEvent(); var exceptionMessage = "Failed to index documents in Elasticsearch"; - when(uploadRangeIndexService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); + when(uploadRangeService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); when(documentConverter.convert(List.of(resourceEvent))).thenReturn(Map.of("key", List.of(SearchDocumentBody.of(null, IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX)))); when(elasticRepository.indexResources(any())).thenThrow(new RuntimeException(exceptionMessage)); @@ -150,7 +151,7 @@ void process_shouldThrowReindexException_whenExceptionOccursDuringIndexing() { // Act & Assert assertThrows(ReindexException.class, () -> service.process(event)); - verify(uploadRangeIndexService).fetchRecordRange(event); + verify(uploadRangeService).fetchRecordRange(event); verify(documentConverter).convert(List.of(resourceEvent)); verify(elasticRepository).indexResources(any()); verify(reindexStatusService).updateReindexUploadFailed(event.getEntityType()); diff --git a/src/test/java/org/folio/search/service/reindex/StagingMigrationServiceTest.java b/src/test/java/org/folio/search/service/reindex/StagingMigrationServiceTest.java index e96851a7b..6035c243d 100644 --- a/src/test/java/org/folio/search/service/reindex/StagingMigrationServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/StagingMigrationServiceTest.java @@ -63,6 +63,7 @@ void setUp() { } @Test + @SuppressWarnings("checkstyle:MethodLength") void migrateAllStagingTables_fullReindex() { // Arrange when(jdbcTemplate.update(contains("staging_instance"), any(Timestamp.class))).thenReturn(10); From cad37f6817f92a5f21f1df0bda67f6a6cb972d5d Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Fri, 13 Feb 2026 17:07:46 +0100 Subject: [PATCH 5/5] Remove duplicate annotation --- src/main/java/org/folio/search/SearchApplication.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/folio/search/SearchApplication.java b/src/main/java/org/folio/search/SearchApplication.java index fbb562e16..b2ee2e92b 100644 --- a/src/main/java/org/folio/search/SearchApplication.java +++ b/src/main/java/org/folio/search/SearchApplication.java @@ -2,14 +2,12 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.cache.annotation.EnableCaching; import org.springframework.resilience.annotation.EnableResilientMethods; import org.springframework.scheduling.annotation.EnableScheduling; /** * Folio search application. */ -@EnableCaching @EnableScheduling @EnableResilientMethods @SpringBootApplication