From 671d7a5d424309ebcf4dc181eab5adcba44445ac Mon Sep 17 00:00:00 2001 From: Anders Jensen-Urstad Date: Wed, 5 Mar 2025 14:29:00 +0100 Subject: [PATCH 1/5] wip/experimental(elasticsearch): Use secondary index for Item and ImageObject --- .../groovy/whelk/rest/api/SearchUtils.groovy | 7 ++- .../groovy/whelk/rest/api/SearchUtils2.java | 2 +- secret.properties.in | 1 + whelk-core/src/main/groovy/whelk/Whelk.groovy | 6 +- .../whelk/component/ElasticSearch.groovy | 55 +++++++++++++------ .../main/groovy/whelk/search/ESQuery.groovy | 26 +++++---- .../main/groovy/whelk/search2/EsBoost.java | 3 + .../main/groovy/whelk/search2/EsMappings.java | 12 +++- .../groovy/whelk/search2/QueryParams.java | 6 ++ .../main/groovy/whelk/search2/QueryUtil.java | 11 +++- .../src/main/groovy/whelk/search2/Stats.java | 2 +- 11 files changed, 92 insertions(+), 39 deletions(-) diff --git a/rest/src/main/groovy/whelk/rest/api/SearchUtils.groovy b/rest/src/main/groovy/whelk/rest/api/SearchUtils.groovy index 6444a6d188..7c5e7f3503 100644 --- a/rest/src/main/groovy/whelk/rest/api/SearchUtils.groovy +++ b/rest/src/main/groovy/whelk/rest/api/SearchUtils.groovy @@ -66,6 +66,7 @@ class SearchUtils { String addStats = getReservedQueryParameter('_stats', queryParameters) String suggest = getReservedQueryParameter('_suggest', queryParameters) String spell = getReservedQueryParameter('_spell', queryParameters) + String searchMainOnly = getReservedQueryParameter('_searchMainOnly', queryParameters) if (queryParameters['p'] && !object) { throw new InvalidQueryException("Parameter 'p' can only be used together with 'o'") @@ -89,6 +90,7 @@ class SearchUtils { '_stats' : addStats, '_suggest' : suggest, '_spell': spell, + '_searchMainOnly': searchMainOnly, ] Map results = queryElasticSearch( @@ -125,6 +127,7 @@ class SearchUtils { String addStats = pageParams['_stats'] String suggest = pageParams['_suggest'] String spell = pageParams['_spell'] + String searchMainOnly = pageParams['_searchMainOnly'] lens = lens ?: 'cards' log.debug("Querying ElasticSearch") @@ -136,7 +139,7 @@ class SearchUtils { // TODO Only manipulate `_limit` in one place queryParameters['_limit'] = [limit.toString()] - Map esResult = esQuery.doQuery(queryParameters, suggest, spell) + Map esResult = esQuery.doQuery(queryParameters, suggest, spell, searchMainOnly) Lookup lookup = new Lookup() List mappings = [] @@ -803,7 +806,7 @@ class SearchUtils { * Return a list of reserved query params */ private List getReservedParameters() { - return ['q', 'p', 'o', 'value', '_limit', '_offset', '_suggest', '_spell'] + return ['q', 'p', 'o', 'value', '_limit', '_offset', '_suggest', '_spell', '_searchMainOnly'] } /* diff --git a/rest/src/main/groovy/whelk/rest/api/SearchUtils2.java b/rest/src/main/groovy/whelk/rest/api/SearchUtils2.java index ebb928c74d..0a81312703 100644 --- a/rest/src/main/groovy/whelk/rest/api/SearchUtils2.java +++ b/rest/src/main/groovy/whelk/rest/api/SearchUtils2.java @@ -65,7 +65,7 @@ Map doSearch(Map queryParameters) throws Inval Map esQueryDsl = getEsQueryDsl(qTree, queryParams, appParams.statsRepr); - QueryResult queryRes = new QueryResult(queryUtil.query(esQueryDsl), queryParams.debug); + QueryResult queryRes = new QueryResult(queryUtil.query(esQueryDsl, queryParams.searchMainOnly), queryParams.debug); Map partialCollectionView = getPartialCollectionView(queryRes, qTree, queryParams, appParams); diff --git a/secret.properties.in b/secret.properties.in index a809c33098..cd3f6bd59e 100644 --- a/secret.properties.in +++ b/secret.properties.in @@ -13,6 +13,7 @@ sqlMaxPoolSize = 4 elasticHost = localhost:9200 elasticIndex = libris_local +elasticIndexSecondary = libris_local_secondary elasticUser = elastic elasticPassword = elastic diff --git a/whelk-core/src/main/groovy/whelk/Whelk.groovy b/whelk-core/src/main/groovy/whelk/Whelk.groovy index 15268f2e00..ef33e61aee 100644 --- a/whelk-core/src/main/groovy/whelk/Whelk.groovy +++ b/whelk-core/src/main/groovy/whelk/Whelk.groovy @@ -357,7 +357,8 @@ class Whelk { removedLinks.each { link -> String id = storage.getSystemIdByIri(link.iri) if (id) { - elastic.decrementReverseLinks(id, link.relation) + Document doc = storage.load(id) + elastic.decrementReverseLinks(id, link.relation, elastic.getIndexForDoc(doc)) } } @@ -377,7 +378,7 @@ class Whelk { reindexAffectedReverseIntegral(doc) } else { // just update link counter - elastic.incrementReverseLinks(id, link.relation) + elastic.incrementReverseLinks(id, link.relation, elastic.getIndexForDoc(doc)) } } } @@ -561,6 +562,7 @@ class Whelk { assertNoDependers(doc) } storage.remove(id, changedIn, changedBy) + // TODO use getThingType here? indexAsyncOrSync { elastic.remove(id) if (features.isEnabled(INDEX_BLANK_WORKS)) { diff --git a/whelk-core/src/main/groovy/whelk/component/ElasticSearch.groovy b/whelk-core/src/main/groovy/whelk/component/ElasticSearch.groovy index 0e9799d8c5..b0930bcee5 100644 --- a/whelk-core/src/main/groovy/whelk/component/ElasticSearch.groovy +++ b/whelk-core/src/main/groovy/whelk/component/ElasticSearch.groovy @@ -42,6 +42,7 @@ class ElasticSearch { public int maxTermsCount = 65536 // Elasticsearch default (fallback value) String defaultIndex = null + String defaultIndexSecondary = null private List elasticHosts private String elasticUser private String elasticPassword @@ -56,14 +57,16 @@ class ElasticSearch { this( props.getProperty("elasticHost"), props.getProperty("elasticIndex"), + props.getProperty("elasticIndexSecondary"), props.getProperty("elasticUser"), props.getProperty("elasticPassword") ) } - ElasticSearch(String elasticHost, String elasticIndex, String elasticUser, String elasticPassword) { + ElasticSearch(String elasticHost, String elasticIndex, String elasticIndexSecondary, String elasticUser, String elasticPassword) { this.elasticHosts = getElasticHosts(elasticHost) this.defaultIndex = elasticIndex + this.defaultIndexSecondary = elasticIndexSecondary this.elasticUser = elasticUser this.elasticPassword = elasticPassword @@ -126,17 +129,23 @@ class ElasticSearch { } String getIndexName() { defaultIndex } + String getSecondaryIndexName() { defaultIndexSecondary } /** * Get ES mappings for associated index * */ - Map getMappings() { + Map getMappings(String index = null) { + String indexToCheck = indexName + if (index) { + indexToCheck = index + } + Map response try { - response = mapper.readValue(client.performRequest('GET', "/${indexName}/_mappings", ''), Map) + response = mapper.readValue(client.performRequest('GET', "/${indexToCheck}/_mappings", ''), Map) } catch (UnexpectedHttpStatusException e) { - log.warn("Got unexpected status code ${e.statusCode} when getting ES mappings: ${e.message}", e) + log.warn("Got unexpected status code ${e.statusCode} when getting ES mappings for ${indexToCheck}: ${e.message}", e) return [:] } @@ -147,7 +156,7 @@ class ElasticSearch { if (keys.size() == 1 && response[(keys[0])].containsKey('mappings')) { return response[(keys[0])]['mappings'] } else { - log.warn("Couldn't get mappings from ES index ${indexName}, response was ${response}.") + log.warn("Couldn't get mappings from ES index ${indexToCheck}, response was ${response}.") return [:] } } @@ -279,8 +288,15 @@ class ElasticSearch { } } + String getIndexForDoc(Document doc) { + if (doc.getThingType() in ["Item", "ImageObject"]) { + return secondaryIndexName + } + return indexName + } + String createActionRow(Document doc) { - def action = ["index" : [ "_index" : indexName, + def action = ["index" : [ "_index" : getIndexForDoc(doc), "_id" : toElasticId(doc.getShortId()) ]] return mapper.writeValueAsString(action) } @@ -291,11 +307,11 @@ class ElasticSearch { try { String response = client.performRequest( 'PUT', - "/${indexName}/_doc/${toElasticId(doc.getShortId())}", + "/${getIndexForDoc(doc)}/_doc/${toElasticId(doc.getShortId())}", getShapeForIndex(doc, whelk)) if (log.isDebugEnabled()) { Map responseMap = mapper.readValue(response, Map) - log.debug("Indexed the document ${doc.getShortId()} as ${indexName}/_doc/${responseMap['_id']} as version ${responseMap['_version']}") + log.debug("Indexed the document ${doc.getShortId()} as ${getIndexForDoc(doc)}/_doc/${responseMap['_id']} as version ${responseMap['_version']}") } } catch (Exception e) { if (!isBadRequest(e)) { @@ -308,15 +324,15 @@ class ElasticSearch { } } - void incrementReverseLinks(String shortId, String relation) { - updateReverseLinkCounter(shortId, relation, 1) + void incrementReverseLinks(String shortId, String relation, String indexToUse) { + updateReverseLinkCounter(shortId, relation, indexToUse, 1) } - void decrementReverseLinks(String shortId, String relation) { - updateReverseLinkCounter(shortId, relation, -1) + void decrementReverseLinks(String shortId, String relation, String indexToUse) { + updateReverseLinkCounter(shortId, relation, indexToUse, -1) } - private void updateReverseLinkCounter(String shortId, String relation, int deltaCount) { + private void updateReverseLinkCounter(String shortId, String relation, String indexToUse, int deltaCount) { // An indexed document will always have reverseLinks.totalItems set to an integer, // and reverseLinks.totalItemsByRelation set to a map, but reverseLinks.totalItemsByRelation['foo'] // doesn't necessarily exist at this time; hence the null check before trying to update the link counter. @@ -334,7 +350,7 @@ class ElasticSearch { try { client.performRequest( 'POST', - "/${indexName}/_update/${toElasticId(shortId)}", + "/${indexToUse}/_update/${toElasticId(shortId)}", body) } catch (Exception e) { @@ -348,7 +364,7 @@ class ElasticSearch { } else { log.warn("Failed to update reverse link counter ($deltaCount) for $shortId: $e, placing in retry queue.", e) - indexingRetryQueue.add({ -> updateReverseLinkCounter(shortId, relation, deltaCount) }) + indexingRetryQueue.add({ -> updateReverseLinkCounter(shortId, relation, indexToUse, deltaCount) }) } } } @@ -360,7 +376,7 @@ class ElasticSearch { def dsl = ["query":["term":["_id":toElasticId(identifier)]]] try { def response = client.performRequest('POST', - "/${indexName}/_delete_by_query", + "/${indexName},${secondaryIndexName}/_delete_by_query", JsonOutput.toJson(dsl)) Map responseMap = mapper.readValue(response, Map) @@ -584,8 +600,11 @@ class ElasticSearch { isnis.findAll{ it.size() == 16 }.collect { Unicode.formatIsni(it) } } - Map query(Map jsonDsl) { - return performQuery(jsonDsl, getQueryUrl()) + Map query(Map jsonDsl, boolean searchMainOnly = false) { + if (searchMainOnly) { + return performQuery(jsonDsl, getQueryUrl()) + } + return performQuery(jsonDsl, getQueryUrl([], "${indexName},${secondaryIndexName}")) } Map queryIds(Map jsonDsl) { diff --git a/whelk-core/src/main/groovy/whelk/search/ESQuery.groovy b/whelk-core/src/main/groovy/whelk/search/ESQuery.groovy index 7f6d7415df..1bee7de80d 100644 --- a/whelk-core/src/main/groovy/whelk/search/ESQuery.groovy +++ b/whelk-core/src/main/groovy/whelk/search/ESQuery.groovy @@ -32,7 +32,7 @@ class ESQuery { private static final int DEFAULT_PAGE_SIZE = 50 private static final List RESERVED_PARAMS = [ - 'q', 'o', '_limit', '_offset', '_sort', '_statsrepr', '_site_base_uri', '_debug', '_boost', '_lens', '_stats', '_suggest', '_site', '_spell' + 'q', 'o', '_limit', '_offset', '_sort', '_statsrepr', '_site_base_uri', '_debug', '_boost', '_lens', '_stats', '_suggest', '_site', '_spell', '_searchMainOnly' ] public static final String AND_PREFIX = 'and-' public static final String AND_MATCHES_PREFIX = 'and-matches-' @@ -86,11 +86,12 @@ class ESQuery { void initFieldMappings(Whelk whelk) { if (whelk.elastic) { Map mappings = whelk.elastic.getMappings() - this.keywordFields = getKeywordFields(mappings) - this.dateFields = getFieldsOfType('date', mappings) - this.nestedFields = getFieldsOfType('nested', mappings) - this.nestedNotInParentFields = nestedFields - getFieldsWithSetting('include_in_parent', true, mappings) - this.numericExtractorFields = getFieldsWithAnalyzer('numeric_extractor', mappings) + Map mappingsSecondary = whelk.elastic.getMappings(whelk.elastic.secondaryIndexName) + this.keywordFields = getKeywordFields(mappings) + getKeywordFields(mappingsSecondary) + this.dateFields = getFieldsOfType('date', mappings) + getFieldsOfType('date', mappingsSecondary) + this.nestedFields = getFieldsOfType('nested', mappings) + getFieldsOfType('nested', mappingsSecondary) + this.nestedNotInParentFields = nestedFields - (getFieldsWithSetting('include_in_parent', true, mappings) + getFieldsWithSetting('include_in_parent', true, mappingsSecondary)) + this.numericExtractorFields = getFieldsWithAnalyzer('numeric_extractor', mappings) + getFieldsWithAnalyzer('numeric_extractor', mappingsSecondary) if (DocumentUtil.getAtPath(mappings, ['properties', '_sortKeyByLang', 'properties', 'sv', 'fields', 'trigram'], null)) { ENABLE_SPELL_CHECK = true @@ -109,9 +110,14 @@ class ESQuery { } @CompileStatic(TypeCheckingMode.SKIP) - Map doQuery(Map queryParameters, String suggest = null, String spell = null) { + Map doQuery(Map queryParameters, String suggest = null, String spell = null, String searchMainOnly = null) { Map esQuery = getESQuery(queryParameters, suggest, spell) - Map esResponse = whelk.elastic.query(esQuery) + Map esResponse + if (searchMainOnly == 'true') { + esResponse = whelk.elastic.query(esQuery, true) + } else { + esResponse = whelk.elastic.query(esQuery) + } return collectQueryResults(esResponse, esQuery, queryParameters, { def d = it."_source"; d."_id" = it."_id"; return d }) } @@ -1078,11 +1084,11 @@ class ESQuery { } } - static Set getFieldsOfType(String type, Map mappings) { + static Set getFieldsOfType(String type, Map mappings) { getFieldsWithSetting('type', type, mappings) } - static Set getFieldsWithAnalyzer(String analyzer, Map mappings) { + static Set getFieldsWithAnalyzer(String analyzer, Map mappings) { getFieldsWithSetting('analyzer', analyzer, mappings) } diff --git a/whelk-core/src/main/groovy/whelk/search2/EsBoost.java b/whelk-core/src/main/groovy/whelk/search2/EsBoost.java index 4d5745264b..0b010e58a3 100644 --- a/whelk-core/src/main/groovy/whelk/search2/EsBoost.java +++ b/whelk-core/src/main/groovy/whelk/search2/EsBoost.java @@ -281,6 +281,9 @@ Map collectBoostFields() { } private Map collectBoostFields(Map lens, int boost) { + System.out.println("LENS"); + System.out.println(lens); + System.out.println("BOOST " + boost); Map boostFields = new HashMap<>(); for (String key : getPropertiesToShow(lens)) { diff --git a/whelk-core/src/main/groovy/whelk/search2/EsMappings.java b/whelk-core/src/main/groovy/whelk/search2/EsMappings.java index 521c984950..beff58e275 100644 --- a/whelk-core/src/main/groovy/whelk/search2/EsMappings.java +++ b/whelk-core/src/main/groovy/whelk/search2/EsMappings.java @@ -20,13 +20,21 @@ public class EsMappings { private final boolean isSpellCheckAvailable; - public EsMappings(Map mappings) { + public EsMappings(Map mappings, Map mappingsSecondary) { this.keywordFields = getKeywordFields(mappings); this.dateFields = getFieldsOfType("date", mappings); this.nestedFields = getFieldsOfType("nested", mappings); + this.numericExtractorFields = getFieldsWithAnalyzer("numeric_extractor", mappings); + + if (!mappingsSecondary.isEmpty()) { + this.keywordFields.addAll(getKeywordFields(mappingsSecondary)); + this.dateFields.addAll(getFieldsOfType("date", mappingsSecondary)); + this.nestedFields.addAll(getFieldsOfType("nested", mappingsSecondary)); + this.numericExtractorFields.addAll(getFieldsWithAnalyzer("numeric_extractor", mappingsSecondary)); + } + this.nestedNotInParentFields = new HashSet<>(nestedFields); this.nestedNotInParentFields.removeAll(getFieldsWithSetting("include_in_parent", true, mappings)); - this.numericExtractorFields = getFieldsWithAnalyzer("numeric_extractor", mappings); // TODO: temporary feature flag, to be removed // this feature only works after a full reindex has been done, so we have to detect that diff --git a/whelk-core/src/main/groovy/whelk/search2/QueryParams.java b/whelk-core/src/main/groovy/whelk/search2/QueryParams.java index a65d6dd740..6394d9acf6 100644 --- a/whelk-core/src/main/groovy/whelk/search2/QueryParams.java +++ b/whelk-core/src/main/groovy/whelk/search2/QueryParams.java @@ -30,6 +30,7 @@ public static class ApiParams { public static final String APP_CONFIG = "_appConfig"; public static final String BOOST = "_boost"; public static final String STATS = "_stats"; + public static final String SEARCH_MAIN_ONLY = "_searchMainOnly"; } public static class Debug { @@ -52,6 +53,7 @@ public static class Debug { public final String i; public final boolean skipStats; + public final boolean searchMainOnly; public QueryParams(Map apiParameters) throws InvalidQueryException { this.sortBy = Sort.fromString(getOptionalSingleNonEmpty(ApiParams.SORT, apiParameters).orElse("")); @@ -67,6 +69,7 @@ public QueryParams(Map apiParameters) throws InvalidQueryExcep this.q = getOptionalSingle(ApiParams.QUERY, apiParameters).orElse(""); this.i = getOptionalSingle(ApiParams.SIMPLE_FREETEXT, apiParameters).orElse(""); this.skipStats = getOptionalSingle(ApiParams.STATS, apiParameters).map("false"::equalsIgnoreCase).isPresent(); + this.searchMainOnly = getOptionalSingle(ApiParams.SEARCH_MAIN_ONLY, apiParameters).map("true"::equalsIgnoreCase).isPresent(); } public Map getNonQueryParams() { @@ -105,6 +108,9 @@ public Map getNonQueryParams(int offset) { if (skipStats) { params.put(ApiParams.STATS, "false"); } + if (searchMainOnly) { + params.put(ApiParams.SEARCH_MAIN_ONLY, "true"); + } return params; } diff --git a/whelk-core/src/main/groovy/whelk/search2/QueryUtil.java b/whelk-core/src/main/groovy/whelk/search2/QueryUtil.java index cee52dcbd5..19b5aa0195 100644 --- a/whelk-core/src/main/groovy/whelk/search2/QueryUtil.java +++ b/whelk-core/src/main/groovy/whelk/search2/QueryUtil.java @@ -31,12 +31,17 @@ public class QueryUtil { public QueryUtil(Whelk whelk) { this.whelk = whelk; - this.esMappings = new EsMappings(whelk.elastic != null ? whelk.elastic.getMappings() : Collections.emptyMap()); + if (whelk.elastic != null) { + this.esMappings = new EsMappings(whelk.elastic.getMappings(), whelk.elastic.getMappings(whelk.elastic.getSecondaryIndexName())); + } else { + this.esMappings = new EsMappings(Collections.emptyMap(), Collections.emptyMap()); + } + //this.esMappings = new EsMappings(whelk.elastic != null ? whelk.elastic.getMappings() : Collections.emptyMap()); this.esBoost = new EsBoost(whelk.getJsonld()); } - public Map query(Map queryDsl) { - return whelk.elastic.query(queryDsl); + public Map query(Map queryDsl, boolean searchMainOnly) { + return whelk.elastic.query(queryDsl, searchMainOnly); } public boolean esIsConfigured() { diff --git a/whelk-core/src/main/groovy/whelk/search2/Stats.java b/whelk-core/src/main/groovy/whelk/search2/Stats.java index 1d1866f893..c81dc9e4ee 100644 --- a/whelk-core/src/main/groovy/whelk/search2/Stats.java +++ b/whelk-core/src/main/groovy/whelk/search2/Stats.java @@ -236,7 +236,7 @@ private List> getCuratedPredicateLinks() { if (curatedPredicates.isEmpty()) { return Collections.emptyList(); } - QueryResult queryRes = new QueryResult(queryUtil.query(getCuratedPredicateEsQueryDsl(o, curatedPredicates))); + QueryResult queryRes = new QueryResult(queryUtil.query(getCuratedPredicateEsQueryDsl(o, curatedPredicates), queryParams.searchMainOnly)); return predicateLinks(queryRes.pAggs, o, queryParams.getNonQueryParams(0)); } From 5823cc7d1e1db4cde282221afbc9f9ed4a781c3c Mon Sep 17 00:00:00 2001 From: Anders Jensen-Urstad Date: Wed, 5 Mar 2025 14:33:56 +0100 Subject: [PATCH 2/5] Remove debug stuff --- whelk-core/src/main/groovy/whelk/search2/EsBoost.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/whelk-core/src/main/groovy/whelk/search2/EsBoost.java b/whelk-core/src/main/groovy/whelk/search2/EsBoost.java index 0b010e58a3..4d5745264b 100644 --- a/whelk-core/src/main/groovy/whelk/search2/EsBoost.java +++ b/whelk-core/src/main/groovy/whelk/search2/EsBoost.java @@ -281,9 +281,6 @@ Map collectBoostFields() { } private Map collectBoostFields(Map lens, int boost) { - System.out.println("LENS"); - System.out.println(lens); - System.out.println("BOOST " + boost); Map boostFields = new HashMap<>(); for (String key : getPropertiesToShow(lens)) { From 5d61ec44aab0e237aefeaaaef14cb13fd97d332b Mon Sep 17 00:00:00 2001 From: Anders Jensen-Urstad Date: Wed, 5 Mar 2025 14:44:09 +0100 Subject: [PATCH 3/5] Cleanup --- whelk-core/src/main/groovy/whelk/search2/QueryUtil.java | 1 - 1 file changed, 1 deletion(-) diff --git a/whelk-core/src/main/groovy/whelk/search2/QueryUtil.java b/whelk-core/src/main/groovy/whelk/search2/QueryUtil.java index 19b5aa0195..5b9342b625 100644 --- a/whelk-core/src/main/groovy/whelk/search2/QueryUtil.java +++ b/whelk-core/src/main/groovy/whelk/search2/QueryUtil.java @@ -36,7 +36,6 @@ public QueryUtil(Whelk whelk) { } else { this.esMappings = new EsMappings(Collections.emptyMap(), Collections.emptyMap()); } - //this.esMappings = new EsMappings(whelk.elastic != null ? whelk.elastic.getMappings() : Collections.emptyMap()); this.esBoost = new EsBoost(whelk.getJsonld()); } From 12fd48a21edcf5c735127ee5cb74eb3efd54ef62 Mon Sep 17 00:00:00 2001 From: Anders Jensen-Urstad Date: Wed, 5 Mar 2025 14:48:22 +0100 Subject: [PATCH 4/5] nestedNotInParentFields fix --- whelk-core/src/main/groovy/whelk/search2/EsMappings.java | 1 + 1 file changed, 1 insertion(+) diff --git a/whelk-core/src/main/groovy/whelk/search2/EsMappings.java b/whelk-core/src/main/groovy/whelk/search2/EsMappings.java index beff58e275..5575fbf458 100644 --- a/whelk-core/src/main/groovy/whelk/search2/EsMappings.java +++ b/whelk-core/src/main/groovy/whelk/search2/EsMappings.java @@ -35,6 +35,7 @@ public EsMappings(Map mappings, Map mappingsSecondary) { this.nestedNotInParentFields = new HashSet<>(nestedFields); this.nestedNotInParentFields.removeAll(getFieldsWithSetting("include_in_parent", true, mappings)); + this.nestedNotInParentFields.removeAll(getFieldsWithSetting("include_in_parent", true, mappingsSecondary)); // TODO: temporary feature flag, to be removed // this feature only works after a full reindex has been done, so we have to detect that From 8281476c8f9dd7da6c2e51a918cc01d719e193b8 Mon Sep 17 00:00:00 2001 From: Anders Jensen-Urstad Date: Thu, 13 Mar 2025 15:05:07 +0100 Subject: [PATCH 5/5] Use both indices in SearchAfterScroll --- .../src/main/groovy/whelk/component/ElasticSearch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/whelk-core/src/main/groovy/whelk/component/ElasticSearch.groovy b/whelk-core/src/main/groovy/whelk/component/ElasticSearch.groovy index b0930bcee5..9e3d2ae2d7 100644 --- a/whelk-core/src/main/groovy/whelk/component/ElasticSearch.groovy +++ b/whelk-core/src/main/groovy/whelk/component/ElasticSearch.groovy @@ -847,7 +847,7 @@ class ElasticSearch { String queryPath() { isPitApiAvailable // point in time is created on index and then index cannot be specified here ? getQueryUrlWithoutIndex(filterPath) - : getQueryUrl(filterPath) + : getQueryUrl(filterPath, "${indexName},${secondaryIndexName}") } @Override @@ -878,7 +878,7 @@ class ElasticSearch { private String createPointInTime(String keepAlive = "1m") { try { - return performRequest('POST', "/$indexName/_pit?keep_alive=$keepAlive").id + return performRequest('POST', "/$indexName,$secondaryIndexName/_pit?keep_alive=$keepAlive").id } catch (Exception e) { log.warn("Failed to create Point In Time: $e")