From 57727fa85dd403723ec51fb606b1a3191bdabbd3 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 16 Aug 2021 11:55:13 +0530 Subject: [PATCH 01/33] Update schema.json --- schemas/asset/1.0/schema.json | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/schemas/asset/1.0/schema.json b/schemas/asset/1.0/schema.json index 36ed2fcf0..ef5faf2cf 100644 --- a/schemas/asset/1.0/schema.json +++ b/schemas/asset/1.0/schema.json @@ -1246,6 +1246,24 @@ "items": { "type": "string" } - } + }, + "taxonomyPathsTest4": { + "type": "array", + "items": { + "type": "object" + } + }, + "taxonomyPaths_v2": { + "type": "array", + "items": { + "type": "object" + } + }, + "competencies_v2": { + "type": "array", + "items": { + "type": "object" + } + } } } From 31fef292693d1a6a506ec946f5ee39554e402497 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 16 Aug 2021 11:58:49 +0530 Subject: [PATCH 02/33] Update schema.json --- schemas/collection/1.0/schema.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/schemas/collection/1.0/schema.json b/schemas/collection/1.0/schema.json index acc1b2569..8cc3547ea 100644 --- a/schemas/collection/1.0/schema.json +++ b/schemas/collection/1.0/schema.json @@ -1246,6 +1246,24 @@ "items": { "type": "string" } + }, + "taxonomyPathsTest4": { + "type": "array", + "items": { + "type": "object" + } + }, + "taxonomyPaths_v2": { + "type": "array", + "items": { + "type": "object" + } + }, + "competencies_v2": { + "type": "array", + "items": { + "type": "object" + } } } } From 78614b1a615a60ca606ccbfba7ae814042aec19b Mon Sep 17 00:00:00 2001 From: pritha-tarento <42533384+pritha-tarento@users.noreply.github.com> Date: Mon, 16 Aug 2021 12:10:21 +0530 Subject: [PATCH 03/33] Update SearchProcessor.java --- .../main/java/org/sunbird/search/processor/SearchProcessor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 7b79d14ac..32f65d12b 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -307,6 +307,8 @@ private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) { formQuery(properties, queryBuilder, boolQuery, totalOperation); List multiFilterProperties = searchDTO.getMultiFilterProperties(); + System.out.println("multiFilterProperties:: "+multiFilterProperties); + if(multiFilterProperties!=null) formQuery(multiFilterProperties, queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR); From d9df49c51831509d845ffdb0d1b6a123e2584466 Mon Sep 17 00:00:00 2001 From: pritha-tarento <42533384+pritha-tarento@users.noreply.github.com> Date: Mon, 16 Aug 2021 12:12:04 +0530 Subject: [PATCH 04/33] Update SearchActor.java --- .../src/main/java/org/sunbird/actors/SearchActor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java index c38b74b8a..e5f171083 100644 --- a/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java +++ b/search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java @@ -242,8 +242,10 @@ private SearchDTO getSearchDTO(Request request) throws Exception { searchObj.setSortBy(sortBy); searchObj.setFacets(facets); searchObj.setProperties(properties); - if(multiFilters!=null) + if(multiFilters!=null){ multiFilterProperties.addAll(getSearchFilterProperties(multiFilters, wordChainsRequest)); + searchObj.setMultiFilterProperties(multiFilterProperties); + } // Added Implicit Filter Properties To Support Collection content tagging to reuse by tenants. setImplicitFilters(filters, searchObj); searchObj.setLimit(limit); From 8bb9e328502592bfee6527357046f11035e89efe Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 16 Aug 2021 12:29:57 +0530 Subject: [PATCH 05/33] Update schema.json --- schemas/content/1.0/schema.json | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/schemas/content/1.0/schema.json b/schemas/content/1.0/schema.json index ca889a75c..3b87dcaf3 100644 --- a/schemas/content/1.0/schema.json +++ b/schemas/content/1.0/schema.json @@ -1347,6 +1347,24 @@ "items": { "type": "string" } - } + }, + "taxonomyPathsTest4": { + "type": "array", + "items": { + "type": "object" + } + }, + "taxonomyPaths_v2": { + "type": "array", + "items": { + "type": "object" + } + }, + "competencies_v2": { + "type": "array", + "items": { + "type": "object" + } + } } } From 866d3290a3b3eb78938231be411d96ae2eba414a Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 16 Aug 2021 15:54:30 +0530 Subject: [PATCH 06/33] Update schema.json --- schemas/asset/1.0/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/asset/1.0/schema.json b/schemas/asset/1.0/schema.json index ef5faf2cf..195df877d 100644 --- a/schemas/asset/1.0/schema.json +++ b/schemas/asset/1.0/schema.json @@ -1247,7 +1247,7 @@ "type": "string" } }, - "taxonomyPathsTest4": { + "taxonomyPathsTest5": { "type": "array", "items": { "type": "object" From a2fe464dc12d584e10d7cefd717efc48ad1c48f9 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 16 Aug 2021 15:56:54 +0530 Subject: [PATCH 07/33] Update schema.json --- schemas/collection/1.0/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/collection/1.0/schema.json b/schemas/collection/1.0/schema.json index 8cc3547ea..0150d34aa 100644 --- a/schemas/collection/1.0/schema.json +++ b/schemas/collection/1.0/schema.json @@ -1247,7 +1247,7 @@ "type": "string" } }, - "taxonomyPathsTest4": { + "taxonomyPathsTest5": { "type": "array", "items": { "type": "object" From afa910082a8915d8326daf3ed3565513b6527c02 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 16 Aug 2021 15:57:33 +0530 Subject: [PATCH 08/33] Update schema.json --- schemas/content/1.0/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/content/1.0/schema.json b/schemas/content/1.0/schema.json index 3b87dcaf3..71aa9b27a 100644 --- a/schemas/content/1.0/schema.json +++ b/schemas/content/1.0/schema.json @@ -1348,7 +1348,7 @@ "type": "string" } }, - "taxonomyPathsTest4": { + "taxonomyPathsTest5": { "type": "array", "items": { "type": "object" From 05b748426e44969b407b25f17981f394b5bf24d6 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 16 Aug 2021 19:59:51 +0530 Subject: [PATCH 09/33] Update schema.json --- schemas/asset/1.0/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/asset/1.0/schema.json b/schemas/asset/1.0/schema.json index 195df877d..bebca5878 100644 --- a/schemas/asset/1.0/schema.json +++ b/schemas/asset/1.0/schema.json @@ -1247,7 +1247,7 @@ "type": "string" } }, - "taxonomyPathsTest5": { + "taxonomyPathsTest6": { "type": "array", "items": { "type": "object" From ba4df72dc983e97b0a14156989f8d6f75d019cad Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 16 Aug 2021 20:00:32 +0530 Subject: [PATCH 10/33] Update schema.json --- schemas/collection/1.0/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/collection/1.0/schema.json b/schemas/collection/1.0/schema.json index 0150d34aa..951412da8 100644 --- a/schemas/collection/1.0/schema.json +++ b/schemas/collection/1.0/schema.json @@ -1247,7 +1247,7 @@ "type": "string" } }, - "taxonomyPathsTest5": { + "taxonomyPathsTest6": { "type": "array", "items": { "type": "object" From 02e9dc40e87f57ac0198e8f4133b694520ff7148 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 16 Aug 2021 20:01:27 +0530 Subject: [PATCH 11/33] Update schema.json --- schemas/content/1.0/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/content/1.0/schema.json b/schemas/content/1.0/schema.json index 71aa9b27a..4c45fe3c0 100644 --- a/schemas/content/1.0/schema.json +++ b/schemas/content/1.0/schema.json @@ -1348,7 +1348,7 @@ "type": "string" } }, - "taxonomyPathsTest5": { + "taxonomyPathsTest6": { "type": "array", "items": { "type": "object" From 2571afc9aa6b1a1649123123b08e103758ca5e8c Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 23 Aug 2021 12:38:59 +0530 Subject: [PATCH 12/33] Added nested field aggregation --- .../search/processor/SearchProcessor.java | 60 +++++++++++++------ 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 32f65d12b..3d3f4f353 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -271,28 +271,52 @@ private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByList, - SearchSourceBuilder searchSourceBuilder) { + @SuppressWarnings("unchecked") + private void setAggregations(List> groupByList, + SearchSourceBuilder searchSourceBuilder) { TermsAggregationBuilder termBuilder = null; if (groupByList != null && !groupByList.isEmpty()) { - for (Map groupByMap : groupByList) { - String groupByParent = (String) groupByMap.get("groupByParent"); - termBuilder = AggregationBuilders.terms(groupByParent) - .field(groupByParent + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit); - List groupByChildList = (List) groupByMap.get("groupByChildList"); - if (groupByChildList != null && !groupByChildList.isEmpty()) { - for (String childGroupBy : groupByChildList) { - termBuilder.subAggregation(AggregationBuilders.terms(childGroupBy) - .field(childGroupBy + SearchConstants.RAW_FIELD_EXTENSION) - .size(ElasticSearchUtil.defaultResultLimit)); - } + HashMap> nestedAggregation = new HashMap<>(); + for (Map groupByMap : groupByList) { + String groupByParent = (String) groupByMap.get("groupByParent"); + if (!groupByParent.contains(".")) { + termBuilder = AggregationBuilders.terms(groupByParent) + .field(groupByParent + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit); + List groupByChildList = (List) groupByMap.get("groupByChildList"); + if (groupByChildList != null && !groupByChildList.isEmpty()) { + for (String childGroupBy : groupByChildList) { + termBuilder.subAggregation(AggregationBuilders.terms(childGroupBy) + .field(childGroupBy + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit)); } - searchSourceBuilder.aggregation(termBuilder); - } + } + searchSourceBuilder.aggregation(termBuilder); + } else { + if (nestedAggregation.get(groupByParent.split("\\.")[0]) != null) { + nestedAggregation.get(groupByParent.split("\\.")[0]).add(groupByParent.split("\\.")[1]); + } else { + List nestedAggrList = new ArrayList<>(); + nestedAggrList.add(groupByParent.split("\\.")[1]); + nestedAggregation.put(groupByParent.split("\\.")[0], nestedAggrList); + } + } + } + + if (!nestedAggregation.isEmpty()) { + for (Map.Entry> mapData : nestedAggregation.entrySet()) { + AggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested(mapData.getKey(), mapData.getKey()); + for (String nestedValue : mapData.getValue()) { + termBuilder = AggregationBuilders.terms(nestedValue) + .field(mapData.getKey() + "." + nestedValue + SearchConstants.RAW_FIELD_EXTENSION) + .size(ElasticSearchUtil.defaultResultLimit); + nestedAggregationBuilder.subAggregation(termBuilder); + } + searchSourceBuilder.aggregation(nestedAggregationBuilder); + } + } } - } + } /** * @param searchDTO From 7e0bc6a34193edac5a2b2aeab92cd1af815db0d5 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 23 Aug 2021 12:51:46 +0530 Subject: [PATCH 13/33] Nested fields aggregations. --- .../search/client/ElasticSearchUtil.java | 90 +++++++++++-------- 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java index e0b96b19c..fc4420758 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java @@ -613,43 +613,57 @@ private static SearchSourceBuilder buildJsonForWildCardQuery(String textKeyWord, @SuppressWarnings("unchecked") public static Object getCountFromAggregation(Aggregations aggregations, List> groupByList, - IESResultTransformer transformer) { - - Map countMap = new HashMap(); - if (aggregations != null) { - for (Map aggregationsMap : groupByList) { - Map parentCountMap = new HashMap(); - String groupByParent = (String) aggregationsMap.get("groupByParent"); - Terms terms = aggregations.get(groupByParent); - List> parentGroupList = new ArrayList>(); - List buckets = (List) terms.getBuckets(); - for (Bucket bucket : buckets) { - Map parentCountObject = new HashMap(); - parentCountObject.put("count", bucket.getDocCount()); - List groupByChildList = (List) aggregationsMap.get("groupByChildList"); - Aggregations subAggregations = bucket.getAggregations(); - if (null != groupByChildList && !groupByChildList.isEmpty() && null != subAggregations) { - Map groupByChildMap = new HashMap(); - for (String groupByChild : groupByChildList) { - Terms subTerms = subAggregations.get(groupByChild); - List childBuckets = (List) subTerms.getBuckets(); - Map childCountMap = new HashMap(); - for (Bucket childBucket : childBuckets) { - childCountMap.put(childBucket.getKeyAsString(), childBucket.getDocCount()); - groupByChildMap.put(groupByChild, childCountMap); - } - } - parentCountObject.putAll(groupByChildMap); - } - parentCountMap.put(bucket.getKeyAsString(), parentCountObject); - parentGroupList.add(parentCountMap); - } - - countMap.put(groupByParent, parentCountMap); - } - } - return transformer.getTransformedObject(countMap); - } + IESResultTransformer transformer) { + Map countMap = new HashMap(); + if (aggregations != null) { + for (Map aggregationsMap : groupByList) { + Map parentCountMap = new HashMap(); + String groupByParent = (String) aggregationsMap.get("groupByParent"); + Terms terms = null; + List buckets = null; + if (groupByParent.contains(".")) { + System.out.println("Enters in the nested aggregation"); + terms = aggregations.get(groupByParent.split("\\.")[0]); + try { + System.out.println("Aggregation values"); + System.out.println(mapper.writeValueAsString(aggregations.getAsMap())); + System.out.println(mapper.writeValueAsString(terms)); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + buckets = (List) terms.getBucketByKey(groupByParent.split("\\.")[1]); + } else { + terms = aggregations.get(groupByParent); + buckets = (List) terms.getBuckets(); + } + List> parentGroupList = new ArrayList>(); + for (Bucket bucket : buckets) { + Map parentCountObject = new HashMap(); + parentCountObject.put("count", bucket.getDocCount()); + List groupByChildList = (List) aggregationsMap.get("groupByChildList"); + Aggregations subAggregations = bucket.getAggregations(); + if (null != groupByChildList && !groupByChildList.isEmpty() && null != subAggregations) { + Map groupByChildMap = new HashMap(); + for (String groupByChild : groupByChildList) { + Terms subTerms = subAggregations.get(groupByChild); + List childBuckets = (List) subTerms.getBuckets(); + Map childCountMap = new HashMap(); + for (Bucket childBucket : childBuckets) { + childCountMap.put(childBucket.getKeyAsString(), childBucket.getDocCount()); + groupByChildMap.put(groupByChild, childCountMap); + } + } + parentCountObject.putAll(groupByChildMap); + } + parentCountMap.put(bucket.getKeyAsString(), parentCountObject); + parentGroupList.add(parentCountMap); + } + + countMap.put(groupByParent, parentCountMap); + } + } + return transformer.getTransformedObject(countMap); + } private static void registerShutdownHook() { Runtime.getRuntime().addShutdownHook(new Thread() { @@ -711,4 +725,4 @@ public static void bulkDeleteDocumentById(String indexName, String documentType, } } -} \ No newline at end of file +} From 67ada25841ba75e1ff173ed712f0d21573068de2 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 23 Aug 2021 14:38:05 +0530 Subject: [PATCH 14/33] Update SearchProcessor.java --- .../org/sunbird/search/processor/SearchProcessor.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 3d3f4f353..c03af7f86 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -63,6 +63,7 @@ public SearchProcessor(String indexName) { @SuppressWarnings({ "unchecked", "rawtypes" }) public Future> processSearch(SearchDTO searchDTO, boolean includeResults) throws Exception { + System.out.println("Enter into process search"); List> groupByFinalList = new ArrayList>(); SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true); Future searchResponse = ElasticSearchUtil.search( @@ -263,6 +264,7 @@ private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByList, SearchSourceBuilder searchSourceBuilder) { + System.out.println("Enter into aggregations ..."); + try { + System.out.println(mapper.writeValueAsString(groupByList)); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } TermsAggregationBuilder termBuilder = null; if (groupByList != null && !groupByList.isEmpty()) { HashMap> nestedAggregation = new HashMap<>(); @@ -293,6 +301,7 @@ private void setAggregations(List> groupByList, } searchSourceBuilder.aggregation(termBuilder); } else { + System.out.println("Nested field ..." + groupByParent); if (nestedAggregation.get(groupByParent.split("\\.")[0]) != null) { nestedAggregation.get(groupByParent.split("\\.")[0]).add(groupByParent.split("\\.")[1]); } else { From 60780897f86aba154f9e92f1c5b6561a74c44356 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 23 Aug 2021 14:42:42 +0530 Subject: [PATCH 15/33] Update ElasticSearchUtil.java --- .../java/org/sunbird/search/client/ElasticSearchUtil.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java index fc4420758..1b9a16705 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java @@ -615,6 +615,13 @@ private static SearchSourceBuilder buildJsonForWildCardQuery(String textKeyWord, public static Object getCountFromAggregation(Aggregations aggregations, List> groupByList, IESResultTransformer transformer) { Map countMap = new HashMap(); + System.out.println("Enter into aggregation count method::"); + try { + System.out.println(mapper.writeValueAsString(groupByList)); + System.out.println(mapper.writeValueAsString(aggregations)); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } if (aggregations != null) { for (Map aggregationsMap : groupByList) { Map parentCountMap = new HashMap(); From 1bcbcd6ca5d145cd05160ba52124332f2ecd2c82 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 23 Aug 2021 17:26:50 +0530 Subject: [PATCH 16/33] Create ElasticSearchUtil.java --- .../java/org/sunbird/search/client/ElasticSearchUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java index 1b9a16705..8f242b229 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java @@ -619,7 +619,7 @@ public static Object getCountFromAggregation(Aggregations aggregations, List) terms.getBucketByKey(groupByParent.split("\\.")[1]); From 125c20b6ddf78b64f770125fa2fa506202c33486 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 23 Aug 2021 17:31:16 +0530 Subject: [PATCH 17/33] Update SearchProcessor.java --- .../main/java/org/sunbird/search/processor/SearchProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index c03af7f86..06a99e6e8 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -279,7 +279,7 @@ private void setAggregations(List> groupByList, System.out.println("Enter into aggregations ..."); try { System.out.println(mapper.writeValueAsString(groupByList)); - } catch (JsonProcessingException e) { + } catch (Exception e) { e.printStackTrace(); } TermsAggregationBuilder termBuilder = null; From 1686f1fae046cdc444c3bc4335222a455a43eaba Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 23 Aug 2021 22:28:42 +0530 Subject: [PATCH 18/33] Update ElasticSearchUtil.java --- .../org/sunbird/search/client/ElasticSearchUtil.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java index 8f242b229..938ad123c 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java @@ -49,6 +49,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.sunbird.common.Platform; import org.sunbird.common.exception.ServerException; @@ -618,7 +619,6 @@ public static Object getCountFromAggregation(Aggregations aggregations, List buckets = null; if (groupByParent.contains(".")) { System.out.println("Enters in the nested aggregation"); - terms = aggregations.get(groupByParent.split("\\.")[0]); try { System.out.println("Aggregation values"); System.out.println(mapper.writeValueAsString(aggregations.getAsMap())); - System.out.println(mapper.writeValueAsString(terms)); } catch (Exception e) { e.printStackTrace(); } - buckets = (List) terms.getBucketByKey(groupByParent.split("\\.")[1]); + ParsedNested parsedNested = aggregations.get(groupByParent.split("\\.")[0]); + terms = parsedNested.getAggregations().get(groupByParent.split("\\.")[1]); } else { terms = aggregations.get(groupByParent); - buckets = (List) terms.getBuckets(); } + buckets = (List)terms.getBuckets(); List> parentGroupList = new ArrayList>(); for (Bucket bucket : buckets) { Map parentCountObject = new HashMap(); From 66e9294be69e5f38897fe6faa3792cc8d7246a09 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Mon, 23 Aug 2021 22:44:28 +0530 Subject: [PATCH 19/33] Update ElasticSearchUtil.java --- .../java/org/sunbird/search/client/ElasticSearchUtil.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java index 938ad123c..276c8c8ae 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java @@ -49,7 +49,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested; +import org.elasticsearch.search.aggregations.bucket.nested.Nested; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.sunbird.common.Platform; import org.sunbird.common.exception.ServerException; @@ -636,8 +636,8 @@ public static Object getCountFromAggregation(Aggregations aggregations, List Date: Wed, 1 Sep 2021 17:46:28 +0530 Subject: [PATCH 20/33] Update ElasticSearchUtil.java --- .../sunbird/search/client/ElasticSearchUtil.java | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java index 276c8c8ae..4929d205c 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java @@ -615,13 +615,7 @@ private static SearchSourceBuilder buildJsonForWildCardQuery(String textKeyWord, @SuppressWarnings("unchecked") public static Object getCountFromAggregation(Aggregations aggregations, List> groupByList, IESResultTransformer transformer) { - Map countMap = new HashMap(); - System.out.println("Enter into aggregation count method::"); - try { - System.out.println(mapper.writeValueAsString(groupByList)); - } catch (Exception e) { - e.printStackTrace(); - } + Map countMap = new HashMap(); if (aggregations != null) { for (Map aggregationsMap : groupByList) { Map parentCountMap = new HashMap(); @@ -629,13 +623,7 @@ public static Object getCountFromAggregation(Aggregations aggregations, List buckets = null; if (groupByParent.contains(".")) { - System.out.println("Enters in the nested aggregation"); - try { - System.out.println("Aggregation values"); - System.out.println(mapper.writeValueAsString(aggregations.getAsMap())); - } catch (Exception e) { - e.printStackTrace(); - } + Nested nested = aggregations.get(groupByParent.split("\\.")[0]); terms = nested.getAggregations().get(groupByParent.split("\\.")[1]); } else { From 747b2ee731e5f2da02811b5ddb6473d3f285707e Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Wed, 1 Sep 2021 17:46:35 +0530 Subject: [PATCH 21/33] Update SearchProcessor.java --- .../org/sunbird/search/processor/SearchProcessor.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java index 06a99e6e8..20ebdaf49 100644 --- a/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java +++ b/search-api/search-core/src/main/java/org/sunbird/search/processor/SearchProcessor.java @@ -63,7 +63,6 @@ public SearchProcessor(String indexName) { @SuppressWarnings({ "unchecked", "rawtypes" }) public Future> processSearch(SearchDTO searchDTO, boolean includeResults) throws Exception { - System.out.println("Enter into process search"); List> groupByFinalList = new ArrayList>(); SearchSourceBuilder query = processSearchQuery(searchDTO, groupByFinalList, true); Future searchResponse = ElasticSearchUtil.search( @@ -264,7 +263,6 @@ private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List> groupByList, SearchSourceBuilder searchSourceBuilder) { - System.out.println("Enter into aggregations ..."); - try { - System.out.println(mapper.writeValueAsString(groupByList)); - } catch (Exception e) { - e.printStackTrace(); - } + TermsAggregationBuilder termBuilder = null; if (groupByList != null && !groupByList.isEmpty()) { HashMap> nestedAggregation = new HashMap<>(); @@ -301,7 +294,6 @@ private void setAggregations(List> groupByList, } searchSourceBuilder.aggregation(termBuilder); } else { - System.out.println("Nested field ..." + groupByParent); if (nestedAggregation.get(groupByParent.split("\\.")[0]) != null) { nestedAggregation.get(groupByParent.split("\\.")[0]).add(groupByParent.split("\\.")[1]); } else { From cef9991441935bbe4022e3651b5494578d5bbca6 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Wed, 1 Sep 2021 17:51:14 +0530 Subject: [PATCH 22/33] Update schema.json --- schemas/asset/1.0/schema.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/schemas/asset/1.0/schema.json b/schemas/asset/1.0/schema.json index bebca5878..86456578b 100644 --- a/schemas/asset/1.0/schema.json +++ b/schemas/asset/1.0/schema.json @@ -1247,12 +1247,6 @@ "type": "string" } }, - "taxonomyPathsTest6": { - "type": "array", - "items": { - "type": "object" - } - }, "taxonomyPaths_v2": { "type": "array", "items": { From 825c6d87171d4f9fc81dcf3da23f4216d4b0d8b3 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Wed, 1 Sep 2021 17:52:18 +0530 Subject: [PATCH 23/33] Update schema.json --- schemas/collection/1.0/schema.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/schemas/collection/1.0/schema.json b/schemas/collection/1.0/schema.json index 951412da8..5d48095f7 100644 --- a/schemas/collection/1.0/schema.json +++ b/schemas/collection/1.0/schema.json @@ -1247,12 +1247,6 @@ "type": "string" } }, - "taxonomyPathsTest6": { - "type": "array", - "items": { - "type": "object" - } - }, "taxonomyPaths_v2": { "type": "array", "items": { From 7e44f24a46d65e5b8fa4753999a6424fa54e3a7c Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Wed, 1 Sep 2021 17:53:07 +0530 Subject: [PATCH 24/33] Update schema.json --- schemas/content/1.0/schema.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/schemas/content/1.0/schema.json b/schemas/content/1.0/schema.json index 4c45fe3c0..cd3b0122d 100644 --- a/schemas/content/1.0/schema.json +++ b/schemas/content/1.0/schema.json @@ -1348,12 +1348,6 @@ "type": "string" } }, - "taxonomyPathsTest6": { - "type": "array", - "items": { - "type": "object" - } - }, "taxonomyPaths_v2": { "type": "array", "items": { From 30a4b7f62bbff0db154b1b545f31b8d79366ece8 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Wed, 1 Sep 2021 20:46:13 +0530 Subject: [PATCH 25/33] Adding Schema for testing --- schemas/content/1.0/schema.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/schemas/content/1.0/schema.json b/schemas/content/1.0/schema.json index cd3b0122d..e7cd15f69 100644 --- a/schemas/content/1.0/schema.json +++ b/schemas/content/1.0/schema.json @@ -1359,6 +1359,12 @@ "items": { "type": "object" } + }, + "test_competencies_v2": { + "type": "array", + "items": { + "type": "object" + } } } } From f569254009fd9bcb724df025cf7c708f77b2a4a7 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Wed, 1 Sep 2021 20:49:57 +0530 Subject: [PATCH 26/33] Added test schema --- schemas/collection/1.0/schema.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/schemas/collection/1.0/schema.json b/schemas/collection/1.0/schema.json index 5d48095f7..0568a35f0 100644 --- a/schemas/collection/1.0/schema.json +++ b/schemas/collection/1.0/schema.json @@ -1258,6 +1258,12 @@ "items": { "type": "object" } + }, + "test_competencies_v2": { + "type": "array", + "items": { + "type": "object" + } } } } From 7790fb3a0116e3e6ef8798301c465dfbd5f97ed5 Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Wed, 1 Sep 2021 20:50:36 +0530 Subject: [PATCH 27/33] Added test schema --- schemas/asset/1.0/schema.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/schemas/asset/1.0/schema.json b/schemas/asset/1.0/schema.json index 86456578b..42544bc36 100644 --- a/schemas/asset/1.0/schema.json +++ b/schemas/asset/1.0/schema.json @@ -1258,6 +1258,12 @@ "items": { "type": "object" } + }, + "test_competencies_v2": { + "type": "array", + "items": { + "type": "object" + } } } } From 0ad324e9cf0f17e690f8b530a9e6ed07cc521349 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Thu, 2 Sep 2021 11:15:04 +0530 Subject: [PATCH 28/33] Update schema.json --- schemas/content/1.0/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/content/1.0/schema.json b/schemas/content/1.0/schema.json index e7cd15f69..69f373db6 100644 --- a/schemas/content/1.0/schema.json +++ b/schemas/content/1.0/schema.json @@ -1354,7 +1354,7 @@ "type": "object" } }, - "competencies_v2": { + "competencies_v3": { "type": "array", "items": { "type": "object" From c98beb615fa55eac8179f40b6c621cf0c30e5f40 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Thu, 2 Sep 2021 11:15:35 +0530 Subject: [PATCH 29/33] Update schema.json --- schemas/collection/1.0/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/collection/1.0/schema.json b/schemas/collection/1.0/schema.json index 0568a35f0..1ac3facae 100644 --- a/schemas/collection/1.0/schema.json +++ b/schemas/collection/1.0/schema.json @@ -1253,7 +1253,7 @@ "type": "object" } }, - "competencies_v2": { + "competencies_v3": { "type": "array", "items": { "type": "object" From 1f63a5ddd28166315719b443dceea6c2e01e2ade Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Thu, 2 Sep 2021 11:16:24 +0530 Subject: [PATCH 30/33] Update schema.json --- schemas/asset/1.0/schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/asset/1.0/schema.json b/schemas/asset/1.0/schema.json index 42544bc36..8186bad99 100644 --- a/schemas/asset/1.0/schema.json +++ b/schemas/asset/1.0/schema.json @@ -1253,7 +1253,7 @@ "type": "object" } }, - "competencies_v2": { + "competencies_v3": { "type": "array", "items": { "type": "object" From dd8eb6f06c1bbc250e7c6f81cbc3d2f3ba9cdba1 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Thu, 2 Sep 2021 13:01:52 +0530 Subject: [PATCH 31/33] Update schema.json --- schemas/content/1.0/schema.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/schemas/content/1.0/schema.json b/schemas/content/1.0/schema.json index 69f373db6..39e50c3e3 100644 --- a/schemas/content/1.0/schema.json +++ b/schemas/content/1.0/schema.json @@ -1359,12 +1359,6 @@ "items": { "type": "object" } - }, - "test_competencies_v2": { - "type": "array", - "items": { - "type": "object" - } } } } From 3c447d817e11e8451b86cdadb7ca0e9754b4ab53 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Thu, 2 Sep 2021 13:24:15 +0530 Subject: [PATCH 32/33] Update schema.json --- schemas/collection/1.0/schema.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/schemas/collection/1.0/schema.json b/schemas/collection/1.0/schema.json index 1ac3facae..279ce7bdb 100644 --- a/schemas/collection/1.0/schema.json +++ b/schemas/collection/1.0/schema.json @@ -1258,12 +1258,6 @@ "items": { "type": "object" } - }, - "test_competencies_v2": { - "type": "array", - "items": { - "type": "object" - } } } } From 2dac5376e2aa19d42a32ddc1a73cb992d415ec38 Mon Sep 17 00:00:00 2001 From: Amit Kumar Yadav <36166686+amit-kumaryadav@users.noreply.github.com> Date: Thu, 2 Sep 2021 13:24:55 +0530 Subject: [PATCH 33/33] Update schema.json --- schemas/asset/1.0/schema.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/schemas/asset/1.0/schema.json b/schemas/asset/1.0/schema.json index 8186bad99..f4fe326b1 100644 --- a/schemas/asset/1.0/schema.json +++ b/schemas/asset/1.0/schema.json @@ -1258,12 +1258,6 @@ "items": { "type": "object" } - }, - "test_competencies_v2": { - "type": "array", - "items": { - "type": "object" - } } } }