Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
57727fa
Update schema.json
amit-kumaryadav Aug 16, 2021
31fef29
Update schema.json
amit-kumaryadav Aug 16, 2021
78614b1
Update SearchProcessor.java
pc0202 Aug 16, 2021
d9df49c
Update SearchActor.java
pc0202 Aug 16, 2021
8bb9e32
Update schema.json
amit-kumaryadav Aug 16, 2021
866d329
Update schema.json
amit-kumaryadav Aug 16, 2021
a2fe464
Update schema.json
amit-kumaryadav Aug 16, 2021
afa9100
Update schema.json
amit-kumaryadav Aug 16, 2021
05b7484
Update schema.json
amit-kumaryadav Aug 16, 2021
ba4df72
Update schema.json
amit-kumaryadav Aug 16, 2021
02e9dc4
Update schema.json
amit-kumaryadav Aug 16, 2021
6c472dd
Merge pull request #5 from pritha-tarento/index-validation
Haritest Aug 23, 2021
2571afc
Added nested field aggregation
amit-kumaryadav Aug 23, 2021
7e0bc6a
Nested fields aggregations.
amit-kumaryadav Aug 23, 2021
67ada25
Update SearchProcessor.java
amit-kumaryadav Aug 23, 2021
6078089
Update ElasticSearchUtil.java
amit-kumaryadav Aug 23, 2021
1bcbcd6
Create ElasticSearchUtil.java
amit-kumaryadav Aug 23, 2021
125c20b
Update SearchProcessor.java
amit-kumaryadav Aug 23, 2021
1686f1f
Update ElasticSearchUtil.java
amit-kumaryadav Aug 23, 2021
66e9294
Update ElasticSearchUtil.java
amit-kumaryadav Aug 23, 2021
e3b9d10
Update ElasticSearchUtil.java
amit-kumaryadav Sep 1, 2021
747b2ee
Update SearchProcessor.java
amit-kumaryadav Sep 1, 2021
cef9991
Update schema.json
amit-kumaryadav Sep 1, 2021
825c6d8
Update schema.json
amit-kumaryadav Sep 1, 2021
7e44f24
Update schema.json
amit-kumaryadav Sep 1, 2021
7e96271
Merge pull request #6 from sunbird-cb/aggregation_on_nested_fields
karthik-tarento Sep 1, 2021
5420a1e
Merge pull request #7 from sunbird-cb/newfields_proposal
karthik-tarento Sep 1, 2021
30a4b7f
Adding Schema for testing
karthik-tarento Sep 1, 2021
f569254
Added test schema
karthik-tarento Sep 1, 2021
7790fb3
Added test schema
karthik-tarento Sep 1, 2021
0ad324e
Update schema.json
amit-kumaryadav Sep 2, 2021
c98beb6
Update schema.json
amit-kumaryadav Sep 2, 2021
1f63a5d
Update schema.json
amit-kumaryadav Sep 2, 2021
dd8eb6f
Update schema.json
amit-kumaryadav Sep 2, 2021
3c447d8
Update schema.json
amit-kumaryadav Sep 2, 2021
2dac537
Update schema.json
amit-kumaryadav Sep 2, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion schemas/asset/1.0/schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -1246,6 +1246,18 @@
"items": {
"type": "string"
}
}
},
"taxonomyPaths_v2": {
"type": "array",
"items": {
"type": "object"
}
},
"competencies_v3": {
"type": "array",
"items": {
"type": "object"
}
}
}
}
12 changes: 12 additions & 0 deletions schemas/collection/1.0/schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -1246,6 +1246,18 @@
"items": {
"type": "string"
}
},
"taxonomyPaths_v2": {
"type": "array",
"items": {
"type": "object"
}
},
"competencies_v3": {
"type": "array",
"items": {
"type": "object"
}
}
}
}
14 changes: 13 additions & 1 deletion schemas/content/1.0/schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -1347,6 +1347,18 @@
"items": {
"type": "string"
}
}
},
"taxonomyPaths_v2": {
"type": "array",
"items": {
"type": "object"
}
},
"competencies_v3": {
"type": "array",
"items": {
"type": "object"
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -242,8 +242,10 @@ private SearchDTO getSearchDTO(Request request) throws Exception {
searchObj.setSortBy(sortBy);
searchObj.setFacets(facets);
searchObj.setProperties(properties);
if(multiFilters!=null)
if(multiFilters!=null){
multiFilterProperties.addAll(getSearchFilterProperties(multiFilters, wordChainsRequest));
searchObj.setMultiFilterProperties(multiFilterProperties);
}
// Added Implicit Filter Properties To Support Collection content tagging to reuse by tenants.
setImplicitFilters(filters, searchObj);
searchObj.setLimit(limit);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.nested.Nested;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.sunbird.common.Platform;
import org.sunbird.common.exception.ServerException;
Expand Down Expand Up @@ -613,43 +614,50 @@ private static SearchSourceBuilder buildJsonForWildCardQuery(String textKeyWord,

@SuppressWarnings("unchecked")
public static Object getCountFromAggregation(Aggregations aggregations, List<Map<String, Object>> groupByList,
IESResultTransformer transformer) {

Map<String, Object> countMap = new HashMap<String, Object>();
if (aggregations != null) {
for (Map<String, Object> aggregationsMap : groupByList) {
Map<String, Object> parentCountMap = new HashMap<String, Object>();
String groupByParent = (String) aggregationsMap.get("groupByParent");
Terms terms = aggregations.get(groupByParent);
List<Map<String, Object>> parentGroupList = new ArrayList<Map<String, Object>>();
List<Bucket> buckets = (List<Bucket>) terms.getBuckets();
for (Bucket bucket : buckets) {
Map<String, Object> parentCountObject = new HashMap<String, Object>();
parentCountObject.put("count", bucket.getDocCount());
List<String> groupByChildList = (List<String>) aggregationsMap.get("groupByChildList");
Aggregations subAggregations = bucket.getAggregations();
if (null != groupByChildList && !groupByChildList.isEmpty() && null != subAggregations) {
Map<String, Object> groupByChildMap = new HashMap<String, Object>();
for (String groupByChild : groupByChildList) {
Terms subTerms = subAggregations.get(groupByChild);
List<Bucket> childBuckets = (List<Bucket>) subTerms.getBuckets();
Map<String, Long> childCountMap = new HashMap<String, Long>();
for (Bucket childBucket : childBuckets) {
childCountMap.put(childBucket.getKeyAsString(), childBucket.getDocCount());
groupByChildMap.put(groupByChild, childCountMap);
}
}
parentCountObject.putAll(groupByChildMap);
}
parentCountMap.put(bucket.getKeyAsString(), parentCountObject);
parentGroupList.add(parentCountMap);
}

countMap.put(groupByParent, parentCountMap);
}
}
return transformer.getTransformedObject(countMap);
}
IESResultTransformer transformer) {
Map<String, Object> countMap = new HashMap<String, Object>();
if (aggregations != null) {
for (Map<String, Object> aggregationsMap : groupByList) {
Map<String, Object> parentCountMap = new HashMap<String, Object>();
String groupByParent = (String) aggregationsMap.get("groupByParent");
Terms terms = null;
List<Bucket> buckets = null;
if (groupByParent.contains(".")) {

Nested nested = aggregations.get(groupByParent.split("\\.")[0]);
terms = nested.getAggregations().get(groupByParent.split("\\.")[1]);
} else {
terms = aggregations.get(groupByParent);
}
buckets = (List<Bucket>)terms.getBuckets();
List<Map<String, Object>> parentGroupList = new ArrayList<Map<String, Object>>();
for (Bucket bucket : buckets) {
Map<String, Object> parentCountObject = new HashMap<String, Object>();
parentCountObject.put("count", bucket.getDocCount());
List<String> groupByChildList = (List<String>) aggregationsMap.get("groupByChildList");
Aggregations subAggregations = bucket.getAggregations();
if (null != groupByChildList && !groupByChildList.isEmpty() && null != subAggregations) {
Map<String, Object> groupByChildMap = new HashMap<String, Object>();
for (String groupByChild : groupByChildList) {
Terms subTerms = subAggregations.get(groupByChild);
List<Bucket> childBuckets = (List<Bucket>) subTerms.getBuckets();
Map<String, Long> childCountMap = new HashMap<String, Long>();
for (Bucket childBucket : childBuckets) {
childCountMap.put(childBucket.getKeyAsString(), childBucket.getDocCount());
groupByChildMap.put(groupByChild, childCountMap);
}
}
parentCountObject.putAll(groupByChildMap);
}
parentCountMap.put(bucket.getKeyAsString(), parentCountObject);
parentGroupList.add(parentCountMap);
}

countMap.put(groupByParent, parentCountMap);
}
}
return transformer.getTransformedObject(countMap);
}

private static void registerShutdownHook() {
Runtime.getRuntime().addShutdownHook(new Thread() {
Expand Down Expand Up @@ -711,4 +719,4 @@ public static void bulkDeleteDocumentById(String indexName, String documentType,
}
}

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -271,28 +271,53 @@ private SearchSourceBuilder processSearchQuery(SearchDTO searchDTO, List<Map<Str
* @param searchSourceBuilder
* @return
*/
@SuppressWarnings("unchecked")
private void setAggregations(List<Map<String, Object>> groupByList,
SearchSourceBuilder searchSourceBuilder) {
@SuppressWarnings("unchecked")
private void setAggregations(List<Map<String, Object>> groupByList,
SearchSourceBuilder searchSourceBuilder) {

TermsAggregationBuilder termBuilder = null;
if (groupByList != null && !groupByList.isEmpty()) {
for (Map<String, Object> groupByMap : groupByList) {
String groupByParent = (String) groupByMap.get("groupByParent");
termBuilder = AggregationBuilders.terms(groupByParent)
.field(groupByParent + SearchConstants.RAW_FIELD_EXTENSION)
.size(ElasticSearchUtil.defaultResultLimit);
List<String> groupByChildList = (List<String>) groupByMap.get("groupByChildList");
if (groupByChildList != null && !groupByChildList.isEmpty()) {
for (String childGroupBy : groupByChildList) {
termBuilder.subAggregation(AggregationBuilders.terms(childGroupBy)
.field(childGroupBy + SearchConstants.RAW_FIELD_EXTENSION)
.size(ElasticSearchUtil.defaultResultLimit));
}
HashMap<String, List<String>> nestedAggregation = new HashMap<>();
for (Map<String, Object> groupByMap : groupByList) {
String groupByParent = (String) groupByMap.get("groupByParent");
if (!groupByParent.contains(".")) {
termBuilder = AggregationBuilders.terms(groupByParent)
.field(groupByParent + SearchConstants.RAW_FIELD_EXTENSION)
.size(ElasticSearchUtil.defaultResultLimit);
List<String> groupByChildList = (List<String>) groupByMap.get("groupByChildList");
if (groupByChildList != null && !groupByChildList.isEmpty()) {
for (String childGroupBy : groupByChildList) {
termBuilder.subAggregation(AggregationBuilders.terms(childGroupBy)
.field(childGroupBy + SearchConstants.RAW_FIELD_EXTENSION)
.size(ElasticSearchUtil.defaultResultLimit));
}
searchSourceBuilder.aggregation(termBuilder);
}
}
searchSourceBuilder.aggregation(termBuilder);
} else {
if (nestedAggregation.get(groupByParent.split("\\.")[0]) != null) {
nestedAggregation.get(groupByParent.split("\\.")[0]).add(groupByParent.split("\\.")[1]);
} else {
List<String> nestedAggrList = new ArrayList<>();
nestedAggrList.add(groupByParent.split("\\.")[1]);
nestedAggregation.put(groupByParent.split("\\.")[0], nestedAggrList);
}
}
}

if (!nestedAggregation.isEmpty()) {
for (Map.Entry<String, List<String>> mapData : nestedAggregation.entrySet()) {
AggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested(mapData.getKey(), mapData.getKey());
for (String nestedValue : mapData.getValue()) {
termBuilder = AggregationBuilders.terms(nestedValue)
.field(mapData.getKey() + "." + nestedValue + SearchConstants.RAW_FIELD_EXTENSION)
.size(ElasticSearchUtil.defaultResultLimit);
nestedAggregationBuilder.subAggregation(termBuilder);
}
searchSourceBuilder.aggregation(nestedAggregationBuilder);
}
}
}
}
}

/**
* @param searchDTO
Expand All @@ -307,6 +332,8 @@ private QueryBuilder prepareSearchQuery(SearchDTO searchDTO) {
formQuery(properties, queryBuilder, boolQuery, totalOperation);

List<Map> multiFilterProperties = searchDTO.getMultiFilterProperties();
System.out.println("multiFilterProperties:: "+multiFilterProperties);

if(multiFilterProperties!=null)
formQuery(multiFilterProperties, queryBuilder, boolQuery, SearchConstants.SEARCH_OPERATION_OR);

Expand Down