diff --git a/README.md b/README.md index db696f36..cfbc96a2 100644 --- a/README.md +++ b/README.md @@ -264,13 +264,13 @@ val results = client.search(SQLQuery(sqlQuery)) "aggs": { "restaurant_name": { "terms": { - "field": "restaurant_name.keyword", + "field": "restaurant_name", "size": 1000 }, "aggs": { "restaurant_city": { "terms": { - "field": "restaurant_city.keyword", + "field": "restaurant_city", "size": 1000 }, "aggs": { @@ -296,7 +296,7 @@ val results = client.search(SQLQuery(sqlQuery)) "aggs": { "menu_category": { "terms": { - "field": "menus.category.keyword", + "field": "menus.category", "size": 1000 }, "aggs": { @@ -307,7 +307,7 @@ val results = client.search(SQLQuery(sqlQuery)) "aggs": { "dish_name": { "terms": { - "field": "menus.dishes.name.keyword", + "field": "menus.dishes.name", "size": 1000 }, "aggs": { @@ -339,7 +339,7 @@ val results = client.search(SQLQuery(sqlQuery)) }, "ingredient_name": { "terms": { - "field": "menus.dishes.ingredients.name.keyword", + "field": "menus.dishes.ingredients.name", "size": 1000 }, "aggs": { @@ -787,18 +787,18 @@ ThisBuild / resolvers ++= Seq( // For Elasticsearch 6 // Using Jest client -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es6-jest-client" % 0.13.0 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es6-jest-client" % 0.13.1 // Or using Rest High Level client -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es6-rest-client" % 0.13.0 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es6-rest-client" % 0.13.1 // For Elasticsearch 7 -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es7-rest-client" % 0.13.0 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es7-rest-client" % 0.13.1 // For Elasticsearch 8 -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es8-java-client" % 0.13.0 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es8-java-client" % 0.13.1 // For Elasticsearch 9 -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es9-java-client" % 0.13.0 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es9-java-client" % 0.13.1 ``` ### **Quick Example** diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 51b3766b..8ed1fb5e 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -17,6 +17,7 @@ package app.softnetwork.elastic.sql.bridge import app.softnetwork.elastic.sql.PainlessContext +import app.softnetwork.elastic.sql.`type`.SQLTemporal import app.softnetwork.elastic.sql.query.{ Asc, Bucket, @@ -31,6 +32,8 @@ import app.softnetwork.elastic.sql.query.{ } import app.softnetwork.elastic.sql.function._ import app.softnetwork.elastic.sql.function.aggregate._ +import app.softnetwork.elastic.sql.function.time.DateTrunc +import app.softnetwork.elastic.sql.time.TimeUnit import com.sksamuel.elastic4s.ElasticApi.{ avgAgg, bucketSelectorAggregation, @@ -44,11 +47,14 @@ import com.sksamuel.elastic4s.ElasticApi.{ valueCountAgg } import com.sksamuel.elastic4s.requests.script.Script +import com.sksamuel.elastic4s.requests.searches.DateHistogramInterval import com.sksamuel.elastic4s.requests.searches.aggs.{ Aggregation, CardinalityAggregation, + DateHistogramAggregation, ExtendedStatsAggregation, FilterAggregation, + HistogramOrder, NestedAggregation, StatsAggregation, TermsAggregation, @@ -93,7 +99,10 @@ object ElasticAggregation { import sqlAgg._ val sourceField = identifier.path - val direction = bucketsDirection.get(identifier.identifierName) + val direction = + bucketsDirection + .get(identifier.identifierName) + .orElse(bucketsDirection.get(identifier.aliasOrName)) val field = fieldAlias match { case Some(alias) => alias.alias @@ -113,8 +122,8 @@ object ElasticAggregation { s"${aggType}_distinct_${sourceField.replace(".", "_")}" else { aggType match { - case th: TopHitsAggregation => - s"${th.topHits.sql.toLowerCase}_${sourceField.replace(".", "_")}" + case th: WindowFunction => + s"${th.window.sql.toLowerCase}_${sourceField.replace(".", "_")}" case _ => s"${aggType}_${sourceField.replace(".", "_")}" @@ -145,16 +154,21 @@ object ElasticAggregation { val _agg = aggType match { case COUNT => + val field = + sourceField match { + case "*" | "_id" | "_index" | "_type" => "_index" + case _ => sourceField + } if (distinct) - cardinalityAgg(aggName, sourceField) + cardinalityAgg(aggName, field) else { - valueCountAgg(aggName, sourceField) + valueCountAgg(aggName, field) } case MIN => aggWithFieldOrScript(minAgg, (name, s) => minAgg(name, sourceField).script(s)) case MAX => aggWithFieldOrScript(maxAgg, (name, s) => maxAgg(name, sourceField).script(s)) case AVG => aggWithFieldOrScript(avgAgg, (name, s) => avgAgg(name, sourceField).script(s)) case SUM => aggWithFieldOrScript(sumAgg, (name, s) => sumAgg(name, sourceField).script(s)) - case th: TopHitsAggregation => + case th: WindowFunction => val limit = { th match { case _: LastValue => 1 @@ -167,27 +181,31 @@ object ElasticAggregation { .fetchSource( th.identifier.name +: th.fields .filterNot(_.isScriptField) + .filterNot(_.sourceField == th.identifier.name) .map(_.sourceField) + .distinct .toArray, Array.empty ) .copy( scripts = th.fields .filter(_.isScriptField) + .groupBy(_.sourceField) + .map(_._2.head) .map(f => f.sourceField -> Script(f.painless(None)).lang("painless")) .toMap ) .size(limit) sortBy th.orderBy.sorts.map(sort => sort.order match { case Some(Desc) => - th.topHits match { - case LAST_VALUE => FieldSort(sort.field).asc() - case _ => FieldSort(sort.field).desc() + th.window match { + case LAST_VALUE => FieldSort(sort.field.aliasOrName).asc() + case _ => FieldSort(sort.field.aliasOrName).desc() } case _ => - th.topHits match { - case LAST_VALUE => FieldSort(sort.field).desc() - case _ => FieldSort(sort.field).asc() + th.window match { + case LAST_VALUE => FieldSort(sort.field.aliasOrName).desc() + case _ => FieldSort(sort.field.aliasOrName).asc() } } ) @@ -263,82 +281,193 @@ object ElasticAggregation { having: Option[Criteria], nested: Option[NestedElement], allElasticAggregations: Seq[ElasticAggregation] - ): Option[TermsAggregation] = { - buckets.reverse.foldLeft(Option.empty[TermsAggregation]) { (current, bucket) => + ): Option[Aggregation] = { + buckets.reverse.foldLeft(Option.empty[Aggregation]) { (current, bucket) => // Determine the bucketPath of the current bucket val currentBucketPath = bucket.identifier.path - var agg = { - bucketsDirection.get(bucket.identifier.identifierName) match { - case Some(direction) => - termsAgg(bucket.name, s"$currentBucketPath.keyword") - .order(Seq(direction match { - case Asc => TermsOrder("_key", asc = true) - case _ => TermsOrder("_key", asc = false) - })) - case None => - termsAgg(bucket.name, s"$currentBucketPath.keyword") + val aggScript = + if (bucket.shouldBeScripted) { + val context = PainlessContext() + val painless = bucket.painless(Some(context)) + Some(Script(s"$context$painless").lang("painless")) + } else { + None + } + + var agg: Aggregation = { + bucket.out match { + case _: SQLTemporal => + val functions = bucket.identifier.functions + val interval: Option[DateHistogramInterval] = + if (functions.size == 1) { + functions.head match { + case trunc: DateTrunc => + trunc.unit match { + case TimeUnit.YEARS => Option(DateHistogramInterval.Year) + case TimeUnit.QUARTERS => Option(DateHistogramInterval.Quarter) + case TimeUnit.MONTHS => Option(DateHistogramInterval.Month) + case TimeUnit.WEEKS => Option(DateHistogramInterval.Week) + case TimeUnit.DAYS => Option(DateHistogramInterval.Day) + case TimeUnit.HOURS => Option(DateHistogramInterval.Hour) + case TimeUnit.MINUTES => Option(DateHistogramInterval.Minute) + case TimeUnit.SECONDS => Option(DateHistogramInterval.Second) + case _ => None + } + case _ => None + } + } else { + None + } + + aggScript match { + case Some(script) => + // Scripted date histogram + bucketsDirection.get(bucket.identifier.identifierName) match { + case Some(direction) => + DateHistogramAggregation(bucket.name, calendarInterval = interval) + .script(script) + .minDocCount(1) + .order(direction match { + case Asc => HistogramOrder("_key", asc = true) + case _ => HistogramOrder("_key", asc = false) + }) + case _ => + DateHistogramAggregation(bucket.name, calendarInterval = interval) + .script(script) + .minDocCount(1) + } + case _ => + // Standard date histogram + bucketsDirection.get(bucket.identifier.identifierName) match { + case Some(direction) => + DateHistogramAggregation(bucket.name, calendarInterval = interval) + .field(currentBucketPath) + .minDocCount(1) + .order(direction match { + case Asc => HistogramOrder("_key", asc = true) + case _ => HistogramOrder("_key", asc = false) + }) + case _ => + DateHistogramAggregation(bucket.name, calendarInterval = interval) + .field(currentBucketPath) + .minDocCount(1) + } + } + + case _ => + aggScript match { + case Some(script) => + // Scripted terms aggregation + bucketsDirection.get(bucket.identifier.identifierName) match { + case Some(direction) => + TermsAggregation(bucket.name) + .script(script) + .minDocCount(1) + .order(Seq(direction match { + case Asc => TermsOrder("_key", asc = true) + case _ => TermsOrder("_key", asc = false) + })) + case _ => + TermsAggregation(bucket.name) + .script(script) + .minDocCount(1) + } + case _ => + // Standard terms aggregation + bucketsDirection.get(bucket.identifier.identifierName) match { + case Some(direction) => + termsAgg(bucket.name, currentBucketPath) + .minDocCount(1) + .order(Seq(direction match { + case Asc => TermsOrder("_key", asc = true) + case _ => TermsOrder("_key", asc = false) + })) + case _ => + termsAgg(bucket.name, currentBucketPath) + .minDocCount(1) + } + } } } - bucket.size.foreach(s => agg = agg.size(s)) - having match { - case Some(criteria) => - criteria.includes(bucket, not = false, BucketIncludesExcludes()) match { - case BucketIncludesExcludes(_, Some(regex)) if regex.nonEmpty => - agg = agg.includeRegex(regex) - case BucketIncludesExcludes(values, _) if values.nonEmpty => - agg = agg.includeExactValues(values.toArray) - case _ => - } - criteria.excludes(bucket, not = false, BucketIncludesExcludes()) match { - case BucketIncludesExcludes(_, Some(regex)) if regex.nonEmpty => - agg = agg.excludeRegex(regex) - case BucketIncludesExcludes(values, _) if values.nonEmpty => - agg = agg.excludeExactValues(values.toArray) + agg match { + case termsAgg: TermsAggregation => + bucket.size.foreach(s => agg = termsAgg.size(s)) + having match { + case Some(criteria) => + criteria.includes(bucket, not = false, BucketIncludesExcludes()) match { + case BucketIncludesExcludes(_, Some(regex)) if regex.nonEmpty => + agg = termsAgg.includeRegex(regex) + case BucketIncludesExcludes(values, _) if values.nonEmpty => + agg = termsAgg.includeExactValues(values.toArray) + case _ => + } + criteria.excludes(bucket, not = false, BucketIncludesExcludes()) match { + case BucketIncludesExcludes(_, Some(regex)) if regex.nonEmpty => + agg = termsAgg.excludeRegex(regex) + case BucketIncludesExcludes(values, _) if values.nonEmpty => + agg = termsAgg.excludeExactValues(values.toArray) + case _ => + } case _ => } case _ => } current match { - case Some(subAgg) => Some(agg.copy(subaggs = Seq(subAgg))) + case Some(subAgg) => + agg match { + case termsAgg: TermsAggregation => + agg = termsAgg.subaggs(Seq(subAgg)) + case dateHistogramAgg: DateHistogramAggregation => + agg = dateHistogramAgg.subaggs(Seq(subAgg)) + case _ => + } + Some(agg) case None => - val aggregationsWithOrder: Seq[TermsOrder] = aggregationsDirection.toSeq.map { kv => - kv._2 match { - case Asc => TermsOrder(kv._1, asc = true) - case _ => TermsOrder(kv._1, asc = false) + val subaggs = + having match { + case Some(criteria) => + val script = metricSelectorForBucket( + criteria, + nested, + allElasticAggregations + ) + + if (script.nonEmpty) { + val bucketSelector = + bucketSelectorAggregation( + "having_filter", + Script(script), + extractMetricsPathForBucket( + criteria, + nested, + allElasticAggregations + ) + ) + aggregations :+ bucketSelector + } else { + aggregations + } + case None => + aggregations } - } - val withAggregationOrders = - if (aggregationsWithOrder.nonEmpty) - agg.order(aggregationsWithOrder) - else - agg - val withHaving = having match { - case Some(criteria) => - val script = metricSelectorForBucket( - criteria, - nested, - allElasticAggregations - ) - if (script.nonEmpty) { - val bucketSelector = - bucketSelectorAggregation( - "having_filter", - Script(script), - extractMetricsPathForBucket( - criteria, - nested, - allElasticAggregations - ) - ) - withAggregationOrders.copy(subaggs = aggregations :+ bucketSelector) - } else { - withAggregationOrders.copy(subaggs = aggregations) + agg match { + case termsAgg: TermsAggregation => + val aggregationsWithOrder: Seq[TermsOrder] = aggregationsDirection.toSeq.map { kv => + kv._2 match { + case Asc => TermsOrder(kv._1, asc = true) + case _ => TermsOrder(kv._1, asc = false) + } } - case None => withAggregationOrders.copy(subaggs = aggregations) + if (aggregationsWithOrder.nonEmpty) + agg = termsAgg.order(aggregationsWithOrder).copy(subaggs = subaggs) + else + agg = termsAgg.copy(subaggs = subaggs) + case dateHistogramAggregation: DateHistogramAggregation => + agg = dateHistogramAggregation.copy(subaggs = subaggs) } - Some(withHaving) + Some(agg) } } } diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index b29f1ce2..68e6c2b7 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -16,7 +16,13 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLDouble, SQLTemporal, SQLVarchar} +import app.softnetwork.elastic.sql.`type`.{ + SQLBigInt, + SQLDouble, + SQLNumeric, + SQLTemporal, + SQLVarchar +} import app.softnetwork.elastic.sql.function.aggregate.COUNT import app.softnetwork.elastic.sql.function.geo.{Distance, Meters} import app.softnetwork.elastic.sql.operator._ @@ -34,7 +40,7 @@ import com.sksamuel.elastic4s.requests.searches.aggs.{ } import com.sksamuel.elastic4s.requests.searches.queries.compound.BoolQuery import com.sksamuel.elastic4s.requests.searches.queries.{InnerHit, Query} -import com.sksamuel.elastic4s.requests.searches.sort.FieldSort +import com.sksamuel.elastic4s.requests.searches.sort.{FieldSort, ScriptSort, ScriptSortType} import com.sksamuel.elastic4s.requests.searches.{ MultiSearchRequest, SearchBodyBuilderFn, @@ -457,7 +463,7 @@ package object bridge { _search } - _search = scriptFields.filterNot(_.aggregation) match { + _search = scriptFields.filterNot(_.isAggregation) match { case Nil => _search case _ => _search scriptfields scriptFields.map { field => @@ -478,17 +484,55 @@ package object bridge { _search = orderBy match { case Some(o) if aggregates.isEmpty && buckets.isEmpty => - _search sortBy o.sorts.map(sort => - sort.order match { - case Some(Desc) => FieldSort(sort.field).desc() - case _ => FieldSort(sort.field).asc() + _search sortBy o.sorts.map { sort => + if (sort.isScriptSort) { + val context = PainlessContext() + val painless = sort.field.painless(Some(context)) + val painlessScript = s"$context$painless" + val script = + sort.out match { + case _: SQLTemporal if !painless.endsWith("toEpochMilli()") => + val parts = painlessScript.split(";").toSeq + if (parts.size > 1) { + val lastPart = parts.last.trim.stripPrefix("return ") + if (lastPart.split(" ").toSeq.size == 1) { + val newLastPart = + s"""($lastPart != null) ? $lastPart.toInstant().toEpochMilli() : null""" + s"${parts.dropRight(1).mkString(";")}; return $newLastPart" + } else { + painlessScript + } + } else { + s"$painlessScript.toInstant().toEpochMilli()" + } + case _ => painlessScript + } + val scriptSort = + ScriptSort( + script = Script(script = script) + .lang("painless") + .scriptType(Source), + scriptSortType = sort.field.out match { + case _: SQLTemporal | _: SQLNumeric => ScriptSortType.Number + case _ => ScriptSortType.String + } + ) + sort.order match { + case Some(Desc) => scriptSort.desc() + case _ => scriptSort.asc() + } + } else { + sort.order match { + case Some(Desc) => FieldSort(sort.field.aliasOrName).desc() + case _ => FieldSort(sort.field.aliasOrName).asc() + } } - ) + } case _ => _search } if (allAggregations.nonEmpty && fields.isEmpty) { - _search size 0 + _search size 0 fetchSource false } else { limit match { case Some(l) => _search limit l.limit from l.offset.map(_.offset).getOrElse(0) @@ -512,7 +556,7 @@ package object bridge { implicit def expressionToQuery(expression: GenericExpression): Query = { import expression._ - if (aggregation) + if (isAggregation) return matchAllQuery() if ( identifier.functions.nonEmpty && (identifier.functions.size > 1 || (identifier.functions.head match { diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 8a41ae9d..f0a07bd1 100644 --- a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -526,12 +526,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "Country": { | "terms": { - | "field": "Country.keyword", + | "field": "Country", | "exclude": ["USA"], + | "min_doc_count": 1, | "order": { | "_key": "asc" | } @@ -539,8 +540,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "aggs": { | "City": { | "terms": { - | "field": "City.keyword", + | "field": "City", | "exclude": ["Berlin"], + | "min_doc_count": 1, | "order": { | "cnt": "desc" | } @@ -709,7 +711,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | }, | "size": 0, | "min_score": 1.0, - | "_source": true, + | "_source": false, | "aggs": { | "inner_products": { | "nested": { @@ -793,8 +795,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "aggs": { | "cat": { | "terms": { - | "field": "products.category.keyword", - | "size": 10 + | "field": "products.category", + | "size": 10, + | "min_doc_count": 1 | }, | "aggs": { | "min_price": { @@ -848,7 +851,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "ct": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); param1" | } | } | }, @@ -871,6 +874,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll(";", "; ") .replaceAll("\\|\\|", " || ") .replaceAll("ChronoUnit", " ChronoUnit") + .replaceAll("==", " == ") } it should "filter with date time and interval" in { @@ -1005,11 +1009,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "userId": { | "terms": { - | "field": "userId.keyword" + | "field": "userId", + | "min_doc_count": 1 | }, | "aggs": { | "lastSeen": { @@ -1049,12 +1054,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "Country": { | "terms": { - | "field": "Country.keyword", + | "field": "Country", | "exclude": ["USA"], + | "min_doc_count": 1, | "order": { | "_key": "asc" | } @@ -1062,8 +1068,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "aggs": { | "City": { | "terms": { - | "field": "City.keyword", - | "exclude": ["Berlin"] + | "field": "City", + | "exclude": ["Berlin"], + | "min_doc_count": 1 | }, | "aggs": { | "cnt": { @@ -1114,14 +1121,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "Country": { | "terms": { - | "field": "Country.keyword", - | "exclude": [ - | "USA" - | ], + | "field": "Country", + | "exclude": ["USA"], + | "min_doc_count": 1, | "order": { | "_key": "asc" | } @@ -1129,10 +1135,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "aggs": { | "City": { | "terms": { - | "field": "City.keyword", - | "exclude": [ - | "Berlin" - | ] + | "field": "City", + | "exclude": ["Berlin"], + | "min_doc_count": 1 | }, | "aggs": { | "cnt": { @@ -1189,11 +1194,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "identifier": { | "terms": { - | "field": "identifier.keyword", + | "field": "identifier", + | "min_doc_count": 1, | "order": { | "ct": "desc" | } @@ -1209,7 +1215,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "field": "createdAt", | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (param1 == null) ? null : LocalDate.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"))" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : LocalDate.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"))" | } | } | } @@ -1260,49 +1266,49 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.withDayOfYear(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfYear(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value); def param2 = param1 != null ? param1.withMonth((((param1.getMonthValue() - 1) / 3) * 3) + 1).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS) : null; def param3 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param3.format(param2)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = param1 != null ? param1.withMonth((((param1.getMonthValue() - 1) / 3) * 3) + 1).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS) : null; def param3 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param3.format(param2)" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.with(DayOfWeek.SUNDAY).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.with(DayOfWeek.SUNDAY).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "d": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.HOURS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.HOURS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "m2": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.MINUTES)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.MINUTES)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.SECONDS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.SECONDS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | } | }, @@ -1356,11 +1362,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "identifier": { | "terms": { - | "field": "identifier.keyword", + | "field": "identifier", + | "min_doc_count": 1, | "order": { | "ct": "desc" | } @@ -1376,7 +1383,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "field": "createdAt", | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).truncatedTo(ChronoUnit.MINUTES).get(ChronoField.YEAR)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).truncatedTo(ChronoUnit.MINUTES).get(ChronoField.YEAR)" | } | } | } @@ -1428,7 +1435,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss XXX\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss XXX\"); (param1 == null) ? null : param2.format(param1)" | } | } | }, @@ -1474,7 +1481,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "diff": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('updatedAt') || doc['updatedAt'].empty ? null : doc['updatedAt'].value); def param2 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param2)" + | "source": "def param1 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param2 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param2)" | } | } | }, @@ -1513,18 +1520,19 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "identifier": { | "terms": { - | "field": "identifier.keyword" + | "field": "identifier", + | "min_doc_count": 1 | }, | "aggs": { | "max_diff": { | "max": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('updatedAt') || doc['updatedAt'].empty ? null : doc['updatedAt'].value); def param2 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); def param3 = (param2 == null) ? null : ZonedDateTime.parse(param2, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param3)" + | "source": "def param1 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param2 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param3 = (param2 == null) ? null : ZonedDateTime.parse(param2, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param3)" | } | } | } @@ -1576,7 +1584,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.plus(10, ChronoUnit.DAYS)); param1" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.plus(10, ChronoUnit.DAYS)); param1" | } | } | }, @@ -1627,7 +1635,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.minus(10, ChronoUnit.DAYS)); param1" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.minus(10, ChronoUnit.DAYS)); param1" | } | } | }, @@ -1678,7 +1686,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.plus(10, ChronoUnit.DAYS)); param1" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.plus(10, ChronoUnit.DAYS)); param1" | } | } | }, @@ -1729,7 +1737,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.minus(10, ChronoUnit.DAYS)); param1" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.minus(10, ChronoUnit.DAYS)); param1" | } | } | }, @@ -1772,7 +1780,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "flag": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); param1 == null" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); param1 == null" | } | } | }, @@ -1810,7 +1818,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "flag": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); param1 != null" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); param1 != null" | } | } | }, @@ -1910,7 +1918,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); param1 != null ? param1 : param2" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); (param1 != null ? param1 : param2)" | } | } | }, @@ -1944,6 +1952,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("ChronoUnit", " ChronoUnit") .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll(":ZonedDateTime", " : ZonedDateTime") + .replaceAll(";\\(param", "; (param") } it should "handle nullif function as script field" in { @@ -1960,7 +1969,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); param3 != null ? param3 : param4" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); (param3 != null ? param3 : param4)" | } | } | }, @@ -2002,6 +2011,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll(",DateTimeFormatter", ", DateTimeFormatter") .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll(":ZonedDateTime", " : ZonedDateTime") + .replaceAll(";\\(param", "; (param") } it should "handle cast function as script field" in { @@ -2018,7 +2028,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { param3 != null ? param3 : param4 } catch (Exception e) { return null; }" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { (param3 != null ? param3 : param4) } catch (Exception e) { return null; }" | } | }, | "c2": { @@ -2086,9 +2096,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll(":ZonedDateTime", " : ZonedDateTime") .replaceAll("try \\{", "try { ") .replaceAll("} catch", " } catch") + .replaceAll(";\\(param", "; (param") } - it should "handle case function as script field" in { + it should "handle case function as script field" in { // 40 val select: ElasticSearchRequest = SQLQuery(caseWhen) val query = select.query @@ -2102,7 +2113,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS); def param3 = param1 == null ? false : (param1.isAfter(param2)); def param4 = (!doc.containsKey('lastSeen') || doc['lastSeen'].empty ? null : doc['lastSeen'].value.plus(2, ChronoUnit.DAYS)); def param5 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); param3 ? param1 : param4 != null ? param4 : param5" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS); def param3 = param1 == null ? false : (param1.isAfter(param2)); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.plus(2, ChronoUnit.DAYS)); def param5 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); param3 ? param1 : param4 != null ? param4 : param5" | } | } | }, @@ -2155,7 +2166,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (!doc.containsKey('lastSeen') || doc['lastSeen'].empty ? null : doc['lastSeen'].value.toLocalDate().plus(2, ChronoUnit.DAYS)); def param4 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param1.isEqual(param2) ? param2 : param1 != null && param1.isEqual(param3) ? param3 : param4" + | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate().plus(2, ChronoUnit.DAYS)); def param4 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param1.isEqual(param2) ? param2 : param1 != null && param1.isEqual(param3) ? param3 : param4" | } | } | }, @@ -2210,91 +2221,91 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "dom": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" | } | }, | "dow": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_WEEK)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_WEEK)); param1" | } | }, | "doy": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" | } | }, | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" | } | }, | "minutes": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" | } | }, | "s": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" | } | }, | "nano": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" | } | }, | "micro": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" | } | }, | "milli": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" | } | }, | "epoch": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" | } | }, | "off": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" | } | } | }, @@ -2334,7 +2345,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().get(ChronoField.YEAR); (param1 == null) ? null : (param1 * (param2 - 10)) > 10000" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().get(ChronoField.YEAR); (param1 == null) ? null : (param1 * (param2 - 10)) > 10000" | } | } | } @@ -2345,37 +2356,37 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "add": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : (param1 + 1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : (param1 + 1)" | } | }, | "sub": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : (param1 - 1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : (param1 - 1)" | } | }, | "mul": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : (param1 * 2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : (param1 * 2)" | } | }, | "div": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : (param1 / 2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : (param1 / 2)" | } | }, | "mod": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : (param1 % 2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : (param1 % 2)" | } | }, | "identifier_mul_identifier2_minus_10": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); def param2 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); def lv0 = ((param1 == null || param2 == null) ? null : (param1 * param2)); (lv0 == null) ? null : (lv0 - 10)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); def lv0 = ((param1 == null || param2 == null) ? null : (param1 * param2)); (lv0 == null) ? null : (lv0 - 10)" | } | } | }, @@ -2423,7 +2434,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1) > 100.0" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1) > 100.0" | } | } | } @@ -2434,109 +2445,109 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "abs_identifier_plus_1_0_mul_2": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); ((param1 == null) ? null : Math.abs(param1) + 1.0) * ((double) 2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); ((param1 == null) ? null : Math.abs(param1) + 1.0) * ((double) 2)" | } | }, | "ceil_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.ceil(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.ceil(param1)" | } | }, | "floor_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.floor(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.floor(param1)" | } | }, | "sqrt_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1)" | } | }, | "exp_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.exp(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.exp(param1)" | } | }, | "log_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.log(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.log(param1)" | } | }, | "log10_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.log10(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.log10(param1)" | } | }, | "pow_identifier_3": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.pow(param1, 3)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.pow(param1, 3)" | } | }, | "round_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); def param2 = Math.pow(10, 0); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 0); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" | } | }, | "round_identifier_2": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); def param2 = Math.pow(10, 2); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 2); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" | } | }, | "sign_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : (param1 > 0 ? 1 : (param1 < 0 ? -1 : 0))" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : (param1 > 0 ? 1 : (param1 < 0 ? -1 : 0))" | } | }, | "cos_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.cos(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.cos(param1)" | } | }, | "acos_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.acos(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.acos(param1)" | } | }, | "sin_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.sin(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sin(param1)" | } | }, | "asin_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.asin(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.asin(param1)" | } | }, | "tan_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.tan(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.tan(param1)" | } | }, | "atan_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan(param1)" | } | }, | "atan2_identifier_3_0": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan2(param1, 3.0)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan2(param1, 3.0)" | } | } | }, @@ -2579,7 +2590,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("\\(double\\)(\\d)", "(double) $1") } - it should "handle string function as script field and condition" in { + it should "handle string function as script field and condition" in { // 45 val select: ElasticSearchRequest = SQLQuery(string) val query = select.query @@ -2593,7 +2604,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.trim().length() > 10" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.trim().length() > 10" | } | } | } @@ -2604,85 +2615,85 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "len": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.length()" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.length()" | } | }, | "low": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.toLowerCase()" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.toLowerCase()" | } | }, | "upp": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.toUpperCase()" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.toUpperCase()" | } | }, | "sub": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(3, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(3, param1.length()))" | } | }, | "tr": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.trim()" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.trim()" | } | }, | "ltr": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.replaceAll(\"^\\\\s+\",\"\")" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.replaceAll(\"^\\\\s+\",\"\")" | } | }, | "rtr": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.replaceAll(\"\\\\s+$\",\"\")" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.replaceAll(\"\\\\s+$\",\"\")" | } | }, | "con": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : String.valueOf(param1) + \"_test\" + String.valueOf(1)" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : String.valueOf(param1) + \"_test\" + String.valueOf(1)" | } | }, | "l": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(5, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(5, param1.length()))" | } | }, | "r": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(param1.length() - Math.min(3, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(param1.length() - Math.min(3, param1.length()))" | } | }, | "rep": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.replace(\"el\", \"le\")" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.replace(\"el\", \"le\")" | } | }, | "rev": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : new StringBuilder(param1).reverse().toString()" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : new StringBuilder(param1).reverse().toString()" | } | }, | "pos": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.indexOf(\"soft\", 0) + 1" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.indexOf(\"soft\", 0) + 1" | } | }, | "reg": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : java.util.regex.Pattern.compile(\"soft\", java.util.regex.Pattern.CASE_INSENSITIVE | java.util.regex.Pattern.MULTILINE).matcher(param1).find()" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : java.util.regex.Pattern.compile(\"soft\", java.util.regex.Pattern.CASE_INSENSITIVE | java.util.regex.Pattern.MULTILINE).matcher(param1).find()" | } | } | }, @@ -2746,19 +2757,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "script_fields": { - | "hire_date": { - | "script": { - | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('hire_date') || doc['hire_date'].empty ? null : doc['hire_date'].value.toLocalDate()); param1" - | } - | } - | }, - | "_source": true, + | "_source": false, | "aggs": { | "dept": { | "terms": { - | "field": "department.keyword" + | "field": "department", + | "min_doc_count": 1 | }, | "aggs": { | "cnt": { @@ -2884,7 +2888,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "ld": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.toLocalDate()); (param1 == null) ? null : param1.withDayOfMonth(param1.lengthOfMonth())" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); (param1 == null) ? null : param1.withDayOfMonth(param1.lengthOfMonth())" | } | } | }, @@ -2940,91 +2944,91 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" | } | }, | "wd": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (param1 == null) ? null : (param1.get(ChronoField.DAY_OF_WEEK) + 6) % 7" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : (param1.get(ChronoField.DAY_OF_WEEK) + 6) % 7" | } | }, | "yd": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" | } | }, | "d": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" | } | }, | "minutes": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" | } | }, | "s": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" | } | }, | "nano": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" | } | }, | "micro": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" | } | }, | "milli": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" | } | }, | "epoch": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" | } | }, | "off": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" | } | } | }, @@ -3080,7 +3084,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)) >= 4000000.0", + | "source": "(def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)) >= 4000000.0", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3104,7 +3108,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('fromLocation') || doc['fromLocation'].empty ? null : doc['fromLocation']); def arg1 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null || arg1 == null) ? null : arg0.arcDistance(arg1.lat, arg1.lon)) < 2000000.0" + | "source": "(def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); def arg1 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null || arg1 == null) ? null : arg0.arcDistance(arg1.lat, arg1.lon)) < 2000000.0" | } | } | }, @@ -3123,7 +3127,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "d1": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "source": "(def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3133,7 +3137,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "d2": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('fromLocation') || doc['fromLocation'].empty ? null : doc['fromLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "source": "(def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3185,7 +3189,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("lat,arg", "lat, arg") } - it should "handle between with temporal" in { + it should "handle between with temporal" in { // 50 val select: ElasticSearchRequest = SQLQuery(betweenTemporal) val query = select.query @@ -3210,7 +3214,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())) == false)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())) == false)" | } | } | }, @@ -3300,7 +3304,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('comments.replies.lastUpdated') || doc['comments.replies.lastUpdated'].empty ? null : doc['comments.replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-10\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" + | "source": "def param1 = (doc['comments.replies.lastUpdated'].size() == 0 ? null : doc['comments.replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-10\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" | } | } | } @@ -3398,7 +3402,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('replies.lastUpdated') || doc['replies.lastUpdated'].empty ? null : doc['replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-10\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" + | "source": "def param1 = (doc['replies.lastUpdated'].size() == 0 ? null : doc['replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-10\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" | } | } | }, @@ -3505,7 +3509,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); param1 == null ? false : (param1.isBefore(param2))" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); param1 == null ? false : (param1.isBefore(param2))" | } | } | } @@ -3603,7 +3607,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "avg_popularity": { | "avg": { @@ -3637,7 +3641,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "comments": { | "nested": { @@ -3698,7 +3702,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "comments": { | "nested": { diff --git a/build.sbt b/build.sbt index 38774a28..cdf10c17 100644 --- a/build.sbt +++ b/build.sbt @@ -19,7 +19,7 @@ ThisBuild / organization := "app.softnetwork" name := "softclient4es" -ThisBuild / version := "0.13.0" +ThisBuild / version := "0.13.1" ThisBuild / scalaVersion := scala213 diff --git a/core/src/main/scala/app/softnetwork/elastic/client/AggregateApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/AggregateApi.scala index fb4dddb6..caa3325c 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/AggregateApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/AggregateApi.scala @@ -130,59 +130,41 @@ trait SingleValueAggregateApi // Execute the search search(sqlQuery) .flatMap { response => - // Parse the response - val parseResult = ElasticResult.fromTry(parseResponse(response)) - - parseResult match { - // Case 1: Parse successful - process the results - case ElasticSuccess(results) => - val aggregationResults = results.flatMap { result => - response.aggregations.map { case (name, aggregation) => - // Attempt to process each aggregation - val aggregationResult = ElasticResult.attempt { - val value = findAggregation(name, result).orNull match { - case b: Boolean => BooleanValue(b) - case n: Number => NumericValue(n) - case s: String => StringValue(s) - case t: Temporal => TemporalValue(t) - case m: Map[_, Any] => ObjectValue(m.map(kv => kv._1.toString -> kv._2)) - case s: Seq[_] if aggregation.multivalued => - getAggregateValue(s, aggregation.distinct) - case _ => EmptyValue - } - - SingleValueAggregateResult(name, aggregation.aggType, value) - } - - // Convert failures to results with errors - aggregationResult match { - case ElasticSuccess(result) => result - case ElasticFailure(error) => - SingleValueAggregateResult( - name, - aggregation.aggType, - EmptyValue, - error = Some(s"Failed to process aggregation: ${error.message}") - ) - } - }.toSeq + val results = response.results + val aggregationResults = results.flatMap { result => + response.aggregations.map { case (name, aggregation) => + // Attempt to process each aggregation + val aggregationResult = ElasticResult.attempt { + val value = findAggregation(name, result).orNull match { + case b: Boolean => BooleanValue(b) + case n: Number => NumericValue(n) + case s: String => StringValue(s) + case t: Temporal => TemporalValue(t) + case m: Map[_, Any] => ObjectValue(m.map(kv => kv._1.toString -> kv._2)) + case s: Seq[_] if aggregation.multivalued => + getAggregateValue(s, aggregation.distinct) + case _ => EmptyValue + } + + SingleValueAggregateResult(name, aggregation.aggType, value) } - ElasticResult.success(aggregationResults) + // Convert failures to results with errors + aggregationResult match { + case ElasticSuccess(result) => result + case ElasticFailure(error) => + SingleValueAggregateResult( + name, + aggregation.aggType, + EmptyValue, + error = Some(s"Failed to process aggregation: ${error.message}") + ) + } + }.toSeq + } - // Case 2: Parse failed - returning empty results with errors - case ElasticFailure(error) => - val errorResults = response.aggregations.map { case (name, aggregation) => - SingleValueAggregateResult( - name, - aggregation.aggType, - EmptyValue, - error = Some(s"Parse error: ${error.message}") - ) - }.toSeq + ElasticResult.success(aggregationResults) - ElasticResult.success(errorResults) - } } .fold( // If search() fails, throw an exception diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala index c69b085a..785882a9 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala @@ -44,11 +44,11 @@ trait ElasticConversion { m: Manifest[T], formats: Formats ): Try[Seq[T]] = { - parseResponse(response).map { rows => - rows.map { row => + Try( + response.results.map { row => convertTo[T](row)(m, formats) } - } + ) } // Formatters for elasticsearch ISO 8601 date/time strings @@ -60,15 +60,17 @@ trait ElasticConversion { * multi-search (msearch/UNION ALL) responses */ def parseResponse( - response: ElasticResponse + results: String, + fieldAliases: Map[String, String], + aggregations: Map[String, ClientAggregation] ): Try[Seq[Map[String, Any]]] = { - val json = mapper.readTree(response.results) + val json = mapper.readTree(results) // Check if it's a multi-search response (array of responses) if (json.isArray) { - parseMultiSearchResponse(json, response.fieldAliases, response.aggregations) + parseMultiSearchResponse(json, fieldAliases, aggregations) } else { // Single search response - parseSingleSearchResponse(json, response.fieldAliases, response.aggregations) + parseSingleSearchResponse(json, fieldAliases, aggregations) } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala index 8db08707..5ea8bee5 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala @@ -19,6 +19,7 @@ package app.softnetwork.elastic.client import akka.NotUsed import akka.actor.ActorSystem import akka.stream.scaladsl.{Sink, Source} +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll.{ ScrollConfig, ScrollMetrics, @@ -28,11 +29,11 @@ import app.softnetwork.elastic.client.scroll.{ UseSearchAfter } import app.softnetwork.elastic.sql.macros.SQLQueryMacros -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLQuery} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLQuery, SQLSearchRequest} import org.json4s.{Formats, JNothing} import org.json4s.jackson.JsonMethods.parse -import scala.concurrent.{ExecutionContext, Promise} +import scala.concurrent.{ExecutionContext, Future, Promise} import scala.language.experimental.macros import scala.util.{Failure, Success} @@ -121,6 +122,14 @@ trait ScrollApi extends ElasticClientHelpers { )(implicit system: ActorSystem): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { sql.request match { case Some(Left(single)) => + if ( + single.windowFunctions.nonEmpty && (single.fields.nonEmpty || single.windowFunctions + .flatMap(_.fields) + .distinct + .size > 1) + ) + return scrollWithWindowEnrichment(sql, single, config) + val sqlRequest = single.copy(score = sql.score) val elasticQuery = ElasticQuery(sqlRequest, collection.immutable.Seq(sqlRequest.sources: _*)) @@ -365,4 +374,73 @@ trait ScrollApi extends ElasticClientHelpers { } } + // ======================================================================== + // WINDOW FUNCTION SEARCH + // ======================================================================== + + /** Scroll with window function enrichment + */ + private def scrollWithWindowEnrichment( + sql: SQLQuery, + request: SQLSearchRequest, + config: ScrollConfig + )(implicit system: ActorSystem): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { + + implicit val ec: ExecutionContext = system.dispatcher + + logger.info(s"🪟 Scrolling with ${request.windowFunctions.size} window functions") + + // Execute window aggregations first + val windowCacheFuture: Future[ElasticResult[WindowCache]] = + Future(executeWindowAggregations(request)) + + // Create base query without window functions + val baseQuery = createBaseQuery(sql, request) + + // Stream and enrich + Source + .futureSource( + windowCacheFuture.map { + case ElasticSuccess(cache) => + scrollWithMetrics( + ElasticQuery( + baseQuery, + collection.immutable.Seq(baseQuery.sources: _*) + ), + baseQuery.fieldAliases, + baseQuery.sqlAggregations, + config, + baseQuery.sorts.nonEmpty + ) + .map { case (doc, metrics) => + val enrichedDoc = enrichDocumentWithWindowValues(doc, cache, request) + (enrichedDoc, metrics) + } + + case ElasticFailure(error) => + logger.error(s"❌ Failed to compute window functions: ${error.message}") + if (config.failOnWindowError.getOrElse(false)) { + // Strict mode: propagate the error + Source.failed( + new RuntimeException(s"Window function computation failed: ${error.message}") + ) + } else { + // Fallback: return base results without enrichment + logger.warn("⚠️ Falling back to base results without window enrichment") + scrollWithMetrics( + ElasticQuery( + baseQuery, + collection.immutable.Seq(baseQuery.sources: _*) + ), + baseQuery.fieldAliases, + baseQuery.sqlAggregations, + config, + baseQuery.sorts.nonEmpty + ) + } + } + ) + .mapMaterializedValue(_ => NotUsed) + } + } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index e2d460af..79bc30b3 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -65,9 +65,18 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { case Some(Left(single)) => val elasticQuery = ElasticQuery( single, - collection.immutable.Seq(single.sources: _*) + collection.immutable.Seq(single.sources: _*), + sql = Some(sql.query) + ) + if ( + single.windowFunctions.nonEmpty && (single.fields.nonEmpty || single.windowFunctions + .flatMap(_.fields) + .distinct + .size > 1) ) - singleSearch(elasticQuery, single.fieldAliases, single.sqlAggregations) + searchWithWindowEnrichment(sql, single) + else + singleSearch(elasticQuery, single.fieldAliases, single.sqlAggregations) case Some(Right(multiple)) => val elasticQueries = ElasticQueries( @@ -76,17 +85,18 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { query, collection.immutable.Seq(query.sources: _*) ) - }.toList + }.toList, + sql = Some(sql.query) ) multiSearch(elasticQueries, multiple.fieldAliases, multiple.sqlAggregations) case None => logger.error( - s"❌ Failed to execute search for query '${sql.query}'" + s"❌ Failed to execute search for query \n${sql.query}" ) ElasticResult.failure( ElasticError( - message = s"SQL query does not contain a valid search request: ${sql.query}", + message = s"SQL query does not contain a valid search request\n${sql.query}", operation = Some("search") ) ) @@ -122,36 +132,60 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { case None => // continue } + val sql = elasticQuery.sql + val query = elasticQuery.query + val indices = elasticQuery.indices.mkString(",") + logger.debug( - s"Searching with query '${elasticQuery.query}' in indices '${elasticQuery.indices.mkString(",")}'" + s"🔍 Searching with query \n${sql.getOrElse(query)}\nin indices '$indices'" ) executeSingleSearch(elasticQuery) match { case ElasticSuccess(Some(response)) => logger.info( - s"✅ Successfully executed search in indices '${elasticQuery.indices.mkString(",")}'" - ) - ElasticResult.success( - ElasticResponse( - elasticQuery.query, - response, - fieldAliases, - aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) - ) + s"✅ Successfully executed search for query \n${sql.getOrElse(query)}\nin indices '$indices'" ) + val aggs = aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) + ElasticResult.fromTry(parseResponse(response, fieldAliases, aggs)) match { + case success @ ElasticSuccess(_) => + logger.info( + s"✅ Successfully parsed search results for query \n${sql.getOrElse(query)}\nin indices '$indices'" + ) + ElasticResult.success( + ElasticResponse( + sql, + query, + success.value, + fieldAliases, + aggs + ) + ) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to parse search results for query \n${sql + .getOrElse(query)}\nin indices '$indices' -> ${error.message}" + ) + ElasticResult.failure( + error.copy( + operation = Some("search"), + index = Some(elasticQuery.indices.mkString(",")) + ) + ) + } case ElasticSuccess(_) => val error = ElasticError( message = - s"Failed to execute search in indices '${elasticQuery.indices.mkString(",")}'", - index = Some(elasticQuery.indices.mkString(",")), + s"Failed to execute search for query \n${sql.getOrElse(query)}\nin indices '$indices'", + index = Some(indices), operation = Some("search") ) logger.error(s"❌ ${error.message}") ElasticResult.failure(error) case ElasticFailure(error) => logger.error( - s"❌ Failed to execute search in indices '${elasticQuery.indices.mkString(",")}': ${error.message}" + s"❌ Failed to execute search for query \n${sql + .getOrElse(query)}\nin indices '$indices' -> ${error.message}" ) ElasticResult.failure( error.copy( @@ -196,34 +230,56 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { ) } + val query = elasticQueries.queries.map(_.query).mkString("\n") + val sql = elasticQueries.sql.orElse( + Option(elasticQueries.queries.flatMap(_.sql).mkString("\nUNION ALL\n")) + ) + logger.debug( - s"Multi-searching with ${elasticQueries.queries.size} queries" + s"🔍 Multi-searching with query \n${sql.getOrElse(query)}" ) executeMultiSearch(elasticQueries) match { case ElasticSuccess(Some(response)) => logger.info( - s"✅ Successfully executed multi-search with ${elasticQueries.queries.size} queries" - ) - ElasticResult.success( - ElasticResponse( - elasticQueries.queries.map(_.query).mkString("\n"), - response, - fieldAliases, - aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) - ) + s"✅ Successfully executed multi-search for query \n${sql.getOrElse(query)}" ) + val aggs = aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) + ElasticResult.fromTry(parseResponse(response, fieldAliases, aggs)) match { + case success @ ElasticSuccess(_) => + logger.info( + s"✅ Successfully parsed multi-search results for query '${sql.getOrElse(query)}'" + ) + ElasticResult.success( + ElasticResponse( + sql, + query, + success.value, + fieldAliases, + aggs + ) + ) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to parse multi-search results for query \n${sql.getOrElse(query)}\n -> ${error.message}" + ) + ElasticResult.failure( + error.copy( + operation = Some("multiSearch") + ) + ) + } case ElasticSuccess(_) => val error = ElasticError( - message = s"Failed to execute multi-search with ${elasticQueries.queries.size} queries", + message = s"Failed to execute multi-search for query \n${sql.getOrElse(query)}", operation = Some("multiSearch") ) logger.error(s"❌ ${error.message}") ElasticResult.failure(error) case ElasticFailure(error) => logger.error( - s"❌ Failed to execute multi-search with ${elasticQueries.queries.size} queries: ${error.message}" + s"❌ Failed to execute multi-search for query \n${sql.getOrElse(query)}\n -> ${error.message}" ) ElasticResult.failure( error.copy( @@ -301,25 +357,50 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { )(implicit ec: ExecutionContext ): Future[ElasticResult[ElasticResponse]] = { + val sql = elasticQuery.sql + val query = elasticQuery.query + val indices = elasticQuery.indices.mkString(",") executeSingleSearchAsync(elasticQuery).flatMap { case ElasticSuccess(Some(response)) => logger.info( - s"✅ Successfully executed asynchronous search for query '${elasticQuery.query}'" + s"✅ Successfully executed asynchronous search for query \n${sql.getOrElse(query)}\nin indices '$indices'" ) - Future.successful( - ElasticResult.success( - ElasticResponse( - elasticQuery.query, - response, - fieldAliases, - aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) + val aggs = aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) + ElasticResult.fromTry(parseResponse(response, fieldAliases, aggs)) match { + case success @ ElasticSuccess(_) => + logger.info( + s"✅ Successfully parsed search results for query \n${sql.getOrElse(query)}\nin indices '$indices'" ) - ) - ) + Future.successful( + ElasticResult.success( + ElasticResponse( + sql, + query, + success.value, + fieldAliases, + aggs + ) + ) + ) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to parse search results for query \n${sql + .getOrElse(query)}\nin indices '$indices' -> ${error.message}" + ) + Future.successful( + ElasticResult.failure( + error.copy( + operation = Some("searchAsync"), + index = Some(indices) + ) + ) + ) + } case ElasticSuccess(_) => val error = ElasticError( - message = s"Failed to execute asynchronous search for query '${elasticQuery.query}'", + message = + s"Failed to execute asynchronous search for query \n${sql.getOrElse(query)}\nin indices '$indices'", index = Some(elasticQuery.indices.mkString(",")), operation = Some("searchAsync") ) @@ -327,7 +408,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { Future.successful(ElasticResult.failure(error)) case ElasticFailure(error) => logger.error( - s"❌ Failed to execute asynchronous search for query '${elasticQuery.query}': ${error.message}" + s"❌ Failed to execute asynchronous search for query \n${sql + .getOrElse(query)}\nin indices '$indices' -> ${error.message}" ) Future.successful( ElasticResult.failure( @@ -358,33 +440,57 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { )(implicit ec: ExecutionContext ): Future[ElasticResult[ElasticResponse]] = { + val query = elasticQueries.queries.map(_.query).mkString("\n") + val sql = elasticQueries.sql.orElse( + Option(elasticQueries.queries.flatMap(_.sql).mkString("\nUNION ALL\n")) + ) + executeMultiSearchAsync(elasticQueries).flatMap { case ElasticSuccess(Some(response)) => logger.info( - s"✅ Successfully executed asynchronous multi-search with ${elasticQueries.queries.size} queries" + s"✅ Successfully executed asynchronous multi-search for query \n${sql.getOrElse(query)}" ) - Future.successful( - ElasticResult.success( - ElasticResponse( - elasticQueries.queries.map(_.query).mkString("\n"), - response, - fieldAliases, - aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) + val aggs = aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) + ElasticResult.fromTry(parseResponse(response, fieldAliases, aggs)) match { + case success @ ElasticSuccess(_) => + logger.info( + s"✅ Successfully parsed multi-search results for query '${sql.getOrElse(query)}'" ) - ) - ) + Future.successful( + ElasticResult.success( + ElasticResponse( + sql, + query, + success.value, + fieldAliases, + aggs + ) + ) + ) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to parse multi-search results for query \n${sql.getOrElse(query)}\n -> ${error.message}" + ) + Future.successful( + ElasticResult.failure( + error.copy( + operation = Some("multiSearchAsync") + ) + ) + ) + } case ElasticSuccess(_) => val error = ElasticError( message = - s"Failed to execute asynchronous multi-search with ${elasticQueries.queries.size} queries", + s"Failed to execute asynchronous multi-search for query \n${sql.getOrElse(query)}", operation = Some("multiSearchAsync") ) logger.error(s"❌ ${error.message}") Future.successful(ElasticResult.failure(error)) case ElasticFailure(error) => logger.error( - s"❌ Failed to execute asynchronous multi-search with ${elasticQueries.queries.size} queries: ${error.message}" + s"❌ Failed to execute asynchronous multi-search for query \n${sql.getOrElse(query)}\n -> ${error.message}" ) Future.successful( ElasticResult.failure( @@ -985,4 +1091,327 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { ) } + // ======================================================================== + // WINDOW FUNCTION SEARCH + // ======================================================================== + + /** Search with window function enrichment + * + * Strategy: + * 1. Execute aggregation query to compute window values 2. Execute main query (without window + * functions) 3. Enrich results with window values + */ + private def searchWithWindowEnrichment( + sql: SQLQuery, + request: SQLSearchRequest + ): ElasticResult[ElasticResponse] = { + + logger.info(s"🪟 Detected ${request.windowFunctions.size} window functions") + + for { + // Step 1: Execute window aggregations + windowCache <- executeWindowAggregations(request) + + // Step 2: Execute base query (without window functions) + baseResponse <- executeBaseQuery(sql, request) + + // Step 3: Enrich results + enrichedResponse <- enrichResponseWithWindowValues(baseResponse, windowCache, request) + + } yield enrichedResponse + } + + // ======================================================================== + // WINDOW AGGREGATION EXECUTION + // ======================================================================== + + /** Execute aggregation queries for all window functions Returns a cache of partition key -> + * window values + */ + protected def executeWindowAggregations( + request: SQLSearchRequest + ): ElasticResult[WindowCache] = { + + // Build aggregation request + val aggRequest = buildWindowAggregationRequest(request) + val sql = aggRequest.sql + + logger.info( + s"🔍 Executing window aggregation query:\n$sql" + ) + + // Execute aggregation using existing search infrastructure + val elasticQuery = ElasticQuery( + aggRequest, + collection.immutable.Seq(aggRequest.sources: _*), + sql = Some(sql) + ) + + for { + // Use singleSearch to execute aggregation + aggResponse <- singleSearch( + elasticQuery, + aggRequest.fieldAliases, + aggRequest.sqlAggregations + ) + + // Parse aggregation results into cache + cache <- parseWindowAggregationsToCache(aggResponse, request) + + } yield cache + } + + /** Build aggregation request for window functions + */ + private def buildWindowAggregationRequest( + request: SQLSearchRequest + ): SQLSearchRequest = { + + // Create modified request with: + // - Only window buckets in GROUP BY + // - Only window aggregations in SELECT + // - No LIMIT (need all partitions) + // - Same WHERE clause (to match base query filtering) + request + .copy( + select = request.select.copy(fields = request.windowFields), + groupBy = request.groupBy.map(_.copy(buckets = request.windowBuckets)), + orderBy = None, // Not needed for aggregations + limit = None // Need all buckets + ) + .update() + } + + /** Parse aggregation response into window cache Uses your existing + * ElasticConversion.parseResponse + */ + private def parseWindowAggregationsToCache( + response: ElasticResponse, + request: SQLSearchRequest + ): ElasticResult[WindowCache] = { + + logger.info( + s"🔍 Parsing window aggregations to cache for query \n${response.sql.getOrElse(response.query)}" + ) + + val aggRows = response.results + + logger.info(s"✅ Parsed ${aggRows.size} aggregation buckets") + + // Build cache: partition key -> window values + val cache = aggRows.map { row => + val partitionKey = extractPartitionKey(row, request) + val windowValues = extractWindowValues(row, response.aggregations) + + partitionKey -> windowValues + }.toMap + + ElasticResult.success(WindowCache(cache)) + } + + // ======================================================================== + // BASE QUERY EXECUTION + // ======================================================================== + + /** Execute base query without window functions + */ + private def executeBaseQuery( + sql: SQLQuery, + request: SQLSearchRequest + ): ElasticResult[ElasticResponse] = { + + val baseQuery = createBaseQuery(sql, request) + + logger.info(s"🔍 Executing base query without window functions ${baseQuery.sql}") + + singleSearch( + ElasticQuery( + baseQuery, + collection.immutable.Seq(baseQuery.sources: _*), + sql = Some(baseQuery.sql) + ), + baseQuery.fieldAliases, + baseQuery.sqlAggregations + ) + } + + /** Create base query by removing window functions from SELECT + */ + protected def createBaseQuery( + sql: SQLQuery, + request: SQLSearchRequest + ): SQLSearchRequest = { + + // Remove window function fields from SELECT + val baseFields = request.select.fields.filterNot(_.windows.nonEmpty) + + // Create modified request + val baseRequest = request + .copy( + select = request.select.copy(fields = baseFields) + ) + .copy(score = sql.score) + .update() + + baseRequest + } + + /** Extract partition key from aggregation row + */ + private def extractPartitionKey( + row: Map[String, Any], + request: SQLSearchRequest + ): PartitionKey = { + + // Get all partition fields from window functions + val partitionFields = request.windowFunctions + .flatMap(_.partitionBy) + .map(_.aliasOrName) + .distinct + + if (partitionFields.isEmpty) { + return PartitionKey(Map("__global__" -> true)) + } + + val keyValues = partitionFields.flatMap { field => + row.get(field).map(field -> _) + }.toMap + + PartitionKey(keyValues) + } + + /** Extract window function values from aggregation row + */ + private def extractWindowValues( + row: Map[String, Any], + aggregations: Map[String, ClientAggregation] + ): WindowValues = { + + val values = aggregations + .filter(_._2.window) + .map { wf => + val fieldName = wf._1 + + val aggType = wf._2.aggType + + val sourceField = wf._2.sourceField + + // Get value from row (already processed by ElasticConversion) + val value = row.get(fieldName).orElse { + logger.warn(s"⚠️ Window function '$fieldName' not found in aggregation result") + None + } + + val validatedValue = + value match { + case Some(m: Map[String, Any]) => + m.get(sourceField) match { + case Some(v) => + aggType match { + case AggregationType.ArrayAgg => + v match { + case l: List[_] => + Some(l) + case other => + logger.warn( + s"⚠️ Expected List for ARRAY_AGG '$fieldName', got ${other.getClass.getSimpleName}" + ) + Some(List(other)) // Wrap into a List + } + case _ => Some(v) + } + case None => + None + } + case other => + other + } + + fieldName -> validatedValue + } + .collect { case (name, Some(value)) => + name -> value + } + + WindowValues(values) + } + + // ======================================================================== + // RESULT ENRICHMENT + // ======================================================================== + + /** Enrich response with window values + */ + private def enrichResponseWithWindowValues( + response: ElasticResponse, + cache: WindowCache, + request: SQLSearchRequest + ): ElasticResult[ElasticResponse] = { + + val baseRows = response.results + // Enrich each row + val enrichedRows = baseRows.map { row => + enrichDocumentWithWindowValues(row, cache, request) + } + + ElasticResult.success(response.copy(results = enrichedRows)) + } + + /** Enrich a single document with window values + */ + protected def enrichDocumentWithWindowValues( + doc: Map[String, Any], + cache: WindowCache, + request: SQLSearchRequest + ): Map[String, Any] = { + + if (request.windowFunctions.isEmpty) { + return doc + } + + // Build partition key from document + val partitionKey = extractPartitionKey(doc, request) + + // Lookup window values + cache.get(partitionKey) match { + case Some(windowValues) => + // Merge document with window values + doc ++ windowValues.values + + case None => + logger.warn(s"⚠️ No window values found for partition: ${partitionKey.values}") + + // Add null values for missing window functions + val nullValues = request.windowFunctions.map { wf => + wf.identifier.aliasOrName -> null + }.toMap + + doc ++ nullValues + } + } + + // ======================================================================== + // HELPER CASE CLASSES + // ======================================================================== + + /** Partition key for window function cache + */ + protected case class PartitionKey(values: Map[String, Any]) { + override def hashCode(): Int = values.hashCode() + override def equals(obj: Any): Boolean = obj match { + case other: PartitionKey => values == other.values + case _ => false + } + } + + /** Window function values for a partition + */ + protected case class WindowValues(values: Map[String, Any]) + + /** Cache of partition key -> window values + */ + protected case class WindowCache(cache: Map[PartitionKey, WindowValues]) { + def get(key: PartitionKey): Option[WindowValues] = cache.get(key) + def size: Int = cache.size + } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/package.scala index 412ca2ac..4eb2d5ef 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/package.scala @@ -34,23 +34,22 @@ package object client extends SerializationApi { */ type JSONQuery = String - /** Type alias for JSON results - */ - type JSONResults = String - /** Elastic response case class + * @param sql + * - the SQL query if any * @param query * - the JSON query * @param results - * - the JSON results + * - the results as a sequence of rows * @param fieldAliases * - the field aliases used * @param aggregations * - the aggregations expected */ case class ElasticResponse( + sql: Option[String] = None, query: JSONQuery, - results: JSONResults, + results: Seq[Map[String, Any]], fieldAliases: Map[String, String], aggregations: Map[String, ClientAggregation] ) @@ -69,9 +68,14 @@ package object client extends SerializationApi { * @param types * - the target types @deprecated types are deprecated in ES 7+ */ - case class ElasticQuery(query: JSONQuery, indices: Seq[String], types: Seq[String] = Seq.empty) + case class ElasticQuery( + query: JSONQuery, + indices: Seq[String], + types: Seq[String] = Seq.empty, + sql: Option[String] = None + ) - case class ElasticQueries(queries: List[ElasticQuery]) + case class ElasticQueries(queries: List[ElasticQuery], sql: Option[String] = None) /** Retry configuration */ @@ -137,7 +141,9 @@ package object client extends SerializationApi { case class ClientAggregation( aggName: String, aggType: AggregationType.AggregationType, - distinct: Boolean + distinct: Boolean, + sourceField: String, + window: Boolean ) { def multivalued: Boolean = aggType == AggregationType.ArrayAgg def singleValued: Boolean = !multivalued @@ -155,6 +161,12 @@ package object client extends SerializationApi { case _: ArrayAgg => AggregationType.ArrayAgg case _ => throw new IllegalArgumentException(s"Unsupported aggregation type: ${agg.aggType}") } - ClientAggregation(agg.aggName, aggType, agg.distinct) + ClientAggregation( + agg.aggName, + aggType, + agg.distinct, + agg.sourceField, + agg.aggType.isInstanceOf[WindowFunction] + ) } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/scroll/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/scroll/package.scala index 36640ea0..72e2989b 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/scroll/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/scroll/package.scala @@ -27,7 +27,8 @@ package object scroll { maxDocuments: Option[Long] = None, // Optional maximum number of documents to retrieve preferSearchAfter: Boolean = true, // Prefer search_after over scroll when possible metrics: ScrollMetrics = ScrollMetrics(), // Initial scroll metrics - retryConfig: RetryConfig = RetryConfig() // Retry configuration + retryConfig: RetryConfig = RetryConfig(), // Retry configuration + failOnWindowError: Option[Boolean] = None ) /** Scroll strategy based on query type diff --git a/core/src/test/scala/app/softnetwork/elastic/client/ElasticConversionSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/ElasticConversionSpec.scala index 02a43f0d..620a403c 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/ElasticConversionSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/ElasticConversionSpec.scala @@ -51,7 +51,7 @@ class ElasticConversionSpec extends AnyFlatSpec with Matchers with ElasticConver | } |}""".stripMargin - parseResponse(ElasticResponse("", results, Map.empty, Map.empty)) match { + parseResponse(results, Map.empty, Map.empty) match { case Success(rows) => rows.foreach(println) // Map(name -> Laptop, price -> 999.99, category -> Electronics, tags -> List(computer, portable), _id -> 1, _index -> products, _score -> 1.0) @@ -86,7 +86,9 @@ class ElasticConversionSpec extends AnyFlatSpec with Matchers with ElasticConver | } |}""".stripMargin parseResponse( - ElasticResponse("", results, Map.empty, Map.empty) + results, + Map.empty, + Map.empty ) match { case Success(rows) => rows.foreach(println) @@ -180,16 +182,15 @@ class ElasticConversionSpec extends AnyFlatSpec with Matchers with ElasticConver |}""".stripMargin parseResponse( - ElasticResponse( - "", - results, - Map.empty, - Map( - "top_products" -> ClientAggregation( - "top_products", - aggType = AggregationType.ArrayAgg, - distinct = false - ) + results, + Map.empty, + Map( + "top_products" -> ClientAggregation( + "top_products", + aggType = AggregationType.ArrayAgg, + distinct = false, + "name", + window = true ) ) ) match { @@ -287,7 +288,7 @@ class ElasticConversionSpec extends AnyFlatSpec with Matchers with ElasticConver | } | } |}""".stripMargin - parseResponse(ElasticResponse("", results, Map.empty, Map.empty)) match { + parseResponse(results, Map.empty, Map.empty) match { case Success(rows) => rows.foreach(println) // Map(country -> France, country_doc_count -> 100, city -> Paris, city_doc_count -> 60, product -> Laptop, product_doc_count -> 30, total_sales -> 29997.0, avg_price -> 999.9) @@ -334,7 +335,7 @@ class ElasticConversionSpec extends AnyFlatSpec with Matchers with ElasticConver | } | } |}""".stripMargin - parseResponse(ElasticResponse("", results, Map.empty, Map.empty)) match { + parseResponse(results, Map.empty, Map.empty) match { case Success(rows) => rows.foreach(println) // Map(date -> 2024-01-01T00:00:00.000Z, doc_count -> 100, total_sales -> 50000.0) @@ -634,16 +635,15 @@ class ElasticConversionSpec extends AnyFlatSpec with Matchers with ElasticConver |}""".stripMargin parseResponse( - ElasticResponse( - "", - results, - Map.empty, - Map( - "employees" -> ClientAggregation( - aggName = "employees", - aggType = AggregationType.ArrayAgg, - distinct = false - ) + results, + Map.empty, + Map( + "employees" -> ClientAggregation( + aggName = "employees", + aggType = AggregationType.ArrayAgg, + distinct = false, + "name", + window = true ) ) ) match { diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index be7d57ed..b2bc13bb 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -17,6 +17,7 @@ package app.softnetwork.elastic.sql.bridge import app.softnetwork.elastic.sql.PainlessContext +import app.softnetwork.elastic.sql.`type`.SQLTemporal import app.softnetwork.elastic.sql.query.{ Asc, Bucket, @@ -31,6 +32,8 @@ import app.softnetwork.elastic.sql.query.{ } import app.softnetwork.elastic.sql.function._ import app.softnetwork.elastic.sql.function.aggregate._ +import app.softnetwork.elastic.sql.function.time.DateTrunc +import app.softnetwork.elastic.sql.time.TimeUnit import com.sksamuel.elastic4s.ElasticApi.{ avgAgg, bucketSelectorAggregation, @@ -44,11 +47,14 @@ import com.sksamuel.elastic4s.ElasticApi.{ valueCountAgg } import com.sksamuel.elastic4s.script.Script +import com.sksamuel.elastic4s.searches.DateHistogramInterval import com.sksamuel.elastic4s.searches.aggs.{ Aggregation, CardinalityAggregation, + DateHistogramAggregation, ExtendedStatsAggregation, FilterAggregation, + HistogramOrder, NestedAggregation, StatsAggregation, TermsAggregation, @@ -93,7 +99,10 @@ object ElasticAggregation { import sqlAgg._ val sourceField = identifier.path - val direction = bucketsDirection.get(identifier.identifierName) + val direction = + bucketsDirection + .get(identifier.identifierName) + .orElse(bucketsDirection.get(identifier.aliasOrName)) val field = fieldAlias match { case Some(alias) => alias.alias @@ -113,8 +122,8 @@ object ElasticAggregation { s"${aggType}_distinct_${sourceField.replace(".", "_")}" else { aggType match { - case th: TopHitsAggregation => - s"${th.topHits.sql.toLowerCase}_${sourceField.replace(".", "_")}" + case th: WindowFunction => + s"${th.window.sql.toLowerCase}_${sourceField.replace(".", "_")}" case _ => s"${aggType}_${sourceField.replace(".", "_")}" @@ -145,16 +154,21 @@ object ElasticAggregation { val _agg = aggType match { case COUNT => + val field = + sourceField match { + case "*" | "_id" | "_index" | "_type" => "_index" + case _ => sourceField + } if (distinct) - cardinalityAgg(aggName, sourceField) + cardinalityAgg(aggName, field) else { - valueCountAgg(aggName, sourceField) + valueCountAgg(aggName, field) } case MIN => aggWithFieldOrScript(minAgg, (name, s) => minAgg(name, sourceField).script(s)) case MAX => aggWithFieldOrScript(maxAgg, (name, s) => maxAgg(name, sourceField).script(s)) case AVG => aggWithFieldOrScript(avgAgg, (name, s) => avgAgg(name, sourceField).script(s)) case SUM => aggWithFieldOrScript(sumAgg, (name, s) => sumAgg(name, sourceField).script(s)) - case th: TopHitsAggregation => + case th: WindowFunction => val limit = { th match { case _: LastValue => 1 @@ -167,27 +181,31 @@ object ElasticAggregation { .fetchSource( th.identifier.name +: th.fields .filterNot(_.isScriptField) + .filterNot(_.sourceField == th.identifier.name) .map(_.sourceField) + .distinct .toArray, Array.empty ) .copy( scripts = th.fields .filter(_.isScriptField) + .groupBy(_.sourceField) + .map(_._2.head) .map(f => f.sourceField -> Script(f.painless(None)).lang("painless")) .toMap ) .size(limit) sortBy th.orderBy.sorts.map(sort => sort.order match { case Some(Desc) => - th.topHits match { - case LAST_VALUE => FieldSort(sort.field).asc() - case _ => FieldSort(sort.field).desc() + th.window match { + case LAST_VALUE => FieldSort(sort.field.aliasOrName).asc() + case _ => FieldSort(sort.field.aliasOrName).desc() } case _ => - th.topHits match { - case LAST_VALUE => FieldSort(sort.field).desc() - case _ => FieldSort(sort.field).asc() + th.window match { + case LAST_VALUE => FieldSort(sort.field.aliasOrName).desc() + case _ => FieldSort(sort.field.aliasOrName).asc() } } ) @@ -260,82 +278,193 @@ object ElasticAggregation { having: Option[Criteria], nested: Option[NestedElement], allElasticAggregations: Seq[ElasticAggregation] - ): Option[TermsAggregation] = { - buckets.reverse.foldLeft(Option.empty[TermsAggregation]) { (current, bucket) => + ): Option[Aggregation] = { + buckets.reverse.foldLeft(Option.empty[Aggregation]) { (current, bucket) => // Determine the bucketPath of the current bucket val currentBucketPath = bucket.identifier.path - var agg = { - bucketsDirection.get(bucket.identifier.identifierName) match { - case Some(direction) => - termsAgg(bucket.name, s"$currentBucketPath.keyword") - .order(Seq(direction match { - case Asc => TermsOrder("_key", asc = true) - case _ => TermsOrder("_key", asc = false) - })) - case None => - termsAgg(bucket.name, s"$currentBucketPath.keyword") + val aggScript = + if (bucket.shouldBeScripted) { + val context = PainlessContext() + val painless = bucket.painless(Some(context)) + Some(Script(s"$context$painless").lang("painless")) + } else { + None + } + + var agg: Aggregation = { + bucket.out match { + case _: SQLTemporal => + val functions = bucket.identifier.functions + val interval: Option[DateHistogramInterval] = + if (functions.size == 1) { + functions.head match { + case trunc: DateTrunc => + trunc.unit match { + case TimeUnit.YEARS => Option(DateHistogramInterval.Year) + case TimeUnit.QUARTERS => Option(DateHistogramInterval.Quarter) + case TimeUnit.MONTHS => Option(DateHistogramInterval.Month) + case TimeUnit.WEEKS => Option(DateHistogramInterval.Week) + case TimeUnit.DAYS => Option(DateHistogramInterval.Day) + case TimeUnit.HOURS => Option(DateHistogramInterval.Hour) + case TimeUnit.MINUTES => Option(DateHistogramInterval.Minute) + case TimeUnit.SECONDS => Option(DateHistogramInterval.Second) + case _ => None + } + case _ => None + } + } else { + None + } + + aggScript match { + case Some(script) => + // Scripted date histogram + bucketsDirection.get(bucket.identifier.identifierName) match { + case Some(direction) => + DateHistogramAggregation(bucket.name, interval = interval) + .script(script) + .minDocCount(1) + .order(direction match { + case Asc => HistogramOrder("_key", asc = true) + case _ => HistogramOrder("_key", asc = false) + }) + case _ => + DateHistogramAggregation(bucket.name, interval = interval) + .script(script) + .minDocCount(1) + } + case _ => + // Standard date histogram + bucketsDirection.get(bucket.identifier.identifierName) match { + case Some(direction) => + DateHistogramAggregation(bucket.name, interval = interval) + .field(currentBucketPath) + .minDocCount(1) + .order(direction match { + case Asc => HistogramOrder("_key", asc = true) + case _ => HistogramOrder("_key", asc = false) + }) + case _ => + DateHistogramAggregation(bucket.name, interval = interval) + .field(currentBucketPath) + .minDocCount(1) + } + } + + case _ => + aggScript match { + case Some(script) => + // Scripted terms aggregation + bucketsDirection.get(bucket.identifier.identifierName) match { + case Some(direction) => + TermsAggregation(bucket.name) + .script(script) + .minDocCount(1) + .order(Seq(direction match { + case Asc => TermsOrder("_key", asc = true) + case _ => TermsOrder("_key", asc = false) + })) + case _ => + TermsAggregation(bucket.name) + .script(script) + .minDocCount(1) + } + case _ => + // Standard terms aggregation + bucketsDirection.get(bucket.identifier.identifierName) match { + case Some(direction) => + termsAgg(bucket.name, currentBucketPath) + .minDocCount(1) + .order(Seq(direction match { + case Asc => TermsOrder("_key", asc = true) + case _ => TermsOrder("_key", asc = false) + })) + case _ => + termsAgg(bucket.name, currentBucketPath) + .minDocCount(1) + } + } } } - bucket.size.foreach(s => agg = agg.size(s)) - having match { - case Some(criteria) => - criteria.includes(bucket, not = false, BucketIncludesExcludes()) match { - case BucketIncludesExcludes(_, Some(regex)) if regex.nonEmpty => - agg = agg.include(regex) - case BucketIncludesExcludes(values, _) if values.nonEmpty => - agg = agg.include(values.toArray) - case _ => - } - criteria.excludes(bucket, not = false, BucketIncludesExcludes()) match { - case BucketIncludesExcludes(_, Some(regex)) if regex.nonEmpty => - agg = agg.exclude(regex) - case BucketIncludesExcludes(values, _) if values.nonEmpty => - agg = agg.exclude(values.toArray) + agg match { + case termsAgg: TermsAggregation => + bucket.size.foreach(s => agg = termsAgg.size(s)) + having match { + case Some(criteria) => + criteria.includes(bucket, not = false, BucketIncludesExcludes()) match { + case BucketIncludesExcludes(_, Some(regex)) if regex.nonEmpty => + agg = termsAgg.include(regex) + case BucketIncludesExcludes(values, _) if values.nonEmpty => + agg = termsAgg.include(values.toArray) + case _ => + } + criteria.excludes(bucket, not = false, BucketIncludesExcludes()) match { + case BucketIncludesExcludes(_, Some(regex)) if regex.nonEmpty => + agg = termsAgg.exclude(regex) + case BucketIncludesExcludes(values, _) if values.nonEmpty => + agg = termsAgg.exclude(values.toArray) + case _ => + } case _ => } case _ => } current match { - case Some(subAgg) => Some(agg.copy(subaggs = Seq(subAgg))) + case Some(subAgg) => + agg match { + case termsAgg: TermsAggregation => + agg = termsAgg.subaggs(Seq(subAgg)) + case dateHistogramAgg: DateHistogramAggregation => + agg = dateHistogramAgg.subaggs(Seq(subAgg)) + case _ => + } + Some(agg) case None => - val aggregationsWithOrder: Seq[TermsOrder] = aggregationsDirection.toSeq.map { kv => - kv._2 match { - case Asc => TermsOrder(kv._1, asc = true) - case _ => TermsOrder(kv._1, asc = false) + val subaggs = + having match { + case Some(criteria) => + val script = metricSelectorForBucket( + criteria, + nested, + allElasticAggregations + ) + + if (script.nonEmpty) { + val bucketSelector = + bucketSelectorAggregation( + "having_filter", + Script(script), + extractMetricsPathForBucket( + criteria, + nested, + allElasticAggregations + ) + ) + aggregations :+ bucketSelector + } else { + aggregations + } + case None => + aggregations } - } - val withAggregationOrders = - if (aggregationsWithOrder.nonEmpty) - agg.order(aggregationsWithOrder) - else - agg - val withHaving = having match { - case Some(criteria) => - val script = metricSelectorForBucket( - criteria, - nested, - allElasticAggregations - ) - if (script.nonEmpty) { - val bucketSelector = - bucketSelectorAggregation( - "having_filter", - Script(script), - extractMetricsPathForBucket( - criteria, - nested, - allElasticAggregations - ) - ) - withAggregationOrders.copy(subaggs = aggregations :+ bucketSelector) - } else { - withAggregationOrders.copy(subaggs = aggregations) + agg match { + case termsAgg: TermsAggregation => + val aggregationsWithOrder: Seq[TermsOrder] = aggregationsDirection.toSeq.map { kv => + kv._2 match { + case Asc => TermsOrder(kv._1, asc = true) + case _ => TermsOrder(kv._1, asc = false) + } } - case None => withAggregationOrders.copy(subaggs = aggregations) + if (aggregationsWithOrder.nonEmpty) + agg = termsAgg.order(aggregationsWithOrder).copy(subaggs = subaggs) + else + agg = termsAgg.copy(subaggs = subaggs) + case dateHistogramAggregation: DateHistogramAggregation => + agg = dateHistogramAggregation.copy(subaggs = subaggs) } - Some(withHaving) + Some(agg) } } } diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index ba3d7dae..c0a30662 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -16,7 +16,13 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLDouble, SQLTemporal, SQLVarchar} +import app.softnetwork.elastic.sql.`type`.{ + SQLBigInt, + SQLDouble, + SQLNumeric, + SQLTemporal, + SQLVarchar +} import app.softnetwork.elastic.sql.function.aggregate.COUNT import app.softnetwork.elastic.sql.function.geo.{Distance, Meters} import app.softnetwork.elastic.sql.operator._ @@ -35,7 +41,7 @@ import com.sksamuel.elastic4s.searches.aggs.{ } import com.sksamuel.elastic4s.searches.queries.{BoolQuery, InnerHit, Query} import com.sksamuel.elastic4s.searches.{MultiSearchRequest, SearchRequest} -import com.sksamuel.elastic4s.searches.sort.FieldSort +import com.sksamuel.elastic4s.searches.sort.{FieldSort, ScriptSort, ScriptSortType} import scala.language.implicitConversions @@ -404,6 +410,7 @@ package object bridge { request.aggregates.map( ElasticAggregation(_, request.having.flatMap(_.criteria), request.sorts) ) + // request.orderBy.map(_.sorts).getOrElse(Seq.empty) ).minScore(request.score) implicit def requestToSearchRequest(request: SQLSearchRequest): SearchRequest = { @@ -453,7 +460,7 @@ package object bridge { _search } - _search = scriptFields.filterNot(_.aggregation) match { + _search = scriptFields.filterNot(_.isAggregation) match { case Nil => _search case _ => _search scriptfields scriptFields.map { field => @@ -474,17 +481,55 @@ package object bridge { _search = orderBy match { case Some(o) if aggregates.isEmpty && buckets.isEmpty => - _search sortBy o.sorts.map(sort => - sort.order match { - case Some(Desc) => FieldSort(sort.field).desc() - case _ => FieldSort(sort.field).asc() + _search sortBy o.sorts.map { sort => + if (sort.isScriptSort) { + val context = PainlessContext() + val painless = sort.field.painless(Some(context)) + val painlessScript = s"$context$painless" + val script = + sort.out match { + case _: SQLTemporal if !painless.endsWith("toEpochMilli()") => + val parts = painlessScript.split(";").toSeq + if (parts.size > 1) { + val lastPart = parts.last.trim.stripPrefix("return ") + if (lastPart.split(" ").toSeq.size == 1) { + val newLastPart = + s"""($lastPart != null) ? $lastPart.toInstant().toEpochMilli() : null""" + s"${parts.dropRight(1).mkString(";")}; return $newLastPart" + } else { + painlessScript + } + } else { + s"$painlessScript.toInstant().toEpochMilli()" + } + case _ => painlessScript + } + val scriptSort = + ScriptSort( + script = Script(script = script) + .lang("painless") + .scriptType(Source), + scriptSortType = sort.field.out match { + case _: SQLTemporal | _: SQLNumeric => ScriptSortType.Number + case _ => ScriptSortType.String + } + ) + sort.order match { + case Some(Desc) => scriptSort.desc() + case _ => scriptSort.asc() + } + } else { + sort.order match { + case Some(Desc) => FieldSort(sort.field.aliasOrName).desc() + case _ => FieldSort(sort.field.aliasOrName).asc() + } } - ) + } case _ => _search } if (allAggregations.nonEmpty || buckets.nonEmpty) { - _search size 0 + _search size 0 fetchSource false } else { limit match { case Some(l) => _search limit l.limit from l.offset.map(_.offset).getOrElse(0) @@ -508,7 +553,7 @@ package object bridge { implicit def expressionToQuery(expression: GenericExpression): Query = { import expression._ - if (aggregation) + if (isAggregation) return matchAllQuery() if ( identifier.functions.nonEmpty && (identifier.functions.size > 1 || (identifier.functions.head match { diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index d28b409f..133cde4c 100644 --- a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -526,12 +526,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "Country": { | "terms": { - | "field": "Country.keyword", + | "field": "Country", | "exclude": "USA", + | "min_doc_count": 1, | "order": { | "_key": "asc" | } @@ -539,8 +540,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "aggs": { | "City": { | "terms": { - | "field": "City.keyword", + | "field": "City", | "exclude": "Berlin", + | "min_doc_count": 1, | "order": { | "cnt": "desc" | } @@ -709,7 +711,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | }, | "size": 0, | "min_score": 1.0, - | "_source": true, + | "_source": false, | "aggs": { | "inner_products": { | "nested": { @@ -793,8 +795,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "aggs": { | "cat": { | "terms": { - | "field": "products.category.keyword", - | "size": 10 + | "field": "products.category", + | "size": 10, + | "min_doc_count": 1 | }, | "aggs": { | "min_price": { @@ -848,7 +851,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "ct": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); param1" | } | } | }, @@ -871,6 +874,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll(";", "; ") .replaceAll("\\|\\|", " || ") .replaceAll("ChronoUnit", " ChronoUnit") + .replaceAll("==", " == ") } it should "filter with date time and interval" in { @@ -1005,11 +1009,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "userId": { | "terms": { - | "field": "userId.keyword" + | "field": "userId", + | "min_doc_count": 1 | }, | "aggs": { | "lastSeen": { @@ -1049,12 +1054,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "Country": { | "terms": { - | "field": "Country.keyword", + | "field": "Country", | "exclude": "USA", + | "min_doc_count": 1, | "order": { | "_key": "asc" | } @@ -1062,8 +1068,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "aggs": { | "City": { | "terms": { - | "field": "City.keyword", - | "exclude": "Berlin" + | "field": "City", + | "exclude": "Berlin", + | "min_doc_count": 1 | }, | "aggs": { | "cnt": { @@ -1114,12 +1121,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "Country": { | "terms": { - | "field": "Country.keyword", + | "field": "Country", | "exclude": "USA", + | "min_doc_count": 1, | "order": { | "_key": "asc" | } @@ -1127,8 +1135,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "aggs": { | "City": { | "terms": { - | "field": "City.keyword", - | "exclude": "Berlin" + | "field": "City", + | "exclude": "Berlin", + | "min_doc_count": 1 | }, | "aggs": { | "cnt": { @@ -1185,11 +1194,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "identifier": { | "terms": { - | "field": "identifier.keyword", + | "field": "identifier", + | "min_doc_count": 1, | "order": { | "ct": "desc" | } @@ -1205,7 +1215,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "field": "createdAt", | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (param1 == null) ? null : LocalDate.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"))" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : LocalDate.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"))" | } | } | } @@ -1256,49 +1266,49 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.withDayOfYear(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfYear(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value); def param2 = param1 != null ? param1.withMonth((((param1.getMonthValue() - 1) / 3) * 3) + 1).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS) : null; def param3 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param3.format(param2)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = param1 != null ? param1.withMonth((((param1.getMonthValue() - 1) / 3) * 3) + 1).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS) : null; def param3 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param3.format(param2)" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.with(DayOfWeek.SUNDAY).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.with(DayOfWeek.SUNDAY).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "d": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.HOURS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.HOURS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "m2": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.MINUTES)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.MINUTES)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.SECONDS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.SECONDS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | } | }, @@ -1352,11 +1362,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "identifier": { | "terms": { - | "field": "identifier.keyword", + | "field": "identifier", + | "min_doc_count": 1, | "order": { | "ct": "desc" | } @@ -1372,7 +1383,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "field": "createdAt", | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).truncatedTo(ChronoUnit.MINUTES).get(ChronoField.YEAR)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).truncatedTo(ChronoUnit.MINUTES).get(ChronoField.YEAR)" | } | } | } @@ -1424,7 +1435,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss XXX\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss XXX\"); (param1 == null) ? null : param2.format(param1)" | } | } | }, @@ -1470,7 +1481,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "diff": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('updatedAt') || doc['updatedAt'].empty ? null : doc['updatedAt'].value); def param2 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param2)" + | "source": "def param1 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param2 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param2)" | } | } | }, @@ -1509,18 +1520,19 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "identifier": { | "terms": { - | "field": "identifier.keyword" + | "field": "identifier", + | "min_doc_count": 1 | }, | "aggs": { | "max_diff": { | "max": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('updatedAt') || doc['updatedAt'].empty ? null : doc['updatedAt'].value); def param2 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); def param3 = (param2 == null) ? null : ZonedDateTime.parse(param2, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param3)" + | "source": "def param1 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param2 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param3 = (param2 == null) ? null : ZonedDateTime.parse(param2, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param3)" | } | } | } @@ -1572,7 +1584,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.plus(10, ChronoUnit.DAYS)); param1" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.plus(10, ChronoUnit.DAYS)); param1" | } | } | }, @@ -1623,7 +1635,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.minus(10, ChronoUnit.DAYS)); param1" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.minus(10, ChronoUnit.DAYS)); param1" | } | } | }, @@ -1674,7 +1686,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.plus(10, ChronoUnit.DAYS)); param1" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.plus(10, ChronoUnit.DAYS)); param1" | } | } | }, @@ -1725,7 +1737,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.minus(10, ChronoUnit.DAYS)); param1" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.minus(10, ChronoUnit.DAYS)); param1" | } | } | }, @@ -1768,7 +1780,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "flag": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); param1 == null" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); param1 == null" | } | } | }, @@ -1806,7 +1818,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "flag": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); param1 != null" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); param1 != null" | } | } | }, @@ -1906,7 +1918,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); param1 != null ? param1 : param2" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); (param1 != null ? param1 : param2)" | } | } | }, @@ -1940,6 +1952,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("ChronoUnit", " ChronoUnit") .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll(":ZonedDateTime", " : ZonedDateTime") + .replaceAll(";\\(param", "; (param") } it should "handle nullif function as script field" in { @@ -1956,7 +1969,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); param3 != null ? param3 : param4" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); (param3 != null ? param3 : param4)" | } | } | }, @@ -1998,6 +2011,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll(",DateTimeFormatter", ", DateTimeFormatter") .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll(":ZonedDateTime", " : ZonedDateTime") + .replaceAll(";\\(param", "; (param") } it should "handle cast function as script field" in { @@ -2014,7 +2028,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { param3 != null ? param3 : param4 } catch (Exception e) { return null; }" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { (param3 != null ? param3 : param4) } catch (Exception e) { return null; }" | } | }, | "c2": { @@ -2082,6 +2096,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll(":ZonedDateTime", " : ZonedDateTime") .replaceAll("try \\{", "try { ") .replaceAll("} catch", " } catch") + .replaceAll(";\\(param", "; (param") } it should "handle case function as script field" in { // 40 @@ -2098,7 +2113,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS); def param3 = param1 == null ? false : (param1.isAfter(param2)); def param4 = (!doc.containsKey('lastSeen') || doc['lastSeen'].empty ? null : doc['lastSeen'].value.plus(2, ChronoUnit.DAYS)); def param5 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); param3 ? param1 : param4 != null ? param4 : param5" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS); def param3 = param1 == null ? false : (param1.isAfter(param2)); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.plus(2, ChronoUnit.DAYS)); def param5 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); param3 ? param1 : param4 != null ? param4 : param5" | } | } | }, @@ -2151,7 +2166,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (!doc.containsKey('lastSeen') || doc['lastSeen'].empty ? null : doc['lastSeen'].value.toLocalDate().plus(2, ChronoUnit.DAYS)); def param4 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param1.isEqual(param2) ? param2 : param1 != null && param1.isEqual(param3) ? param3 : param4" + | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate().plus(2, ChronoUnit.DAYS)); def param4 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param1.isEqual(param2) ? param2 : param1 != null && param1.isEqual(param3) ? param3 : param4" | } | } | }, @@ -2206,91 +2221,91 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "dom": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" | } | }, | "dow": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_WEEK)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_WEEK)); param1" | } | }, | "doy": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" | } | }, | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" | } | }, | "minutes": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" | } | }, | "s": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" | } | }, | "nano": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" | } | }, | "micro": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" | } | }, | "milli": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" | } | }, | "epoch": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" | } | }, | "off": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" | } | } | }, @@ -2330,7 +2345,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().get(ChronoField.YEAR); (param1 == null) ? null : (param1 * (param2 - 10)) > 10000" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().get(ChronoField.YEAR); (param1 == null) ? null : (param1 * (param2 - 10)) > 10000" | } | } | } @@ -2341,37 +2356,37 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "add": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : (param1 + 1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : (param1 + 1)" | } | }, | "sub": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : (param1 - 1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : (param1 - 1)" | } | }, | "mul": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : (param1 * 2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : (param1 * 2)" | } | }, | "div": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : (param1 / 2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : (param1 / 2)" | } | }, | "mod": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : (param1 % 2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : (param1 % 2)" | } | }, | "identifier_mul_identifier2_minus_10": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); def param2 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); def lv0 = ((param1 == null || param2 == null) ? null : (param1 * param2)); (lv0 == null) ? null : (lv0 - 10)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); def lv0 = ((param1 == null || param2 == null) ? null : (param1 * param2)); (lv0 == null) ? null : (lv0 - 10)" | } | } | }, @@ -2419,7 +2434,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1) > 100.0" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1) > 100.0" | } | } | } @@ -2430,109 +2445,109 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "abs_identifier_plus_1_0_mul_2": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); ((param1 == null) ? null : Math.abs(param1) + 1.0) * ((double) 2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); ((param1 == null) ? null : Math.abs(param1) + 1.0) * ((double) 2)" | } | }, | "ceil_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.ceil(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.ceil(param1)" | } | }, | "floor_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.floor(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.floor(param1)" | } | }, | "sqrt_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1)" | } | }, | "exp_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.exp(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.exp(param1)" | } | }, | "log_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.log(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.log(param1)" | } | }, | "log10_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.log10(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.log10(param1)" | } | }, | "pow_identifier_3": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.pow(param1, 3)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.pow(param1, 3)" | } | }, | "round_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); def param2 = Math.pow(10, 0); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 0); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" | } | }, | "round_identifier_2": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); def param2 = Math.pow(10, 2); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 2); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" | } | }, | "sign_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : (param1 > 0 ? 1 : (param1 < 0 ? -1 : 0))" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : (param1 > 0 ? 1 : (param1 < 0 ? -1 : 0))" | } | }, | "cos_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.cos(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.cos(param1)" | } | }, | "acos_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.acos(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.acos(param1)" | } | }, | "sin_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.sin(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sin(param1)" | } | }, | "asin_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.asin(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.asin(param1)" | } | }, | "tan_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.tan(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.tan(param1)" | } | }, | "atan_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan(param1)" | } | }, | "atan2_identifier_3_0": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan2(param1, 3.0)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan2(param1, 3.0)" | } | } | }, @@ -2589,7 +2604,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.trim().length() > 10" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.trim().length() > 10" | } | } | } @@ -2600,85 +2615,85 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "len": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.length()" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.length()" | } | }, | "low": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.toLowerCase()" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.toLowerCase()" | } | }, | "upp": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.toUpperCase()" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.toUpperCase()" | } | }, | "sub": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(3, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(3, param1.length()))" | } | }, | "tr": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.trim()" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.trim()" | } | }, | "ltr": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.replaceAll(\"^\\\\s+\",\"\")" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.replaceAll(\"^\\\\s+\",\"\")" | } | }, | "rtr": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.replaceAll(\"\\\\s+$\",\"\")" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.replaceAll(\"\\\\s+$\",\"\")" | } | }, | "con": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : String.valueOf(param1) + \"_test\" + String.valueOf(1)" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : String.valueOf(param1) + \"_test\" + String.valueOf(1)" | } | }, | "l": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(5, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(5, param1.length()))" | } | }, | "r": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(param1.length() - Math.min(3, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(param1.length() - Math.min(3, param1.length()))" | } | }, | "rep": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.replace(\"el\", \"le\")" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.replace(\"el\", \"le\")" | } | }, | "rev": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : new StringBuilder(param1).reverse().toString()" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : new StringBuilder(param1).reverse().toString()" | } | }, | "pos": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : param1.indexOf(\"soft\", 0) + 1" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.indexOf(\"soft\", 0) + 1" | } | }, | "reg": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (param1 == null) ? null : java.util.regex.Pattern.compile(\"soft\", java.util.regex.Pattern.CASE_INSENSITIVE | java.util.regex.Pattern.MULTILINE).matcher(param1).find()" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : java.util.regex.Pattern.compile(\"soft\", java.util.regex.Pattern.CASE_INSENSITIVE | java.util.regex.Pattern.MULTILINE).matcher(param1).find()" | } | } | }, @@ -2742,19 +2757,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "script_fields": { - | "hire_date": { - | "script": { - | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('hire_date') || doc['hire_date'].empty ? null : doc['hire_date'].value.toLocalDate()); param1" - | } - | } - | }, - | "_source": true, + | "_source": false, | "aggs": { | "dept": { | "terms": { - | "field": "department.keyword" + | "field": "department", + | "min_doc_count": 1 | }, | "aggs": { | "cnt": { @@ -2880,7 +2888,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "ld": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.toLocalDate()); (param1 == null) ? null : param1.withDayOfMonth(param1.lengthOfMonth())" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); (param1 == null) ? null : param1.withDayOfMonth(param1.lengthOfMonth())" | } | } | }, @@ -2936,91 +2944,91 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" | } | }, | "wd": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (param1 == null) ? null : (param1.get(ChronoField.DAY_OF_WEEK) + 6) % 7" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : (param1.get(ChronoField.DAY_OF_WEEK) + 6) % 7" | } | }, | "yd": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" | } | }, | "d": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" | } | }, | "minutes": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" | } | }, | "s": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" | } | }, | "nano": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" | } | }, | "micro": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" | } | }, | "milli": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" | } | }, | "epoch": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" | } | }, | "off": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" | } | } | }, @@ -3076,7 +3084,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)) >= 4000000.0", + | "source": "(def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)) >= 4000000.0", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3100,7 +3108,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('fromLocation') || doc['fromLocation'].empty ? null : doc['fromLocation']); def arg1 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null || arg1 == null) ? null : arg0.arcDistance(arg1.lat, arg1.lon)) < 2000000.0" + | "source": "(def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); def arg1 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null || arg1 == null) ? null : arg0.arcDistance(arg1.lat, arg1.lon)) < 2000000.0" | } | } | }, @@ -3119,7 +3127,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "d1": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "source": "(def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3129,7 +3137,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "d2": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('fromLocation') || doc['fromLocation'].empty ? null : doc['fromLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "source": "(def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3206,7 +3214,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())) == false)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())) == false)" | } | } | }, @@ -3296,7 +3304,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('comments.replies.lastUpdated') || doc['comments.replies.lastUpdated'].empty ? null : doc['comments.replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-10\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" + | "source": "def param1 = (doc['comments.replies.lastUpdated'].size() == 0 ? null : doc['comments.replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-10\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" | } | } | } @@ -3394,7 +3402,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('replies.lastUpdated') || doc['replies.lastUpdated'].empty ? null : doc['replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-10\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" + | "source": "def param1 = (doc['replies.lastUpdated'].size() == 0 ? null : doc['replies.lastUpdated'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-10\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); param1 == null ? false : (param1.isBefore(param2.withDayOfMonth(param2.lengthOfMonth())))" | } | } | }, @@ -3501,7 +3509,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); param1 == null ? false : (param1.isBefore(param2))" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); param1 == null ? false : (param1.isBefore(param2))" | } | } | } @@ -3599,7 +3607,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "avg_popularity": { | "avg": { @@ -3633,7 +3641,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "comments": { | "nested": { @@ -3694,7 +3702,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | }, | "size": 0, - | "_source": true, + | "_source": false, | "aggs": { | "comments": { | "nested": { @@ -3729,4 +3737,20 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { |}""".stripMargin.replaceAll("\\s+", "") } + it should "test" in { + val query = + """SELECT + | category, + | SUM(amount) AS totalSales, + | COUNT(*) AS orderCount, + | DATE_TRUNC(sales_date, MONTH) as salesMonth + | FROM orders + | GROUP BY DATE_TRUNC(sales_date, MONTH), category + | ORDER BY DATE_TRUNC(sales_date, MONTH) DESC, category ASC""".stripMargin.replaceAll( + "\n", + " " + ) + val select: ElasticSearchRequest = SQLQuery(query) + println(select.query) + } } diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala index 5d036251..af2cf47f 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala @@ -263,15 +263,11 @@ trait JestScrollApi extends ScrollApi with JestClientHelpers { aggregations: Map[String, SQLAggregation] ): Seq[Map[String, Any]] = { val jsonString = jsonObject.toString - val sqlResponse = - ElasticResponse( - "", - jsonString, - fieldAliases, - aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) - ) - - parseResponse(sqlResponse) match { + parseResponse( + jsonString, + fieldAliases, + aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) + ) match { case Success(rows) => rows case Failure(ex) => logger.error(s"Failed to parse Jest scroll response: ${ex.getMessage}", ex) @@ -286,9 +282,8 @@ trait JestScrollApi extends ScrollApi with JestClientHelpers { fieldAliases: Map[String, String] ): Seq[Map[String, Any]] = { val jsonString = jsonObject.toString - val sqlResponse = ElasticResponse("", jsonString, fieldAliases, Map.empty) - parseResponse(sqlResponse) match { + parseResponse(jsonString, fieldAliases, Map.empty) match { case Success(rows) => rows case Failure(ex) => logger.error(s"Failed to parse Jest search after response: ${ex.getMessage}", ex) diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientWindowFunctionSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientWindowFunctionSpec.scala new file mode 100644 index 00000000..886b7fe0 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientWindowFunctionSpec.scala @@ -0,0 +1,3 @@ +package app.softnetwork.elastic.client + +class JestClientWindowFunctionSpec extends WindowFunctionSpec diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index a22aaf8f..9758537f 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -1345,15 +1345,11 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel aggregations: Map[String, SQLAggregation] ): Seq[Map[String, Any]] = { val jsonString = response.toString - val sqlResponse = - ElasticResponse( - "", - jsonString, - fieldAliases, - aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) - ) - - parseResponse(sqlResponse) match { + parseResponse( + jsonString, + fieldAliases, + aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) + ) match { case Success(rows) => logger.debug(s"Parsed ${rows.size} rows from response") rows @@ -1370,9 +1366,7 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel fieldAliases: Map[String, String] ): Seq[Map[String, Any]] = { val jsonString = response.toString - val sqlResponse = ElasticResponse("", jsonString, fieldAliases, Map.empty) - - parseResponse(sqlResponse) match { + parseResponse(jsonString, fieldAliases, Map.empty) match { case Success(rows) => rows case Failure(ex) => logger.error(s"Failed to parse search after response: ${ex.getMessage}", ex) diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientWindowFunctionSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientWindowFunctionSpec.scala new file mode 100644 index 00000000..4df1d284 --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientWindowFunctionSpec.scala @@ -0,0 +1,3 @@ +package app.softnetwork.elastic.client + +class RestHighLevelClientWindowFunctionSpec extends WindowFunctionSpec diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 30f5564e..63356f2e 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -1516,15 +1516,12 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel aggregations: Map[String, SQLAggregation] ): Seq[Map[String, Any]] = { val jsonString = response.toString - val sqlResponse = - ElasticResponse( - "", - jsonString, - fieldAliases, - aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) - ) - parseResponse(sqlResponse) match { + parseResponse( + jsonString, + fieldAliases, + aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) + ) match { case Success(rows) => logger.debug(s"Parsed ${rows.size} rows from response") rows @@ -1541,9 +1538,8 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel fieldAliases: Map[String, String] ): Seq[Map[String, Any]] = { val jsonString = response.toString - val sqlResponse = ElasticResponse("", jsonString, fieldAliases, Map.empty) - parseResponse(sqlResponse) match { + parseResponse(jsonString, fieldAliases, Map.empty) match { case Success(rows) => rows case Failure(ex) => logger.error(s"Failed to parse search after response: ${ex.getMessage}", ex) diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientWindowFunctionSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientWindowFunctionSpec.scala new file mode 100644 index 00000000..4df1d284 --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientWindowFunctionSpec.scala @@ -0,0 +1,3 @@ +package app.softnetwork.elastic.client + +class RestHighLevelClientWindowFunctionSpec extends WindowFunctionSpec diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 612a76d1..ec367082 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -1453,15 +1453,12 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { case Left(l) => convertToJson(l) case Right(r) => convertToJson(r) } - val sqlResponse = - ElasticResponse( - "", - jsonString, - fieldAliases, - aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) - ) - parseResponse(sqlResponse) match { + parseResponse( + jsonString, + fieldAliases, + aggregations.map(kv => kv._1 -> implicitly[ClientAggregation](kv._2)) + ) match { case Success(rows) => logger.debug(s"Parsed ${rows.size} rows from response (hits + aggregations)") rows @@ -1478,9 +1475,8 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { fieldAliases: Map[String, String] ): Seq[Map[String, Any]] = { val jsonString = convertToJson(response) - val sqlResponse = ElasticResponse("", jsonString, fieldAliases, Map.empty) - parseResponse(sqlResponse) match { + parseResponse(jsonString, fieldAliases, Map.empty) match { case Success(rows) => logger.debug(s"Parsed ${rows.size} hits from response") rows diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientWindowFunctionSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientWindowFunctionSpec.scala new file mode 100644 index 00000000..ad5e6daa --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientWindowFunctionSpec.scala @@ -0,0 +1,3 @@ +package app.softnetwork.elastic.client + +class JavaClientWindowFunctionSpec extends WindowFunctionSpec diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index f6c3a4e3..a6fc8e2e 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -1446,10 +1446,12 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { case Left(l) => convertToJson(l) case Right(r) => convertToJson(r) } - val sqlResponse = - ElasticResponse("", jsonString, fieldAliases, aggregations.map(kv => kv._1 -> kv._2)) - parseResponse(sqlResponse) match { + parseResponse( + jsonString, + fieldAliases, + aggregations.map(kv => kv._1 -> kv._2) + ) match { case Success(rows) => logger.debug(s"Parsed ${rows.size} rows from response (hits + aggregations)") rows @@ -1466,9 +1468,8 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { fieldAliases: Map[String, String] ): Seq[Map[String, Any]] = { val jsonString = convertToJson(response) - val sqlResponse = ElasticResponse("", jsonString, fieldAliases, Map.empty) - parseResponse(sqlResponse) match { + parseResponse(jsonString, fieldAliases, Map.empty) match { case Success(rows) => logger.debug(s"Parsed ${rows.size} hits from response") rows diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientWindowFunctionSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientWindowFunctionSpec.scala new file mode 100644 index 00000000..ad5e6daa --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientWindowFunctionSpec.scala @@ -0,0 +1,3 @@ +package app.softnetwork.elastic.client + +class JavaClientWindowFunctionSpec extends WindowFunctionSpec diff --git a/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala b/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala index 3c3cd97e..35a4cd3d 100644 --- a/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala +++ b/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala @@ -17,6 +17,7 @@ package app.softnetwork.elastic.sql.macros import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import app.softnetwork.elastic.sql.function.aggregate.COUNT import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query.SQLSearchRequest @@ -199,7 +200,12 @@ trait SQLQueryValidator { // Check if any field is a wildcard (*) val hasWildcard = parsedQuery.select.fields.exists { field => - field.identifier.name == "*" + field.identifier.name == "*" && (field.aggregateFunction match { + case Some(COUNT) => + false + case _ => + true + }) } if (hasWildcard) { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index 68611f9f..1b1f0abf 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -23,6 +23,8 @@ package object aggregate { sealed trait AggregateFunction extends Function { def multivalued: Boolean = false + + override def isAggregation: Boolean = true } case object COUNT extends Expr("COUNT") with AggregateFunction @@ -35,17 +37,17 @@ package object aggregate { case object SUM extends Expr("SUM") with AggregateFunction - sealed trait TopHits extends TokenRegex + sealed trait Window extends TokenRegex - case object FIRST_VALUE extends Expr("FIRST_VALUE") with TopHits { + case object FIRST_VALUE extends Expr("FIRST_VALUE") with Window { override val words: List[String] = List(sql, "FIRST") } - case object LAST_VALUE extends Expr("LAST_VALUE") with TopHits { + case object LAST_VALUE extends Expr("LAST_VALUE") with Window { override val words: List[String] = List(sql, "LAST") } - case object ARRAY_AGG extends Expr("ARRAY_AGG") with TopHits { + case object ARRAY_AGG extends Expr("ARRAY_AGG") with Window { override val words: List[String] = List(sql, "ARRAY") } @@ -53,14 +55,14 @@ package object aggregate { case object PARTITION_BY extends Expr("PARTITION BY") with TokenRegex - sealed trait TopHitsAggregation + sealed trait WindowFunction extends AggregateFunction with FunctionWithIdentifier with Updateable { def partitionBy: Seq[Identifier] - def withPartitionBy(partitionBy: Seq[Identifier]): TopHitsAggregation + def withPartitionBy(partitionBy: Seq[Identifier]): WindowFunction def orderBy: OrderBy - def topHits: TopHits + def window: Window def limit: Option[Limit] lazy val buckets: Seq[Bucket] = partitionBy.map(identifier => Bucket(identifier, None)) @@ -73,22 +75,22 @@ package object aggregate { val partitionByStr = if (partitionBy.nonEmpty) s"$PARTITION_BY ${partitionBy.mkString(", ")}" else "" - s"$topHits($identifier) $OVER ($partitionByStr$orderBy)" + s"$window($identifier) $OVER ($partitionByStr$orderBy)" } override def toSQL(base: String): String = sql def fields: Seq[Field] - def withFields(fields: Seq[Field]): TopHitsAggregation + def withFields(fields: Seq[Field]): WindowFunction - def update(request: SQLSearchRequest): TopHitsAggregation = { + def update(request: SQLSearchRequest): WindowFunction = { val updated = this .withPartitionBy(partitionBy = partitionBy.map(_.update(request))) updated.withFields( fields = request.select.fields .filterNot(field => - field.aggregation || request.bucketNames.keys.toSeq + field.isAggregation || request.bucketNames.keys.toSeq .contains(field.identifier.identifierName) ) .filterNot(f => request.excludes.contains(f.sourceField)) @@ -101,13 +103,13 @@ package object aggregate { partitionBy: Seq[Identifier] = Seq.empty, orderBy: OrderBy, fields: Seq[Field] = Seq.empty - ) extends TopHitsAggregation { + ) extends WindowFunction { override def limit: Option[Limit] = Some(Limit(1, None)) - override def topHits: TopHits = FIRST_VALUE - override def withPartitionBy(partitionBy: Seq[Identifier]): TopHitsAggregation = + override def window: Window = FIRST_VALUE + override def withPartitionBy(partitionBy: Seq[Identifier]): WindowFunction = this.copy(partitionBy = partitionBy) - override def withFields(fields: Seq[Field]): TopHitsAggregation = this.copy(fields = fields) - override def update(request: SQLSearchRequest): TopHitsAggregation = super + override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) + override def update(request: SQLSearchRequest): WindowFunction = super .update(request) .asInstanceOf[FirstValue] .copy( @@ -121,13 +123,13 @@ package object aggregate { partitionBy: Seq[Identifier] = Seq.empty, orderBy: OrderBy, fields: Seq[Field] = Seq.empty - ) extends TopHitsAggregation { + ) extends WindowFunction { override def limit: Option[Limit] = Some(Limit(1, None)) - override def topHits: TopHits = LAST_VALUE - override def withPartitionBy(partitionBy: Seq[Identifier]): TopHitsAggregation = + override def window: Window = LAST_VALUE + override def withPartitionBy(partitionBy: Seq[Identifier]): WindowFunction = this.copy(partitionBy = partitionBy) - override def withFields(fields: Seq[Field]): TopHitsAggregation = this.copy(fields = fields) - override def update(request: SQLSearchRequest): TopHitsAggregation = super + override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) + override def update(request: SQLSearchRequest): WindowFunction = super .update(request) .asInstanceOf[LastValue] .copy( @@ -142,12 +144,12 @@ package object aggregate { orderBy: OrderBy, fields: Seq[Field] = Seq.empty, limit: Option[Limit] = None - ) extends TopHitsAggregation { - override def topHits: TopHits = ARRAY_AGG - override def withPartitionBy(partitionBy: Seq[Identifier]): TopHitsAggregation = + ) extends WindowFunction { + override def window: Window = ARRAY_AGG + override def withPartitionBy(partitionBy: Seq[Identifier]): WindowFunction = this.copy(partitionBy = partitionBy) - override def withFields(fields: Seq[Field]): TopHitsAggregation = this - override def update(request: SQLSearchRequest): TopHitsAggregation = super + override def withFields(fields: Seq[Field]): WindowFunction = this + override def update(request: SQLSearchRequest): WindowFunction = super .update(request) .asInstanceOf[ArrayAgg] .copy( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index 9670a497..0c432619 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -138,9 +138,9 @@ package object cond { callArgs .take(values.length - 1) .map { arg => - s"${arg.trim} != null ? ${arg.trim}" // TODO check when value is nullable and has functions + s"(${arg.trim} != null ? ${arg.trim}" // TODO check when value is nullable and has functions } - .mkString(" : ") + s" : ${callArgs.last}" + .mkString(" : ") + s" : ${callArgs.last})" } override def nullable: Boolean = values.forall(_.nullable) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala index 4f2d46fb..6f450160 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala @@ -152,7 +152,7 @@ package object geo { identifiers.zipWithIndex .map { case (a, i) => val name = a.name - s"def arg$i = (!doc.containsKey('$name') || doc['$name'].empty ? ${a.nullValue} : doc['$name']);" + s"def arg$i = (doc['$name'].size() == 0 ? ${a.nullValue} : doc['$name']);" } .mkString(" ") diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index 0daf4517..687c20d6 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -93,7 +93,9 @@ package object function { lazy val aggregateFunction: Option[AggregateFunction] = aggregations.headOption - lazy val aggregation: Boolean = aggregateFunction.isDefined + override def isAggregation: Boolean = aggregateFunction.isDefined + + override def hasAggregation: Boolean = functions.exists(_.hasAggregation) override def in: SQLType = functions.lastOption.map(_.in).getOrElse(super.in) @@ -137,6 +139,9 @@ package object function { case f => f } } + + override def shouldBeScripted: Boolean = functions.exists(_.shouldBeScripted) + } trait FunctionN[In <: SQLType, Out <: SQLType] extends Function with PainlessScript { @@ -275,6 +280,8 @@ package object function { override def args: List[PainlessScript] = List(left, right) override def nullable: Boolean = left.nullable || right.nullable + + override def shouldBeScripted: Boolean = left.shouldBeScripted || right.shouldBeScripted } trait TransformFunction[In <: SQLType, Out <: SQLType] extends FunctionN[In, Out] { @@ -310,6 +317,8 @@ package object function { s"$base${painless(context)}" } } + + override def shouldBeScripted: Boolean = true } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 8f4d183b..82fee992 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -23,7 +23,6 @@ import app.softnetwork.elastic.sql.{ Identifier, LiteralParam, PainlessContext, - PainlessParam, PainlessScript, StringValue, TokenRegex @@ -269,6 +268,9 @@ package object time { case _ => super.toPainlessCall(callArgs, context) } } + + override def shouldBeScripted: Boolean = false + } case object Extract extends Expr("EXTRACT") with TokenRegex with PainlessScript { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala index 24e199c9..d97f4bc3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -129,4 +129,7 @@ case class ArithmeticExpression( expr } + override def hasAggregation: Boolean = left.hasAggregation || right.hasAggregation + + override def shouldBeScripted: Boolean = true } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index bac4accb..bb3a9ace 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -67,6 +67,9 @@ package object sql { def nullable: Boolean = !system def dateMathScript: Boolean = false def isTemporal: Boolean = out.isInstanceOf[SQLTemporal] + def isAggregation: Boolean = false + def hasAggregation: Boolean = isAggregation + def shouldBeScripted: Boolean = false } trait TokenValue extends Token { @@ -623,7 +626,7 @@ package object sql { def hasBucket: Boolean = bucket.isDefined def allMetricsPath: Map[String, String] = { - if (aggregation) { + if (isAggregation) { val metricName = aliasOrName Map(metricName -> metricName) } else { @@ -675,7 +678,7 @@ package object sql { } def paramName: String = - if (aggregation && functions.size == 1) s"params.$aliasOrName" + if (isAggregation && functions.size == 1) s"params.$aliasOrName" else if (path.nonEmpty) s"doc['$path'].value" else "" @@ -733,8 +736,7 @@ package object sql { def checkNotNull: String = if (path.isEmpty) "" else - s"(!doc.containsKey('$path') || doc['$path'].empty ? $nullValue : doc['$path'].value${painlessMethods - .mkString("")})" + s"(doc['$path'].size() == 0 ? $nullValue : doc['$path'].value${painlessMethods.mkString("")})" override def painless(context: Option[PainlessContext]): String = { val base = @@ -762,7 +764,7 @@ package object sql { override def param: String = paramName private[this] var _nullable = - this.name.nonEmpty && (!aggregation || functions.size > 1) + this.name.nonEmpty && (!isAggregation || functions.size > 1) protected def nullable_=(b: Boolean): Unit = { _nullable = b diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/GroupByParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/GroupByParser.scala index 4b7670b4..3feb6669 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/GroupByParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/GroupByParser.scala @@ -16,12 +16,21 @@ package app.softnetwork.elastic.sql.parser +import app.softnetwork.elastic.sql.Identifier import app.softnetwork.elastic.sql.query.{Bucket, GroupBy} trait GroupByParser { self: Parser with WhereParser => - def bucket: PackratParser[Bucket] = (long | identifier) ^^ { i => + def bucketWithFunction: PackratParser[Identifier] = + identifierWithArithmeticExpression | + identifierWithTransformation | + identifierWithAggregation | + identifierWithIntervalFunction | + identifierWithFunction | + identifier + + def bucket: PackratParser[Bucket] = (long | bucketWithFunction) ^^ { i => Bucket(i) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/OrderByParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/OrderByParser.scala index 884f0616..74ea9c9d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/OrderByParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/OrderByParser.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic.sql.parser +import app.softnetwork.elastic.sql.Identifier import app.softnetwork.elastic.sql.function.Function import app.softnetwork.elastic.sql.query.{Asc, Desc, FieldSort, OrderBy} @@ -29,17 +30,17 @@ trait OrderByParser { private def fieldName: PackratParser[String] = """\b(?!(?i)limit\b)[a-zA-Z_][a-zA-Z0-9_]*""".r ^^ (f => f) - def fieldWithFunction: PackratParser[(String, List[Function])] = - rep1sep(sql_function, start) ~ start.? ~ fieldName ~ rep1(end) ^^ { case f ~ _ ~ n ~ _ => - (n, f) - } + def fieldWithFunction: PackratParser[Identifier] = + identifierWithArithmeticExpression | + identifierWithTransformation | + identifierWithAggregation | + identifierWithIntervalFunction | + identifierWithFunction | + identifier def sort: PackratParser[FieldSort] = - (fieldWithFunction | fieldName) ~ (asc | desc).? ^^ { case f ~ o => - f match { - case i: (String, List[Function]) => FieldSort(i._1, o, i._2) - case s: String => FieldSort(s, o, List.empty) - } + fieldWithFunction ~ (asc | desc).? ^^ { case f ~ o => + FieldSort(f, o) } def orderBy: PackratParser[OrderBy] = OrderBy.regex ~ rep1sep(sort, separator) ^^ { case _ ~ s => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala index 752fcaa8..92ead92d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala @@ -22,7 +22,7 @@ trait SelectParser { self: Parser with WhereParser => def field: PackratParser[Field] = - (identifierWithTopHits | + (identifierWithWindowFunction | identifierWithArithmeticExpression | identifierWithTransformation | identifierWithAggregation | diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala index 8a962efd..f32e2cd9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala @@ -37,10 +37,16 @@ package object aggregate { def aggregate_function: PackratParser[AggregateFunction] = count | min | max | avg | sum + def aggWithFunction: PackratParser[Identifier] = + identifierWithArithmeticExpression | + identifierWithTransformation | + identifierWithIntervalFunction | + identifierWithFunction | + identifier + def identifierWithAggregation: PackratParser[Identifier] = - aggregate_function ~ start ~ (identifierWithFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { - case a ~ _ ~ i ~ _ => - i.withFunctions(a +: i.functions) + aggregate_function ~ start ~ aggWithFunction ~ end ^^ { case a ~ _ ~ i ~ _ => + i.withFunctions(a +: i.functions) } def partition_by: PackratParser[Seq[Identifier]] = @@ -55,26 +61,26 @@ package object aggregate { start ~ identifier ~ end ~ over.? ^^ { case _ ~ id ~ _ ~ o => o match { case Some((pb, ob)) => (id, pb, ob) - case None => (id, Seq.empty, OrderBy(Seq(FieldSort(id.name, order = None)))) + case None => (id, Seq.empty, OrderBy(Seq(FieldSort(id, order = None)))) } } - def first_value: PackratParser[TopHitsAggregation] = + def first_value: PackratParser[WindowFunction] = FIRST_VALUE.regex ~ top_hits ^^ { case _ ~ top => FirstValue(top._1, top._2, top._3) } - def last_value: PackratParser[TopHitsAggregation] = + def last_value: PackratParser[WindowFunction] = LAST_VALUE.regex ~ top_hits ^^ { case _ ~ top => LastValue(top._1, top._2, top._3) } - def array_agg: PackratParser[TopHitsAggregation] = + def array_agg: PackratParser[WindowFunction] = ARRAY_AGG.regex ~ top_hits ^^ { case _ ~ top => ArrayAgg(top._1, top._2, top._3, limit = None) } - def identifierWithTopHits: PackratParser[Identifier] = + def identifierWithWindowFunction: PackratParser[Identifier] = (first_value | last_value | array_agg) ^^ { th => th.identifier.withFunctions(th +: th.identifier.functions) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala index 9544f8a3..710d4af5 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala @@ -16,9 +16,17 @@ package app.softnetwork.elastic.sql.query -import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} import app.softnetwork.elastic.sql.operator._ -import app.softnetwork.elastic.sql.{Expr, Identifier, LongValue, TokenRegex, Updateable} +import app.softnetwork.elastic.sql.{ + Expr, + Identifier, + LongValue, + PainlessContext, + PainlessScript, + TokenRegex, + Updateable +} case object GroupBy extends Expr("GROUP BY") with TokenRegex @@ -45,7 +53,8 @@ case class GroupBy(buckets: Seq[Bucket]) extends Updateable { case class Bucket( identifier: Identifier, size: Option[Int] = None -) extends Updateable { +) extends Updateable + with PainlessScript { override def sql: String = s"$identifier" def update(request: SQLSearchRequest): Bucket = { identifier.functions.headOption match { @@ -86,6 +95,20 @@ case class Bucket( case None => "" // Root level } } + + override def out: SQLType = identifier.out + + override def shouldBeScripted: Boolean = identifier.shouldBeScripted + + /** Generate painless script for this token + * + * @param context + * the painless context + * @return + * the painless script + */ + override def painless(context: Option[PainlessContext]): String = + identifier.painless(context) } object MetricSelectorScript { @@ -118,7 +141,7 @@ object MetricSelectorScript { case _: MultiMatchCriteria => "1 == 1" - case e: Expression if e.aggregation => + case e: Expression if e.isAggregation => // NO FILTERING: the script is generated for all metrics val painless = e.painless(None) e.maybeValue match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala index 94e111ac..952a08ad 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala @@ -28,17 +28,20 @@ case object Desc extends Expr("DESC") with SortOrder case object Asc extends Expr("ASC") with SortOrder case class FieldSort( - field: String, - order: Option[SortOrder], - functions: List[Function] = List.empty + field: Identifier, + order: Option[SortOrder] ) extends FunctionChain with Updateable { + lazy val functions: List[Function] = field.functions lazy val direction: SortOrder = order.getOrElse(Asc) - lazy val name: String = toSQL(field) + lazy val name: String = field.identifierName override def sql: String = s"$name $direction" override def update(request: SQLSearchRequest): FieldSort = this.copy( - field = Identifier(field).update(request).name + field = field.update(request) ) + def isScriptSort: Boolean = functions.nonEmpty && !hasAggregation && field.fieldAlias.isEmpty + + def isBucketScript: Boolean = functions.nonEmpty && !isAggregation && hasAggregation } case class OrderBy(sorts: Seq[FieldSort]) extends Updateable { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala index 15c174b4..b711a54f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.query -import app.softnetwork.elastic.sql.function.aggregate.TopHitsAggregation +import app.softnetwork.elastic.sql.function.aggregate.WindowFunction import app.softnetwork.elastic.sql.{asString, Token} case class SQLSearchRequest( @@ -38,7 +38,7 @@ case class SQLSearchRequest( lazy val bucketNames: Map[String, Bucket] = buckets.flatMap { b => val name = b.identifier.identifierName "\\d+".r.findFirstIn(name) match { - case Some(n) => + case Some(n) if name.trim.split(" ").length == 1 => val identifier = select.fields(n.toInt - 1).identifier val updated = b.copy(identifier = select.fields(n.toInt - 1).identifier) Map( @@ -56,7 +56,7 @@ case class SQLSearchRequest( lazy val nestedFields: Map[String, Seq[Field]] = select.fields - .filterNot(_.aggregation) + .filterNot(_.isAggregation) .filter(_.nested) .groupBy(_.identifier.innerHitsName.getOrElse("")) lazy val nested: Seq[NestedElement] = @@ -114,25 +114,31 @@ case class SQLSearchRequest( ) } - lazy val scriptFields: Seq[Field] = select.fields.filter(_.isScriptField) + lazy val scriptFields: Seq[Field] = { + if (aggregates.nonEmpty) + Seq.empty + else + select.fields.filter(_.isScriptField) + } lazy val fields: Seq[String] = { - if (aggregates.isEmpty && buckets.isEmpty) + if (aggregates.isEmpty && buckets.isEmpty && bucketScripts.isEmpty) select.fields .filterNot(_.isScriptField) .filterNot(_.nested) .map(_.sourceField) .filterNot(f => excludes.contains(f)) + .distinct else Seq.empty } - lazy val topHitsFields: Seq[Field] = select.fields.filter(_.topHits.nonEmpty) + lazy val windowFields: Seq[Field] = select.fields.filter(_.isWindow) - lazy val topHitsAggs: Seq[TopHitsAggregation] = topHitsFields.flatMap(_.topHits) + lazy val windowFunctions: Seq[WindowFunction] = windowFields.flatMap(_.windows) lazy val aggregates: Seq[Field] = - select.fields.filter(_.aggregation).filterNot(_.topHits.isDefined) ++ topHitsFields + select.fields.filter(_.isAggregation).filterNot(_.windows.isDefined) ++ windowFields lazy val sqlAggregations: Map[String, SQLAggregation] = aggregates.flatMap(f => SQLAggregation.fromField(f, this)).map(a => a.aggName -> a).toMap @@ -141,16 +147,18 @@ case class SQLSearchRequest( lazy val sources: Seq[String] = from.tables.map(_.name) - lazy val topHitsBuckets: Seq[Bucket] = topHitsAggs + lazy val windowBuckets: Seq[Bucket] = windowFunctions .flatMap(_.bucketNames) .filterNot(bucket => groupBy.map(_.bucketNames).getOrElse(Map.empty).keys.toSeq.contains(bucket._1) ) .toMap .values + .groupBy(_.identifier.aliasOrName) + .map(_._2.head) .toSeq - lazy val buckets: Seq[Bucket] = groupBy.map(_.buckets).getOrElse(Seq.empty) ++ topHitsBuckets + lazy val buckets: Seq[Bucket] = groupBy.map(_.buckets).getOrElse(Seq.empty) ++ windowBuckets override def validate(): Either[String, Unit] = { for { @@ -172,7 +180,8 @@ case class SQLSearchRequest( _ <- { // validate that non-aggregated fields are not present when group by is present if (groupBy.isDefined) { - val nonAggregatedFields = select.fields.filterNot(f => f.aggregation || f.isScriptField) + val nonAggregatedFields = + select.fields.filterNot(f => f.hasAggregation) val invalidFields = nonAggregatedFields.filterNot(f => buckets.exists(b => b.name == f.fieldAlias.map(_.alias).getOrElse(f.sourceField.replace(".", "_")) @@ -191,4 +200,6 @@ case class SQLSearchRequest( } } yield () } + + lazy val bucketScripts: Seq[Field] = select.fields.filter(_.isBucketScript) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala index 6499405a..f30944fa 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.query -import app.softnetwork.elastic.sql.function.aggregate.{AggregateFunction, TopHitsAggregation} +import app.softnetwork.elastic.sql.function.aggregate.{AggregateFunction, WindowFunction} import app.softnetwork.elastic.sql.function.{Function, FunctionChain, FunctionUtils} import app.softnetwork.elastic.sql.{ asString, @@ -40,7 +40,8 @@ case class Field( with FunctionChain with PainlessScript with DateMathScript { - def isScriptField: Boolean = functions.nonEmpty && !aggregation && identifier.bucket.isEmpty + def isScriptField: Boolean = + functions.nonEmpty && !hasAggregation && identifier.bucket.isEmpty override def sql: String = s"$identifier${asString(fieldAlias)}" lazy val sourceField: String = { if (identifier.nested) { @@ -64,19 +65,21 @@ case class Field( override def functions: List[Function] = identifier.functions - lazy val topHits: Option[TopHitsAggregation] = - functions.collectFirst { case th: TopHitsAggregation => th } + lazy val windows: Option[WindowFunction] = + functions.collectFirst { case th: WindowFunction => th } + + def isWindow: Boolean = windows.isDefined def update(request: SQLSearchRequest): Field = { - topHits match { + windows match { case Some(th) => - val topHitsAggregation = th.update(request) - val identifier = topHitsAggregation.identifier + val windowFunction = th.update(request) + val identifier = windowFunction.identifier identifier.functions match { case _ :: tail => - this.copy(identifier = identifier.withFunctions(functions = topHitsAggregation +: tail)) + this.copy(identifier = identifier.withFunctions(functions = windowFunction +: tail)) case _ => - this.copy(identifier = identifier.withFunctions(functions = List(topHitsAggregation))) + this.copy(identifier = identifier.withFunctions(functions = List(windowFunction))) } case None => this.copy(identifier = identifier.update(request)) } @@ -93,6 +96,8 @@ case class Field( lazy val nested: Boolean = identifier.nested lazy val path: String = identifier.path + + def isBucketScript: Boolean = functions.nonEmpty && !isAggregation && hasAggregation } case object Except extends Expr("except") with TokenRegex @@ -162,8 +167,8 @@ object SQLAggregation { s"${aggType}_distinct_${sourceField.replace(".", "_")}" else { aggType match { - case th: TopHitsAggregation => - s"${th.topHits.sql.toLowerCase}_${sourceField.replace(".", "_")}" + case th: WindowFunction => + s"${th.window.sql.toLowerCase}_${sourceField.replace(".", "_")}" case _ => s"${aggType}_${sourceField.replace(".", "_")}" diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 939ec421..41af2cfb 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -353,9 +353,9 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { } } - override lazy val aggregation: Boolean = maybeValue match { - case Some(v: FunctionChain) => identifier.aggregation || v.aggregation - case _ => identifier.aggregation + override lazy val isAggregation: Boolean = maybeValue match { + case Some(v: FunctionChain) => identifier.isAggregation || v.isAggregation + case _ => identifier.isAggregation } def hasBucket: Boolean = identifier.hasBucket || maybeValue.exists { @@ -421,7 +421,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { maybeValue.map(v => v.out).getOrElse(SQLTypes.Any) match { case SQLTypes.Varchar => return s"$param.compareTo(${painlessValue(context)}) < 0" - case _: SQLTemporal if !aggregation && !hasBucket => + case _: SQLTemporal if !isAggregation && !hasBucket => return s"$param.isBefore(${painlessValue(context)})" case _ => } @@ -429,7 +429,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { maybeValue.map(v => v.out).getOrElse(SQLTypes.Any) match { case SQLTypes.Varchar => return s"$param.compareTo(${painlessValue(context)}) > 0" - case _: SQLTemporal if !aggregation && !hasBucket => + case _: SQLTemporal if !isAggregation && !hasBucket => return s"$param.isAfter(${painlessValue(context)})" case _ => } @@ -437,7 +437,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { maybeValue.map(v => v.out).getOrElse(SQLTypes.Any) match { case SQLTypes.Varchar => return s"$param.compareTo(${painlessValue(context)}) == 0" - case _: SQLTemporal if !aggregation && !hasBucket => + case _: SQLTemporal if !isAggregation && !hasBucket => return s"$param.isEqual(${painlessValue(context)})" case _ => } @@ -445,7 +445,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { maybeValue.map(v => v.out).getOrElse(SQLTypes.Any) match { case SQLTypes.Varchar => return s"$param.compareTo(${painlessValue(context)}) != 0" - case _: SQLTemporal if !aggregation && !hasBucket => + case _: SQLTemporal if !isAggregation && !hasBucket => return s"$param.isEqual(${painlessValue(context)}) == false" case _ => } @@ -453,7 +453,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { maybeValue.map(v => v.out).getOrElse(SQLTypes.Any) match { case SQLTypes.Varchar => return s"$param.compareTo(${painlessValue(context)}) >= 0" - case _: SQLTemporal if !aggregation && !hasBucket => + case _: SQLTemporal if !isAggregation && !hasBucket => return s"$param.isBefore(${painlessValue(context)}) == false" case _ => } @@ -461,7 +461,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { maybeValue.map(v => v.out).getOrElse(SQLTypes.Any) match { case SQLTypes.Varchar => return s"$param.compareTo(${painlessValue(context)}) <= 0" - case _: SQLTemporal if !aggregation && !hasBucket => + case _: SQLTemporal if !isAggregation && !hasBucket => return s"$param.isAfter(${painlessValue(context)}) == false" case _ => } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/EmployeeData.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/EmployeeData.scala new file mode 100644 index 00000000..a98d3cba --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/EmployeeData.scala @@ -0,0 +1,240 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.Source +import app.softnetwork.elastic.client.bulk._ +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticSuccess} +import app.softnetwork.elastic.model.window.Employee +import app.softnetwork.persistence.generateUUID +import org.json4s.Formats +import org.scalatest.Suite + +import scala.language.implicitConversions + +trait EmployeeData { _: Suite => + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + implicit def formats: Formats + + def client: ElasticClientApi + + /** Load employees + */ + def loadEmployees(): Unit = { + + implicit val bulkOptions: BulkOptions = BulkOptions( + defaultIndex = "emp", + logEvery = 5 + ) + + val employees = getEmployees.zipWithIndex.map { case (emp, idx) => + serialization.write(emp.copy(id = s"emp_${idx + 1}")) + }.toList + + implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = + Source.fromIterator(() => list.iterator) + + client.bulk[String](employees, identity, idKey = Some("id")) match { + case ElasticSuccess(response) => + println(s"✅ Bulk indexing completed:") + println(s" - Total items: ${response.metrics.totalDocuments}") + println(s" - Successful: ${response.successCount}") + println(s" - Failed: ${response.failedCount}") + println(s" - Took: ${response.metrics.durationMs}ms") + + // Afficher les erreurs éventuelles + val failures = response.failedDocuments + if (failures.nonEmpty) { + println(s" ⚠️ ${failures.size} documents failed:") + failures.foreach { item => + println(s" - Document ${item.id}: ${item.error.message}") + } + } + + case ElasticFailure(error) => + error.cause.foreach(t => t.printStackTrace()) + fail(s"❌ Bulk indexing failed: ${error.message}") + } + } + + // ======================================================================== + // HELPER METHODS + // ======================================================================== + + private def getEmployees: Seq[Employee] = Seq( + Employee( + "Alice Johnson", + "Engineering", + "New York", + 95000, + "2019-03-15", + "Senior", + List("Java", "Python", "Scala") + ), + Employee( + "Bob Smith", + "Engineering", + "New York", + 120000, + "2018-01-10", + "Lead", + List("Scala", "Spark", "Kafka") + ), + Employee( + "Charlie Brown", + "Engineering", + "San Francisco", + 85000, + "2020-06-20", + "Mid", + List("Python", "Django") + ), + Employee( + "Diana Prince", + "Engineering", + "San Francisco", + 110000, + "2017-09-05", + "Senior", + List("Go", "Kubernetes", "Docker") + ), + Employee( + "Eve Davis", + "Engineering", + "New York", + 75000, + "2021-02-12", + "Junior", + List("JavaScript", "React") + ), + Employee( + "Frank Miller", + "Sales", + "New York", + 80000, + "2019-07-22", + "Mid", + List("Salesforce", "CRM") + ), + Employee( + "Grace Lee", + "Sales", + "Chicago", + 90000, + "2018-11-30", + "Senior", + List("Negotiation", "B2B") + ), + Employee( + "Henry Wilson", + "Sales", + "Chicago", + 70000, + "2020-04-18", + "Junior", + List("Cold Calling") + ), + Employee( + "Iris Chen", + "Sales", + "New York", + 95000, + "2017-03-08", + "Lead", + List("Strategy", "Analytics") + ), + Employee( + "Jack Taylor", + "Marketing", + "San Francisco", + 78000, + "2019-10-01", + "Mid", + List("SEO", "Content") + ), + Employee( + "Karen White", + "Marketing", + "San Francisco", + 88000, + "2018-05-15", + "Senior", + List("Brand", "Digital") + ), + Employee( + "Leo Martinez", + "Marketing", + "Chicago", + 65000, + "2021-01-20", + "Junior", + List("Social Media") + ), + Employee( + "Maria Garcia", + "HR", + "New York", + 72000, + "2019-08-12", + "Mid", + List("Recruiting", "Onboarding") + ), + Employee("Nathan King", "HR", "Chicago", 68000, "2020-11-05", "Junior", List("Payroll")), + Employee( + "Olivia Scott", + "HR", + "New York", + 85000, + "2017-12-01", + "Senior", + List("Policy", "Compliance") + ), + Employee( + "Paul Anderson", + "Engineering", + "Remote", + 105000, + "2016-04-10", + "Senior", + List("Rust", "Systems") + ), + Employee("Quinn Roberts", "Sales", "Remote", 92000, "2019-02-28", "Senior", List("Enterprise")), + Employee( + "Rachel Green", + "Marketing", + "Remote", + 81000, + "2020-09-10", + "Mid", + List("Analytics", "PPC") + ), + Employee( + "Sam Turner", + "Engineering", + "San Francisco", + 130000, + "2015-06-01", + "Principal", + List("Architecture", "Leadership") + ), + Employee("Tina Brooks", "Sales", "Chicago", 75000, "2021-03-15", "Junior", List("B2C")) + ) +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala new file mode 100644 index 00000000..e2e1d87e --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala @@ -0,0 +1,1273 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.stream.scaladsl.Sink +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticSuccess} +import app.softnetwork.elastic.client.scroll.ScrollConfig +import app.softnetwork.elastic.client.spi.ElasticClientFactory +import app.softnetwork.elastic.model.window._ +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import java.time.LocalDate +import scala.concurrent.Await +import scala.concurrent.duration._ + +trait WindowFunctionSpec + extends AnyFlatSpecLike + with ElasticDockerTestKit + with Matchers + with EmployeeData { + + lazy val log: Logger = LoggerFactory getLogger getClass.getName + + override def client: ElasticClientApi = ElasticClientFactory.create(elasticConfig) + + override def beforeAll(): Unit = { + super.beforeAll() + + val mapping = + """{ + | "properties": { + | "name": { + | "type": "text", + | "fields": { + | "keyword": { + | "type": "keyword" + | } + | } + | }, + | "department": { + | "type": "keyword" + | }, + | "location": { + | "type": "keyword" + | }, + | "salary": { + | "type": "integer" + | }, + | "hire_date": { + | "type": "date", + | "format": "yyyy-MM-dd" + | }, + | "level": { + | "type": "keyword" + | }, + | "skills": { + | "type": "keyword" + | } + | } + |}""".stripMargin + + client.createIndex("emp").get shouldBe true + + client.setMapping("emp", mapping).get shouldBe true + + loadEmployees() + } + + override def afterAll(): Unit = { + client.deleteIndex("emp") + // system.terminate() + super.afterAll() + } + + "Index mapping" should "have correct field types" in { + client.getMapping("emp") match { + case ElasticSuccess(mapping) => + log.info(s"📋 Mapping: $mapping") + + mapping should include("hire_date") + mapping should include("\"type\":\"date\"") + mapping should include("\"format\":\"yyyy-MM-dd\"") + + case ElasticFailure(error) => fail(s"Failed to get mapping: ${error.message}") + } + } + + "Sample document" should "have hire_date as string" in { + val results = client.searchAs[Employee](""" + SELECT + name, + department, + location, + salary, + hire_date, + level, + skills, + id + FROM emp + WHERE name.keyword = 'Sam Turner' + """) + + results match { + case ElasticSuccess(employees) => + employees should have size 1 + val sam = employees.head + + sam.name shouldBe "Sam Turner" + sam.hire_date shouldBe "2015-06-01" + + log.info(s"✅ Sam Turner hire_date: ${sam.hire_date}") + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + // ======================================================================== + // BASIC WINDOW FUNCTION TESTS + // ======================================================================== + + "FIRST_VALUE window function" should "return first salary per department" in { + val results = client.searchAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + location, + level, + FIRST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS first_salary + FROM emp + ORDER BY department, hire_date + """ + ) + + results match { + case ElasticSuccess(employees) => + employees should not be empty + + // Engineering: first hire = Sam Turner (2015-06-01, $130k) + val engineering = employees.filter(_.department == "Engineering") + engineering.foreach { emp => + emp.first_salary shouldBe Some(130000) + } + + // Sales: first hire = Iris Chen (2017-03-08, $95k) + val sales = employees.filter(_.department == "Sales") + sales.foreach { emp => + emp.first_salary shouldBe Some(95000) + } + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + "LAST_VALUE window function" should "return last salary per department" in { + val results = client.searchAs[EmployeeWithWindow](""" + SELECT + department, + name, + salary, + hire_date, + location, + level, + LAST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS last_salary + FROM emp + ORDER BY department, hire_date + """) + + results match { + case ElasticSuccess(employees) => + employees should not be empty + + // Engineering: last hire = Eve Davis (2021-02-12, $75k) + val engineering = employees.filter(_.department == "Engineering") + engineering.foreach { emp => + emp.last_salary shouldBe Some(75000) + } + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + /*"ROW_NUMBER window function" should "assign sequential numbers per partition" in { + val results = client.searchAs[EmployeeWithWindow](""" + SELECT + department, + name, + salary, + hire_date, + ROW_NUMBER() OVER ( + PARTITION BY department + ORDER BY salary DESC + ) AS row_number + FROM emp + ORDER BY department, row_number + """) + + results match { + case ElasticSuccess(employees) => + employees.groupBy(_.department).foreach { case (dept, emps) => + val rowNumbers = emps.flatMap(_.row_number).sorted + rowNumbers shouldBe (1 to emps.size).toList + + info(s"$dept: ${emps.size} employees numbered 1 to ${emps.size}") + } + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + "RANK window function" should "handle ties correctly" in { + val results = client.searchAs[EmployeeWithWindow](""" + SELECT + department, + name, + salary, + hire_date, + RANK() OVER ( + PARTITION BY department + ORDER BY salary DESC + ) AS rank + FROM emp + ORDER BY department, rank + """) + + results match { + case ElasticSuccess(employees) => + employees.groupBy(_.department).foreach { case (dept, emps) => + val ranks = emps.flatMap(_.rank) + ranks.head shouldBe 1 // Top earner always rank 1 + + info(s"$dept top earner: ${emps.head.name} (${emps.head.salary})") + } + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + }*/ + + // ======================================================================== + // TESTS WITH FILTERS + // ======================================================================== + + "Window function with WHERE clause" should "apply filters before computation" in { + val results = client.searchAs[EmployeeWithWindow](""" + SELECT + department, + name, + salary, + hire_date, + FIRST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS first_salary + FROM emp + WHERE salary > 80000 + ORDER BY department, hire_date + """) + + results match { + case ElasticSuccess(employees) => + employees.foreach { emp => + emp.salary should be > 80000 + } + + // Engineering avec filtre: first = Paul Anderson (2016-04-10, $105k) + val engineering = employees.filter(_.department == "Engineering") + engineering should not be empty + engineering.foreach { emp => + emp.first_salary shouldBe Some(130000) + } + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + "Window function with department filter" should "compute only for filtered data" in { + val results = client.searchAs[EmployeeWithWindow](""" + SELECT + department, + name, + salary, + hire_date, + location, + FIRST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS first_salary + FROM emp + WHERE department IN ('Engineering', 'Sales') + ORDER BY department, hire_date + """) + + results match { + case ElasticSuccess(employees) => + employees.foreach { emp => + emp.department should (be("Engineering") or be("Sales")) + emp.first_salary shouldBe defined + } + + val departments = employees.map(_.department).distinct + departments should contain only ("Engineering", "Sales") + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + // ======================================================================== + // TESTS WITH GLOBAL WINDOW + // ======================================================================== + + "Global window function" should "use same value for all rows" in { + val results = client.searchAs[EmployeeWithGlobalWindow](""" + SELECT + name, + salary, + hire_date, + FIRST_VALUE(salary) OVER (ORDER BY hire_date ASC) AS first_ever_salary, + LAST_VALUE(salary) OVER ( + ORDER BY hire_date ASC + ) AS last_ever_salary + FROM emp + ORDER BY hire_date + LIMIT 20 + """) + + results match { + case ElasticSuccess(employees) => + employees should have size 20 + + // Premier embauché: Sam Turner (2015-06-01, $130k) + employees.foreach { emp => + emp.first_ever_salary shouldBe Some(130000) + } + + // Dernier embauché: Tina Brooks (2021-03-15, $75k) + employees.foreach { emp => + emp.last_ever_salary shouldBe Some(75000) + } + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + // ======================================================================== + // TESTS WITH MULTIPLE PARTITIONS + // ======================================================================== + + "Multiple partition keys" should "compute independently" in { + val results = client.searchAs[EmployeeMultiPartition](""" + SELECT + department, + location, + name, + salary, + hire_date, + FIRST_VALUE(salary) OVER ( + PARTITION BY department, location + ORDER BY hire_date ASC + ) AS first_in_dept_loc + FROM emp + WHERE department IN ('Engineering', 'Sales') + ORDER BY department, location, hire_date + """) + + results match { + case ElasticSuccess(employees) => + employees + .groupBy(e => (e.department, e.location)) + .foreach { case ((dept, loc), emps) => + val firstValues = emps.flatMap(_.first_in_dept_loc).distinct + firstValues should have size 1 + + info(s"$dept @ $loc: first_salary = ${firstValues.head}") + } + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + // ======================================================================== + // TESTS WITH LIMIT + // ======================================================================== + + "Window function with LIMIT" should "return correct number of results" in { + val results = client.searchAs[EmployeeWithWindow](""" + SELECT + department, + name, + salary, + hire_date, + FIRST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS first_salary + FROM emp + ORDER BY salary DESC + LIMIT 10 + """) + + results match { + case ElasticSuccess(employees) => + employees should have size 10 + + // Top salary: Sam Turner ($130k) + employees.head.name shouldBe "Sam Turner" + employees.head.salary shouldBe 130000 + + // Tous les salaires >= $80k + employees.foreach { emp => + emp.salary should be >= 80000 + } + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + // ======================================================================== + // TESTS WITH AGGREGATIONS + // ======================================================================== + + "Window function with aggregations" should "combine GROUP BY and OVER" in { + val results = client.searchAs[DepartmentStats](""" + SELECT + department, + AVG(salary) AS avg_salary, + MAX(salary) AS max_salary, + MIN(salary) AS min_salary, + COUNT(*) AS employee_count + FROM emp + GROUP BY department + """) + + results match { + case ElasticSuccess(departments) => + departments should not be empty + + val engineering = departments.find(_.department == "Engineering") + engineering shouldBe defined + engineering.get.max_salary shouldBe 130000 // Sam Turner + engineering.get.min_salary shouldBe 75000 // Eve Davis + engineering.get.employee_count shouldBe 7 + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + // ======================================================================== + // SCROLL TESTS + // ======================================================================== + + "Scroll with FIRST_VALUE" should "stream all employees with window enrichment" in { + val config = ScrollConfig(scrollSize = 5) + + val futureResults = client + .scrollAs[EmployeeWithWindow]( + """SELECT + department, + name, + salary, + hire_date, + location, + level, + FIRST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS first_salary + FROM emp + ORDER BY department, hire_date + """, + config + ) + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + + results should have size 20 + + // Vérifier la cohérence par département + results.map(_._1).groupBy(_.department).foreach { case (dept, emps) => + val firstSalaries = emps.flatMap(_.first_salary).distinct + firstSalaries should have size 1 + + info(s"$dept: first_salary = ${firstSalaries.head}") + } + } + + "Scroll with multiple window functions" should "enrich with multiple columns" in { + val config = ScrollConfig(scrollSize = 3, logEvery = 5) + + val futureResults = client + .scrollAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + location, + level, + FIRST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS first_salary, + LAST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS last_salary + FROM emp + ORDER BY department, hire_date + """, + config + ) + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + + results should have size 20 + + results.foreach { case (emp, _) => + emp.first_salary shouldBe defined + emp.last_salary shouldBe defined + } + } + + // ======================================================================== + // TESTS DE PERFORMANCE + // ======================================================================== + + "Window functions performance" should "maintain good throughput" in { + val config = ScrollConfig(scrollSize = 5, logEvery = 10) + + val startTime = System.currentTimeMillis() + + val futureResults = client + .scrollAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + FIRST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS first_salary, + LAST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS last_salary + FROM emp + """, + config + ) + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + val duration = System.currentTimeMillis() - startTime + + results should have size 20 + duration should be < 5000L + + info(s"Scrolled ${results.size} documents with 3 window functions in ${duration}ms") + } + + // ======================================================================== + // TEST WITH MINIMAL CASE CLASS + // ======================================================================== + + "Minimal case class" should "work with partial SELECT" in { + val results = client.searchAs[EmployeeMinimal](""" + SELECT + name, + department, + salary + FROM emp + WHERE salary > 100000 + ORDER BY salary DESC + """) + + results match { + case ElasticSuccess(employees) => + employees should not be empty + employees.foreach { emp => + emp.salary should be > 100000 + } + + // Top 3: Sam Turner, Bob Smith, Diana Prince + employees.take(3).map(_.name) should contain allOf ( + "Sam Turner", "Bob Smith", "Diana Prince" + ) + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + // ======================================================================== + // SCROLL: FIRST_VALUE + // ======================================================================== + + "Scroll with FIRST_VALUE" should "stream all employees with first salary per department" in { + val config = ScrollConfig(scrollSize = 5, logEvery = 10) + + val futureResults = client + .scrollAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + location, + level, + FIRST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS first_salary + FROM emp + ORDER BY department, hire_date + """, + config + ) + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + + results should have size 20 + + // Vérifier la cohérence par département + val employees = results.map(_._1) + + val engineering = employees.filter(_.department == "Engineering") + engineering.foreach { emp => + emp.first_salary shouldBe Some(130000) // Sam Turner (2015-06-01) + } + + val sales = employees.filter(_.department == "Sales") + sales.foreach { emp => + emp.first_salary shouldBe Some(95000) // Iris Chen (2017-03-08) + } + + val marketing = employees.filter(_.department == "Marketing") + marketing.foreach { emp => + emp.first_salary shouldBe Some(88000) // Karen White (2018-05-15) + } + + info(s"✅ Scrolled ${results.size} employees with FIRST_VALUE") + } + + "Scroll with FIRST_VALUE and small batches" should "handle pagination correctly" in { + val config = ScrollConfig(scrollSize = 2, logEvery = 5) + + val futureResults = client + .scrollAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + FIRST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS first_salary + FROM emp + WHERE department = 'Engineering' + ORDER BY hire_date + """, + config + ) + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + + results should have size 7 // 7 engineers + + val employees = results.map(_._1) + employees.foreach { emp => + emp.department shouldBe "Engineering" + emp.first_salary shouldBe Some(130000) + } + + // Vérifier l'ordre chronologique + val hireDates = employees.map(_.hire_date) + hireDates shouldBe hireDates.sorted + + info(s"✅ Scrolled ${results.size} engineers in batches of 2") + } + + // ======================================================================== + // SCROLL: LAST_VALUE + // ======================================================================== + + "Scroll with LAST_VALUE" should "stream all employees with last salary per department" in { + val config = ScrollConfig(scrollSize = 4, logEvery = 8) + + val futureResults = client + .scrollAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + location, + LAST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS last_salary + FROM emp + ORDER BY department, hire_date + """, + config + ) + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + + results should have size 20 + + val employees = results.map(_._1) + + val engineering = employees.filter(_.department == "Engineering") + engineering.foreach { emp => + emp.last_salary shouldBe Some(75000) // Eve Davis (2021-02-12) + } + + val sales = employees.filter(_.department == "Sales") + sales.foreach { emp => + emp.last_salary shouldBe Some(75000) // Tina Brooks (2021-03-15) + } + + info(s"✅ Scrolled ${results.size} employees with LAST_VALUE") + } + + "Scroll with LAST_VALUE and filter" should "apply WHERE before window computation" in { + val config = ScrollConfig(scrollSize = 3) + + val futureResults = client + .scrollAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + LAST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS last_salary + FROM emp + WHERE salary > 80000 + ORDER BY department, hire_date + """, + config + ) + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + + val employees = results.map(_._1) + + employees.foreach { emp => + emp.salary should be > 80000 + } + + // Engineering avec filtre: last = Diana Prince (2017-09-05, $110k) + val engineering = employees.filter(_.department == "Engineering") + engineering.foreach { emp => + emp.last_salary shouldBe Some(85000) + } + + info(s"✅ Scrolled ${employees.size} high-salary employees with LAST_VALUE") + } + + // ======================================================================== + // SCROLL: ROW_NUMBER + // ======================================================================== + + /*"Scroll with ROW_NUMBER" should "assign sequential numbers per partition" in { + val config = ScrollConfig(scrollSize = 5) + + val futureResults = client + .scrollAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + ROW_NUMBER() OVER ( + PARTITION BY department + ORDER BY salary DESC + ) AS row_number + FROM emp + ORDER BY department, row_number + """, + config + ) + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + + results should have size 20 + + val employees = results.map(_._1) + + employees.groupBy(_.department).foreach { case (dept, emps) => + val rowNumbers = emps.flatMap(_.row_number).sorted + rowNumbers shouldBe (1 to emps.size).toList + + // Top earner (row_number = 1) + val topEarner = emps.find(_.row_number.contains(1)).get + + dept match { + case "Engineering" => topEarner.name shouldBe "Sam Turner" + case "Sales" => topEarner.name shouldBe "Iris Chen" + case "Marketing" => topEarner.name shouldBe "Karen White" + case "HR" => topEarner.name shouldBe "Olivia Scott" + case _ => // OK + } + + info(s"$dept: ${emps.size} employees, top earner = ${topEarner.name}") + } + } + + "Scroll with ROW_NUMBER and LIMIT simulation" should "get top N per department" in { + val config = ScrollConfig(scrollSize = 10) + + val futureResults = client + .scrollAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + ROW_NUMBER() OVER ( + PARTITION BY department + ORDER BY salary DESC + ) AS row_number + FROM emp + ORDER BY department, row_number + """, + config + ) + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + + val employees = results.map(_._1) + + // Filtrer les top 2 par département + val top2PerDept = employees + .filter(_.row_number.exists(_ <= 2)) + .groupBy(_.department) + + top2PerDept.foreach { case (dept, emps) => + emps should have size 2 + info(s"$dept top 2: ${emps.map(e => s"${e.name} ($${e.salary})").mkString(", ")}") + } + } + + // ======================================================================== + // SCROLL: RANK + // ======================================================================== + + "Scroll with RANK" should "handle ties correctly" in { + val config = ScrollConfig(scrollSize = 5) + + val futureResults = client + .scrollAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + RANK() OVER ( + PARTITION BY department + ORDER BY salary DESC + ) AS rank + FROM emp + ORDER BY department, rank + """, + config + ) + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + + results should have size 20 + + val employees = results.map(_._1) + + employees.groupBy(_.department).foreach { case (dept, emps) => + val ranks = emps.flatMap(_.rank) + ranks.head shouldBe 1 // Top earner always rank 1 + + val topEarner = emps.head + info(s"$dept rank 1: ${topEarner.name} ($${topEarner.salary})") + } + }*/ + + // ======================================================================== + // SCROLL: GLOBAL WINDOW + // ======================================================================== + + "Scroll with global window" should "use same value for all rows" in { + val config = ScrollConfig(scrollSize = 7) + + val futureResults = client + .scrollAs[EmployeeWithGlobalWindow]( + """ + SELECT + name, + salary, + hire_date, + FIRST_VALUE(salary) OVER (ORDER BY hire_date ASC) AS first_ever_salary, + LAST_VALUE(salary) OVER ( + ORDER BY hire_date ASC + ) AS last_ever_salary + FROM emp + ORDER BY hire_date + """, + config + ) + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + + results should have size 20 + + val employees = results.map(_._1) + + // Premier embauché: Sam Turner (2015-06-01, $130k) + employees.foreach { emp => + emp.first_ever_salary shouldBe Some(130000) + } + + // Dernier embauché: Tina Brooks (2021-03-15, $75k) + employees.foreach { emp => + emp.last_ever_salary shouldBe Some(75000) + } + + // Vérifier l'ordre chronologique + val hireDates = employees.map(_.hire_date) + hireDates shouldBe hireDates.sorted + + info(s"✅ Global window: first = $$130k, last = $$75k") + } + + // ======================================================================== + // SCROLL: MULTIPLE PARTITIONS + // ======================================================================== + + "Scroll with multiple partition keys" should "compute independently" in { + val config = ScrollConfig(scrollSize = 4) + + val futureResults = client + .scrollAs[EmployeeMultiPartition]( + """ + SELECT + department, + location, + name, + salary, + hire_date, + FIRST_VALUE(salary) OVER ( + PARTITION BY department, location + ORDER BY hire_date ASC + ) AS first_in_dept_loc + FROM emp + WHERE department IN ('Engineering', 'Sales') + ORDER BY department, location, hire_date + """, + config + ) + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + + val employees = results.map(_._1) + + employees + .groupBy(e => (e.department, e.location)) + .foreach { case ((dept, loc), emps) => + val firstValues = emps.flatMap(_.first_in_dept_loc).distinct + firstValues should have size 1 + + info(s"$dept @ $loc: first_salary = ${firstValues.head}, ${emps.size} employees") + } + + // Engineering @ New York: Alice (2019-03-15, $95k) ou Bob (2018-01-10, $120k) + val engNY = employees.filter(e => e.department == "Engineering" && e.location == "New York") + engNY.foreach { emp => + emp.first_in_dept_loc shouldBe Some(120000) // Bob Smith + } + } + + // ======================================================================== + // SCROLL WITH COMPLEX FILTERS + // ======================================================================== + + "Scroll with complex WHERE clause" should "apply all filters before window" in { + val config = ScrollConfig(scrollSize = 5) + + val futureResults = client + .scrollAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + location, + level, + FIRST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS first_salary + FROM emp + WHERE salary > 80000 + AND hire_date >= '2018-01-01' + AND department IN ('Engineering', 'Sales') + ORDER BY department, hire_date + """, + config + ) + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + + val employees = results.map(_._1) + + employees.foreach { emp => + emp.salary should be > 80000 + emp.hire_date should be >= LocalDate.of(2018, 1, 1) + emp.department should (be("Engineering") or be("Sales")) + } + + info(s"✅ Filtered scroll: ${employees.size} employees matching criteria") + } + + // ======================================================================== + // SCROLL: PERFORMANCE AND MONITORING + // ======================================================================== + + "Scroll with performance monitoring" should "track progress and timing" in { + val config = ScrollConfig( + scrollSize = 5, + logEvery = 5 + ) + + val startTime = System.currentTimeMillis() + + val futureResults = client + .scrollAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + FIRST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS first_salary, + LAST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS last_salary + FROM emp + """, + config + ) + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + val duration = System.currentTimeMillis() - startTime + + results should have size 20 + duration should be < 5000L + + val employees = results.map(_._1) + + // Vérifier que toutes les colonnes window sont présentes + employees.foreach { emp => + emp.first_salary shouldBe defined + emp.last_salary shouldBe defined + } + + info(s"✅ Scrolled ${results.size} docs with 4 window functions in ${duration}ms") + info(s" Throughput: ${results.size * 1000 / duration} docs/sec") + } + + // ======================================================================== + // SCROLL: STREAMING WITH TRANSFORMATION + // ======================================================================== + + "Scroll with stream transformation" should "process results on-the-fly" in { + val config = ScrollConfig(scrollSize = 3) + + val futureResults = client + .scrollAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + FIRST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS first_salary + FROM emp + ORDER BY department, salary DESC + """, + config + ) + .map { case (emp, scrollId) => + // Transformation: calculer le % vs premier salaire + val pctVsFirst = emp.first_salary.map { first => + (emp.salary.toDouble / first * 100).round.toInt + } + + (emp, pctVsFirst, scrollId) + } + .filter { case (emp, pct, _) => + // Ne garder que les top earners (row_number <= 3) + emp.row_number.exists(_ <= 3) + } + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + + results.foreach { case (emp, pctVsFirst, _) => + info( + s"${emp.department} - ${emp.name}: $${emp.salary} (${pctVsFirst.getOrElse("N/A")}% vs first)" + ) + } + + // Chaque département devrait avoir max 3 employés + val countPerDept = results.map(_._1).groupBy(_.department).mapValues(_.size) + countPerDept.values.foreach { count => + count should be <= 3 + } + } + + // ======================================================================== + // SCROLL WITH DOWNSTREAM AGGREGATION + // ======================================================================== + + "Scroll with downstream aggregation" should "compute stats from stream" in { + val config = ScrollConfig(scrollSize = 4) + + val futureStats = client + .scrollAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + FIRST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS first_salary + FROM emp + """, + config + ) + .map(_._1) + .runFold(Map.empty[String, (Int, Int, Int)]) { case (acc, emp) => + // Calculer min/max/count par département + val (min, max, count) = acc.getOrElse(emp.department, (Int.MaxValue, 0, 0)) + + acc + (emp.department -> ( + math.min(min, emp.salary), + math.max(max, emp.salary), + count + 1 + )) + } + + val stats = Await.result(futureStats, 30.seconds) + + stats should not be empty + + stats.foreach { case (dept, (min, max, count)) => + info(s"$dept: $count employees, salary range: $$${min} - $$${max}") + } + + // Engineering: 7 employees, $75k - $130k + stats("Engineering") shouldBe (75000, 130000, 7) + + // Sales: 6 employees, $70k - $95k + stats("Sales") shouldBe (70000, 95000, 6) + } + + // ======================================================================== + // SCROLL: ERROR HANDLING + // ======================================================================== + + "Scroll with error handling" should "handle failures gracefully" in { + val config = ScrollConfig(scrollSize = 5) + + val futureResults = client + .scrollAs[EmployeeWithWindow]( + """ + SELECT + department, + name, + salary, + hire_date, + FIRST_VALUE(salary) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + ) AS first_salary + FROM emp + """, + config + ) + .recover { case ex => + info(s"⚠️ Scroll error: ${ex.getMessage}") + (EmployeeWithWindow("", "", 0, LocalDate.now), config.metrics) + } + .filter(_._1.department.nonEmpty) // Filtrer les erreurs + .runWith(Sink.seq) + + val results = Await.result(futureResults, 30.seconds) + + results should not be empty + results.foreach { case (emp, _) => + emp.department should not be empty + } + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/model/window/package.scala b/testkit/src/main/scala/app/softnetwork/elastic/model/window/package.scala new file mode 100644 index 00000000..1074e96b --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/model/window/package.scala @@ -0,0 +1,86 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.model + +import java.time.LocalDate + +package object window { + + case class Employee( + name: String, + department: String, + location: String, + salary: Int, + hire_date: String, + level: String, + skills: List[String], + id: String = "" + ) + + case class EmployeeWithWindow( + department: String, + name: String, + salary: Int, + hire_date: LocalDate, + location: Option[String] = None, + level: Option[String] = None, + skills: Option[List[String]] = None, + first_salary: Option[Int] = None, + last_salary: Option[Int] = None, + rank: Option[Int] = None, + row_number: Option[Int] = None + ) + + case class DepartmentStats( + department: String, + avg_salary: Double, + max_salary: Int, + min_salary: Int, + employee_count: Long + ) + + case class DepartmentWithWindow( + department: String, + location: Option[String] = None, + avg_salary: Option[Double] = None, + top_earners: Option[List[String]] = None, + first_hire_date: Option[String] = None + ) + + case class EmployeeMinimal( + name: String, + department: String, + salary: Int + ) + + case class EmployeeWithGlobalWindow( + name: String, + salary: Int, + hire_date: String, + first_ever_salary: Option[Int] = None, + last_ever_salary: Option[Int] = None + ) + + case class EmployeeMultiPartition( + department: String, + location: String, + name: String, + salary: Int, + hire_date: String, + first_in_dept_loc: Option[Int] = None + ) +}