From 1f5bd9599749f81c310d8b9a91f34373eab73e6b Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Tue, 29 Nov 2022 11:24:34 +0100 Subject: [PATCH 01/21] Consider-none-equivalent-value-for-missing --- CHANGELOG.md | 1 + .../service/weather/WeatherSource.scala | 248 +++++++++++++++--- .../weather/WeatherSourceWrapper.scala | 10 +- 3 files changed, 221 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 550526c62e..0c59574802 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added Kafka sink for runtime events, re-implemented RuntimeEventListener in akka typed [#242](https://github.com/ie3-institute/simona/issues/242) - Added listeners to DBFS tests to check the result output and check the handling of failed power flows [#269](https://github.com/ie3-institute/simona/issues/269) - Added DBFS test with participant load and added testing for FinishGridSimulationTrigger [#281](https://github.com/ie3-institute/simona/issues/281) +- Added an interpolation for missing weather data [#188](https://github.com/ie3-institute/simona/issues/188) ### Changed - Re-organizing test resources into their respective packages [#105](https://github.com/ie3-institute/simona/issues/105) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index e840a082ef..48a8460107 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -6,45 +6,39 @@ package edu.ie3.simona.service.weather -import edu.ie3.datamodel.io.factory.timeseries.{ - CosmoIdCoordinateFactory, - IconIdCoordinateFactory, - IdCoordinateFactory -} +import edu.ie3.datamodel.io.factory.timeseries.{CosmoIdCoordinateFactory, IconIdCoordinateFactory, IdCoordinateFactory} import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.source.IdCoordinateSource import edu.ie3.datamodel.io.source.csv.CsvIdCoordinateSource import edu.ie3.datamodel.models.StandardUnits -import edu.ie3.datamodel.models.value.WeatherValue +import edu.ie3.datamodel.models.timeseries.individual.{IndividualTimeSeries, TimeBasedValue} +import edu.ie3.datamodel.models.value.{SolarIrradianceValue, WeatherValue} import edu.ie3.simona.config.SimonaConfig import edu.ie3.simona.config.SimonaConfig.BaseCsvParams import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource._ -import edu.ie3.simona.exceptions.{ - InvalidConfigParameterException, - ServiceException -} +import edu.ie3.simona.exceptions.{InvalidConfigParameterException, ServiceException} import edu.ie3.simona.ontology.messages.services.WeatherMessage.WeatherData -import edu.ie3.simona.service.weather.WeatherSource.{ - AgentCoordinates, - WeightedCoordinates -} +import edu.ie3.simona.service.weather.WeatherSource.{AgentCoordinates, WeightedCoordinates} import edu.ie3.simona.util.ConfigUtil.CsvConfigUtil.checkBaseCsvParams -import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{ - checkCouchbaseParams, - checkInfluxDb1xParams, - checkSqlParams -} +import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{checkCouchbaseParams, checkInfluxDb1xParams, checkSqlParams} import edu.ie3.simona.util.ParsableEnumeration +import edu.ie3.simona.util.TickUtil.RichZonedDateTime import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} import edu.ie3.util.quantities.PowerSystemUnits +import edu.ie3.util.quantities.interfaces.Irradiance import org.locationtech.jts.geom.{Coordinate, Point} +import tech.units.indriya.ComparableQuantity import tech.units.indriya.quantity.Quantities import tech.units.indriya.unit.Units import java.time.ZonedDateTime +import java.time.temporal.ChronoUnit import javax.measure.Quantity -import javax.measure.quantity.{Dimensionless, Length} +import javax.measure.quantity.{Dimensionless, Length, Speed, Temperature} +import scala.annotation.tailrec +import scala.concurrent.duration.Duration import scala.jdk.CollectionConverters._ +import scala.jdk.OptionConverters.RichOptional import scala.util.{Failure, Success, Try} trait WeatherSource { @@ -510,22 +504,212 @@ object WeatherSource { val EMPTY_WEATHER_DATA: WeatherData = WeatherData( Quantities.getQuantity(0d, StandardUnits.SOLAR_IRRADIANCE), Quantities.getQuantity(0d, StandardUnits.SOLAR_IRRADIANCE), - Quantities.getQuantity(0d, Units.KELVIN).to(StandardUnits.TEMPERATURE), + Quantities.getQuantity(288.15d, Units.KELVIN).to(StandardUnits.TEMPERATURE), Quantities.getQuantity(0d, StandardUnits.WIND_VELOCITY) ) - def toWeatherData( - weatherValue: WeatherValue + /** Methode to get weather data from a time series. This method automatically + * interpolates missing values. + * @param timeSeries + * with weather values + * @param dateTime + * timestamp in question + * @return + * weather data object + */ + def getWeatherData( + timeSeries: IndividualTimeSeries[WeatherValue], + dateTime: ZonedDateTime ): WeatherData = { - WeatherData( - weatherValue.getSolarIrradiance.getDiffuseIrradiance - .orElse(EMPTY_WEATHER_DATA.diffIrr), - weatherValue.getSolarIrradiance.getDirectIrradiance - .orElse(EMPTY_WEATHER_DATA.dirIrr), - weatherValue.getTemperature.getTemperature - .orElse(EMPTY_WEATHER_DATA.temp), - weatherValue.getWind.getVelocity.orElse(EMPTY_WEATHER_DATA.windVel) - ) + + // gets a value option + val valueOption: Option[WeatherValue] = + timeSeries.getValue(dateTime).toScala + + valueOption match { + case Some(value) => + // found values are used to create a weather data object + // missing values are interpolated + + WeatherData( + value.getSolarIrradiance.getDiffuseIrradiance.toScala + .getOrElse(interpolateValue(timeSeries, dateTime, "diffIrr")), + value.getSolarIrradiance.getDirectIrradiance.toScala + .getOrElse(interpolateValue(timeSeries, dateTime, "dirIrr")), + value.getTemperature.getTemperature.toScala + .getOrElse(interpolateValue(timeSeries, dateTime, "temp")), + value.getWind.getVelocity.toScala + .getOrElse(interpolateValue(timeSeries, dateTime, "windVel")) + ) + case None => + // if no values are found all values are interpolated + + WeatherData( + Quantities.getQuantity( + interpolateValue(timeSeries, dateTime, "diffIrr").getValue, + StandardUnits.SOLAR_IRRADIANCE + ), + Quantities.getQuantity( + interpolateValue(timeSeries, dateTime, "dirIrr").getValue, + StandardUnits.SOLAR_IRRADIANCE + ), + Quantities.getQuantity( + interpolateValue(timeSeries, dateTime, "temp").getValue, + StandardUnits.TEMPERATURE + ), + Quantities.getQuantity( + interpolateValue(timeSeries, dateTime, "windVel").getValue, + StandardUnits.WIND_VELOCITY + ) + ) + } + } + + /** Method for interpolation of weather values. + * @param timeSeries + * with weather data + * @param dateTime + * timestamp for which an interpolation is needed + * @param get + * string defining the searched value + * @return + * new quantity + */ + def interpolateValue( + timeSeries: IndividualTimeSeries[WeatherValue], + dateTime: ZonedDateTime, + get: String + ): ComparableQuantity[_] = { + // gets two options for values + val previousOption: Option[(ComparableQuantity[_], ZonedDateTime)] = + getNewValue( + timeSeries, + dateTime, + dateTime.minusHours(2), + get, + backwards = true + ) + val nextOption: Option[(ComparableQuantity[_], ZonedDateTime)] = + getNewValue( + timeSeries, + dateTime, + dateTime.plusHours(2), + get, + backwards = false + ) + + (previousOption, nextOption) match { + case (Some(preVal), Some(nextVal)) => + // only if a previous and a next value are found + // found values are weighted with their time difference and used to calculate a new value + + val diffToPreviousValue: Long = + ChronoUnit.SECONDS.between(preVal._2, dateTime) + val diffToNextValue: Long = + ChronoUnit.SECONDS.between(dateTime, nextVal._2) + + preVal._1 + .multiply(diffToPreviousValue) + .add(nextVal._1.multiply(diffToNextValue)) + .divide(diffToPreviousValue + diffToNextValue) + case (_, _) => + // if at least one value could not be found, the value found in the EMPTY_WEATHER_DATA object is returned + get match { + case "diffIrr" => + EMPTY_WEATHER_DATA.diffIrr + case "dirIrr" => + EMPTY_WEATHER_DATA.dirIrr + case "temp" => + EMPTY_WEATHER_DATA.temp + case "windVel" => + EMPTY_WEATHER_DATA.windVel + } + } + } + + /** Recursive method to find a new value for a specific weather data. + * + * @param timeSeries + * with weather values + * @param lastTime + * last timestamp + * @param maxDateTime + * after with the recursion ends + * @param get + * string defining the searched value + * @param backwards + * defines if the recursion is backwards or forwards + * @return + * an option for a quantity and a time + */ + @tailrec + def getNewValue( + timeSeries: IndividualTimeSeries[WeatherValue], + lastTime: ZonedDateTime, + maxDateTime: ZonedDateTime, + get: String, + backwards: Boolean + ): Option[(ComparableQuantity[_], ZonedDateTime)] = { + // if one is true, then the recursion ends + if (backwards && lastTime.isBefore(maxDateTime)) { + return None + } else if (!backwards && lastTime.isAfter(maxDateTime)) { + return None + } + + // gets the new value option + val timeBasedValue: Option[TimeBasedValue[WeatherValue]] = + if (backwards) { + timeSeries.getPreviousTimeBasedValue(lastTime).toScala + } else { + timeSeries.getNextTimeBasedValue(lastTime).toScala + } + + timeBasedValue match { + case Some(value) => + val weatherValue: WeatherValue = value.getValue + + // gets the searched value as an option + val quantity: Option[ComparableQuantity[_]] = get match { + case "diffIrr" => + weatherValue.getSolarIrradiance.getDiffuseIrradiance.toScala + case "dirIrr" => + weatherValue.getSolarIrradiance.getDirectIrradiance.toScala + case "temp" => + weatherValue.getTemperature.getTemperature.toScala + case "windVel" => + weatherValue.getWind.getVelocity.toScala + } + + quantity match { + case Some(data) => + // returning the found value + Some(data, value.getTime) + case None => + // recursion with found time as a new timestamp + getNewValue( + timeSeries, + value.getTime, + maxDateTime, + get, + backwards + ) + } + case None => + val newTime: ZonedDateTime = if (backwards) { + lastTime.minusMinutes(15) + } else { + lastTime.plusMinutes(15) + } + + getNewValue( + timeSeries, + newTime, + maxDateTime, + get, + backwards + ) + } } /** Weather package private case class to combine the provided agent diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala index 5f3df78758..362e465f6d 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala @@ -37,7 +37,7 @@ import edu.ie3.simona.ontology.messages.services.WeatherMessage.WeatherData import edu.ie3.simona.service.weather.WeatherSource.{ EMPTY_WEATHER_DATA, WeatherScheme, - toWeatherData + getWeatherData } import edu.ie3.simona.service.weather.WeatherSourceWrapper.WeightSum import edu.ie3.simona.service.weather.{WeatherSource => SimonaWeatherSource} @@ -102,12 +102,10 @@ private[weather] final case class WeatherSourceWrapper private ( ) .asScala .toMap - val weatherDataMap = results.flatMap { case (point, timeSeries) => + val weatherDataMap = results.map { case (point, timeSeries) => // change temperature scale for the upcoming calculations - timeSeries - .getValue(dateTime) - .toScala - .map(weatherValue => point -> toWeatherData(weatherValue)) + + point -> getWeatherData(timeSeries, dateTime) } weatherDataMap.foldLeft((EMPTY_WEATHER_DATA, WeightSum.EMPTY_WEIGHT_SUM)) { From e4db83482310a4184a1fdd894f3f3a9618b3eff0 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Tue, 29 Nov 2022 11:27:45 +0100 Subject: [PATCH 02/21] fmt --- .../service/weather/WeatherSource.scala | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 48a8460107..3591d7ee2d 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -6,26 +6,41 @@ package edu.ie3.simona.service.weather -import edu.ie3.datamodel.io.factory.timeseries.{CosmoIdCoordinateFactory, IconIdCoordinateFactory, IdCoordinateFactory} +import edu.ie3.datamodel.io.factory.timeseries.{ + CosmoIdCoordinateFactory, + IconIdCoordinateFactory, + IdCoordinateFactory +} import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.source.IdCoordinateSource import edu.ie3.datamodel.io.source.csv.CsvIdCoordinateSource import edu.ie3.datamodel.models.StandardUnits -import edu.ie3.datamodel.models.timeseries.individual.{IndividualTimeSeries, TimeBasedValue} -import edu.ie3.datamodel.models.value.{SolarIrradianceValue, WeatherValue} +import edu.ie3.datamodel.models.timeseries.individual.{ + IndividualTimeSeries, + TimeBasedValue +} +import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.simona.config.SimonaConfig import edu.ie3.simona.config.SimonaConfig.BaseCsvParams import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource._ -import edu.ie3.simona.exceptions.{InvalidConfigParameterException, ServiceException} +import edu.ie3.simona.exceptions.{ + InvalidConfigParameterException, + ServiceException +} import edu.ie3.simona.ontology.messages.services.WeatherMessage.WeatherData -import edu.ie3.simona.service.weather.WeatherSource.{AgentCoordinates, WeightedCoordinates} +import edu.ie3.simona.service.weather.WeatherSource.{ + AgentCoordinates, + WeightedCoordinates +} import edu.ie3.simona.util.ConfigUtil.CsvConfigUtil.checkBaseCsvParams -import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{checkCouchbaseParams, checkInfluxDb1xParams, checkSqlParams} +import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{ + checkCouchbaseParams, + checkInfluxDb1xParams, + checkSqlParams +} import edu.ie3.simona.util.ParsableEnumeration -import edu.ie3.simona.util.TickUtil.RichZonedDateTime import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} import edu.ie3.util.quantities.PowerSystemUnits -import edu.ie3.util.quantities.interfaces.Irradiance import org.locationtech.jts.geom.{Coordinate, Point} import tech.units.indriya.ComparableQuantity import tech.units.indriya.quantity.Quantities @@ -34,9 +49,8 @@ import tech.units.indriya.unit.Units import java.time.ZonedDateTime import java.time.temporal.ChronoUnit import javax.measure.Quantity -import javax.measure.quantity.{Dimensionless, Length, Speed, Temperature} +import javax.measure.quantity.{Dimensionless, Length} import scala.annotation.tailrec -import scala.concurrent.duration.Duration import scala.jdk.CollectionConverters._ import scala.jdk.OptionConverters.RichOptional import scala.util.{Failure, Success, Try} From 8b5c484e80f90497754c7c5b7eadbce9642975ce Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Tue, 29 Nov 2022 11:40:25 +0100 Subject: [PATCH 03/21] Fixing some problems --- .../service/weather/WeatherSource.scala | 57 ++++++++++++------- .../weather/WeatherSourceWrapper.scala | 1 - 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 3591d7ee2d..012c6db974 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -544,35 +544,53 @@ object WeatherSource { case Some(value) => // found values are used to create a weather data object // missing values are interpolated - WeatherData( value.getSolarIrradiance.getDiffuseIrradiance.toScala - .getOrElse(interpolateValue(timeSeries, dateTime, "diffIrr")), + .getOrElse( + Quantities.getQuantity( + interpolateValue(timeSeries, dateTime, "diffIrr"), + StandardUnits.SOLAR_IRRADIANCE + ) + ), value.getSolarIrradiance.getDirectIrradiance.toScala - .getOrElse(interpolateValue(timeSeries, dateTime, "dirIrr")), + .getOrElse( + Quantities.getQuantity( + interpolateValue(timeSeries, dateTime, "dirIrr"), + StandardUnits.SOLAR_IRRADIANCE + ) + ), value.getTemperature.getTemperature.toScala - .getOrElse(interpolateValue(timeSeries, dateTime, "temp")), + .getOrElse( + Quantities.getQuantity( + interpolateValue(timeSeries, dateTime, "temp"), + StandardUnits.TEMPERATURE + ) + ), value.getWind.getVelocity.toScala - .getOrElse(interpolateValue(timeSeries, dateTime, "windVel")) + .getOrElse( + Quantities.getQuantity( + interpolateValue(timeSeries, dateTime, "windVel"), + StandardUnits.WIND_VELOCITY + ) + ) ) case None => // if no values are found all values are interpolated - WeatherData( Quantities.getQuantity( - interpolateValue(timeSeries, dateTime, "diffIrr").getValue, + interpolateValue(timeSeries, dateTime, "diffIrr"), StandardUnits.SOLAR_IRRADIANCE ), Quantities.getQuantity( - interpolateValue(timeSeries, dateTime, "dirIrr").getValue, + interpolateValue(timeSeries, dateTime, "dirIrr"), StandardUnits.SOLAR_IRRADIANCE ), Quantities.getQuantity( - interpolateValue(timeSeries, dateTime, "temp").getValue, + interpolateValue(timeSeries, dateTime, "temp"), StandardUnits.TEMPERATURE ), Quantities.getQuantity( - interpolateValue(timeSeries, dateTime, "windVel").getValue, + interpolateValue(timeSeries, dateTime, "windVel"), StandardUnits.WIND_VELOCITY ) ) @@ -593,9 +611,9 @@ object WeatherSource { timeSeries: IndividualTimeSeries[WeatherValue], dateTime: ZonedDateTime, get: String - ): ComparableQuantity[_] = { + ): Double = { // gets two options for values - val previousOption: Option[(ComparableQuantity[_], ZonedDateTime)] = + val previousOption: Option[(Double, ZonedDateTime)] = getNewValue( timeSeries, dateTime, @@ -603,7 +621,7 @@ object WeatherSource { get, backwards = true ) - val nextOption: Option[(ComparableQuantity[_], ZonedDateTime)] = + val nextOption: Option[(Double, ZonedDateTime)] = getNewValue( timeSeries, dateTime, @@ -622,13 +640,10 @@ object WeatherSource { val diffToNextValue: Long = ChronoUnit.SECONDS.between(dateTime, nextVal._2) - preVal._1 - .multiply(diffToPreviousValue) - .add(nextVal._1.multiply(diffToNextValue)) - .divide(diffToPreviousValue + diffToNextValue) + (preVal._1 * diffToPreviousValue + nextVal._1 * diffToNextValue) / (diffToPreviousValue + diffToNextValue) case (_, _) => // if at least one value could not be found, the value found in the EMPTY_WEATHER_DATA object is returned - get match { + val defaultValue = get match { case "diffIrr" => EMPTY_WEATHER_DATA.diffIrr case "dirIrr" => @@ -638,6 +653,8 @@ object WeatherSource { case "windVel" => EMPTY_WEATHER_DATA.windVel } + + defaultValue.getValue.doubleValue() } } @@ -663,7 +680,7 @@ object WeatherSource { maxDateTime: ZonedDateTime, get: String, backwards: Boolean - ): Option[(ComparableQuantity[_], ZonedDateTime)] = { + ): Option[(Double, ZonedDateTime)] = { // if one is true, then the recursion ends if (backwards && lastTime.isBefore(maxDateTime)) { return None @@ -698,7 +715,7 @@ object WeatherSource { quantity match { case Some(data) => // returning the found value - Some(data, value.getTime) + Some(data.getValue.doubleValue(), value.getTime) case None => // recursion with found time as a new timestamp getNewValue( diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala index 362e465f6d..747c36bcb2 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala @@ -52,7 +52,6 @@ import tech.units.indriya.unit.Units import java.time.ZonedDateTime import javax.measure.Quantity import scala.jdk.CollectionConverters.{IterableHasAsJava, MapHasAsScala} -import scala.jdk.OptionConverters.RichOptional import scala.util.{Failure, Success, Try} /** This class provides an implementation of the SIMONA trait From de89bacf7e3b1a7bf1207fdc05aea824c397de8b Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Tue, 29 Nov 2022 12:11:52 +0100 Subject: [PATCH 04/21] Fixing problem with recursion. --- .../service/weather/WeatherSource.scala | 106 +++++++++--------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 012c6db974..0e0e20754a 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -683,63 +683,63 @@ object WeatherSource { ): Option[(Double, ZonedDateTime)] = { // if one is true, then the recursion ends if (backwards && lastTime.isBefore(maxDateTime)) { - return None + None } else if (!backwards && lastTime.isAfter(maxDateTime)) { - return None - } - - // gets the new value option - val timeBasedValue: Option[TimeBasedValue[WeatherValue]] = - if (backwards) { - timeSeries.getPreviousTimeBasedValue(lastTime).toScala - } else { - timeSeries.getNextTimeBasedValue(lastTime).toScala - } - - timeBasedValue match { - case Some(value) => - val weatherValue: WeatherValue = value.getValue - - // gets the searched value as an option - val quantity: Option[ComparableQuantity[_]] = get match { - case "diffIrr" => - weatherValue.getSolarIrradiance.getDiffuseIrradiance.toScala - case "dirIrr" => - weatherValue.getSolarIrradiance.getDirectIrradiance.toScala - case "temp" => - weatherValue.getTemperature.getTemperature.toScala - case "windVel" => - weatherValue.getWind.getVelocity.toScala - } - - quantity match { - case Some(data) => - // returning the found value - Some(data.getValue.doubleValue(), value.getTime) - case None => - // recursion with found time as a new timestamp - getNewValue( - timeSeries, - value.getTime, - maxDateTime, - get, - backwards - ) - } - case None => - val newTime: ZonedDateTime = if (backwards) { - lastTime.minusMinutes(15) + None + } else { + // gets the new value option + val timeBasedValue: Option[TimeBasedValue[WeatherValue]] = + if (backwards) { + timeSeries.getPreviousTimeBasedValue(lastTime).toScala } else { - lastTime.plusMinutes(15) + timeSeries.getNextTimeBasedValue(lastTime).toScala } - getNewValue( - timeSeries, - newTime, - maxDateTime, - get, - backwards - ) + timeBasedValue match { + case Some(value) => + val weatherValue: WeatherValue = value.getValue + + // gets the searched value as an option + val quantity: Option[ComparableQuantity[_]] = get match { + case "diffIrr" => + weatherValue.getSolarIrradiance.getDiffuseIrradiance.toScala + case "dirIrr" => + weatherValue.getSolarIrradiance.getDirectIrradiance.toScala + case "temp" => + weatherValue.getTemperature.getTemperature.toScala + case "windVel" => + weatherValue.getWind.getVelocity.toScala + } + + quantity match { + case Some(data) => + // returning the found value + Some(data.getValue.doubleValue(), value.getTime) + case None => + // recursion with found time as a new timestamp + getNewValue( + timeSeries, + value.getTime, + maxDateTime, + get, + backwards + ) + } + case None => + val newTime: ZonedDateTime = if (backwards) { + lastTime.minusMinutes(15) + } else { + lastTime.plusMinutes(15) + } + + getNewValue( + timeSeries, + newTime, + maxDateTime, + get, + backwards + ) + } } } From c25d7b822f280d1fbf3bfc27fd9ceab2d2db92be Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Tue, 29 Nov 2022 16:34:24 +0100 Subject: [PATCH 05/21] Fixing problem with recursion. --- .../service/weather/WeatherSource.scala | 27 ++++++++++++------- .../weather/WeatherSourceWrapper.scala | 2 +- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 0e0e20754a..6aaee8558b 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -518,7 +518,7 @@ object WeatherSource { val EMPTY_WEATHER_DATA: WeatherData = WeatherData( Quantities.getQuantity(0d, StandardUnits.SOLAR_IRRADIANCE), Quantities.getQuantity(0d, StandardUnits.SOLAR_IRRADIANCE), - Quantities.getQuantity(288.15d, Units.KELVIN).to(StandardUnits.TEMPERATURE), + Quantities.getQuantity(0d, Units.KELVIN).to(StandardUnits.TEMPERATURE), Quantities.getQuantity(0d, StandardUnits.WIND_VELOCITY) ) @@ -619,7 +619,8 @@ object WeatherSource { dateTime, dateTime.minusHours(2), get, - backwards = true + backwards = true, + 8 ) val nextOption: Option[(Double, ZonedDateTime)] = getNewValue( @@ -627,7 +628,8 @@ object WeatherSource { dateTime, dateTime.plusHours(2), get, - backwards = false + backwards = false, + 8 ) (previousOption, nextOption) match { @@ -649,7 +651,7 @@ object WeatherSource { case "dirIrr" => EMPTY_WEATHER_DATA.dirIrr case "temp" => - EMPTY_WEATHER_DATA.temp + Quantities.getQuantity(0d, StandardUnits.TEMPERATURE) case "windVel" => EMPTY_WEATHER_DATA.windVel } @@ -679,12 +681,15 @@ object WeatherSource { lastTime: ZonedDateTime, maxDateTime: ZonedDateTime, get: String, - backwards: Boolean + backwards: Boolean, + remainingNumberOfTries: Int ): Option[(Double, ZonedDateTime)] = { // if one is true, then the recursion ends - if (backwards && lastTime.isBefore(maxDateTime)) { - None - } else if (!backwards && lastTime.isAfter(maxDateTime)) { + if ( + (remainingNumberOfTries == 0) + || (backwards && lastTime.isBefore(maxDateTime)) + || (!backwards && lastTime.isAfter(maxDateTime)) + ) { None } else { // gets the new value option @@ -722,7 +727,8 @@ object WeatherSource { value.getTime, maxDateTime, get, - backwards + backwards, + remainingNumberOfTries - 1 ) } case None => @@ -737,7 +743,8 @@ object WeatherSource { newTime, maxDateTime, get, - backwards + backwards, + remainingNumberOfTries - 1 ) } } diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala index 747c36bcb2..a597781f2f 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala @@ -160,7 +160,7 @@ private[weather] final case class WeatherSourceWrapper private ( ) } val (tempContrib, tempWeight) = currentWeather.temp match { - case EMPTY_WEATHER_DATA.temp => (EMPTY_WEATHER_DATA.temp, 0d) + case EMPTY_WEATHER_DATA.temp => (EMPTY_WEATHER_DATA.temp, 288.15d) case nonEmptyTemp => calculateContrib( nonEmptyTemp.to(Units.KELVIN), From 80cb59bdb4fbdcad5d619c2eb3edafcb58effefb Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Thu, 1 Dec 2022 12:46:22 +0100 Subject: [PATCH 06/21] Changing from recursive to iterative approach. --- .../service/weather/WeatherSource.scala | 270 ++++++++---------- .../weather/WeatherSourceWrapper.scala | 2 +- 2 files changed, 118 insertions(+), 154 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 6aaee8558b..536b47e2a2 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -50,7 +50,6 @@ import java.time.ZonedDateTime import java.time.temporal.ChronoUnit import javax.measure.Quantity import javax.measure.quantity.{Dimensionless, Length} -import scala.annotation.tailrec import scala.jdk.CollectionConverters._ import scala.jdk.OptionConverters.RichOptional import scala.util.{Failure, Success, Try} @@ -544,56 +543,29 @@ object WeatherSource { case Some(value) => // found values are used to create a weather data object // missing values are interpolated + val weatherData: WeatherData = interpolateValue(timeSeries, dateTime) + WeatherData( value.getSolarIrradiance.getDiffuseIrradiance.toScala .getOrElse( - Quantities.getQuantity( - interpolateValue(timeSeries, dateTime, "diffIrr"), - StandardUnits.SOLAR_IRRADIANCE - ) + weatherData.diffIrr ), value.getSolarIrradiance.getDirectIrradiance.toScala .getOrElse( - Quantities.getQuantity( - interpolateValue(timeSeries, dateTime, "dirIrr"), - StandardUnits.SOLAR_IRRADIANCE - ) + weatherData.dirIrr ), value.getTemperature.getTemperature.toScala .getOrElse( - Quantities.getQuantity( - interpolateValue(timeSeries, dateTime, "temp"), - StandardUnits.TEMPERATURE - ) + weatherData.temp ), value.getWind.getVelocity.toScala .getOrElse( - Quantities.getQuantity( - interpolateValue(timeSeries, dateTime, "windVel"), - StandardUnits.WIND_VELOCITY - ) + weatherData.windVel ) ) case None => // if no values are found all values are interpolated - WeatherData( - Quantities.getQuantity( - interpolateValue(timeSeries, dateTime, "diffIrr"), - StandardUnits.SOLAR_IRRADIANCE - ), - Quantities.getQuantity( - interpolateValue(timeSeries, dateTime, "dirIrr"), - StandardUnits.SOLAR_IRRADIANCE - ), - Quantities.getQuantity( - interpolateValue(timeSeries, dateTime, "temp"), - StandardUnits.TEMPERATURE - ), - Quantities.getQuantity( - interpolateValue(timeSeries, dateTime, "windVel"), - StandardUnits.WIND_VELOCITY - ) - ) + interpolateValue(timeSeries, dateTime) } } @@ -602,152 +574,144 @@ object WeatherSource { * with weather data * @param dateTime * timestamp for which an interpolation is needed - * @param get - * string defining the searched value * @return - * new quantity + * interpolated weather data */ def interpolateValue( timeSeries: IndividualTimeSeries[WeatherValue], - dateTime: ZonedDateTime, - get: String - ): Double = { - // gets two options for values - val previousOption: Option[(Double, ZonedDateTime)] = - getNewValue( - timeSeries, - dateTime, - dateTime.minusHours(2), - get, - backwards = true, - 8 - ) - val nextOption: Option[(Double, ZonedDateTime)] = - getNewValue( - timeSeries, - dateTime, - dateTime.plusHours(2), - get, - backwards = false, - 8 - ) - - (previousOption, nextOption) match { - case (Some(preVal), Some(nextVal)) => - // only if a previous and a next value are found - // found values are weighted with their time difference and used to calculate a new value - - val diffToPreviousValue: Long = - ChronoUnit.SECONDS.between(preVal._2, dateTime) - val diffToNextValue: Long = - ChronoUnit.SECONDS.between(dateTime, nextVal._2) - - (preVal._1 * diffToPreviousValue + nextVal._1 * diffToNextValue) / (diffToPreviousValue + diffToNextValue) - case (_, _) => - // if at least one value could not be found, the value found in the EMPTY_WEATHER_DATA object is returned - val defaultValue = get match { - case "diffIrr" => - EMPTY_WEATHER_DATA.diffIrr - case "dirIrr" => - EMPTY_WEATHER_DATA.dirIrr - case "temp" => - Quantities.getQuantity(0d, StandardUnits.TEMPERATURE) - case "windVel" => - EMPTY_WEATHER_DATA.windVel - } + dateTime: ZonedDateTime + ): WeatherData = { + if (timeSeries.getEntries.size() < 3) { + EMPTY_WEATHER_DATA + } else { + val previousOption: Map[String, Option[(ComparableQuantity[_], Long)]] = + getNewValue(timeSeries, dateTime, previous = true) + val nextValueOption: Map[String, Option[(ComparableQuantity[_], Long)]] = + getNewValue(timeSeries, dateTime, previous = false) + + val dataMap: Map[String, Double] = previousOption.map { + case (str, preOption) => + val nextOption: Option[(ComparableQuantity[_], Long)] = + nextValueOption(str) + + (preOption, nextOption) match { + case (Some(preVal), Some(nextVal)) => + val firstValue: Double = preVal._1.getValue.doubleValue() + val firstWeight: Long = preVal._2 + val secondValue: Double = nextVal._1.getValue.doubleValue() + val secondWeight: Long = nextVal._2 + + str -> (firstValue * firstWeight + secondValue * secondWeight) / (firstWeight + firstWeight) + case (_, _) => + val x: ComparableQuantity[_] = str match { + case "diffIrr" => EMPTY_WEATHER_DATA.diffIrr + case "dirIrr" => EMPTY_WEATHER_DATA.dirIrr + case "temp" => + Quantities.getQuantity(15d, StandardUnits.TEMPERATURE) + case "windVel" => EMPTY_WEATHER_DATA.windVel + } + str -> x.getValue.doubleValue() + } + } - defaultValue.getValue.doubleValue() + WeatherData( + Quantities + .getQuantity(dataMap("diffIrr"), StandardUnits.SOLAR_IRRADIANCE), + Quantities + .getQuantity(dataMap("dirIrr"), StandardUnits.SOLAR_IRRADIANCE), + Quantities.getQuantity(dataMap("temp"), StandardUnits.TEMPERATURE), + Quantities.getQuantity(dataMap("windVel"), StandardUnits.WIND_VELOCITY) + ) } } - /** Recursive method to find a new value for a specific weather data. + /** Method to find a new value for weather data. * * @param timeSeries * with weather values - * @param lastTime - * last timestamp - * @param maxDateTime - * after with the recursion ends - * @param get - * string defining the searched value - * @param backwards - * defines if the recursion is backwards or forwards + * @param dateTime + * starting time + * @param previous + * determines if previous ore next value is searched * @return - * an option for a quantity and a time + * map containing option for quantity and weight for each value */ - @tailrec def getNewValue( timeSeries: IndividualTimeSeries[WeatherValue], - lastTime: ZonedDateTime, - maxDateTime: ZonedDateTime, - get: String, - backwards: Boolean, - remainingNumberOfTries: Int - ): Option[(Double, ZonedDateTime)] = { - // if one is true, then the recursion ends - if ( - (remainingNumberOfTries == 0) - || (backwards && lastTime.isBefore(maxDateTime)) - || (!backwards && lastTime.isAfter(maxDateTime)) - ) { - None - } else { - // gets the new value option - val timeBasedValue: Option[TimeBasedValue[WeatherValue]] = - if (backwards) { - timeSeries.getPreviousTimeBasedValue(lastTime).toScala + dateTime: ZonedDateTime, + previous: Boolean + ): Map[String, Option[(ComparableQuantity[_], Long)]] = { + var currentTime: ZonedDateTime = dateTime + + val optionMap: Map[String, Option[(ComparableQuantity[_], Long)]] = Map( + "diffIrr" -> None, + "dirIrr" -> None, + "temp" -> None, + "windVel" -> None + ) + var run: Boolean = true + + while (run) { + val timeBasedValue: Option[TimeBasedValue[WeatherValue]] = { + if (previous) { + timeSeries.getPreviousTimeBasedValue(currentTime).toScala } else { - timeSeries.getNextTimeBasedValue(lastTime).toScala + timeSeries.getNextTimeBasedValue(currentTime).toScala } + } timeBasedValue match { case Some(value) => - val weatherValue: WeatherValue = value.getValue - - // gets the searched value as an option - val quantity: Option[ComparableQuantity[_]] = get match { - case "diffIrr" => - weatherValue.getSolarIrradiance.getDiffuseIrradiance.toScala - case "dirIrr" => - weatherValue.getSolarIrradiance.getDirectIrradiance.toScala - case "temp" => - weatherValue.getTemperature.getTemperature.toScala - case "windVel" => - weatherValue.getWind.getVelocity.toScala + val valueTime: ZonedDateTime = value.getTime + val currentWeight: Long = + ChronoUnit.SECONDS.between(valueTime, dateTime) + val actualValue: WeatherValue = value.getValue + + val (diffIrr, dirIrr, temp, windVel) = ( + actualValue.getSolarIrradiance.getDiffuseIrradiance.toScala, + actualValue.getSolarIrradiance.getDirectIrradiance.toScala, + actualValue.getTemperature.getTemperature.toScala, + actualValue.getWind.getVelocity.toScala + ) + + (optionMap("diffIrr"), diffIrr) match { + case (None, Some(x)) => + optionMap + ("diffIrr" -> Some(x, currentWeight)) } - quantity match { - case Some(data) => - // returning the found value - Some(data.getValue.doubleValue(), value.getTime) - case None => - // recursion with found time as a new timestamp - getNewValue( - timeSeries, - value.getTime, - maxDateTime, - get, - backwards, - remainingNumberOfTries - 1 - ) + (optionMap("dirIrr"), dirIrr) match { + case (None, Some(x)) => + optionMap + ("dirIrr" -> Some(x, currentWeight)) } - case None => - val newTime: ZonedDateTime = if (backwards) { - lastTime.minusMinutes(15) + + (optionMap("temp"), temp) match { + case (None, Some(x)) => + optionMap + ("temp" -> Some(x, currentWeight)) + } + + (optionMap("windVel"), windVel) match { + case (None, Some(x)) => + optionMap + ("windVel" -> Some(x, currentWeight)) + } + + if (valueTime != currentTime) { + currentTime = valueTime } else { - lastTime.plusMinutes(15) + currentTime = currentTime.minusMinutes(15) } + case None => + currentTime = currentTime.minusMinutes(15) + } - getNewValue( - timeSeries, - newTime, - maxDateTime, - get, - backwards, - remainingNumberOfTries - 1 - ) + // determine if limit is reached + run = if (previous) { + currentTime.isAfter(dateTime.minusHours(2)) + } else { + currentTime.isBefore(dateTime.plusHours(2)) } } + + optionMap } /** Weather package private case class to combine the provided agent diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala index a597781f2f..747c36bcb2 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala @@ -160,7 +160,7 @@ private[weather] final case class WeatherSourceWrapper private ( ) } val (tempContrib, tempWeight) = currentWeather.temp match { - case EMPTY_WEATHER_DATA.temp => (EMPTY_WEATHER_DATA.temp, 288.15d) + case EMPTY_WEATHER_DATA.temp => (EMPTY_WEATHER_DATA.temp, 0d) case nonEmptyTemp => calculateContrib( nonEmptyTemp.to(Units.KELVIN), From 16f3d54afe3492172e255e70be8214379420110b Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Mon, 5 Dec 2022 14:29:56 +0100 Subject: [PATCH 07/21] Improving code, adding tests. --- .../messages/services/WeatherMessage.scala | 19 ++ .../service/weather/WeatherSource.scala | 264 ++++++++++++------ .../service/weather/WeatherSourceSpec.scala | 256 ++++++++++++++++- .../weather/WeatherSourceWrapperSpec.scala | 6 +- .../edu/ie3/simona/test/common/UnitSpec.scala | 4 +- 5 files changed, 450 insertions(+), 99 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala b/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala index 27a9e88e7f..3e2e39845f 100644 --- a/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala +++ b/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala @@ -75,4 +75,23 @@ object WeatherMessage { windVel: ComparableQuantity[Speed] ) extends SecondaryData + /** Container class for the weather data option. It is similar to the normal + * weather data but instead of concrete quantities, this container holds + * options for quantities with their weight. + * + * @param diffIrr + * Diffuse irradiance on the horizontal pane + * @param dirIrr + * Direct irradiance on the horizontal pane + * @param temp + * Temperature + * @param windVel + * Wind velocity + */ + final case class WeatherDataOption( + diffIrr: Option[(ComparableQuantity[Irradiance], Long)], + dirIrr: Option[(ComparableQuantity[Irradiance], Long)], + temp: Option[(ComparableQuantity[Temperature], Long)], + windVel: Option[(ComparableQuantity[Speed], Long)] + ) } diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 536b47e2a2..33fa8552c9 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -15,10 +15,7 @@ import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.source.IdCoordinateSource import edu.ie3.datamodel.io.source.csv.CsvIdCoordinateSource import edu.ie3.datamodel.models.StandardUnits -import edu.ie3.datamodel.models.timeseries.individual.{ - IndividualTimeSeries, - TimeBasedValue -} +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.simona.config.SimonaConfig import edu.ie3.simona.config.SimonaConfig.BaseCsvParams @@ -27,7 +24,10 @@ import edu.ie3.simona.exceptions.{ InvalidConfigParameterException, ServiceException } -import edu.ie3.simona.ontology.messages.services.WeatherMessage.WeatherData +import edu.ie3.simona.ontology.messages.services.WeatherMessage.{ + WeatherData, + WeatherDataOption +} import edu.ie3.simona.service.weather.WeatherSource.{ AgentCoordinates, WeightedCoordinates @@ -41,6 +41,7 @@ import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{ import edu.ie3.simona.util.ParsableEnumeration import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} import edu.ie3.util.quantities.PowerSystemUnits +import edu.ie3.util.quantities.interfaces.Irradiance import org.locationtech.jts.geom.{Coordinate, Point} import tech.units.indriya.ComparableQuantity import tech.units.indriya.quantity.Quantities @@ -49,7 +50,7 @@ import tech.units.indriya.unit.Units import java.time.ZonedDateTime import java.time.temporal.ChronoUnit import javax.measure.Quantity -import javax.measure.quantity.{Dimensionless, Length} +import javax.measure.quantity.{Dimensionless, Length, Speed, Temperature} import scala.jdk.CollectionConverters._ import scala.jdk.OptionConverters.RichOptional import scala.util.{Failure, Success, Try} @@ -584,88 +585,90 @@ object WeatherSource { if (timeSeries.getEntries.size() < 3) { EMPTY_WEATHER_DATA } else { - val previousOption: Map[String, Option[(ComparableQuantity[_], Long)]] = - getNewValue(timeSeries, dateTime, previous = true) - val nextValueOption: Map[String, Option[(ComparableQuantity[_], Long)]] = - getNewValue(timeSeries, dateTime, previous = false) - - val dataMap: Map[String, Double] = previousOption.map { - case (str, preOption) => - val nextOption: Option[(ComparableQuantity[_], Long)] = - nextValueOption(str) - - (preOption, nextOption) match { - case (Some(preVal), Some(nextVal)) => - val firstValue: Double = preVal._1.getValue.doubleValue() - val firstWeight: Long = preVal._2 - val secondValue: Double = nextVal._1.getValue.doubleValue() - val secondWeight: Long = nextVal._2 - - str -> (firstValue * firstWeight + secondValue * secondWeight) / (firstWeight + firstWeight) - case (_, _) => - val x: ComparableQuantity[_] = str match { - case "diffIrr" => EMPTY_WEATHER_DATA.diffIrr - case "dirIrr" => EMPTY_WEATHER_DATA.dirIrr - case "temp" => - Quantities.getQuantity(15d, StandardUnits.TEMPERATURE) - case "windVel" => EMPTY_WEATHER_DATA.windVel - } - str -> x.getValue.doubleValue() - } - } + // find previous and next values + val previousOption: WeatherDataOption = + getPreviousValue(timeSeries, dateTime) + val nextOption: WeatherDataOption = getNextValue(timeSeries, dateTime) + + // weight found values or return EMPTY_WEATHER_DATA + val diffIrr: ComparableQuantity[Irradiance] = + (previousOption.diffIrr, nextOption.diffIrr) match { + case (Some(preVal), Some(nextVal)) => + preVal._1 + .multiply(preVal._2) + .add(nextVal._1.multiply(nextVal._2)) + .divide(preVal._2 + nextVal._2) + case (_, _) => + EMPTY_WEATHER_DATA.diffIrr + } + + val dirIrr: ComparableQuantity[Irradiance] = + (previousOption.dirIrr, nextOption.dirIrr) match { + case (Some(preVal), Some(nextVal)) => + preVal._1 + .multiply(preVal._2) + .add(nextVal._1.multiply(nextVal._2)) + .divide(preVal._2 + nextVal._2) + case (_, _) => + EMPTY_WEATHER_DATA.dirIrr + } + val temp: ComparableQuantity[Temperature] = + (previousOption.temp, nextOption.temp) match { + case (Some(preVal), Some(nextVal)) => + preVal._1 + .multiply(preVal._2) + .add(nextVal._1.multiply(nextVal._2)) + .divide(preVal._2 + nextVal._2) + case (_, _) => + EMPTY_WEATHER_DATA.temp + } + + val windVel: ComparableQuantity[Speed] = + (previousOption.windVel, nextOption.windVel) match { + case (Some(preVal), Some(nextVal)) => + preVal._1 + .multiply(preVal._2) + .add(nextVal._1.multiply(nextVal._2)) + .divide(preVal._2 + nextVal._2) + case (_, _) => + EMPTY_WEATHER_DATA.windVel + } + + // returning new WeatherData WeatherData( - Quantities - .getQuantity(dataMap("diffIrr"), StandardUnits.SOLAR_IRRADIANCE), - Quantities - .getQuantity(dataMap("dirIrr"), StandardUnits.SOLAR_IRRADIANCE), - Quantities.getQuantity(dataMap("temp"), StandardUnits.TEMPERATURE), - Quantities.getQuantity(dataMap("windVel"), StandardUnits.WIND_VELOCITY) + diffIrr, + dirIrr, + temp, + windVel ) } } - /** Method to find a new value for weather data. + /** Method to find previous weather data. * * @param timeSeries - * with weather values + * with weather data * @param dateTime * starting time - * @param previous - * determines if previous ore next value is searched * @return - * map containing option for quantity and weight for each value + * a weather data option */ - def getNewValue( + def getPreviousValue( timeSeries: IndividualTimeSeries[WeatherValue], - dateTime: ZonedDateTime, - previous: Boolean - ): Map[String, Option[(ComparableQuantity[_], Long)]] = { + dateTime: ZonedDateTime + ): WeatherDataOption = { var currentTime: ZonedDateTime = dateTime + var weatherDataOption: WeatherDataOption = + WeatherDataOption(None, None, None, None) - val optionMap: Map[String, Option[(ComparableQuantity[_], Long)]] = Map( - "diffIrr" -> None, - "dirIrr" -> None, - "temp" -> None, - "windVel" -> None - ) - var run: Boolean = true - - while (run) { - val timeBasedValue: Option[TimeBasedValue[WeatherValue]] = { - if (previous) { - timeSeries.getPreviousTimeBasedValue(currentTime).toScala - } else { - timeSeries.getNextTimeBasedValue(currentTime).toScala - } - } - - timeBasedValue match { - case Some(value) => - val valueTime: ZonedDateTime = value.getTime + while (currentTime.isAfter(dateTime.minusHours(2))) { + timeSeries.getPreviousTimeBasedValue(currentTime).toScala match { + case Some(timeBasedValue) => + val valueTime: ZonedDateTime = timeBasedValue.getTime val currentWeight: Long = ChronoUnit.SECONDS.between(valueTime, dateTime) - val actualValue: WeatherValue = value.getValue + val actualValue: WeatherValue = timeBasedValue.getValue val (diffIrr, dirIrr, temp, windVel) = ( actualValue.getSolarIrradiance.getDiffuseIrradiance.toScala, @@ -674,44 +677,119 @@ object WeatherSource { actualValue.getWind.getVelocity.toScala ) - (optionMap("diffIrr"), diffIrr) match { - case (None, Some(x)) => - optionMap + ("diffIrr" -> Some(x, currentWeight)) + // check options and update weather data option + (weatherDataOption.diffIrr, diffIrr) match { + case (None, Some(value)) => + weatherDataOption = + weatherDataOption.copy(diffIrr = Some(value, currentWeight)) + case (_, _) => } - (optionMap("dirIrr"), dirIrr) match { - case (None, Some(x)) => - optionMap + ("dirIrr" -> Some(x, currentWeight)) + (weatherDataOption.dirIrr, dirIrr) match { + case (None, Some(value)) => + weatherDataOption = + weatherDataOption.copy(dirIrr = Some(value, currentWeight)) + case (_, _) => } - (optionMap("temp"), temp) match { - case (None, Some(x)) => - optionMap + ("temp" -> Some(x, currentWeight)) + (weatherDataOption.temp, temp) match { + case (None, Some(value)) => + weatherDataOption = + weatherDataOption.copy(temp = Some(value, currentWeight)) + case (_, _) => } - (optionMap("windVel"), windVel) match { - case (None, Some(x)) => - optionMap + ("windVel" -> Some(x, currentWeight)) + (weatherDataOption.windVel, windVel) match { + case (None, Some(value)) => + weatherDataOption = + weatherDataOption.copy(windVel = Some(value, currentWeight)) + case (_, _) => } - if (valueTime != currentTime) { - currentTime = valueTime + if (currentTime == valueTime) { + currentTime = valueTime.minusMinutes(15) } else { - currentTime = currentTime.minusMinutes(15) + currentTime = valueTime } case None => currentTime = currentTime.minusMinutes(15) } + } + weatherDataOption + } - // determine if limit is reached - run = if (previous) { - currentTime.isAfter(dateTime.minusHours(2)) - } else { - currentTime.isBefore(dateTime.plusHours(2)) + /** Method to find previous weather data. + * + * @param timeSeries + * with weather data + * @param dateTime + * starting time + * @return + * a weather data option + */ + def getNextValue( + timeSeries: IndividualTimeSeries[WeatherValue], + dateTime: ZonedDateTime + ): WeatherDataOption = { + var currentTime: ZonedDateTime = dateTime + var weatherDataOption: WeatherDataOption = + WeatherDataOption(None, None, None, None) + + while (currentTime.isBefore(dateTime.plusHours(2))) { + + timeSeries.getNextTimeBasedValue(currentTime).toScala match { + case Some(timeBasedValue) => + val valueTime: ZonedDateTime = timeBasedValue.getTime + val currentWeight: Long = + ChronoUnit.SECONDS.between(dateTime, valueTime) + val actualValue: WeatherValue = timeBasedValue.getValue + + val (diffIrr, dirIrr, temp, windVel) = ( + actualValue.getSolarIrradiance.getDiffuseIrradiance.toScala, + actualValue.getSolarIrradiance.getDirectIrradiance.toScala, + actualValue.getTemperature.getTemperature.toScala, + actualValue.getWind.getVelocity.toScala + ) + + // check options and update weather data option + (weatherDataOption.diffIrr, diffIrr) match { + case (None, Some(value)) => + weatherDataOption = + weatherDataOption.copy(diffIrr = Some(value, currentWeight)) + case (_, _) => + } + + (weatherDataOption.dirIrr, dirIrr) match { + case (None, Some(value)) => + weatherDataOption = + weatherDataOption.copy(dirIrr = Some(value, currentWeight)) + case (_, _) => + } + + (weatherDataOption.temp, temp) match { + case (None, Some(value)) => + weatherDataOption = + weatherDataOption.copy(temp = Some(value, currentWeight)) + case (_, _) => + } + + (weatherDataOption.windVel, windVel) match { + case (None, Some(value)) => + weatherDataOption = + weatherDataOption.copy(windVel = Some(value, currentWeight)) + case (_, _) => + } + + if (currentTime == valueTime) { + currentTime = valueTime.plusMinutes(15) + } else { + currentTime = valueTime + } + case None => + currentTime = currentTime.plusMinutes(15) } } - - optionMap + weatherDataOption } /** Weather package private case class to combine the provided agent diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala index 71281a6dda..47e15f6481 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala @@ -7,11 +7,25 @@ package edu.ie3.simona.service.weather import edu.ie3.datamodel.io.source.IdCoordinateSource +import edu.ie3.datamodel.models.StandardUnits +import edu.ie3.datamodel.models.timeseries.individual.{ + IndividualTimeSeries, + TimeBasedValue +} +import edu.ie3.datamodel.models.value.{ + SolarIrradianceValue, + TemperatureValue, + WeatherValue, + WindValue +} import edu.ie3.simona.exceptions.ServiceException import edu.ie3.simona.ontology.messages.services.WeatherMessage +import edu.ie3.simona.ontology.messages.services.WeatherMessage.WeatherDataOption import edu.ie3.simona.service.weather.WeatherSource.{ AgentCoordinates, - WeightedCoordinates + WeightedCoordinates, + getNextValue, + getPreviousValue } import edu.ie3.simona.service.weather.WeatherSourceSpec._ import edu.ie3.simona.test.common.UnitSpec @@ -19,9 +33,11 @@ import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} import edu.ie3.util.quantities.{PowerSystemUnits, QuantityUtil} import org.locationtech.jts.geom.Point import tech.units.indriya.quantity.Quantities +import tech.units.indriya.unit.Units +import java.time.ZonedDateTime import java.util -import java.util.Optional +import java.util.{Optional, UUID} import scala.jdk.CollectionConverters._ import scala.jdk.OptionConverters._ import scala.util.{Failure, Success} @@ -296,6 +312,123 @@ class WeatherSourceSpec extends UnitSpec { ) } } + + "interpolate missing weather values correctly" in {} + + "find correct previous values" in { + val cases = Table( + ( + "timeBasedValues", + "expectedDiffIrr", + "expectedDirIrr", + "expectedTemp", + "expectedWindVel" + ), + ( + Set( + timeBasedValue0, + timeBasedValue1, + timeBasedValue2, + timeBasedValue3, + timeBasedValue4 + ), + Some( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 3900 + ), + Some( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 6900 + ), + Some(Quantities.getQuantity(14.85d, StandardUnits.TEMPERATURE), 3900), + Some(Quantities.getQuantity(10d, Units.METRE_PER_SECOND), 3900) + ), + ( + Set(timeBasedValue0, timeBasedValue6), + None, + None, + None, + None + ) + ) + + forAll(cases) { + ( + timeBasedValues, + expectedDiffIrr, + expectedDirIrr, + expectedTemp, + expectedWindVel + ) => + val weatherDataOption: WeatherDataOption = + getPreviousValue(buildTimeSeries(timeBasedValues), time) + + weatherDataOption.diffIrr shouldBe expectedDiffIrr + weatherDataOption.dirIrr shouldBe expectedDirIrr + weatherDataOption.temp shouldBe expectedTemp + weatherDataOption.windVel shouldBe expectedWindVel + } + } + + "find correct next values" in { + val cases = Table( + ( + "timeBasedValues", + "expectedDiffIrr", + "expectedDirIrr", + "expectedTemp", + "expectedWindVel" + ), + ( + Set( + timeBasedValue0, + timeBasedValue1, + timeBasedValue2, + timeBasedValue3, + timeBasedValue4, + timeBasedValue5, + timeBasedValue6 + ), + Some( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 3300 + ), + Some( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 3300 + ), + Some(Quantities.getQuantity(16.85d, StandardUnits.TEMPERATURE), 3300), + Some(Quantities.getQuantity(20d, Units.METRE_PER_SECOND), 7200) + ), + ( + Set(timeBasedValue0, timeBasedValue5), + Some( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 6000 + ), + None, + Some(Quantities.getQuantity(14.85d, StandardUnits.TEMPERATURE), 6000), + None + ) + ) + + forAll(cases) { + ( + timeBasedValues, + expectedDiffIrr, + expectedDirIrr, + expectedTemp, + expectedWindVel + ) => + val weatherDataOption: WeatherDataOption = + getNextValue(buildTimeSeries(timeBasedValues), time) + + weatherDataOption.diffIrr shouldBe expectedDiffIrr + weatherDataOption.dirIrr shouldBe expectedDirIrr + weatherDataOption.temp shouldBe expectedTemp + weatherDataOption.windVel shouldBe expectedWindVel + } + } } } @@ -309,6 +442,125 @@ case object WeatherSourceSpec { private val coordinate144112 = GeoUtils.buildPoint(52.312, 12.875) private val coordinate165125 = GeoUtils.buildPoint(52.25, 12.875) + private val time: ZonedDateTime = ZonedDateTime.now() + + private val solarIrradianceValue0: SolarIrradianceValue = + new SolarIrradianceValue(null, null) + private val solarIrradianceValue1: SolarIrradianceValue = + new SolarIrradianceValue( + null, + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE) + ) + private val solarIrradianceValue2: SolarIrradianceValue = + new SolarIrradianceValue( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE) + ) + + private val temperatureValue0: TemperatureValue = new TemperatureValue(null) + private val temperatureValue1: TemperatureValue = new TemperatureValue( + Quantities.getQuantity(288d, Units.KELVIN) + ) + private val temperatureValue2: TemperatureValue = new TemperatureValue( + Quantities.getQuantity(290d, Units.KELVIN) + ) + + private val windValue0: WindValue = new WindValue(null, null) + private val windValue1: WindValue = + new WindValue(null, Quantities.getQuantity(10, Units.METRE_PER_SECOND)) + private val windValue2: WindValue = + new WindValue(null, Quantities.getQuantity(20, Units.METRE_PER_SECOND)) + + private val timeBasedValue0: TimeBasedValue[WeatherValue] = + buildTimeBasedValue( + time, + new WeatherValue( + coordinate67775, + solarIrradianceValue0, + temperatureValue0, + windValue0 + ) + ) + + private val timeBasedValue1: TimeBasedValue[WeatherValue] = + buildTimeBasedValue( + time.minusMinutes(65), + new WeatherValue( + coordinate67775, + solarIrradianceValue1, + temperatureValue1, + windValue1 + ) + ) + + private val timeBasedValue2: TimeBasedValue[WeatherValue] = + buildTimeBasedValue( + time.minusMinutes(115), + new WeatherValue( + coordinate67775, + solarIrradianceValue2, + temperatureValue1, + windValue2 + ) + ) + + private val timeBasedValue3: TimeBasedValue[WeatherValue] = + buildTimeBasedValue( + time.minusMinutes(121), + new WeatherValue( + coordinate67775, + solarIrradianceValue1, + temperatureValue1, + windValue2 + ) + ) + + private val timeBasedValue4: TimeBasedValue[WeatherValue] = + buildTimeBasedValue( + time.plusMinutes(55), + new WeatherValue( + coordinate67775, + solarIrradianceValue2, + temperatureValue2, + windValue0 + ) + ) + + private val timeBasedValue5: TimeBasedValue[WeatherValue] = + buildTimeBasedValue( + time.plusMinutes(100), + new WeatherValue( + coordinate67775, + solarIrradianceValue1, + temperatureValue1, + windValue0 + ) + ) + + private val timeBasedValue6: TimeBasedValue[WeatherValue] = + buildTimeBasedValue( + time.plusMinutes(120), + new WeatherValue( + coordinate67775, + solarIrradianceValue2, + temperatureValue2, + windValue2 + ) + ) + + def buildTimeBasedValue( + time: ZonedDateTime, + value: WeatherValue + ): TimeBasedValue[WeatherValue] = { + new TimeBasedValue[WeatherValue](UUID.randomUUID(), time, value) + } + + def buildTimeSeries( + set: Set[TimeBasedValue[WeatherValue]] + ): IndividualTimeSeries[WeatherValue] = { + new IndividualTimeSeries[WeatherValue](UUID.randomUUID(), set.asJava) + } + case object DummyWeatherSource extends WeatherSource { override protected val idCoordinateSource: IdCoordinateSource = DummyIdCoordinateSource diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceWrapperSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceWrapperSpec.scala index 96f83968bc..9dbb19d067 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceWrapperSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceWrapperSpec.scala @@ -40,14 +40,14 @@ import scala.jdk.CollectionConverters.{MapHasAsJava, SetHasAsJava} class WeatherSourceWrapperSpec extends UnitSpec { "A weather source wrapper" should { - val ctor = classOf[WeatherSourceWrapper].getDeclaredConstructor( + val actor = classOf[WeatherSourceWrapper].getDeclaredConstructor( classOf[PsdmWeatherSource], classOf[IdCoordinateSource], classOf[Long], classOf[ZonedDateTime] ) - ctor.setAccessible(true) - val source = ctor.newInstance( + actor.setAccessible(true) + val source = actor.newInstance( WeatherSourceWrapperSpec.DummyPsdmWeatherSource, DummyIdCoordinateSource, 360L, diff --git a/src/test/scala/edu/ie3/simona/test/common/UnitSpec.scala b/src/test/scala/edu/ie3/simona/test/common/UnitSpec.scala index f3bc5c2136..6d4dc4f856 100644 --- a/src/test/scala/edu/ie3/simona/test/common/UnitSpec.scala +++ b/src/test/scala/edu/ie3/simona/test/common/UnitSpec.scala @@ -12,6 +12,7 @@ import edu.ie3.simona.test.matchers.QuantityMatchers import edu.ie3.util.scala.quantities.{QuantityUtil => PSQuantityUtil} import org.scalatest._ import org.scalatest.matchers.should +import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.wordspec.AnyWordSpecLike /** Base class to be used with all scala unit tests. All data that should be @@ -31,7 +32,8 @@ trait UnitSpec with Inspectors with PrivateMethodTester with LazyLogging - with TryValues { + with TryValues + with TableDrivenPropertyChecks { /* Set default locale in order to ensure proper number parsing - among others */ Locale.setDefault(Locale.ENGLISH) From e0263876f86cf83f1c664cb0c972b763a543e5e1 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Mon, 5 Dec 2022 14:47:52 +0100 Subject: [PATCH 08/21] Fixing failing tests. --- .../edu/ie3/simona/service/weather/WeatherSourceSpec.scala | 3 ++- src/test/scala/edu/ie3/simona/test/common/UnitSpec.scala | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala index 47e15f6481..6927183dd0 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala @@ -32,6 +32,7 @@ import edu.ie3.simona.test.common.UnitSpec import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} import edu.ie3.util.quantities.{PowerSystemUnits, QuantityUtil} import org.locationtech.jts.geom.Point +import org.scalatest.prop.TableDrivenPropertyChecks import tech.units.indriya.quantity.Quantities import tech.units.indriya.unit.Units @@ -42,7 +43,7 @@ import scala.jdk.CollectionConverters._ import scala.jdk.OptionConverters._ import scala.util.{Failure, Success} -class WeatherSourceSpec extends UnitSpec { +class WeatherSourceSpec extends UnitSpec with TableDrivenPropertyChecks { private val coordinate0 = GeoUtils.buildPoint(51.47, 7.41) "A weather source" should { diff --git a/src/test/scala/edu/ie3/simona/test/common/UnitSpec.scala b/src/test/scala/edu/ie3/simona/test/common/UnitSpec.scala index 6d4dc4f856..0c2b003218 100644 --- a/src/test/scala/edu/ie3/simona/test/common/UnitSpec.scala +++ b/src/test/scala/edu/ie3/simona/test/common/UnitSpec.scala @@ -32,8 +32,7 @@ trait UnitSpec with Inspectors with PrivateMethodTester with LazyLogging - with TryValues - with TableDrivenPropertyChecks { + with TryValues { /* Set default locale in order to ensure proper number parsing - among others */ Locale.setDefault(Locale.ENGLISH) From 727281ca396fbe3921ed5b779d767efbfe40ce10 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Tue, 6 Dec 2022 11:15:39 +0100 Subject: [PATCH 09/21] Adding tests, improving code quality. --- .../service/weather/WeatherSource.scala | 151 +++++++------ .../service/weather/WeatherSourceSpec.scala | 209 +++++++++++++++++- .../edu/ie3/simona/test/common/UnitSpec.scala | 1 - 3 files changed, 273 insertions(+), 88 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 33fa8552c9..d4de28b370 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -535,7 +535,6 @@ object WeatherSource { timeSeries: IndividualTimeSeries[WeatherValue], dateTime: ZonedDateTime ): WeatherData = { - // gets a value option val valueOption: Option[WeatherValue] = timeSeries.getValue(dateTime).toScala @@ -662,48 +661,20 @@ object WeatherSource { var weatherDataOption: WeatherDataOption = WeatherDataOption(None, None, None, None) - while (currentTime.isAfter(dateTime.minusHours(2))) { + for (i <- 0 to 7) { timeSeries.getPreviousTimeBasedValue(currentTime).toScala match { case Some(timeBasedValue) => val valueTime: ZonedDateTime = timeBasedValue.getTime - val currentWeight: Long = - ChronoUnit.SECONDS.between(valueTime, dateTime) - val actualValue: WeatherValue = timeBasedValue.getValue - - val (diffIrr, dirIrr, temp, windVel) = ( - actualValue.getSolarIrradiance.getDiffuseIrradiance.toScala, - actualValue.getSolarIrradiance.getDirectIrradiance.toScala, - actualValue.getTemperature.getTemperature.toScala, - actualValue.getWind.getVelocity.toScala - ) - - // check options and update weather data option - (weatherDataOption.diffIrr, diffIrr) match { - case (None, Some(value)) => - weatherDataOption = - weatherDataOption.copy(diffIrr = Some(value, currentWeight)) - case (_, _) => - } - (weatherDataOption.dirIrr, dirIrr) match { - case (None, Some(value)) => - weatherDataOption = - weatherDataOption.copy(dirIrr = Some(value, currentWeight)) - case (_, _) => - } - - (weatherDataOption.temp, temp) match { - case (None, Some(value)) => - weatherDataOption = - weatherDataOption.copy(temp = Some(value, currentWeight)) - case (_, _) => - } + if (valueTime.isAfter(dateTime.minusHours(2))) { + val currentWeight: Long = + ChronoUnit.SECONDS.between(valueTime, dateTime) - (weatherDataOption.windVel, windVel) match { - case (None, Some(value)) => - weatherDataOption = - weatherDataOption.copy(windVel = Some(value, currentWeight)) - case (_, _) => + weatherDataOption = updateWeatherDataOption( + weatherDataOption, + timeBasedValue.getValue, + currentWeight + ) } if (currentTime == valueTime) { @@ -735,49 +706,20 @@ object WeatherSource { var weatherDataOption: WeatherDataOption = WeatherDataOption(None, None, None, None) - while (currentTime.isBefore(dateTime.plusHours(2))) { - + for (i <- 0 to 7) { timeSeries.getNextTimeBasedValue(currentTime).toScala match { case Some(timeBasedValue) => val valueTime: ZonedDateTime = timeBasedValue.getTime - val currentWeight: Long = - ChronoUnit.SECONDS.between(dateTime, valueTime) - val actualValue: WeatherValue = timeBasedValue.getValue - - val (diffIrr, dirIrr, temp, windVel) = ( - actualValue.getSolarIrradiance.getDiffuseIrradiance.toScala, - actualValue.getSolarIrradiance.getDirectIrradiance.toScala, - actualValue.getTemperature.getTemperature.toScala, - actualValue.getWind.getVelocity.toScala - ) - - // check options and update weather data option - (weatherDataOption.diffIrr, diffIrr) match { - case (None, Some(value)) => - weatherDataOption = - weatherDataOption.copy(diffIrr = Some(value, currentWeight)) - case (_, _) => - } - (weatherDataOption.dirIrr, dirIrr) match { - case (None, Some(value)) => - weatherDataOption = - weatherDataOption.copy(dirIrr = Some(value, currentWeight)) - case (_, _) => - } - - (weatherDataOption.temp, temp) match { - case (None, Some(value)) => - weatherDataOption = - weatherDataOption.copy(temp = Some(value, currentWeight)) - case (_, _) => - } + if (valueTime.isBefore(dateTime.plusHours(2))) { + val currentWeight: Long = + ChronoUnit.SECONDS.between(dateTime, valueTime) - (weatherDataOption.windVel, windVel) match { - case (None, Some(value)) => - weatherDataOption = - weatherDataOption.copy(windVel = Some(value, currentWeight)) - case (_, _) => + weatherDataOption = updateWeatherDataOption( + weatherDataOption, + timeBasedValue.getValue, + currentWeight + ) } if (currentTime == valueTime) { @@ -792,6 +734,63 @@ object WeatherSource { weatherDataOption } + /** Method to find options for new weather data. + * + * @param weatherDataOption + * container class holding the options + * @param value + * possible new values + * @param weight + * weighting for the found values + * @return + * updated weather data option + */ + def updateWeatherDataOption( + weatherDataOption: WeatherDataOption, + value: WeatherValue, + weight: Long + ): WeatherDataOption = { + var option: WeatherDataOption = weatherDataOption + if (weight == 0L) { + weatherDataOption + } else { + // check options and update weather data option + ( + weatherDataOption.diffIrr, + value.getSolarIrradiance.getDiffuseIrradiance.toScala + ) match { + case (None, Some(value)) => + option = option.copy(diffIrr = Some(value, weight)) + case (_, _) => + } + + ( + weatherDataOption.dirIrr, + value.getSolarIrradiance.getDirectIrradiance.toScala + ) match { + case (None, Some(value)) => + option = option.copy(dirIrr = Some(value, weight)) + case (_, _) => + } + + ( + weatherDataOption.temp, + value.getTemperature.getTemperature.toScala + ) match { + case (None, Some(value)) => + option = option.copy(temp = Some(value, weight)) + case (_, _) => + } + + (weatherDataOption.windVel, value.getWind.getVelocity.toScala) match { + case (None, Some(value)) => + option = option.copy(windVel = Some(value, weight)) + case (_, _) => + } + option + } + } + /** Weather package private case class to combine the provided agent * coordinates into one single entity */ diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala index 6927183dd0..75964efa01 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala @@ -20,12 +20,18 @@ import edu.ie3.datamodel.models.value.{ } import edu.ie3.simona.exceptions.ServiceException import edu.ie3.simona.ontology.messages.services.WeatherMessage -import edu.ie3.simona.ontology.messages.services.WeatherMessage.WeatherDataOption +import edu.ie3.simona.ontology.messages.services.WeatherMessage.{ + WeatherData, + WeatherDataOption +} import edu.ie3.simona.service.weather.WeatherSource.{ AgentCoordinates, + EMPTY_WEATHER_DATA, WeightedCoordinates, getNextValue, - getPreviousValue + getPreviousValue, + getWeatherData, + updateWeatherDataOption } import edu.ie3.simona.service.weather.WeatherSourceSpec._ import edu.ie3.simona.test.common.UnitSpec @@ -314,7 +320,68 @@ class WeatherSourceSpec extends UnitSpec with TableDrivenPropertyChecks { } } - "interpolate missing weather values correctly" in {} + "interpolate missing weather values correctly" in { + val cases = Table( + ("timeBasedValues", "time", "expectedWeatherData"), + (Set.empty[TimeBasedValue[WeatherValue]], time, EMPTY_WEATHER_DATA), + ( + Set( + timeBasedValue0, + timeBasedValue1, + timeBasedValue2, + timeBasedValue3, + timeBasedValue4, + timeBasedValue5, + timeBasedValue6 + ), + time.minusHours(3), + EMPTY_WEATHER_DATA + ), + ( + Set( + timeBasedValue0, + timeBasedValue1, + timeBasedValue2, + timeBasedValue3, + timeBasedValue4, + timeBasedValue5, + timeBasedValue6 + ), + time.plusHours(2), + EMPTY_WEATHER_DATA + ), + ( + Set( + timeBasedValue0, + timeBasedValue1, + timeBasedValue2, + timeBasedValue3, + timeBasedValue4, + timeBasedValue5, + timeBasedValue6 + ), + time, + WeatherData( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), + Quantities.getQuantity(0d, StandardUnits.WIND_VELOCITY) + ) + ) + ) + + forAll(cases) { (timeBasedValues, time, expectedWeatherData) => + val timeSeries: IndividualTimeSeries[WeatherValue] = + buildTimeSeries(timeBasedValues) + val weatherData: WeatherData = getWeatherData(timeSeries, time) + + weatherData.diffIrr shouldBe expectedWeatherData.diffIrr + weatherData.dirIrr shouldBe expectedWeatherData.dirIrr + weatherData.temp shouldBe expectedWeatherData.temp + weatherData.windVel shouldBe expectedWeatherData.windVel + } + + } "find correct previous values" in { val cases = Table( @@ -341,7 +408,7 @@ class WeatherSourceSpec extends UnitSpec with TableDrivenPropertyChecks { Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), 6900 ), - Some(Quantities.getQuantity(14.85d, StandardUnits.TEMPERATURE), 3900), + Some(Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), 3900), Some(Quantities.getQuantity(10d, Units.METRE_PER_SECOND), 3900) ), ( @@ -398,8 +465,8 @@ class WeatherSourceSpec extends UnitSpec with TableDrivenPropertyChecks { Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), 3300 ), - Some(Quantities.getQuantity(16.85d, StandardUnits.TEMPERATURE), 3300), - Some(Quantities.getQuantity(20d, Units.METRE_PER_SECOND), 7200) + Some(Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), 6000), + None ), ( Set(timeBasedValue0, timeBasedValue5), @@ -408,7 +475,7 @@ class WeatherSourceSpec extends UnitSpec with TableDrivenPropertyChecks { 6000 ), None, - Some(Quantities.getQuantity(14.85d, StandardUnits.TEMPERATURE), 6000), + Some(Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), 6000), None ) ) @@ -430,6 +497,126 @@ class WeatherSourceSpec extends UnitSpec with TableDrivenPropertyChecks { weatherDataOption.windVel shouldBe expectedWindVel } } + + "update weather data option correctly" in { + val weatherDataOption: WeatherDataOption = + WeatherDataOption(None, None, None, None) + + val cases = Table( + ("givenValue", "givenWeight", "expectedResult"), + (timeBasedValue0.getValue, 200L, weatherDataOption), + ( + timeBasedValue1.getValue, + 300L, + WeatherDataOption( + Some( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 300L + ), + None, + Some( + Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), + 300L + ), + Some(Quantities.getQuantity(10, Units.METRE_PER_SECOND), 300L) + ) + ), + ( + timeBasedValue2.getValue, + 250L, + WeatherDataOption( + Some( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 250L + ), + Some( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 250L + ), + Some( + Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), + 250L + ), + Some(Quantities.getQuantity(20, Units.METRE_PER_SECOND), 250L) + ) + ), + ( + timeBasedValue3.getValue, + 300L, + WeatherDataOption( + Some( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 300L + ), + None, + Some( + Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), + 300L + ), + Some(Quantities.getQuantity(20, Units.METRE_PER_SECOND), 300L) + ) + ), + ( + timeBasedValue4.getValue, + 250L, + WeatherDataOption( + Some( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 250L + ), + Some( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 250L + ), + None, + None + ) + ), + ( + timeBasedValue5.getValue, + 300L, + WeatherDataOption( + Some( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 300L + ), + None, + Some( + Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), + 300L + ), + None + ) + ), + ( + timeBasedValue6.getValue, + 250L, + WeatherDataOption( + Some( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 250L + ), + Some( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 250L + ), + Some( + Quantities.getQuantity(17d, StandardUnits.TEMPERATURE), + 250L + ), + Some(Quantities.getQuantity(20, Units.METRE_PER_SECOND), 250L) + ) + ) + ) + + forAll(cases) { (givenValue, givenWeight, expectedResult) => + val updatedWeatherDataOption = + updateWeatherDataOption(weatherDataOption, givenValue, givenWeight) + + updatedWeatherDataOption shouldBe expectedResult + } + + } } } @@ -460,10 +647,10 @@ case object WeatherSourceSpec { private val temperatureValue0: TemperatureValue = new TemperatureValue(null) private val temperatureValue1: TemperatureValue = new TemperatureValue( - Quantities.getQuantity(288d, Units.KELVIN) + Quantities.getQuantity(288.15d, Units.KELVIN) ) private val temperatureValue2: TemperatureValue = new TemperatureValue( - Quantities.getQuantity(290d, Units.KELVIN) + Quantities.getQuantity(290.15d, Units.KELVIN) ) private val windValue0: WindValue = new WindValue(null, null) @@ -522,7 +709,7 @@ case object WeatherSourceSpec { new WeatherValue( coordinate67775, solarIrradianceValue2, - temperatureValue2, + temperatureValue0, windValue0 ) ) @@ -540,7 +727,7 @@ case object WeatherSourceSpec { private val timeBasedValue6: TimeBasedValue[WeatherValue] = buildTimeBasedValue( - time.plusMinutes(120), + time.plusMinutes(1), new WeatherValue( coordinate67775, solarIrradianceValue2, diff --git a/src/test/scala/edu/ie3/simona/test/common/UnitSpec.scala b/src/test/scala/edu/ie3/simona/test/common/UnitSpec.scala index 0c2b003218..f3bc5c2136 100644 --- a/src/test/scala/edu/ie3/simona/test/common/UnitSpec.scala +++ b/src/test/scala/edu/ie3/simona/test/common/UnitSpec.scala @@ -12,7 +12,6 @@ import edu.ie3.simona.test.matchers.QuantityMatchers import edu.ie3.util.scala.quantities.{QuantityUtil => PSQuantityUtil} import org.scalatest._ import org.scalatest.matchers.should -import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.wordspec.AnyWordSpecLike /** Base class to be used with all scala unit tests. All data that should be From cf970afdfcff87b9cd284e4386ce2028fa6e882c Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Thu, 19 Jan 2023 10:27:13 +0100 Subject: [PATCH 10/21] Adding logging for missing weather values. --- .../service/weather/WeatherSource.scala | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index d4de28b370..7cadd1e0c1 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -6,6 +6,7 @@ package edu.ie3.simona.service.weather +import com.typesafe.scalalogging.LazyLogging import edu.ie3.datamodel.io.factory.timeseries.{ CosmoIdCoordinateFactory, IconIdCoordinateFactory, @@ -294,7 +295,7 @@ trait WeatherSource { ): Array[Long] } -object WeatherSource { +object WeatherSource extends LazyLogging { def apply( dataSourceConfig: SimonaConfig.Simona.Input.Weather.Datasource, @@ -565,6 +566,8 @@ object WeatherSource { ) case None => // if no values are found all values are interpolated + logger.warn(s"No weather value found for timestamp $dateTime.") + interpolateValue(timeSeries, dateTime) } } @@ -582,6 +585,10 @@ object WeatherSource { dateTime: ZonedDateTime ): WeatherData = { if (timeSeries.getEntries.size() < 3) { + logger.info( + s"Not enough entries in time series $timeSeries to interpolate weather data. At least three values are needed, found ${timeSeries.getEntries.size()}." + ) + EMPTY_WEATHER_DATA } else { // find previous and next values @@ -598,6 +605,10 @@ object WeatherSource { .add(nextVal._1.multiply(nextVal._2)) .divide(preVal._2 + nextVal._2) case (_, _) => + logger.warn( + s"Interpolating diffused irradiance value for timestamp $dateTime was not possible. The default value is used." + ) + EMPTY_WEATHER_DATA.diffIrr } @@ -609,6 +620,10 @@ object WeatherSource { .add(nextVal._1.multiply(nextVal._2)) .divide(preVal._2 + nextVal._2) case (_, _) => + logger.warn( + s"Interpolating direct irradiance value for timestamp $dateTime was not possible. The default value is used." + ) + EMPTY_WEATHER_DATA.dirIrr } @@ -620,6 +635,10 @@ object WeatherSource { .add(nextVal._1.multiply(nextVal._2)) .divide(preVal._2 + nextVal._2) case (_, _) => + logger.warn( + s"Interpolating temperature value for timestamp $dateTime was not possible. The default value is used." + ) + EMPTY_WEATHER_DATA.temp } @@ -631,6 +650,10 @@ object WeatherSource { .add(nextVal._1.multiply(nextVal._2)) .divide(preVal._2 + nextVal._2) case (_, _) => + logger.warn( + s"Interpolating wind velocity value for timestamp $dateTime was not possible. The default value is used." + ) + EMPTY_WEATHER_DATA.windVel } @@ -661,7 +684,7 @@ object WeatherSource { var weatherDataOption: WeatherDataOption = WeatherDataOption(None, None, None, None) - for (i <- 0 to 7) { + for (_ <- 0 to 7) { timeSeries.getPreviousTimeBasedValue(currentTime).toScala match { case Some(timeBasedValue) => val valueTime: ZonedDateTime = timeBasedValue.getTime @@ -706,7 +729,7 @@ object WeatherSource { var weatherDataOption: WeatherDataOption = WeatherDataOption(None, None, None, None) - for (i <- 0 to 7) { + for (_ <- 0 to 7) { timeSeries.getNextTimeBasedValue(currentTime).toScala match { case Some(timeBasedValue) => val valueTime: ZonedDateTime = timeBasedValue.getTime @@ -762,6 +785,9 @@ object WeatherSource { case (None, Some(value)) => option = option.copy(diffIrr = Some(value, weight)) case (_, _) => + logger.debug( + s"No diffused irradiance value with the given time difference of $weight found." + ) } ( @@ -771,6 +797,9 @@ object WeatherSource { case (None, Some(value)) => option = option.copy(dirIrr = Some(value, weight)) case (_, _) => + logger.debug( + s"No direct irradiance value with the given time difference of $weight found." + ) } ( @@ -780,12 +809,18 @@ object WeatherSource { case (None, Some(value)) => option = option.copy(temp = Some(value, weight)) case (_, _) => + logger.debug( + s"No temperature value with the given time difference of $weight found." + ) } (weatherDataOption.windVel, value.getWind.getVelocity.toScala) match { case (None, Some(value)) => option = option.copy(windVel = Some(value, weight)) case (_, _) => + logger.debug( + s"No wind velocity value with the given time difference of $weight found." + ) } option } From 4f66dd77e3629b05e3660530414fdafc815da7fc Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Mon, 27 Feb 2023 10:58:49 +0100 Subject: [PATCH 11/21] Fixing codacy issues. --- .../service/weather/WeatherSource.scala | 126 +++++++++--------- .../service/weather/WeatherSourceSpec.scala | 22 ++- 2 files changed, 81 insertions(+), 67 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 7cadd1e0c1..0c89985be1 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -37,6 +37,7 @@ import edu.ie3.simona.util.ConfigUtil.CsvConfigUtil.checkBaseCsvParams import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{ checkCouchbaseParams, checkInfluxDb1xParams, + checkKafkaParams, checkSqlParams } import edu.ie3.simona.util.ParsableEnumeration @@ -580,7 +581,7 @@ object WeatherSource extends LazyLogging { * @return * interpolated weather data */ - def interpolateValue( + private def interpolateValue( timeSeries: IndividualTimeSeries[WeatherValue], dateTime: ZonedDateTime ): WeatherData = { @@ -588,7 +589,6 @@ object WeatherSource extends LazyLogging { logger.info( s"Not enough entries in time series $timeSeries to interpolate weather data. At least three values are needed, found ${timeSeries.getEntries.size()}." ) - EMPTY_WEATHER_DATA } else { // find previous and next values @@ -597,65 +597,30 @@ object WeatherSource extends LazyLogging { val nextOption: WeatherDataOption = getNextValue(timeSeries, dateTime) // weight found values or return EMPTY_WEATHER_DATA - val diffIrr: ComparableQuantity[Irradiance] = - (previousOption.diffIrr, nextOption.diffIrr) match { - case (Some(preVal), Some(nextVal)) => - preVal._1 - .multiply(preVal._2) - .add(nextVal._1.multiply(nextVal._2)) - .divide(preVal._2 + nextVal._2) - case (_, _) => - logger.warn( - s"Interpolating diffused irradiance value for timestamp $dateTime was not possible. The default value is used." - ) - - EMPTY_WEATHER_DATA.diffIrr - } - - val dirIrr: ComparableQuantity[Irradiance] = - (previousOption.dirIrr, nextOption.dirIrr) match { - case (Some(preVal), Some(nextVal)) => - preVal._1 - .multiply(preVal._2) - .add(nextVal._1.multiply(nextVal._2)) - .divide(preVal._2 + nextVal._2) - case (_, _) => - logger.warn( - s"Interpolating direct irradiance value for timestamp $dateTime was not possible. The default value is used." - ) - - EMPTY_WEATHER_DATA.dirIrr - } - - val temp: ComparableQuantity[Temperature] = - (previousOption.temp, nextOption.temp) match { - case (Some(preVal), Some(nextVal)) => - preVal._1 - .multiply(preVal._2) - .add(nextVal._1.multiply(nextVal._2)) - .divide(preVal._2 + nextVal._2) - case (_, _) => - logger.warn( - s"Interpolating temperature value for timestamp $dateTime was not possible. The default value is used." - ) - - EMPTY_WEATHER_DATA.temp - } - - val windVel: ComparableQuantity[Speed] = - (previousOption.windVel, nextOption.windVel) match { - case (Some(preVal), Some(nextVal)) => - preVal._1 - .multiply(preVal._2) - .add(nextVal._1.multiply(nextVal._2)) - .divide(preVal._2 + nextVal._2) - case (_, _) => - logger.warn( - s"Interpolating wind velocity value for timestamp $dateTime was not possible. The default value is used." - ) - - EMPTY_WEATHER_DATA.windVel - } + val diffIrr: ComparableQuantity[Irradiance] = interpolate( + dateTime, + previousOption.diffIrr, + nextOption.diffIrr, + EMPTY_WEATHER_DATA.diffIrr + ) + val dirIrr: ComparableQuantity[Irradiance] = interpolate( + dateTime, + previousOption.dirIrr, + nextOption.dirIrr, + EMPTY_WEATHER_DATA.dirIrr + ) + val temp: ComparableQuantity[Temperature] = interpolate( + dateTime, + previousOption.temp, + nextOption.temp, + EMPTY_WEATHER_DATA.temp + ) + val windVel: ComparableQuantity[Speed] = interpolate( + dateTime, + previousOption.windVel, + nextOption.windVel, + EMPTY_WEATHER_DATA.windVel + ) // returning new WeatherData WeatherData( @@ -667,6 +632,45 @@ object WeatherSource extends LazyLogging { } } + /** Method to interpolate a quantity for a specific timestamp with a previous + * and a next value. + * @param dateTime + * timestamp which is interpolated + * @param previousOption + * option of quantity and time difference + * @param nextOption + * option of quantity and time difference + * @param empty + * weather data + * @return + * option of interpolated quantity + */ + private def interpolate[V <: Quantity[V]]( + dateTime: ZonedDateTime, + previousOption: Option[(ComparableQuantity[V], Long)], + nextOption: Option[(ComparableQuantity[V], Long)], + empty: ComparableQuantity[V] + ): ComparableQuantity[V] = { + (previousOption, nextOption) match { + case (Some(previousValue), Some(nextValue)) => + val time1: Long = previousValue._2 + val time2: Long = nextValue._2 + val interval = time1 + time2 + val quantity1: ComparableQuantity[V] = previousValue._1 + val quantity2: ComparableQuantity[V] = nextValue._1 + + val weightedQuantity1 = quantity1.multiply(time1) + val weightedQuantity2 = quantity2.multiply(time2) + + weightedQuantity1.add(weightedQuantity2).divide(interval) + case (_, _) => + logger.warn( + s"Interpolating value with unit ${empty.getUnit} for timestamp $dateTime was not possible. The default value is used." + ) + empty + } + } + /** Method to find previous weather data. * * @param timeSeries diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala index 75964efa01..afa97fe6fc 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala @@ -632,11 +632,13 @@ case object WeatherSourceSpec { private val time: ZonedDateTime = ZonedDateTime.now() + private val missingValue: Null = null + private val solarIrradianceValue0: SolarIrradianceValue = - new SolarIrradianceValue(null, null) + new SolarIrradianceValue(missingValue, missingValue) private val solarIrradianceValue1: SolarIrradianceValue = new SolarIrradianceValue( - null, + missingValue, Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE) ) private val solarIrradianceValue2: SolarIrradianceValue = @@ -645,7 +647,9 @@ case object WeatherSourceSpec { Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE) ) - private val temperatureValue0: TemperatureValue = new TemperatureValue(null) + private val temperatureValue0: TemperatureValue = new TemperatureValue( + missingValue + ) private val temperatureValue1: TemperatureValue = new TemperatureValue( Quantities.getQuantity(288.15d, Units.KELVIN) ) @@ -653,11 +657,17 @@ case object WeatherSourceSpec { Quantities.getQuantity(290.15d, Units.KELVIN) ) - private val windValue0: WindValue = new WindValue(null, null) + private val windValue0: WindValue = new WindValue(missingValue, missingValue) private val windValue1: WindValue = - new WindValue(null, Quantities.getQuantity(10, Units.METRE_PER_SECOND)) + new WindValue( + missingValue, + Quantities.getQuantity(10, Units.METRE_PER_SECOND) + ) private val windValue2: WindValue = - new WindValue(null, Quantities.getQuantity(20, Units.METRE_PER_SECOND)) + new WindValue( + missingValue, + Quantities.getQuantity(20, Units.METRE_PER_SECOND) + ) private val timeBasedValue0: TimeBasedValue[WeatherValue] = buildTimeBasedValue( From c5305fc956b19c205e5aa72e6943f9e09fc1decb Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Wed, 1 Mar 2023 12:30:33 +0100 Subject: [PATCH 12/21] Fixing codacy issues. --- .../messages/services/WeatherMessage.scala | 23 +- .../service/weather/WeatherSource.scala | 367 ++++++-------- .../service/weather/WeatherSourceSpec.scala | 479 ++++++++---------- 3 files changed, 391 insertions(+), 478 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala b/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala index 3e2e39845f..304f245ea2 100644 --- a/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala +++ b/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala @@ -14,6 +14,7 @@ import edu.ie3.simona.ontology.messages.services.ServiceMessage.{ import edu.ie3.util.quantities.interfaces.Irradiance import tech.units.indriya.ComparableQuantity +import javax.measure.Quantity import javax.measure.quantity.{Speed, Temperature} sealed trait WeatherMessage @@ -89,9 +90,23 @@ object WeatherMessage { * Wind velocity */ final case class WeatherDataOption( - diffIrr: Option[(ComparableQuantity[Irradiance], Long)], - dirIrr: Option[(ComparableQuantity[Irradiance], Long)], - temp: Option[(ComparableQuantity[Temperature], Long)], - windVel: Option[(ComparableQuantity[Speed], Long)] + diffIrr: Option[QuantityWithWeight[Irradiance]], + dirIrr: Option[QuantityWithWeight[Irradiance]], + temp: Option[QuantityWithWeight[Temperature]], + windVel: Option[QuantityWithWeight[Speed]] + ) + + /** Container class for a weather quantity with a weight. It is primarily used + * for interpolation. + * @param quantity + * value + * @param weight + * of the value + * @tparam V + * unit of the quantity + */ + final case class QuantityWithWeight[V <: Quantity[V]]( + quantity: ComparableQuantity[V], + weight: Long ) } diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 0c89985be1..8c6a224d83 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -16,7 +16,10 @@ import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.source.IdCoordinateSource import edu.ie3.datamodel.io.source.csv.CsvIdCoordinateSource import edu.ie3.datamodel.models.StandardUnits -import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries +import edu.ie3.datamodel.models.timeseries.individual.{ + IndividualTimeSeries, + TimeBasedValue +} import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.simona.config.SimonaConfig import edu.ie3.simona.config.SimonaConfig.BaseCsvParams @@ -26,6 +29,7 @@ import edu.ie3.simona.exceptions.{ ServiceException } import edu.ie3.simona.ontology.messages.services.WeatherMessage.{ + QuantityWithWeight, WeatherData, WeatherDataOption } @@ -53,6 +57,7 @@ import java.time.ZonedDateTime import java.time.temporal.ChronoUnit import javax.measure.Quantity import javax.measure.quantity.{Dimensionless, Length, Speed, Temperature} +import scala.collection.immutable.SortedSet import scala.jdk.CollectionConverters._ import scala.jdk.OptionConverters.RichOptional import scala.util.{Failure, Success, Try} @@ -543,97 +548,120 @@ object WeatherSource extends LazyLogging { valueOption match { case Some(value) => - // found values are used to create a weather data object - // missing values are interpolated - val weatherData: WeatherData = interpolateValue(timeSeries, dateTime) - WeatherData( value.getSolarIrradiance.getDiffuseIrradiance.toScala .getOrElse( - weatherData.diffIrr + interpolateValue( + timeSeries, + dateTime, + "diffIrr", + EMPTY_WEATHER_DATA.diffIrr + ) ), value.getSolarIrradiance.getDirectIrradiance.toScala .getOrElse( - weatherData.dirIrr + interpolateValue( + timeSeries, + dateTime, + "dirIrr", + EMPTY_WEATHER_DATA.dirIrr + ) ), value.getTemperature.getTemperature.toScala .getOrElse( - weatherData.temp + interpolateValue( + timeSeries, + dateTime, + "temp", + EMPTY_WEATHER_DATA.temp + ) ), value.getWind.getVelocity.toScala .getOrElse( - weatherData.windVel + interpolateValue( + timeSeries, + dateTime, + "windVel", + EMPTY_WEATHER_DATA.windVel + ) ) ) case None => // if no values are found all values are interpolated logger.warn(s"No weather value found for timestamp $dateTime.") - interpolateValue(timeSeries, dateTime) + WeatherData( + interpolateValue( + timeSeries, + dateTime, + "diffIrr", + EMPTY_WEATHER_DATA.diffIrr + ), + interpolateValue( + timeSeries, + dateTime, + "dirIrr", + EMPTY_WEATHER_DATA.dirIrr + ), + interpolateValue( + timeSeries, + dateTime, + "temp", + EMPTY_WEATHER_DATA.temp + ), + interpolateValue( + timeSeries, + dateTime, + "windVel", + EMPTY_WEATHER_DATA.windVel + ) + ) } } /** Method for interpolation of weather values. + * * @param timeSeries * with weather data * @param dateTime * timestamp for which an interpolation is needed + * @param searchedValue + * string containing the searched value * @return * interpolated weather data */ - private def interpolateValue( + private def interpolateValue[V <: Quantity[V]]( timeSeries: IndividualTimeSeries[WeatherValue], - dateTime: ZonedDateTime - ): WeatherData = { + dateTime: ZonedDateTime, + searchedValue: String, + empty: ComparableQuantity[V] + ): ComparableQuantity[V] = { if (timeSeries.getEntries.size() < 3) { logger.info( s"Not enough entries in time series $timeSeries to interpolate weather data. At least three values are needed, found ${timeSeries.getEntries.size()}." ) - EMPTY_WEATHER_DATA + empty } else { - // find previous and next values - val previousOption: WeatherDataOption = - getPreviousValue(timeSeries, dateTime) - val nextOption: WeatherDataOption = getNextValue(timeSeries, dateTime) + val intervalStart: ZonedDateTime = dateTime.minusHours(2) + val intervalEnd: ZonedDateTime = dateTime.plusHours(2) - // weight found values or return EMPTY_WEATHER_DATA - val diffIrr: ComparableQuantity[Irradiance] = interpolate( + val previous: Option[QuantityWithWeight[V]] = getValue( + timeSeries, dateTime, - previousOption.diffIrr, - nextOption.diffIrr, - EMPTY_WEATHER_DATA.diffIrr - ) - val dirIrr: ComparableQuantity[Irradiance] = interpolate( + intervalStart, dateTime, - previousOption.dirIrr, - nextOption.dirIrr, - EMPTY_WEATHER_DATA.dirIrr - ) - val temp: ComparableQuantity[Temperature] = interpolate( - dateTime, - previousOption.temp, - nextOption.temp, - EMPTY_WEATHER_DATA.temp - ) - val windVel: ComparableQuantity[Speed] = interpolate( - dateTime, - previousOption.windVel, - nextOption.windVel, - EMPTY_WEATHER_DATA.windVel + searchedValue ) + val next: Option[QuantityWithWeight[V]] = + getValue(timeSeries, dateTime, dateTime, intervalEnd, searchedValue) - // returning new WeatherData - WeatherData( - diffIrr, - dirIrr, - temp, - windVel - ) + interpolate(dateTime, previous, next, empty) } } /** Method to interpolate a quantity for a specific timestamp with a previous * and a next value. + * * @param dateTime * timestamp which is interpolated * @param previousOption @@ -645,22 +673,22 @@ object WeatherSource extends LazyLogging { * @return * option of interpolated quantity */ - private def interpolate[V <: Quantity[V]]( + def interpolate[V <: Quantity[V]]( dateTime: ZonedDateTime, - previousOption: Option[(ComparableQuantity[V], Long)], - nextOption: Option[(ComparableQuantity[V], Long)], + previousOption: Option[QuantityWithWeight[V]], + nextOption: Option[QuantityWithWeight[V]], empty: ComparableQuantity[V] ): ComparableQuantity[V] = { (previousOption, nextOption) match { case (Some(previousValue), Some(nextValue)) => - val time1: Long = previousValue._2 - val time2: Long = nextValue._2 - val interval = time1 + time2 - val quantity1: ComparableQuantity[V] = previousValue._1 - val quantity2: ComparableQuantity[V] = nextValue._1 + val previousWeight: Long = previousValue.weight + val nextWeight: Long = nextValue.weight + val interval: Long = previousWeight + nextWeight + val previousQuantity: ComparableQuantity[V] = previousValue.quantity + val nextQuantity: ComparableQuantity[V] = nextValue.quantity - val weightedQuantity1 = quantity1.multiply(time1) - val weightedQuantity2 = quantity2.multiply(time2) + val weightedQuantity1 = previousQuantity.multiply(previousWeight) + val weightedQuantity2 = nextQuantity.multiply(nextWeight) weightedQuantity1.add(weightedQuantity2).divide(interval) case (_, _) => @@ -671,162 +699,105 @@ object WeatherSource extends LazyLogging { } } - /** Method to find previous weather data. - * + /** Method to get an quantity with its weight from an interval of a time + * series. * @param timeSeries - * with weather data - * @param dateTime - * starting time + * given time series + * @param timestamp + * given timestamp + * @param intervalStart + * start of the interval + * @param intervalEnd + * end of the interval + * @param searchedValue + * value that is searched + * @tparam V + * unit of the quantity * @return - * a weather data option + * an option of a quantity with a weight */ - def getPreviousValue( + def getValue[V <: Quantity[V]]( timeSeries: IndividualTimeSeries[WeatherValue], - dateTime: ZonedDateTime - ): WeatherDataOption = { - var currentTime: ZonedDateTime = dateTime - var weatherDataOption: WeatherDataOption = - WeatherDataOption(None, None, None, None) - - for (_ <- 0 to 7) { - timeSeries.getPreviousTimeBasedValue(currentTime).toScala match { - case Some(timeBasedValue) => - val valueTime: ZonedDateTime = timeBasedValue.getTime - - if (valueTime.isAfter(dateTime.minusHours(2))) { - val currentWeight: Long = - ChronoUnit.SECONDS.between(valueTime, dateTime) - - weatherDataOption = updateWeatherDataOption( - weatherDataOption, - timeBasedValue.getValue, - currentWeight - ) - } + timestamp: ZonedDateTime, + intervalStart: ZonedDateTime, + intervalEnd: ZonedDateTime, + searchedValue: String + ): Option[QuantityWithWeight[V]] = { + val values: List[QuantityWithWeight[V]] = + timeSeries.getEntries.asScala.flatMap { weatherValue => + val time: ZonedDateTime = weatherValue.getTime + + // calculates the time difference to the given timestamp + val weight = if (time.isBefore(timestamp)) { + ChronoUnit.SECONDS.between(time, timestamp) + } else { + ChronoUnit.SECONDS.between(timestamp, time) + } - if (currentTime == valueTime) { - currentTime = valueTime.minusMinutes(15) - } else { - currentTime = valueTime - } - case None => - currentTime = currentTime.minusMinutes(15) - } - } - weatherDataOption - } + // check is the found timestamp is in the defined interval + if (time.isAfter(intervalStart) && time.isBefore(intervalEnd)) { + getQuantity(weatherValue.getValue, weight, searchedValue): Option[ + QuantityWithWeight[V] + ] + } else { + // if timestamp is not inside is not inside the interval none is returned + None + } + }.toList - /** Method to find previous weather data. - * - * @param timeSeries - * with weather data - * @param dateTime - * starting time - * @return - * a weather data option - */ - def getNextValue( - timeSeries: IndividualTimeSeries[WeatherValue], - dateTime: ZonedDateTime - ): WeatherDataOption = { - var currentTime: ZonedDateTime = dateTime - var weatherDataOption: WeatherDataOption = - WeatherDataOption(None, None, None, None) - - for (_ <- 0 to 7) { - timeSeries.getNextTimeBasedValue(currentTime).toScala match { - case Some(timeBasedValue) => - val valueTime: ZonedDateTime = timeBasedValue.getTime - - if (valueTime.isBefore(dateTime.plusHours(2))) { - val currentWeight: Long = - ChronoUnit.SECONDS.between(dateTime, valueTime) - - weatherDataOption = updateWeatherDataOption( - weatherDataOption, - timeBasedValue.getValue, - currentWeight - ) - } + if (values.isEmpty) { + None + } else { + // sorting the list to return the value with the least time difference + val sortedSet: Set[QuantityWithWeight[V]] = values.sortBy { x => + x.weight + }.toSet - if (currentTime == valueTime) { - currentTime = valueTime.plusMinutes(15) - } else { - currentTime = valueTime - } - case None => - currentTime = currentTime.plusMinutes(15) - } + sortedSet.headOption } - weatherDataOption } - /** Method to find options for new weather data. - * - * @param weatherDataOption - * container class holding the options - * @param value - * possible new values + /** Method to get a specific value from a "WeatherValue" + * @param weatherValue + * given value * @param weight - * weighting for the found values + * current weight for the value + * @param searchedValue + * value that is searched + * @tparam V + * unit of the quantity * @return - * updated weather data option + * an option for a quantity with its weight */ - def updateWeatherDataOption( - weatherDataOption: WeatherDataOption, - value: WeatherValue, - weight: Long - ): WeatherDataOption = { - var option: WeatherDataOption = weatherDataOption - if (weight == 0L) { - weatherDataOption - } else { - // check options and update weather data option - ( - weatherDataOption.diffIrr, - value.getSolarIrradiance.getDiffuseIrradiance.toScala - ) match { - case (None, Some(value)) => - option = option.copy(diffIrr = Some(value, weight)) - case (_, _) => - logger.debug( - s"No diffused irradiance value with the given time difference of $weight found." - ) - } - - ( - weatherDataOption.dirIrr, - value.getSolarIrradiance.getDirectIrradiance.toScala - ) match { - case (None, Some(value)) => - option = option.copy(dirIrr = Some(value, weight)) - case (_, _) => - logger.debug( - s"No direct irradiance value with the given time difference of $weight found." - ) - } - - ( - weatherDataOption.temp, - value.getTemperature.getTemperature.toScala - ) match { - case (None, Some(value)) => - option = option.copy(temp = Some(value, weight)) - case (_, _) => - logger.debug( - s"No temperature value with the given time difference of $weight found." - ) - } + private def getQuantity[V <: Quantity[V]]( + weatherValue: WeatherValue, + weight: Long, + searchedValue: String + ): Option[QuantityWithWeight[V]] = { + val option = searchedValue match { + case "diffIrr" => + weatherValue.getSolarIrradiance.getDiffuseIrradiance.toScala + case "dirIrr" => + weatherValue.getSolarIrradiance.getDirectIrradiance.toScala + case "temp" => + weatherValue.getTemperature.getTemperature.toScala + case "windVel" => + weatherValue.getWind.getVelocity.toScala + } - (weatherDataOption.windVel, value.getWind.getVelocity.toScala) match { - case (None, Some(value)) => - option = option.copy(windVel = Some(value, weight)) - case (_, _) => - logger.debug( - s"No wind velocity value with the given time difference of $weight found." + option match { + case Some(value) => + if (value.isInstanceOf[V]) { + Some( + QuantityWithWeight( + value.asInstanceOf[ComparableQuantity[V]], + weight + ) ) - } - option + } else { + None + } + case None => None } } diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala index afa97fe6fc..b66b85c891 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala @@ -20,31 +20,29 @@ import edu.ie3.datamodel.models.value.{ } import edu.ie3.simona.exceptions.ServiceException import edu.ie3.simona.ontology.messages.services.WeatherMessage -import edu.ie3.simona.ontology.messages.services.WeatherMessage.{ - WeatherData, - WeatherDataOption -} +import edu.ie3.simona.ontology.messages.services.WeatherMessage.QuantityWithWeight import edu.ie3.simona.service.weather.WeatherSource.{ AgentCoordinates, EMPTY_WEATHER_DATA, WeightedCoordinates, - getNextValue, - getPreviousValue, - getWeatherData, - updateWeatherDataOption + getValue, + interpolate } import edu.ie3.simona.service.weather.WeatherSourceSpec._ import edu.ie3.simona.test.common.UnitSpec import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} +import edu.ie3.util.quantities.interfaces.Irradiance import edu.ie3.util.quantities.{PowerSystemUnits, QuantityUtil} import org.locationtech.jts.geom.Point import org.scalatest.prop.TableDrivenPropertyChecks +import tech.units.indriya.ComparableQuantity import tech.units.indriya.quantity.Quantities import tech.units.indriya.unit.Units import java.time.ZonedDateTime import java.util import java.util.{Optional, UUID} +import javax.measure.quantity.{Speed, Temperature} import scala.jdk.CollectionConverters._ import scala.jdk.OptionConverters._ import scala.util.{Failure, Success} @@ -322,63 +320,36 @@ class WeatherSourceSpec extends UnitSpec with TableDrivenPropertyChecks { "interpolate missing weather values correctly" in { val cases = Table( - ("timeBasedValues", "time", "expectedWeatherData"), - (Set.empty[TimeBasedValue[WeatherValue]], time, EMPTY_WEATHER_DATA), + ("previousOption", "nextOption", "emptyValue", "expectedValue"), + (None, None, EMPTY_WEATHER_DATA.diffIrr, EMPTY_WEATHER_DATA.diffIrr), ( - Set( - timeBasedValue0, - timeBasedValue1, - timeBasedValue2, - timeBasedValue3, - timeBasedValue4, - timeBasedValue5, - timeBasedValue6 - ), - time.minusHours(3), - EMPTY_WEATHER_DATA + Some(QuantityWithWeight(EMPTY_WEATHER_DATA.diffIrr, 0L)), + None, + EMPTY_WEATHER_DATA.diffIrr, + EMPTY_WEATHER_DATA.diffIrr ), ( - Set( - timeBasedValue0, - timeBasedValue1, - timeBasedValue2, - timeBasedValue3, - timeBasedValue4, - timeBasedValue5, - timeBasedValue6 + Some( + QuantityWithWeight( + Quantities.getQuantity(10, StandardUnits.SOLAR_IRRADIANCE), + 10 + ) ), - time.plusHours(2), - EMPTY_WEATHER_DATA - ), - ( - Set( - timeBasedValue0, - timeBasedValue1, - timeBasedValue2, - timeBasedValue3, - timeBasedValue4, - timeBasedValue5, - timeBasedValue6 + Some( + QuantityWithWeight( + Quantities.getQuantity(20, StandardUnits.SOLAR_IRRADIANCE), + 10 + ) ), - time, - WeatherData( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), - Quantities.getQuantity(0d, StandardUnits.WIND_VELOCITY) - ) + EMPTY_WEATHER_DATA.diffIrr, + Quantities.getQuantity(15, StandardUnits.SOLAR_IRRADIANCE) ) ) - forAll(cases) { (timeBasedValues, time, expectedWeatherData) => - val timeSeries: IndividualTimeSeries[WeatherValue] = - buildTimeSeries(timeBasedValues) - val weatherData: WeatherData = getWeatherData(timeSeries, time) - - weatherData.diffIrr shouldBe expectedWeatherData.diffIrr - weatherData.dirIrr shouldBe expectedWeatherData.dirIrr - weatherData.temp shouldBe expectedWeatherData.temp - weatherData.windVel shouldBe expectedWeatherData.windVel + forAll(cases) { (previousOption, nextOption, emptyValue, expectedValue) => + val value: ComparableQuantity[Irradiance] = + interpolate(time, previousOption, nextOption, emptyValue) + value shouldBe expectedValue } } @@ -401,18 +372,32 @@ class WeatherSourceSpec extends UnitSpec with TableDrivenPropertyChecks { timeBasedValue4 ), Some( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 3900 + QuantityWithWeight( + Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE), + 1800 + ) ), Some( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 6900 + QuantityWithWeight( + Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE), + 1800 + ) ), - Some(Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), 3900), - Some(Quantities.getQuantity(10d, Units.METRE_PER_SECOND), 3900) + Some( + QuantityWithWeight( + Quantities.getQuantity(-35d, StandardUnits.TEMPERATURE), + 6000 + ) + ), + Some( + QuantityWithWeight( + Quantities.getQuantity(10d, Units.METRE_PER_SECOND), + 6000 + ) + ) ), ( - Set(timeBasedValue0, timeBasedValue6), + Set(timeBasedValue0), None, None, None, @@ -428,13 +413,45 @@ class WeatherSourceSpec extends UnitSpec with TableDrivenPropertyChecks { expectedTemp, expectedWindVel ) => - val weatherDataOption: WeatherDataOption = - getPreviousValue(buildTimeSeries(timeBasedValues), time) - - weatherDataOption.diffIrr shouldBe expectedDiffIrr - weatherDataOption.dirIrr shouldBe expectedDirIrr - weatherDataOption.temp shouldBe expectedTemp - weatherDataOption.windVel shouldBe expectedWindVel + val timeSeries: IndividualTimeSeries[WeatherValue] = + buildTimeSeries(timeBasedValues) + val intervalStart: ZonedDateTime = time.minusHours(2) + + val diffIrr: Option[QuantityWithWeight[Irradiance]] = getValue( + timeSeries, + time, + intervalStart, + intervalEnd = time, + "diffIrr" + ) + diffIrr shouldBe expectedDiffIrr + + val dirIrr: Option[QuantityWithWeight[Irradiance]] = getValue( + timeSeries, + time, + intervalStart, + intervalEnd = time, + "diffIrr" + ) + dirIrr shouldBe expectedDirIrr + + val temp: Option[QuantityWithWeight[Temperature]] = getValue( + timeSeries, + time, + intervalStart, + intervalEnd = time, + "temp" + ) + temp shouldBe expectedTemp + + val windVel: Option[QuantityWithWeight[Speed]] = getValue( + timeSeries, + time, + intervalStart, + intervalEnd = time, + "windVel" + ) + windVel shouldBe expectedWindVel } } @@ -454,28 +471,48 @@ class WeatherSourceSpec extends UnitSpec with TableDrivenPropertyChecks { timeBasedValue2, timeBasedValue3, timeBasedValue4, - timeBasedValue5, - timeBasedValue6 + timeBasedValue5 ), Some( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 3300 + QuantityWithWeight( + Quantities.getQuantity(60d, StandardUnits.SOLAR_IRRADIANCE), + 1800 + ) ), Some( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 3300 + QuantityWithWeight( + Quantities.getQuantity(60d, StandardUnits.SOLAR_IRRADIANCE), + 1800 + ) ), - Some(Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), 6000), - None + Some( + QuantityWithWeight( + Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), + 1800 + ) + ), + Some( + QuantityWithWeight( + Quantities.getQuantity(20d, StandardUnits.WIND_VELOCITY), + 1800 + ) + ) ), ( - Set(timeBasedValue0, timeBasedValue5), + Set(timeBasedValue0, timeBasedValue4, timeBasedValue5), Some( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 6000 + QuantityWithWeight( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 6000 + ) + ), + Some( + QuantityWithWeight( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + 6000 + ) ), None, - Some(Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), 6000), None ) ) @@ -488,135 +525,48 @@ class WeatherSourceSpec extends UnitSpec with TableDrivenPropertyChecks { expectedTemp, expectedWindVel ) => - val weatherDataOption: WeatherDataOption = - getNextValue(buildTimeSeries(timeBasedValues), time) - - weatherDataOption.diffIrr shouldBe expectedDiffIrr - weatherDataOption.dirIrr shouldBe expectedDirIrr - weatherDataOption.temp shouldBe expectedTemp - weatherDataOption.windVel shouldBe expectedWindVel - } - } - - "update weather data option correctly" in { - val weatherDataOption: WeatherDataOption = - WeatherDataOption(None, None, None, None) - - val cases = Table( - ("givenValue", "givenWeight", "expectedResult"), - (timeBasedValue0.getValue, 200L, weatherDataOption), - ( - timeBasedValue1.getValue, - 300L, - WeatherDataOption( - Some( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 300L - ), - None, - Some( - Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), - 300L - ), - Some(Quantities.getQuantity(10, Units.METRE_PER_SECOND), 300L) + val timeSeries: IndividualTimeSeries[WeatherValue] = + buildTimeSeries(timeBasedValues) + val intervalEnd: ZonedDateTime = time.plusHours(2) + + val diffIrr: Option[QuantityWithWeight[Irradiance]] = getValue( + timeSeries, + time, + intervalStart = time, + intervalEnd, + "diffIrr" ) - ), - ( - timeBasedValue2.getValue, - 250L, - WeatherDataOption( - Some( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 250L - ), - Some( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 250L - ), - Some( - Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), - 250L - ), - Some(Quantities.getQuantity(20, Units.METRE_PER_SECOND), 250L) + diffIrr shouldBe expectedDiffIrr + + val dirIrr: Option[QuantityWithWeight[Irradiance]] = getValue( + timeSeries, + time, + intervalStart = time, + intervalEnd, + "diffIrr" ) - ), - ( - timeBasedValue3.getValue, - 300L, - WeatherDataOption( - Some( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 300L - ), - None, - Some( - Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), - 300L - ), - Some(Quantities.getQuantity(20, Units.METRE_PER_SECOND), 300L) + dirIrr shouldBe expectedDirIrr + + val temp: Option[QuantityWithWeight[Temperature]] = getValue( + timeSeries, + time, + intervalStart = time, + intervalEnd, + "temp" ) - ), - ( - timeBasedValue4.getValue, - 250L, - WeatherDataOption( - Some( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 250L - ), - Some( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 250L - ), - None, - None + temp shouldBe expectedTemp + + val windVel: Option[QuantityWithWeight[Speed]] = getValue( + timeSeries, + time, + intervalStart = time, + intervalEnd, + "windVel" ) - ), - ( - timeBasedValue5.getValue, - 300L, - WeatherDataOption( - Some( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 300L - ), - None, - Some( - Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), - 300L - ), - None - ) - ), - ( - timeBasedValue6.getValue, - 250L, - WeatherDataOption( - Some( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 250L - ), - Some( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 250L - ), - Some( - Quantities.getQuantity(17d, StandardUnits.TEMPERATURE), - 250L - ), - Some(Quantities.getQuantity(20, Units.METRE_PER_SECOND), 250L) - ) - ) - ) - - forAll(cases) { (givenValue, givenWeight, expectedResult) => - val updatedWeatherDataOption = - updateWeatherDataOption(weatherDataOption, givenValue, givenWeight) - - updatedWeatherDataOption shouldBe expectedResult + windVel shouldBe expectedWindVel } - } + } } @@ -634,115 +584,92 @@ case object WeatherSourceSpec { private val missingValue: Null = null - private val solarIrradianceValue0: SolarIrradianceValue = - new SolarIrradianceValue(missingValue, missingValue) - private val solarIrradianceValue1: SolarIrradianceValue = - new SolarIrradianceValue( - missingValue, - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE) - ) - private val solarIrradianceValue2: SolarIrradianceValue = - new SolarIrradianceValue( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE) - ) - - private val temperatureValue0: TemperatureValue = new TemperatureValue( - missingValue - ) - private val temperatureValue1: TemperatureValue = new TemperatureValue( - Quantities.getQuantity(288.15d, Units.KELVIN) - ) - private val temperatureValue2: TemperatureValue = new TemperatureValue( - Quantities.getQuantity(290.15d, Units.KELVIN) - ) - - private val windValue0: WindValue = new WindValue(missingValue, missingValue) - private val windValue1: WindValue = - new WindValue( - missingValue, - Quantities.getQuantity(10, Units.METRE_PER_SECOND) - ) - private val windValue2: WindValue = - new WindValue( - missingValue, - Quantities.getQuantity(20, Units.METRE_PER_SECOND) - ) - private val timeBasedValue0: TimeBasedValue[WeatherValue] = buildTimeBasedValue( - time, + time.minusMinutes(130), new WeatherValue( coordinate67775, - solarIrradianceValue0, - temperatureValue0, - windValue0 + new SolarIrradianceValue(missingValue, missingValue), + new TemperatureValue( + missingValue + ), + new WindValue(missingValue, missingValue) ) ) private val timeBasedValue1: TimeBasedValue[WeatherValue] = buildTimeBasedValue( - time.minusMinutes(65), + time.minusMinutes(100), new WeatherValue( coordinate67775, - solarIrradianceValue1, - temperatureValue1, - windValue1 + new SolarIrradianceValue( + missingValue, + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE) + ), + new TemperatureValue(Quantities.getQuantity(238.15d, Units.KELVIN)), + new WindValue( + missingValue, + Quantities.getQuantity(10, Units.METRE_PER_SECOND) + ) ) ) private val timeBasedValue2: TimeBasedValue[WeatherValue] = buildTimeBasedValue( - time.minusMinutes(115), + time.minusMinutes(30), new WeatherValue( coordinate67775, - solarIrradianceValue2, - temperatureValue1, - windValue2 + new SolarIrradianceValue( + Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE), + Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE) + ), + new TemperatureValue(missingValue), + new WindValue(missingValue, missingValue) ) ) private val timeBasedValue3: TimeBasedValue[WeatherValue] = buildTimeBasedValue( - time.minusMinutes(121), + time.plusMinutes(30), new WeatherValue( coordinate67775, - solarIrradianceValue1, - temperatureValue1, - windValue2 + new SolarIrradianceValue( + missingValue, + Quantities.getQuantity(60d, StandardUnits.SOLAR_IRRADIANCE) + ), + new TemperatureValue(Quantities.getQuantity(288.15d, Units.KELVIN)), + new WindValue( + missingValue, + Quantities.getQuantity(20, Units.METRE_PER_SECOND) + ) ) ) private val timeBasedValue4: TimeBasedValue[WeatherValue] = - buildTimeBasedValue( - time.plusMinutes(55), - new WeatherValue( - coordinate67775, - solarIrradianceValue2, - temperatureValue0, - windValue0 - ) - ) - - private val timeBasedValue5: TimeBasedValue[WeatherValue] = buildTimeBasedValue( time.plusMinutes(100), new WeatherValue( coordinate67775, - solarIrradianceValue1, - temperatureValue1, - windValue0 + new SolarIrradianceValue( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE) + ), + new TemperatureValue(missingValue), + new WindValue(missingValue, missingValue) ) ) - private val timeBasedValue6: TimeBasedValue[WeatherValue] = + private val timeBasedValue5: TimeBasedValue[WeatherValue] = buildTimeBasedValue( - time.plusMinutes(1), + time.plusMinutes(130), new WeatherValue( coordinate67775, - solarIrradianceValue2, - temperatureValue2, - windValue2 + new SolarIrradianceValue( + missingValue, + Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE) + ), + new TemperatureValue(Quantities.getQuantity(288.15d, Units.KELVIN)), + new WindValue(missingValue, missingValue) ) ) From 603a6e1d40860bab177ffe7c9f302ec9c5b5aa80 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Fri, 3 Mar 2023 09:41:50 +0100 Subject: [PATCH 13/21] Some improvements. --- .../messages/services/WeatherMessage.scala | 20 --- .../service/weather/WeatherSource.scala | 150 +++++++----------- 2 files changed, 60 insertions(+), 110 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala b/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala index 304f245ea2..41e511b397 100644 --- a/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala +++ b/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala @@ -76,26 +76,6 @@ object WeatherMessage { windVel: ComparableQuantity[Speed] ) extends SecondaryData - /** Container class for the weather data option. It is similar to the normal - * weather data but instead of concrete quantities, this container holds - * options for quantities with their weight. - * - * @param diffIrr - * Diffuse irradiance on the horizontal pane - * @param dirIrr - * Direct irradiance on the horizontal pane - * @param temp - * Temperature - * @param windVel - * Wind velocity - */ - final case class WeatherDataOption( - diffIrr: Option[QuantityWithWeight[Irradiance]], - dirIrr: Option[QuantityWithWeight[Irradiance]], - temp: Option[QuantityWithWeight[Temperature]], - windVel: Option[QuantityWithWeight[Speed]] - ) - /** Container class for a weather quantity with a weight. It is primarily used * for interpolation. * @param quantity diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 8c6a224d83..877618211b 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -16,10 +16,7 @@ import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.source.IdCoordinateSource import edu.ie3.datamodel.io.source.csv.CsvIdCoordinateSource import edu.ie3.datamodel.models.StandardUnits -import edu.ie3.datamodel.models.timeseries.individual.{ - IndividualTimeSeries, - TimeBasedValue -} +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.simona.config.SimonaConfig import edu.ie3.simona.config.SimonaConfig.BaseCsvParams @@ -30,8 +27,7 @@ import edu.ie3.simona.exceptions.{ } import edu.ie3.simona.ontology.messages.services.WeatherMessage.{ QuantityWithWeight, - WeatherData, - WeatherDataOption + WeatherData } import edu.ie3.simona.service.weather.WeatherSource.{ AgentCoordinates, @@ -41,7 +37,6 @@ import edu.ie3.simona.util.ConfigUtil.CsvConfigUtil.checkBaseCsvParams import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{ checkCouchbaseParams, checkInfluxDb1xParams, - checkKafkaParams, checkSqlParams } import edu.ie3.simona.util.ParsableEnumeration @@ -57,7 +52,6 @@ import java.time.ZonedDateTime import java.time.temporal.ChronoUnit import javax.measure.Quantity import javax.measure.quantity.{Dimensionless, Length, Speed, Temperature} -import scala.collection.immutable.SortedSet import scala.jdk.CollectionConverters._ import scala.jdk.OptionConverters.RichOptional import scala.util.{Failure, Success, Try} @@ -546,77 +540,60 @@ object WeatherSource extends LazyLogging { val valueOption: Option[WeatherValue] = timeSeries.getValue(dateTime).toScala - valueOption match { + val (diffIrr, dirIrr, temp, windVel): ( + Option[ComparableQuantity[Irradiance]], + Option[ComparableQuantity[Irradiance]], + Option[ComparableQuantity[Temperature]], + Option[ComparableQuantity[Speed]] + ) = valueOption match { case Some(value) => - WeatherData( - value.getSolarIrradiance.getDiffuseIrradiance.toScala - .getOrElse( - interpolateValue( - timeSeries, - dateTime, - "diffIrr", - EMPTY_WEATHER_DATA.diffIrr - ) - ), - value.getSolarIrradiance.getDirectIrradiance.toScala - .getOrElse( - interpolateValue( - timeSeries, - dateTime, - "dirIrr", - EMPTY_WEATHER_DATA.dirIrr - ) - ), - value.getTemperature.getTemperature.toScala - .getOrElse( - interpolateValue( - timeSeries, - dateTime, - "temp", - EMPTY_WEATHER_DATA.temp - ) - ), - value.getWind.getVelocity.toScala - .getOrElse( - interpolateValue( - timeSeries, - dateTime, - "windVel", - EMPTY_WEATHER_DATA.windVel - ) - ) + ( + getQuantity(value, "diffIrr"), + getQuantity(value, "diffIrr"), + getQuantity(value, "diffIrr"), + getQuantity(value, "diffIrr") ) case None => // if no values are found all values are interpolated logger.warn(s"No weather value found for timestamp $dateTime.") - WeatherData( - interpolateValue( - timeSeries, - dateTime, - "diffIrr", - EMPTY_WEATHER_DATA.diffIrr - ), - interpolateValue( - timeSeries, - dateTime, - "dirIrr", - EMPTY_WEATHER_DATA.dirIrr - ), - interpolateValue( - timeSeries, - dateTime, - "temp", - EMPTY_WEATHER_DATA.temp - ), - interpolateValue( - timeSeries, - dateTime, - "windVel", - EMPTY_WEATHER_DATA.windVel - ) - ) + (None, None, None, None) } + + WeatherData( + diffIrr.getOrElse( + interpolateValue( + timeSeries, + dateTime, + "diffIrr", + EMPTY_WEATHER_DATA.diffIrr + ) + ), + dirIrr.getOrElse( + interpolateValue( + timeSeries, + dateTime, + "dirIrr", + EMPTY_WEATHER_DATA.dirIrr + ) + ), + temp.getOrElse( + interpolateValue( + timeSeries, + dateTime, + "temp", + EMPTY_WEATHER_DATA.temp + ) + ), + windVel.getOrElse( + interpolateValue( + timeSeries, + dateTime, + "windVel", + EMPTY_WEATHER_DATA.windVel + ) + ) + ) } /** Method for interpolation of weather values. @@ -736,9 +713,12 @@ object WeatherSource extends LazyLogging { // check is the found timestamp is in the defined interval if (time.isAfter(intervalStart) && time.isBefore(intervalEnd)) { - getQuantity(weatherValue.getValue, weight, searchedValue): Option[ - QuantityWithWeight[V] - ] + val option: Option[ComparableQuantity[V]] = + getQuantity(weatherValue.getValue, searchedValue) + option match { + case Some(value) => Some(QuantityWithWeight(value, weight)) + case None => None + } } else { // if timestamp is not inside is not inside the interval none is returned None @@ -760,21 +740,18 @@ object WeatherSource extends LazyLogging { /** Method to get a specific value from a "WeatherValue" * @param weatherValue * given value - * @param weight - * current weight for the value * @param searchedValue * value that is searched * @tparam V * unit of the quantity * @return - * an option for a quantity with its weight + * an option for a quantity */ private def getQuantity[V <: Quantity[V]]( weatherValue: WeatherValue, - weight: Long, searchedValue: String - ): Option[QuantityWithWeight[V]] = { - val option = searchedValue match { + ): Option[ComparableQuantity[V]] = { + (searchedValue match { case "diffIrr" => weatherValue.getSolarIrradiance.getDiffuseIrradiance.toScala case "dirIrr" => @@ -783,17 +760,10 @@ object WeatherSource extends LazyLogging { weatherValue.getTemperature.getTemperature.toScala case "windVel" => weatherValue.getWind.getVelocity.toScala - } - - option match { + }) match { case Some(value) => if (value.isInstanceOf[V]) { - Some( - QuantityWithWeight( - value.asInstanceOf[ComparableQuantity[V]], - weight - ) - ) + Some(value.asInstanceOf[ComparableQuantity[V]]) } else { None } From 3f7ec8ef081ee192ed458f70216b4cfeefdeb34b Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Thu, 4 May 2023 14:47:52 +0200 Subject: [PATCH 14/21] Fixing failing tests. --- .../edu/ie3/simona/service/weather/WeatherSource.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 877618211b..c020f14733 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -549,9 +549,9 @@ object WeatherSource extends LazyLogging { case Some(value) => ( getQuantity(value, "diffIrr"), - getQuantity(value, "diffIrr"), - getQuantity(value, "diffIrr"), - getQuantity(value, "diffIrr") + getQuantity(value, "dirIrr"), + getQuantity(value, "temp"), + getQuantity(value, "windVel") ) case None => // if no values are found all values are interpolated From b8e68cd56dafbedcef37608a1599311fd196c2cf Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Thu, 28 Sep 2023 16:55:42 +0200 Subject: [PATCH 15/21] Adapting to changes. --- .../messages/services/WeatherMessage.scala | 13 +- .../service/weather/WeatherSource.scala | 355 ++++------------ .../weather/WeatherSourceWrapper.scala | 1 - .../weather/WeatherValueInterpolation.scala | 203 ++++++++++ .../QuantitySquantsConversions.scala | 48 +++ .../service/weather/WeatherSourceSpec.scala | 382 +----------------- .../WeatherValueInterpolationSpec.scala | 332 +++++++++++++++ 7 files changed, 665 insertions(+), 669 deletions(-) create mode 100644 src/main/scala/edu/ie3/simona/service/weather/WeatherValueInterpolation.scala create mode 100644 src/main/scala/edu/ie3/util/scala/quantities/QuantitySquantsConversions.scala create mode 100644 src/test/scala/edu/ie3/simona/service/weather/WeatherValueInterpolationSpec.scala diff --git a/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala b/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala index 253e1f51c5..8ae8557c39 100644 --- a/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala +++ b/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala @@ -6,6 +6,7 @@ package edu.ie3.simona.ontology.messages.services +import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.simona.agent.participant.data.Data.SecondaryData import edu.ie3.simona.ontology.messages.services.ServiceMessage.{ ProvisionMessage, @@ -73,17 +74,17 @@ object WeatherMessage { windVel: Velocity ) extends SecondaryData - /** Container class for a weather quantity with a weight. It is primarily used + /** Container class for a weather value with a weight. It is primarily used * for interpolation. - * @param quantity - * value + * @param value + * weather value * @param weight * of the value * @tparam V - * unit of the quantity + * type of value */ - final case class QuantityWithWeight[V <: Quantity[V]]( - quantity: ComparableQuantity[V], + final case class ValueWithWeight[V]( + value: V, weight: Long ) } diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index adba782d5b..373e217c35 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -7,39 +7,45 @@ package edu.ie3.simona.service.weather import com.typesafe.scalalogging.LazyLogging -import edu.ie3.datamodel.io.factory.timeseries.{CosmoIdCoordinateFactory, IconIdCoordinateFactory, IdCoordinateFactory} +import edu.ie3.datamodel.io.connectors.SqlConnector +import edu.ie3.datamodel.io.factory.timeseries.{ + CosmoIdCoordinateFactory, + IconIdCoordinateFactory, IdCoordinateFactory, SqlIdCoordinateFactory -import edu.ie3.datamodel.io.connectors.SqlConnector +} import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.source.IdCoordinateSource import edu.ie3.datamodel.io.source.csv.CsvIdCoordinateSource import edu.ie3.datamodel.io.source.sql.SqlIdCoordinateSource -import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.simona.config.SimonaConfig -import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource._ -import edu.ie3.simona.exceptions.{InvalidConfigParameterException, ServiceException} import edu.ie3.simona.config.SimonaConfig.BaseCsvParams +import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource._ +import edu.ie3.simona.exceptions.{ + InvalidConfigParameterException, + ServiceException +} import edu.ie3.simona.ontology.messages.services.WeatherMessage.WeatherData -import edu.ie3.simona.ontology.messages.services.WeatherMessage.{ - QuantityWithWeight, - WeatherData +import edu.ie3.simona.service.weather.WeatherSource.{ + AgentCoordinates, + WeightedCoordinates } -import edu.ie3.simona.service.weather.WeatherSource.{AgentCoordinates, WeightedCoordinates, logger} +import edu.ie3.simona.service.weather.WeatherValueInterpolation.interpolate import edu.ie3.simona.util.ConfigUtil.CsvConfigUtil.checkBaseCsvParams -import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{checkCouchbaseParams, checkInfluxDb1xParams, checkSqlParams} +import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{ + checkCouchbaseParams, + checkInfluxDb1xParams, + checkSqlParams +} import edu.ie3.simona.util.ParsableEnumeration import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} import edu.ie3.util.quantities.PowerSystemUnits -import edu.ie3.util.quantities.interfaces.Irradiance import edu.ie3.util.scala.io.CsvDataSourceAdapter -import edu.ie3.util.quantities.interfaces.Irradiance -import org.locationtech.jts.geom.{Coordinate, Point} -import tech.units.indriya.ComparableQuantity -import tech.units.indriya.ComparableQuantity +import edu.ie3.util.scala.quantities.QuantitySquantsConversions._ import edu.ie3.util.scala.quantities.WattsPerSquareMeter +import org.locationtech.jts.geom.{Coordinate, Point} import squants.motion.MetersPerSecond import squants.thermal.Kelvin import tech.units.indriya.ComparableQuantity @@ -49,9 +55,6 @@ import tech.units.indriya.unit.Units import java.nio.file.Paths import java.time.ZonedDateTime import javax.measure.quantity.{Dimensionless, Length} -import java.time.temporal.ChronoUnit -import javax.measure.Quantity -import javax.measure.quantity.{Dimensionless, Length, Speed, Temperature} import scala.jdk.CollectionConverters._ import scala.jdk.OptionConverters.RichOptional import scala.util.{Failure, Success, Try} @@ -292,7 +295,7 @@ trait WeatherSource { ): Array[Long] } -object WeatherSource { +object WeatherSource extends LazyLogging { def apply( dataSourceConfig: SimonaConfig.Simona.Input.Weather.Datasource, @@ -557,44 +560,20 @@ object WeatherSource { ): WeatherData = { WeatherData( weatherValue.getSolarIrradiance.getDiffuseIrradiance.toScala match { - case Some(irradiance) => - WattsPerSquareMeter( - irradiance - .to(PowerSystemUnits.WATT_PER_SQUAREMETRE) - .getValue - .doubleValue() - ) - case None => EMPTY_WEATHER_DATA.diffIrr + case Some(irradiance) => irradiance.toSquants + case None => EMPTY_WEATHER_DATA.diffIrr }, weatherValue.getSolarIrradiance.getDirectIrradiance.toScala match { - case Some(irradiance) => - WattsPerSquareMeter( - irradiance - .to(PowerSystemUnits.WATT_PER_SQUAREMETRE) - .getValue - .doubleValue() - ) - case None => EMPTY_WEATHER_DATA.dirIrr + case Some(irradiance) => irradiance.toSquants + case None => EMPTY_WEATHER_DATA.dirIrr }, weatherValue.getTemperature.getTemperature.toScala match { - case Some(temperature) => - Kelvin( - temperature - .to(Units.KELVIN) - .getValue - .doubleValue() - ) - case None => EMPTY_WEATHER_DATA.temp + case Some(temperature) => temperature.toSquants + case None => EMPTY_WEATHER_DATA.temp }, weatherValue.getWind.getVelocity.toScala match { - case Some(windVel) => - MetersPerSecond( - windVel - .to(Units.METRE_PER_SECOND) - .getValue - .doubleValue() - ) - case None => EMPTY_WEATHER_DATA.windVel + case Some(windVel) => windVel.toSquants + case None => EMPTY_WEATHER_DATA.windVel } ) } @@ -603,253 +582,67 @@ object WeatherSource { * interpolates missing values. * * @param timeSeries - * with weather values + * with weather values * @param dateTime - * timestamp in question + * timestamp in question * @return - * weather data object + * weather data object */ def getWeatherData( - timeSeries: IndividualTimeSeries[WeatherValue], - dateTime: ZonedDateTime - ): WeatherData = { + timeSeries: IndividualTimeSeries[WeatherValue], + dateTime: ZonedDateTime + ): WeatherData = { // gets a value option val valueOption: Option[WeatherValue] = timeSeries.getValue(dateTime).toScala - val (diffIrr, dirIrr, temp, windVel): ( - Option[ComparableQuantity[Irradiance]], - Option[ComparableQuantity[Irradiance]], - Option[ComparableQuantity[Temperature]], - Option[ComparableQuantity[Speed]] - ) = valueOption match { + // check which data is missing + val (diffIrr, dirIrr, temp, windVel) = valueOption match { case Some(value) => + val solar = value.getSolarIrradiance ( - getQuantity(value, "diffIrr"), - getQuantity(value, "dirIrr"), - getQuantity(value, "temp"), - getQuantity(value, "windVel") + solar.getDiffuseIrradiance.toScala, + solar.getDirectIrradiance.toScala, + value.getTemperature.getTemperature.toScala, + value.getWind.getVelocity.toScala ) - case None => - // if no values are found all values are interpolated - logger.warn(s"No weather value found for timestamp $dateTime.") - - (None, None, None, None) + case None => (None, None, None, None) } WeatherData( - diffIrr.getOrElse( - interpolateValue( - timeSeries, - dateTime, - "diffIrr", - EMPTY_WEATHER_DATA.diffIrr - ) - ), - dirIrr.getOrElse( - interpolateValue( - timeSeries, - dateTime, - "dirIrr", - EMPTY_WEATHER_DATA.dirIrr - ) - ), - temp.getOrElse( - interpolateValue( - timeSeries, - dateTime, - "temp", - EMPTY_WEATHER_DATA.temp - ) - ), - windVel.getOrElse( - interpolateValue( - timeSeries, - dateTime, - "windVel", - EMPTY_WEATHER_DATA.windVel - ) - ) + diffIrr match { + case Some(irradiance) => irradiance.toSquants + case None => + interpolate( + timeSeries, + dateTime, + "diffIrr", + EMPTY_WEATHER_DATA.diffIrr + ) + }, + dirIrr match { + case Some(irradiance) => irradiance.toSquants + case None => + interpolate(timeSeries, dateTime, "dirIrr", EMPTY_WEATHER_DATA.dirIrr) + }, + temp match { + case Some(temperature) => temperature.toSquants + case None => + interpolate(timeSeries, dateTime, "temp", EMPTY_WEATHER_DATA.temp) + }, + windVel match { + case Some(velocity) => velocity.toSquants + case None => + interpolate( + timeSeries, + dateTime, + "windVel", + EMPTY_WEATHER_DATA.windVel + ) + } ) } - /** Method for interpolation of weather values. - * - * @param timeSeries - * with weather data - * @param dateTime - * timestamp for which an interpolation is needed - * @param searchedValue - * string containing the searched value - * @return - * interpolated weather data - */ - private def interpolateValue[V <: Quantity[V]]( - timeSeries: IndividualTimeSeries[WeatherValue], - dateTime: ZonedDateTime, - searchedValue: String, - empty: ComparableQuantity[V] - ): ComparableQuantity[V] = { - if (timeSeries.getEntries.size() < 3) { - logger.info( - s"Not enough entries in time series $timeSeries to interpolate weather data. At least three values are needed, found ${timeSeries.getEntries.size()}." - ) - empty - } else { - val intervalStart: ZonedDateTime = dateTime.minusHours(2) - val intervalEnd: ZonedDateTime = dateTime.plusHours(2) - - val previous: Option[QuantityWithWeight[V]] = getValue( - timeSeries, - dateTime, - intervalStart, - dateTime, - searchedValue - ) - val next: Option[QuantityWithWeight[V]] = - getValue(timeSeries, dateTime, dateTime, intervalEnd, searchedValue) - - interpolate(dateTime, previous, next, empty) - } - } - - /** Method to interpolate a quantity for a specific timestamp with a previous - * and a next value. - * - * @param dateTime - * timestamp which is interpolated - * @param previousOption - * option of quantity and time difference - * @param nextOption - * option of quantity and time difference - * @param empty - * weather data - * @return - * option of interpolated quantity - */ - def interpolate[V <: Quantity[V]]( - dateTime: ZonedDateTime, - previousOption: Option[QuantityWithWeight[V]], - nextOption: Option[QuantityWithWeight[V]], - empty: ComparableQuantity[V] - ): ComparableQuantity[V] = { - (previousOption, nextOption) match { - case (Some(previousValue), Some(nextValue)) => - val previousWeight: Long = previousValue.weight - val nextWeight: Long = nextValue.weight - val interval: Long = previousWeight + nextWeight - val previousQuantity: ComparableQuantity[V] = previousValue.quantity - val nextQuantity: ComparableQuantity[V] = nextValue.quantity - - val weightedQuantity1 = previousQuantity.multiply(previousWeight) - val weightedQuantity2 = nextQuantity.multiply(nextWeight) - - weightedQuantity1.add(weightedQuantity2).divide(interval) - case (_, _) => - logger.warn( - s"Interpolating value with unit ${empty.getUnit} for timestamp $dateTime was not possible. The default value is used." - ) - empty - } - } - - /** Method to get an quantity with its weight from an interval of a time - * series. - * - * @param timeSeries - * given time series - * @param timestamp - * given timestamp - * @param intervalStart - * start of the interval - * @param intervalEnd - * end of the interval - * @param searchedValue - * value that is searched - * @tparam V - * unit of the quantity - * @return - * an option of a quantity with a weight - */ - def getValue[V <: Quantity[V]]( - timeSeries: IndividualTimeSeries[WeatherValue], - timestamp: ZonedDateTime, - intervalStart: ZonedDateTime, - intervalEnd: ZonedDateTime, - searchedValue: String - ): Option[QuantityWithWeight[V]] = { - val values: List[QuantityWithWeight[V]] = - timeSeries.getEntries.asScala.flatMap { weatherValue => - val time: ZonedDateTime = weatherValue.getTime - - // calculates the time difference to the given timestamp - val weight = if (time.isBefore(timestamp)) { - ChronoUnit.SECONDS.between(time, timestamp) - } else { - ChronoUnit.SECONDS.between(timestamp, time) - } - - // check is the found timestamp is in the defined interval - if (time.isAfter(intervalStart) && time.isBefore(intervalEnd)) { - val option: Option[ComparableQuantity[V]] = - getQuantity(weatherValue.getValue, searchedValue) - option match { - case Some(value) => Some(QuantityWithWeight(value, weight)) - case None => None - } - } else { - // if timestamp is not inside is not inside the interval none is returned - None - } - }.toList - - if (values.isEmpty) { - None - } else { - // sorting the list to return the value with the least time difference - val sortedSet: Set[QuantityWithWeight[V]] = values.sortBy { x => - x.weight - }.toSet - - sortedSet.headOption - } - } - - /** Method to get a specific value from a "WeatherValue" - * - * @param weatherValue - * given value - * @param searchedValue - * value that is searched - * @tparam V - * unit of the quantity - * @return - * an option for a quantity - */ - private def getQuantity[V <: Quantity[V]]( - weatherValue: WeatherValue, - searchedValue: String - ): Option[ComparableQuantity[V]] = { - (searchedValue match { - case "diffIrr" => - weatherValue.getSolarIrradiance.getDiffuseIrradiance.toScala - case "dirIrr" => - weatherValue.getSolarIrradiance.getDirectIrradiance.toScala - case "temp" => - weatherValue.getTemperature.getTemperature.toScala - case "windVel" => - weatherValue.getWind.getVelocity.toScala - }) match { - case Some(value) => - if (value.isInstanceOf[V]) { - Some(value.asInstanceOf[ComparableQuantity[V]]) - } else { - None - } - case None => None - } - } - /** Weather package private case class to combine the provided agent * coordinates into one single entity */ diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala index ae9c2ccc94..d2b89f54a1 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala @@ -49,7 +49,6 @@ import tech.units.indriya.ComparableQuantity import java.nio.file.Path import java.time.ZonedDateTime import javax.measure.quantity.Length - import scala.jdk.CollectionConverters.{IterableHasAsJava, MapHasAsScala} import scala.util.{Failure, Success, Try} diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherValueInterpolation.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherValueInterpolation.scala new file mode 100644 index 0000000000..d0f862da94 --- /dev/null +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherValueInterpolation.scala @@ -0,0 +1,203 @@ +/* + * © 2023. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.simona.service.weather + +import com.typesafe.scalalogging.LazyLogging +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries +import edu.ie3.datamodel.models.value.WeatherValue +import edu.ie3.simona.ontology.messages.services.WeatherMessage.ValueWithWeight +import edu.ie3.util.scala.quantities.QuantitySquantsConversions._ +import squants.Quantity + +import java.time.ZonedDateTime +import java.time.temporal.ChronoUnit +import scala.jdk.CollectionConverters.CollectionHasAsScala +import scala.jdk.OptionConverters.RichOptional + +object WeatherValueInterpolation extends LazyLogging { + + /** Method for interpolating weather values. + * + * @param timeSeries + * with weather data + * @param dateTime + * timestamp for which an interpolation is needed + * @param typeString + * string containing the searched value + * @param empty + * default value, if no other value was found + * @tparam V + * type of value + * @return + * a new quantity + */ + def interpolate[V <: Quantity[V]]( + timeSeries: IndividualTimeSeries[WeatherValue], + dateTime: ZonedDateTime, + typeString: String, + empty: V + ): V = { + getValueOptions[V](timeSeries, dateTime, typeString) match { + case Some((preVal, preWeight, nextVal, nextWeight)) => + val interval: Long = preWeight + nextWeight + + val weightedQuantity1 = preVal * preWeight + val weightedQuantity2 = nextVal * nextWeight + + (weightedQuantity1 + weightedQuantity2) / interval + case None => + logger.warn( + s"Interpolating value with unit ${empty.unit} for timestamp $dateTime was not possible. The default value is used." + ) + empty + } + } + + /** Method for getting interval and values for an interpolation- + * + * @param timeSeries + * with weather data + * @param dateTime + * timestamp for which an interpolation is needed + * @param typeString + * string containing the searched value + * @tparam V + * type of value + * @return + * an option + */ + private def getValueOptions[V]( + timeSeries: IndividualTimeSeries[WeatherValue], + dateTime: ZonedDateTime, + typeString: String + ): Option[(V, Long, V, Long)] = { + if (timeSeries.getEntries.size() < 3) { + logger.info( + s"Not enough entries in time series $timeSeries to interpolate weather data. At least three values are needed, found ${timeSeries.getEntries.size()}." + ) + None + } else { + val intervalStart: ZonedDateTime = dateTime.minusHours(2) + val intervalEnd: ZonedDateTime = dateTime.plusHours(2) + + val previous: Option[ValueWithWeight[V]] = getValue( + timeSeries, + dateTime, + intervalStart, + dateTime, + typeString + ) + val next: Option[ValueWithWeight[V]] = + getValue(timeSeries, dateTime, dateTime, intervalEnd, typeString) + + (previous, next) match { + case (Some(previous), Some(next)) => + Some((previous.value, previous.weight, next.value, next.weight)) + case (_, _) => + logger.warn( + s"Interpolating value $typeString for timestamp $dateTime was not possible. The default value is used." + ) + None + } + } + } + + /** Method to get a weather value with its weight from an interval of a time + * series. + * + * @param timeSeries + * given time series + * @param timestamp + * given timestamp + * @param intervalStart + * start of the interval + * @param intervalEnd + * end of the interval + * @param typeString + * value that is searched + * @return + * an option of a quantity with a weight + */ + private def getValue[V]( + timeSeries: IndividualTimeSeries[WeatherValue], + timestamp: ZonedDateTime, + intervalStart: ZonedDateTime, + intervalEnd: ZonedDateTime, + typeString: String + ): Option[ValueWithWeight[V]] = { + val values: List[ValueWithWeight[V]] = + timeSeries.getEntries.asScala.flatMap { weatherValue => + val time: ZonedDateTime = weatherValue.getTime + + // calculates the time difference to the given timestamp + val weight = if (time.isBefore(timestamp)) { + ChronoUnit.SECONDS.between(time, timestamp) + } else { + ChronoUnit.SECONDS.between(timestamp, time) + } + + // check is the found timestamp is in the defined interval + if (time.isAfter(intervalStart) && time.isBefore(intervalEnd)) { + getValue[V](weatherValue.getValue, typeString).map { value => + ValueWithWeight(value, weight) + } + } else { + // if timestamp is not inside is not inside the interval none is returned + None + } + }.toList + + if (values.isEmpty) { + None + } else { + // sorting the list to return the value with the least time difference + val sortedSet: Set[ValueWithWeight[V]] = values.sortBy { x => + x.weight + }.toSet + + sortedSet.headOption + } + } + + /** Method to get a value from a [[WeatherValue]].. + * + * @param weatherValue + * given value + * @param typeString + * value that is searched + * @return + * an option for a quantity + */ + private def getValue[V]( + weatherValue: WeatherValue, + typeString: String + ): Option[V] = { + typeString match { + case "diffIrr" => + weatherValue.getSolarIrradiance.getDiffuseIrradiance.toScala + .map(v => v.toSquants) + .asInstanceOf[Option[V]] + case "dirIrr" => + weatherValue.getSolarIrradiance.getDirectIrradiance.toScala + .map(v => v.toSquants) + .asInstanceOf[Option[V]] + case "temp" => + weatherValue.getTemperature.getTemperature.toScala + .map(v => v.toSquants) + .asInstanceOf[Option[V]] + case "windVel" => + weatherValue.getWind.getVelocity.toScala + .map(v => v.toSquants) + .asInstanceOf[Option[V]] + case _ => + logger.warn( + s"Getting value of type $typeString is not implemented yet." + ) + None + } + } +} diff --git a/src/main/scala/edu/ie3/util/scala/quantities/QuantitySquantsConversions.scala b/src/main/scala/edu/ie3/util/scala/quantities/QuantitySquantsConversions.scala new file mode 100644 index 0000000000..69487e6125 --- /dev/null +++ b/src/main/scala/edu/ie3/util/scala/quantities/QuantitySquantsConversions.scala @@ -0,0 +1,48 @@ +/* + * © 2023. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.util.scala.quantities + +import edu.ie3.util.quantities.PowerSystemUnits +import edu.ie3.util.quantities.interfaces.Irradiance +import edu.ie3.util.scala.quantities +import squants.{Kelvin, Velocity} +import squants.motion.MetersPerSecond +import tech.units.indriya.ComparableQuantity +import tech.units.indriya.unit.Units +import javax.measure.quantity.{Speed, Temperature} + +object QuantitySquantsConversions { + + implicit class IrradianceConversion( + irradiance: ComparableQuantity[Irradiance] + ) { + def toSquants: quantities.Irradiance = WattsPerSquareMeter( + irradiance + .to(PowerSystemUnits.WATT_PER_SQUAREMETRE) + .getValue + .doubleValue() + ) + } + + implicit class TemperatureConversion(temp: ComparableQuantity[Temperature]) { + def toSquants: squants.thermal.Temperature = Kelvin( + temp + .to(Units.KELVIN) + .getValue + .doubleValue() + ) + } + + implicit class WindVelocityConversion(windVel: ComparableQuantity[Speed]) { + def toSquants: Velocity = MetersPerSecond( + windVel + .to(Units.METRE_PER_SECOND) + .getValue + .doubleValue() + ) + } +} diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala index ace73d6c9d..f7494a1a4e 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala @@ -16,47 +16,24 @@ import edu.ie3.simona.exceptions.{ InvalidConfigParameterException, ServiceException } -import edu.ie3.datamodel.models.StandardUnits -import edu.ie3.datamodel.models.timeseries.individual.{ - IndividualTimeSeries, - TimeBasedValue -} -import edu.ie3.datamodel.models.value.{ - SolarIrradianceValue, - TemperatureValue, - WeatherValue, - WindValue -} -import edu.ie3.simona.exceptions.ServiceException import edu.ie3.simona.ontology.messages.services.WeatherMessage -import edu.ie3.simona.ontology.messages.services.WeatherMessage.QuantityWithWeight import edu.ie3.simona.service.weather.WeatherSource.{ AgentCoordinates, - EMPTY_WEATHER_DATA, - WeightedCoordinates, - getValue, - interpolate + WeightedCoordinates } import edu.ie3.simona.service.weather.WeatherSourceSpec._ import edu.ie3.simona.test.common.UnitSpec import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} import edu.ie3.util.quantities.QuantityUtil import org.locationtech.jts.geom.{Envelope, Point} -import tech.units.indriya.ComparableQuantity -import edu.ie3.util.quantities.interfaces.Irradiance -import edu.ie3.util.quantities.{PowerSystemUnits, QuantityUtil} -import org.locationtech.jts.geom.Point import org.scalatest.prop.TableDrivenPropertyChecks import tech.units.indriya.ComparableQuantity import tech.units.indriya.quantity.Quantities import tech.units.indriya.unit.Units -import java.time.ZonedDateTime import java.util import java.util.Optional import javax.measure.quantity.Length -import java.util.{Optional, UUID} -import javax.measure.quantity.{Speed, Temperature} import scala.jdk.CollectionConverters._ import scala.jdk.OptionConverters._ import scala.util.{Failure, Success, Try} @@ -358,257 +335,6 @@ class WeatherSourceSpec extends UnitSpec with TableDrivenPropertyChecks { } } } - - "interpolate missing weather values correctly" in { - val cases = Table( - ("previousOption", "nextOption", "emptyValue", "expectedValue"), - (None, None, EMPTY_WEATHER_DATA.diffIrr, EMPTY_WEATHER_DATA.diffIrr), - ( - Some(QuantityWithWeight(EMPTY_WEATHER_DATA.diffIrr, 0L)), - None, - EMPTY_WEATHER_DATA.diffIrr, - EMPTY_WEATHER_DATA.diffIrr - ), - ( - Some( - QuantityWithWeight( - Quantities.getQuantity(10, StandardUnits.SOLAR_IRRADIANCE), - 10 - ) - ), - Some( - QuantityWithWeight( - Quantities.getQuantity(20, StandardUnits.SOLAR_IRRADIANCE), - 10 - ) - ), - EMPTY_WEATHER_DATA.diffIrr, - Quantities.getQuantity(15, StandardUnits.SOLAR_IRRADIANCE) - ) - ) - - forAll(cases) { (previousOption, nextOption, emptyValue, expectedValue) => - val value: ComparableQuantity[Irradiance] = - interpolate(time, previousOption, nextOption, emptyValue) - value shouldBe expectedValue - } - - } - - "find correct previous values" in { - val cases = Table( - ( - "timeBasedValues", - "expectedDiffIrr", - "expectedDirIrr", - "expectedTemp", - "expectedWindVel" - ), - ( - Set( - timeBasedValue0, - timeBasedValue1, - timeBasedValue2, - timeBasedValue3, - timeBasedValue4 - ), - Some( - QuantityWithWeight( - Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE), - 1800 - ) - ), - Some( - QuantityWithWeight( - Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE), - 1800 - ) - ), - Some( - QuantityWithWeight( - Quantities.getQuantity(-35d, StandardUnits.TEMPERATURE), - 6000 - ) - ), - Some( - QuantityWithWeight( - Quantities.getQuantity(10d, Units.METRE_PER_SECOND), - 6000 - ) - ) - ), - ( - Set(timeBasedValue0), - None, - None, - None, - None - ) - ) - - forAll(cases) { - ( - timeBasedValues, - expectedDiffIrr, - expectedDirIrr, - expectedTemp, - expectedWindVel - ) => - val timeSeries: IndividualTimeSeries[WeatherValue] = - buildTimeSeries(timeBasedValues) - val intervalStart: ZonedDateTime = time.minusHours(2) - - val diffIrr: Option[QuantityWithWeight[Irradiance]] = getValue( - timeSeries, - time, - intervalStart, - intervalEnd = time, - "diffIrr" - ) - diffIrr shouldBe expectedDiffIrr - - val dirIrr: Option[QuantityWithWeight[Irradiance]] = getValue( - timeSeries, - time, - intervalStart, - intervalEnd = time, - "diffIrr" - ) - dirIrr shouldBe expectedDirIrr - - val temp: Option[QuantityWithWeight[Temperature]] = getValue( - timeSeries, - time, - intervalStart, - intervalEnd = time, - "temp" - ) - temp shouldBe expectedTemp - - val windVel: Option[QuantityWithWeight[Speed]] = getValue( - timeSeries, - time, - intervalStart, - intervalEnd = time, - "windVel" - ) - windVel shouldBe expectedWindVel - } - } - - "find correct next values" in { - val cases = Table( - ( - "timeBasedValues", - "expectedDiffIrr", - "expectedDirIrr", - "expectedTemp", - "expectedWindVel" - ), - ( - Set( - timeBasedValue0, - timeBasedValue1, - timeBasedValue2, - timeBasedValue3, - timeBasedValue4, - timeBasedValue5 - ), - Some( - QuantityWithWeight( - Quantities.getQuantity(60d, StandardUnits.SOLAR_IRRADIANCE), - 1800 - ) - ), - Some( - QuantityWithWeight( - Quantities.getQuantity(60d, StandardUnits.SOLAR_IRRADIANCE), - 1800 - ) - ), - Some( - QuantityWithWeight( - Quantities.getQuantity(15d, StandardUnits.TEMPERATURE), - 1800 - ) - ), - Some( - QuantityWithWeight( - Quantities.getQuantity(20d, StandardUnits.WIND_VELOCITY), - 1800 - ) - ) - ), - ( - Set(timeBasedValue0, timeBasedValue4, timeBasedValue5), - Some( - QuantityWithWeight( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 6000 - ) - ), - Some( - QuantityWithWeight( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - 6000 - ) - ), - None, - None - ) - ) - - forAll(cases) { - ( - timeBasedValues, - expectedDiffIrr, - expectedDirIrr, - expectedTemp, - expectedWindVel - ) => - val timeSeries: IndividualTimeSeries[WeatherValue] = - buildTimeSeries(timeBasedValues) - val intervalEnd: ZonedDateTime = time.plusHours(2) - - val diffIrr: Option[QuantityWithWeight[Irradiance]] = getValue( - timeSeries, - time, - intervalStart = time, - intervalEnd, - "diffIrr" - ) - diffIrr shouldBe expectedDiffIrr - - val dirIrr: Option[QuantityWithWeight[Irradiance]] = getValue( - timeSeries, - time, - intervalStart = time, - intervalEnd, - "diffIrr" - ) - dirIrr shouldBe expectedDirIrr - - val temp: Option[QuantityWithWeight[Temperature]] = getValue( - timeSeries, - time, - intervalStart = time, - intervalEnd, - "temp" - ) - temp shouldBe expectedTemp - - val windVel: Option[QuantityWithWeight[Speed]] = getValue( - timeSeries, - time, - intervalStart = time, - intervalEnd, - "windVel" - ) - windVel shouldBe expectedWindVel - } - } - - } } } @@ -622,112 +348,6 @@ case object WeatherSourceSpec { private val coordinate144112 = GeoUtils.buildPoint(52.312, 12.875) private val coordinate165125 = GeoUtils.buildPoint(52.25, 12.875) - private val time: ZonedDateTime = ZonedDateTime.now() - - private val missingValue: Null = null - - private val timeBasedValue0: TimeBasedValue[WeatherValue] = - buildTimeBasedValue( - time.minusMinutes(130), - new WeatherValue( - coordinate67775, - new SolarIrradianceValue(missingValue, missingValue), - new TemperatureValue( - missingValue - ), - new WindValue(missingValue, missingValue) - ) - ) - - private val timeBasedValue1: TimeBasedValue[WeatherValue] = - buildTimeBasedValue( - time.minusMinutes(100), - new WeatherValue( - coordinate67775, - new SolarIrradianceValue( - missingValue, - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE) - ), - new TemperatureValue(Quantities.getQuantity(238.15d, Units.KELVIN)), - new WindValue( - missingValue, - Quantities.getQuantity(10, Units.METRE_PER_SECOND) - ) - ) - ) - - private val timeBasedValue2: TimeBasedValue[WeatherValue] = - buildTimeBasedValue( - time.minusMinutes(30), - new WeatherValue( - coordinate67775, - new SolarIrradianceValue( - Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE), - Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE) - ), - new TemperatureValue(missingValue), - new WindValue(missingValue, missingValue) - ) - ) - - private val timeBasedValue3: TimeBasedValue[WeatherValue] = - buildTimeBasedValue( - time.plusMinutes(30), - new WeatherValue( - coordinate67775, - new SolarIrradianceValue( - missingValue, - Quantities.getQuantity(60d, StandardUnits.SOLAR_IRRADIANCE) - ), - new TemperatureValue(Quantities.getQuantity(288.15d, Units.KELVIN)), - new WindValue( - missingValue, - Quantities.getQuantity(20, Units.METRE_PER_SECOND) - ) - ) - ) - - private val timeBasedValue4: TimeBasedValue[WeatherValue] = - buildTimeBasedValue( - time.plusMinutes(100), - new WeatherValue( - coordinate67775, - new SolarIrradianceValue( - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), - Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE) - ), - new TemperatureValue(missingValue), - new WindValue(missingValue, missingValue) - ) - ) - - private val timeBasedValue5: TimeBasedValue[WeatherValue] = - buildTimeBasedValue( - time.plusMinutes(130), - new WeatherValue( - coordinate67775, - new SolarIrradianceValue( - missingValue, - Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE) - ), - new TemperatureValue(Quantities.getQuantity(288.15d, Units.KELVIN)), - new WindValue(missingValue, missingValue) - ) - ) - - def buildTimeBasedValue( - time: ZonedDateTime, - value: WeatherValue - ): TimeBasedValue[WeatherValue] = { - new TimeBasedValue[WeatherValue](UUID.randomUUID(), time, value) - } - - def buildTimeSeries( - set: Set[TimeBasedValue[WeatherValue]] - ): IndividualTimeSeries[WeatherValue] = { - new IndividualTimeSeries[WeatherValue](UUID.randomUUID(), set.asJava) - } - case object DummyWeatherSource extends WeatherSource { override protected val idCoordinateSource: IdCoordinateSource = DummyIdCoordinateSource diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherValueInterpolationSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherValueInterpolationSpec.scala new file mode 100644 index 0000000000..51d72fb7fa --- /dev/null +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherValueInterpolationSpec.scala @@ -0,0 +1,332 @@ +/* + * © 2023. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.simona.service.weather + +import edu.ie3.datamodel.models.StandardUnits +import edu.ie3.datamodel.models.timeseries.individual.{ + IndividualTimeSeries, + TimeBasedValue +} +import edu.ie3.datamodel.models.value.{ + SolarIrradianceValue, + TemperatureValue, + WeatherValue, + WindValue +} +import edu.ie3.simona.ontology.messages.services.WeatherMessage.ValueWithWeight +import edu.ie3.simona.service.weather.WeatherValueInterpolationSpec._ +import edu.ie3.simona.test.common.UnitSpec +import edu.ie3.util.geo.GeoUtils +import edu.ie3.util.scala.quantities.WattsPerSquareMeter +import org.scalatest.prop.TableDrivenPropertyChecks +import squants.Kelvin +import squants.motion.MetersPerSecond +import squants.thermal.{Celsius, Temperature} +import tech.units.indriya.quantity.Quantities +import tech.units.indriya.unit.Units + +import java.time.ZonedDateTime +import java.util.UUID +import scala.jdk.CollectionConverters._ + +class WeatherValueInterpolationSpec + extends UnitSpec + with TableDrivenPropertyChecks { + "The WeatherValueInterpolation" should { + val getValue = PrivateMethod[Option[ValueWithWeight[_]]](Symbol("getValue")) + + "find correct previous values" in { + val cases = Table( + ( + "timeBasedValues", + "expectedDiffIrr", + "expectedDirIrr", + "expectedTemp", + "expectedWindVel" + ), + ( + Set( + timeBasedValue0, + timeBasedValue1, + timeBasedValue2, + timeBasedValue3, + timeBasedValue4 + ), + Some(ValueWithWeight(WattsPerSquareMeter(50d), 1800)), + Some(ValueWithWeight(WattsPerSquareMeter(50d), 1800)), + Some(ValueWithWeight(Temperature(238.15d, Kelvin), 6000)), + Some(ValueWithWeight(MetersPerSecond(10d), 6000)) + ), + (Set(timeBasedValue0), None, None, None, None) + ) + + forAll(cases) { + ( + timeBasedValues, + expectedDiffIrr, + expectedDirIrr, + expectedTemp, + expectedWindVel + ) => + val timeSeries: IndividualTimeSeries[WeatherValue] = + buildTimeSeries(timeBasedValues) + val intervalStart: ZonedDateTime = time.minusHours(2) + + WeatherValueInterpolation invokePrivate getValue( + timeSeries, + time, + intervalStart, + time, + "diffIrr" + ) shouldBe expectedDiffIrr + + WeatherValueInterpolation invokePrivate getValue( + timeSeries, + time, + intervalStart, + time, + "diffIrr" + ) shouldBe expectedDirIrr + + WeatherValueInterpolation invokePrivate getValue( + timeSeries, + time, + intervalStart, + time, + "temp" + ) shouldBe expectedTemp + + WeatherValueInterpolation invokePrivate getValue( + timeSeries, + time, + intervalStart, + time, + "windVel" + ) shouldBe expectedWindVel + } + } + + "find correct next values" in { + val cases = Table( + ( + "timeBasedValues", + "expectedDiffIrr", + "expectedDirIrr", + "expectedTemp", + "expectedWindVel" + ), + ( + Set( + timeBasedValue0, + timeBasedValue1, + timeBasedValue2, + timeBasedValue3, + timeBasedValue4, + timeBasedValue5 + ), + Some(ValueWithWeight(WattsPerSquareMeter(60d), 1800)), + Some(ValueWithWeight(WattsPerSquareMeter(60d), 1800)), + Some(ValueWithWeight(Temperature(15d, Celsius), 1800)), + Some(ValueWithWeight(MetersPerSecond(20d), 1800)) + ), + ( + Set(timeBasedValue0, timeBasedValue4, timeBasedValue5), + Some(ValueWithWeight(WattsPerSquareMeter(40d), 6000)), + Some(ValueWithWeight(WattsPerSquareMeter(40d), 6000)), + None, + None + ) + ) + + forAll(cases) { + ( + timeBasedValues, + expectedDiffIrr, + expectedDirIrr, + expectedTemp, + expectedWindVel + ) => + val timeSeries: IndividualTimeSeries[WeatherValue] = + buildTimeSeries(timeBasedValues) + val intervalEnd: ZonedDateTime = time.plusHours(2) + + WeatherValueInterpolation invokePrivate getValue( + timeSeries, + time, + time, + intervalEnd, + "diffIrr" + ) shouldBe expectedDiffIrr + + WeatherValueInterpolation invokePrivate getValue( + timeSeries, + time, + time, + intervalEnd, + "diffIrr" + ) shouldBe expectedDirIrr + + WeatherValueInterpolation invokePrivate getValue( + timeSeries, + time, + time, + intervalEnd, + "temp" + ) shouldBe expectedTemp + + WeatherValueInterpolation invokePrivate getValue( + timeSeries, + time, + time, + intervalEnd, + "windVel" + ) shouldBe expectedWindVel + } + } + + "get a value correctly" in { + val getValue = PrivateMethod[Option[_]](Symbol("getValue")) + + val weatherValue = new WeatherValue( + coordinate67775, + new SolarIrradianceValue( + Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE), + Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE) + ), + new TemperatureValue(Quantities.getQuantity(238.15d, Units.KELVIN)), + new WindValue( + missingValue, + Quantities.getQuantity(10d, Units.METRE_PER_SECOND) + ) + ) + + val cases = Table( + ("typeString", "expectedValue"), + ("diffIrr", Some(WattsPerSquareMeter(50d))), + ("dirIrr", Some(WattsPerSquareMeter(50d))), + ("temp", Some(Temperature(238.15d, Kelvin))), + ("windVel", Some(MetersPerSecond(10d))), + ("other", None) + ) + + forAll(cases) { (typeString, expectedValue) => + WeatherValueInterpolation invokePrivate getValue( + weatherValue, + typeString + ) shouldBe expectedValue + } + } + } +} + +case object WeatherValueInterpolationSpec { + private val coordinate67775 = GeoUtils.buildPoint(51.5, 7.438) + private val time: ZonedDateTime = ZonedDateTime.now() + private val missingValue: Null = null + + private val timeBasedValue0: TimeBasedValue[WeatherValue] = + buildTimeBasedValue( + time.minusMinutes(130), + new WeatherValue( + coordinate67775, + new SolarIrradianceValue(missingValue, missingValue), + new TemperatureValue( + missingValue + ), + new WindValue(missingValue, missingValue) + ) + ) + + private val timeBasedValue1: TimeBasedValue[WeatherValue] = + buildTimeBasedValue( + time.minusMinutes(100), + new WeatherValue( + coordinate67775, + new SolarIrradianceValue( + missingValue, + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE) + ), + new TemperatureValue(Quantities.getQuantity(238.15d, Units.KELVIN)), + new WindValue( + missingValue, + Quantities.getQuantity(10, Units.METRE_PER_SECOND) + ) + ) + ) + + private val timeBasedValue2: TimeBasedValue[WeatherValue] = + buildTimeBasedValue( + time.minusMinutes(30), + new WeatherValue( + coordinate67775, + new SolarIrradianceValue( + Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE), + Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE) + ), + new TemperatureValue(missingValue), + new WindValue(missingValue, missingValue) + ) + ) + + private val timeBasedValue3: TimeBasedValue[WeatherValue] = + buildTimeBasedValue( + time.plusMinutes(30), + new WeatherValue( + coordinate67775, + new SolarIrradianceValue( + missingValue, + Quantities.getQuantity(60d, StandardUnits.SOLAR_IRRADIANCE) + ), + new TemperatureValue(Quantities.getQuantity(288.15d, Units.KELVIN)), + new WindValue( + missingValue, + Quantities.getQuantity(20, Units.METRE_PER_SECOND) + ) + ) + ) + + private val timeBasedValue4: TimeBasedValue[WeatherValue] = + buildTimeBasedValue( + time.plusMinutes(100), + new WeatherValue( + coordinate67775, + new SolarIrradianceValue( + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE), + Quantities.getQuantity(40d, StandardUnits.SOLAR_IRRADIANCE) + ), + new TemperatureValue(missingValue), + new WindValue(missingValue, missingValue) + ) + ) + + private val timeBasedValue5: TimeBasedValue[WeatherValue] = + buildTimeBasedValue( + time.plusMinutes(130), + new WeatherValue( + coordinate67775, + new SolarIrradianceValue( + missingValue, + Quantities.getQuantity(50d, StandardUnits.SOLAR_IRRADIANCE) + ), + new TemperatureValue(Quantities.getQuantity(288.15d, Units.KELVIN)), + new WindValue(missingValue, missingValue) + ) + ) + + def buildTimeBasedValue( + time: ZonedDateTime, + value: WeatherValue + ): TimeBasedValue[WeatherValue] = { + new TimeBasedValue[WeatherValue](UUID.randomUUID(), time, value) + } + + def buildTimeSeries( + set: Set[TimeBasedValue[WeatherValue]] + ): IndividualTimeSeries[WeatherValue] = { + new IndividualTimeSeries[WeatherValue](UUID.randomUUID(), set.asJava) + } +} From e50c58952b416df04e9ee98ac2af9b0f31aa72e1 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Thu, 28 Sep 2023 17:09:38 +0200 Subject: [PATCH 16/21] Fixing ``Codacy`` issue. --- .../service/weather/WeatherSource.scala | 80 +++++++++---------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 373e217c35..cc8beb06c0 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -44,10 +44,11 @@ import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} import edu.ie3.util.quantities.PowerSystemUnits import edu.ie3.util.scala.io.CsvDataSourceAdapter import edu.ie3.util.scala.quantities.QuantitySquantsConversions._ -import edu.ie3.util.scala.quantities.WattsPerSquareMeter +import edu.ie3.util.scala.quantities.{Irradiance, WattsPerSquareMeter} import org.locationtech.jts.geom.{Coordinate, Point} import squants.motion.MetersPerSecond import squants.thermal.Kelvin +import squants.{Temperature, Velocity} import tech.units.indriya.ComparableQuantity import tech.units.indriya.quantity.Quantities import tech.units.indriya.unit.Units @@ -593,54 +594,51 @@ object WeatherSource extends LazyLogging { dateTime: ZonedDateTime ): WeatherData = { // gets a value option - val valueOption: Option[WeatherValue] = - timeSeries.getValue(dateTime).toScala + val valueOption = timeSeries.getValue(dateTime).toScala // check which data is missing - val (diffIrr, dirIrr, temp, windVel) = valueOption match { + val (diffIrr, dirIrr, temp, windVel) = getOptions(valueOption) + + WeatherData( + diffIrr.getOrElse( + interpolate(timeSeries, dateTime, "diffIrr", EMPTY_WEATHER_DATA.diffIrr) + ), + dirIrr.getOrElse( + interpolate(timeSeries, dateTime, "dirIrr", EMPTY_WEATHER_DATA.dirIrr) + ), + temp.getOrElse( + interpolate(timeSeries, dateTime, "temp", EMPTY_WEATHER_DATA.temp) + ), + windVel.getOrElse( + interpolate(timeSeries, dateTime, "windVel", EMPTY_WEATHER_DATA.windVel) + ) + ) + } + + /** Method to get the data of a [[WeatherValue]]. + * + * @param valueOption + * value with data + * @return + * a tuple of options + */ + def getOptions(valueOption: Option[WeatherValue]): ( + Option[Irradiance], + Option[Irradiance], + Option[Temperature], + Option[Velocity] + ) = { + valueOption match { case Some(value) => val solar = value.getSolarIrradiance ( - solar.getDiffuseIrradiance.toScala, - solar.getDirectIrradiance.toScala, - value.getTemperature.getTemperature.toScala, - value.getWind.getVelocity.toScala + solar.getDiffuseIrradiance.toScala.map(v => v.toSquants), + solar.getDirectIrradiance.toScala.map(v => v.toSquants), + value.getTemperature.getTemperature.toScala.map(v => v.toSquants), + value.getWind.getVelocity.toScala.map(v => v.toSquants) ) case None => (None, None, None, None) } - - WeatherData( - diffIrr match { - case Some(irradiance) => irradiance.toSquants - case None => - interpolate( - timeSeries, - dateTime, - "diffIrr", - EMPTY_WEATHER_DATA.diffIrr - ) - }, - dirIrr match { - case Some(irradiance) => irradiance.toSquants - case None => - interpolate(timeSeries, dateTime, "dirIrr", EMPTY_WEATHER_DATA.dirIrr) - }, - temp match { - case Some(temperature) => temperature.toSquants - case None => - interpolate(timeSeries, dateTime, "temp", EMPTY_WEATHER_DATA.temp) - }, - windVel match { - case Some(velocity) => velocity.toSquants - case None => - interpolate( - timeSeries, - dateTime, - "windVel", - EMPTY_WEATHER_DATA.windVel - ) - } - ) } /** Weather package private case class to combine the provided agent From 395ea9db354e0adff8cdb2f9614b33e67713f897 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Mon, 8 Jan 2024 13:47:41 +0100 Subject: [PATCH 17/21] Resolving merge conflicts. --- .../service/weather/WeatherSource.scala | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 331057f4a5..d86033bb1c 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -560,29 +560,6 @@ object WeatherSource extends LazyLogging { MetersPerSecond(0d) ) - def toWeatherData( - weatherValue: WeatherValue - ): WeatherData = { - WeatherData( - weatherValue.getSolarIrradiance.getDiffuseIrradiance.toScala match { - case Some(irradiance) => irradiance.toSquants - case None => EMPTY_WEATHER_DATA.diffIrr - }, - weatherValue.getSolarIrradiance.getDirectIrradiance.toScala match { - case Some(irradiance) => irradiance.toSquants - case None => EMPTY_WEATHER_DATA.dirIrr - }, - weatherValue.getTemperature.getTemperature.toScala match { - case Some(temperature) => temperature.toSquants - case None => EMPTY_WEATHER_DATA.temp - }, - weatherValue.getWind.getVelocity.toScala match { - case Some(windVel) => windVel.toSquants - case None => EMPTY_WEATHER_DATA.windVel - } - ) - } - /** Methode to get weather data from a time series. This method automatically * interpolates missing values. * From 23e042f08e4acf3678f6c1ebfa577cf7eccb7d35 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Tue, 2 Sep 2025 15:06:44 +0200 Subject: [PATCH 18/21] Moving ValueWithWeight to appropriate class --- .../messages/services/WeatherMessage.scala | 14 -------------- .../weather/WeatherValueInterpolation.scala | 16 +++++++++++++++- .../weather/WeatherValueInterpolationSpec.scala | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala b/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala index a008ac39f8..9d9d4ee5f6 100644 --- a/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala +++ b/src/main/scala/edu/ie3/simona/ontology/messages/services/WeatherMessage.scala @@ -6,7 +6,6 @@ package edu.ie3.simona.ontology.messages.services -import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.simona.agent.participant.data.Data.SecondaryData import edu.ie3.simona.ontology.messages.services.ServiceMessage.{ ProvisionMessage, @@ -76,17 +75,4 @@ object WeatherMessage { windVel: Velocity ) extends SecondaryData - /** Container class for a weather value with a weight. It is primarily used - * for interpolation. - * @param value - * weather value - * @param weight - * of the value - * @tparam V - * type of value - */ - final case class ValueWithWeight[V]( - value: V, - weight: Long - ) } diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherValueInterpolation.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherValueInterpolation.scala index d0f862da94..d1926f244a 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherValueInterpolation.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherValueInterpolation.scala @@ -9,7 +9,6 @@ package edu.ie3.simona.service.weather import com.typesafe.scalalogging.LazyLogging import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.simona.ontology.messages.services.WeatherMessage.ValueWithWeight import edu.ie3.util.scala.quantities.QuantitySquantsConversions._ import squants.Quantity @@ -200,4 +199,19 @@ object WeatherValueInterpolation extends LazyLogging { None } } + + /** Container class for a weather value with a weight. It is primarily used + * for interpolation. + * + * @param value + * weather value + * @param weight + * of the value + * @tparam V + * type of value + */ + final case class ValueWithWeight[V]( + value: V, + weight: Long + ) } diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherValueInterpolationSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherValueInterpolationSpec.scala index 51d72fb7fa..8e482ec7ef 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherValueInterpolationSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherValueInterpolationSpec.scala @@ -17,7 +17,7 @@ import edu.ie3.datamodel.models.value.{ WeatherValue, WindValue } -import edu.ie3.simona.ontology.messages.services.WeatherMessage.ValueWithWeight +import edu.ie3.simona.service.weather.WeatherValueInterpolation.ValueWithWeight import edu.ie3.simona.service.weather.WeatherValueInterpolationSpec._ import edu.ie3.simona.test.common.UnitSpec import edu.ie3.util.geo.GeoUtils From 4459ca7142985ea58b42a4c2295624fdd4eec2c8 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Tue, 2 Sep 2025 16:14:32 +0200 Subject: [PATCH 19/21] Merging quantity conversion --- .../service/weather/WeatherSource.scala | 2 +- .../weather/WeatherValueInterpolation.scala | 2 +- .../quantities/QuantityConversionUtils.scala | 40 +++++++++++++++- .../QuantitySquantsConversions.scala | 48 ------------------- 4 files changed, 41 insertions(+), 51 deletions(-) delete mode 100644 src/main/scala/edu/ie3/util/scala/quantities/QuantitySquantsConversions.scala diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 89a92ae572..d0f42d2567 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -37,7 +37,7 @@ import edu.ie3.simona.service.weather.WeatherValueInterpolation.interpolate import edu.ie3.simona.util.ParsableEnumeration import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} import edu.ie3.util.quantities.PowerSystemUnits -import edu.ie3.util.scala.quantities.QuantitySquantsConversions.* +import edu.ie3.util.scala.quantities.QuantityConversionUtils.* import edu.ie3.util.scala.quantities.{Irradiance, WattsPerSquareMeter} import org.locationtech.jts.geom.{Coordinate, Point} import squants.motion.MetersPerSecond diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherValueInterpolation.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherValueInterpolation.scala index 78ade385b7..a95c9b4b39 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherValueInterpolation.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherValueInterpolation.scala @@ -9,7 +9,7 @@ package edu.ie3.simona.service.weather import com.typesafe.scalalogging.LazyLogging import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.util.scala.quantities.QuantitySquantsConversions.* +import edu.ie3.util.scala.quantities.QuantityConversionUtils.* import squants.Quantity import java.time.ZonedDateTime diff --git a/src/main/scala/edu/ie3/util/scala/quantities/QuantityConversionUtils.scala b/src/main/scala/edu/ie3/util/scala/quantities/QuantityConversionUtils.scala index e802806c9b..b346550680 100644 --- a/src/main/scala/edu/ie3/util/scala/quantities/QuantityConversionUtils.scala +++ b/src/main/scala/edu/ie3/util/scala/quantities/QuantityConversionUtils.scala @@ -6,9 +6,11 @@ package edu.ie3.util.scala.quantities +import edu.ie3.util.quantities.PowerSystemUnits import edu.ie3.util.quantities.PowerSystemUnits.* import edu.ie3.util.quantities.interfaces.{ EnergyPrice, + Irradiance, SpecificConductance, SpecificHeatCapacity, SpecificResistance, @@ -16,10 +18,12 @@ import edu.ie3.util.quantities.interfaces.{ import edu.ie3.util.scala.quantities import squants.electro.{Kilovolts, Ohms, Siemens} import squants.energy.{KilowattHours, Kilowatts} +import squants.motion.MetersPerSecond import squants.space.{CubicMeters, SquareMeters} import squants.thermal.Celsius -import squants.{Amperes, Each, Radians} +import squants.{Amperes, Each, Radians, Velocity} import tech.units.indriya.ComparableQuantity +import tech.units.indriya.unit.Units import tech.units.indriya.unit.Units.* import javax.measure.quantity.* @@ -296,4 +300,38 @@ object QuantityConversionUtils { .doubleValue ) } + + /** Implicit class that contains a method to convert a given + * [[ComparableQuantity]] with unit [[WATT_PER_SQUAREMETRE]] into + * [[WattsPerSquareMeter]]. + * + * @param quantity + * To convert. + */ + implicit class IrradianceConversion( + quantity: ComparableQuantity[Irradiance] + ) { + def toSquants: quantities.Irradiance = WattsPerSquareMeter( + quantity + .to(PowerSystemUnits.WATT_PER_SQUAREMETRE) + .getValue + .doubleValue() + ) + } + + /** Implicit class that contains a method to convert a given + * [[ComparableQuantity]] with unit [[METRE_PER_SECOND]] into + * [[MetersPerSecond]]. + * + * @param quantity + * To convert. + */ + implicit class VelocityConversion(quantity: ComparableQuantity[Speed]) { + def toSquants: Velocity = MetersPerSecond( + quantity + .to(Units.METRE_PER_SECOND) + .getValue + .doubleValue() + ) + } } diff --git a/src/main/scala/edu/ie3/util/scala/quantities/QuantitySquantsConversions.scala b/src/main/scala/edu/ie3/util/scala/quantities/QuantitySquantsConversions.scala deleted file mode 100644 index 69487e6125..0000000000 --- a/src/main/scala/edu/ie3/util/scala/quantities/QuantitySquantsConversions.scala +++ /dev/null @@ -1,48 +0,0 @@ -/* - * © 2023. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation - */ - -package edu.ie3.util.scala.quantities - -import edu.ie3.util.quantities.PowerSystemUnits -import edu.ie3.util.quantities.interfaces.Irradiance -import edu.ie3.util.scala.quantities -import squants.{Kelvin, Velocity} -import squants.motion.MetersPerSecond -import tech.units.indriya.ComparableQuantity -import tech.units.indriya.unit.Units -import javax.measure.quantity.{Speed, Temperature} - -object QuantitySquantsConversions { - - implicit class IrradianceConversion( - irradiance: ComparableQuantity[Irradiance] - ) { - def toSquants: quantities.Irradiance = WattsPerSquareMeter( - irradiance - .to(PowerSystemUnits.WATT_PER_SQUAREMETRE) - .getValue - .doubleValue() - ) - } - - implicit class TemperatureConversion(temp: ComparableQuantity[Temperature]) { - def toSquants: squants.thermal.Temperature = Kelvin( - temp - .to(Units.KELVIN) - .getValue - .doubleValue() - ) - } - - implicit class WindVelocityConversion(windVel: ComparableQuantity[Speed]) { - def toSquants: Velocity = MetersPerSecond( - windVel - .to(Units.METRE_PER_SECOND) - .getValue - .doubleValue() - ) - } -} From 850cdb7116bc9dbae31c1bab429444bcfd5512f3 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Tue, 2 Sep 2025 16:30:47 +0200 Subject: [PATCH 20/21] Reverting unnecessary changes --- .../edu/ie3/simona/service/weather/WeatherSourceSpec.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala index ecea7f3565..fa2cc46bd2 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala @@ -18,7 +18,6 @@ import edu.ie3.simona.test.common.UnitSpec import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} import edu.ie3.util.quantities.QuantityUtil import org.locationtech.jts.geom.{Envelope, Point} -import org.scalatest.prop.TableDrivenPropertyChecks import tech.units.indriya.ComparableQuantity import tech.units.indriya.quantity.Quantities import tech.units.indriya.unit.Units @@ -30,7 +29,7 @@ import scala.jdk.CollectionConverters.* import scala.jdk.OptionConverters.* import scala.util.{Failure, Success} -class WeatherSourceSpec extends UnitSpec with TableDrivenPropertyChecks { +class WeatherSourceSpec extends UnitSpec { private val coordinate0 = GeoUtils.buildPoint(51.47, 7.41) "A weather source" should { From 0aedaac073db7bef4002c6657e225d9dd08e17b5 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Tue, 2 Sep 2025 17:15:22 +0200 Subject: [PATCH 21/21] Fixing temperature addition --- .../scala/edu/ie3/simona/service/weather/WeatherSource.scala | 2 +- .../ie3/simona/service/weather/WeatherSourceWrapper.scala | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index d0f42d2567..bb54ddba41 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -353,7 +353,7 @@ object WeatherSource extends LazyLogging { /** Represents an empty weather data object * * For temperature to represent an "empty" quantity, we need to explicitly - * set temperature to absolute zero, so 0°K. When temperature measures the + * set temperature to absolute zero, so 0 K. When temperature measures the * movement of atoms, absolute zero means no movement, which represents the * "empty" concept best. */ diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala index cf6b2244b8..1c481de4a8 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala @@ -44,6 +44,7 @@ import edu.ie3.simona.service.weather.WeatherSource as SimonaWeatherSource import edu.ie3.simona.util.TickUtil.{RichZonedDateTime, TickLong} import edu.ie3.util.DoubleUtils.!~= import edu.ie3.util.interval.ClosedInterval +import squants.thermal.Kelvin import tech.units.indriya.ComparableQuantity import java.nio.file.Paths @@ -153,7 +154,9 @@ private[weather] final case class WeatherSourceWrapper private ( logger.warn(s"Temperature not available at $point.") (averagedWeather.temp, 0d) case nonEmptyTemp => - (averagedWeather.temp + nonEmptyTemp * weight, weight) + // Important: squants temperature addition is bugged. + // Conversion to Kelvin necessary. + (averagedWeather.temp + nonEmptyTemp.in(Kelvin) * weight, weight) } val (windVelocity, windVelWeight) = currentWeather.windVel match {