Adding uniqueness check to CsvIdCoordinateSource and CsvWeatherSource.

Author: staudtMarius

src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoIdCoordinateFactory.java

@@ -6,12 +6,10 @@
 package edu.ie3.datamodel.io.factory.timeseries;
 
 import edu.ie3.datamodel.io.factory.SimpleFactoryData;
-import edu.ie3.util.geo.GeoUtils;
+import edu.ie3.datamodel.models.input.IdCoordinatePair;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
-import org.apache.commons.lang3.tuple.Pair;
-import org.locationtech.jts.geom.Point;
 
 /**
  * Factory, that is able to build coordinate id to coordinate mapping from German Federal Weather
@@ -26,11 +24,11 @@ public class CosmoIdCoordinateFactory extends IdCoordinateFactory {
   private static final String LAT_ROT = "latRot";
 
   @Override
-  protected Pair<Integer, Point> buildModel(SimpleFactoryData data) {
+  protected IdCoordinatePair buildModel(SimpleFactoryData data) {
     int coordinateId = data.getInt(COORDINATE_ID);
     double lat = data.getDouble(LAT_GEO);
     double lon = data.getDouble(LONG_GEO);
-    return Pair.of(coordinateId, GeoUtils.buildPoint(lat, lon));
+    return IdCoordinatePair.of(coordinateId, lat, lon);
   }
 
   @Override

src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconIdCoordinateFactory.java

@@ -6,12 +6,10 @@
 package edu.ie3.datamodel.io.factory.timeseries;
 
 import edu.ie3.datamodel.io.factory.SimpleFactoryData;
-import edu.ie3.util.geo.GeoUtils;
+import edu.ie3.datamodel.models.input.IdCoordinatePair;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
-import org.apache.commons.lang3.tuple.Pair;
-import org.locationtech.jts.geom.Point;
 
 /**
  * Factory, that is able to build coordinate id to coordinate mapping from German Federal Weather
@@ -24,11 +22,11 @@ public class IconIdCoordinateFactory extends IdCoordinateFactory {
   private static final String TYPE = "coordinateType";
 
   @Override
-  protected Pair<Integer, Point> buildModel(SimpleFactoryData data) {
+  protected IdCoordinatePair buildModel(SimpleFactoryData data) {
     int coordinateId = data.getInt(COORDINATE_ID);
     double lat = data.getDouble(LAT);
     double lon = data.getDouble(LONG);
-    return Pair.of(coordinateId, GeoUtils.buildPoint(lat, lon));
+    return IdCoordinatePair.of(coordinateId, lat, lon);
   }
 
   @Override

src/main/java/edu/ie3/datamodel/io/factory/timeseries/IdCoordinateFactory.java

@@ -7,17 +7,16 @@
 
 import edu.ie3.datamodel.io.factory.Factory;
 import edu.ie3.datamodel.io.factory.SimpleFactoryData;
-import org.apache.commons.lang3.tuple.Pair;
-import org.locationtech.jts.geom.Point;
+import edu.ie3.datamodel.models.input.IdCoordinatePair;
 
 /**
  * Abstract class definition for a factory, that is able to build single mapping entries from
  * coordinate identifier to actual coordinate
  */
 public abstract class IdCoordinateFactory
-    extends Factory<Pair, SimpleFactoryData, Pair<Integer, Point>> {
+    extends Factory<IdCoordinatePair, SimpleFactoryData, IdCoordinatePair> {
   protected IdCoordinateFactory() {
-    super(Pair.class);
+    super(IdCoordinatePair.class);
   }
 
   /** @return the field id for the coordinate id */

src/main/java/edu/ie3/datamodel/io/factory/timeseries/SqlIdCoordinateFactory.java

@@ -7,11 +7,11 @@
 
 import edu.ie3.datamodel.exceptions.FactoryException;
 import edu.ie3.datamodel.io.factory.SimpleFactoryData;
+import edu.ie3.datamodel.models.input.IdCoordinatePair;
 import edu.ie3.util.geo.GeoUtils;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
-import org.apache.commons.lang3.tuple.Pair;
 import org.locationtech.jts.geom.Coordinate;
 import org.locationtech.jts.geom.Point;
 import org.locationtech.jts.io.ParseException;
@@ -23,15 +23,15 @@ public class SqlIdCoordinateFactory extends IdCoordinateFactory {
   private final WKBReader reader = new WKBReader();
 
   @Override
-  protected Pair<Integer, Point> buildModel(SimpleFactoryData data) {
+  protected IdCoordinatePair buildModel(SimpleFactoryData data) {
     try {
       int coordinateId = data.getInt(COORDINATE_ID);
       byte[] byteArr = WKBReader.hexToBytes(data.getField(COORDINATE));
 
       Coordinate coordinate = reader.read(byteArr).getCoordinate();
 
       Point point = GeoUtils.buildPoint(coordinate);
-      return Pair.of(coordinateId, point);
+      return IdCoordinatePair.of(coordinateId, point);
 
     } catch (ParseException e) {
       throw new FactoryException(e);

src/main/java/edu/ie3/datamodel/io/source/IdCoordinateSource.java

@@ -27,11 +27,6 @@ public interface IdCoordinateSource {
    */
   Optional<Set<String>> getSourceFields() throws SourceException;
 
-  /** Returns a set of fields that needs to be unique for the source. */
-  default Set<String> getUniqueFields() {
-    return Set.of("");
-  }
-
   /**
    * Get the matching coordinate for the given ID
    *

src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java

@@ -5,13 +5,16 @@
 */
 package edu.ie3.datamodel.io.source.csv;
 
+import edu.ie3.datamodel.exceptions.DuplicateEntitiesException;
 import edu.ie3.datamodel.exceptions.SourceException;
 import edu.ie3.datamodel.exceptions.ValidationException;
 import edu.ie3.datamodel.io.factory.SimpleFactoryData;
 import edu.ie3.datamodel.io.factory.timeseries.IdCoordinateFactory;
 import edu.ie3.datamodel.io.source.IdCoordinateSource;
+import edu.ie3.datamodel.models.input.IdCoordinatePair;
 import edu.ie3.datamodel.utils.Try;
 import edu.ie3.datamodel.utils.Try.*;
+import edu.ie3.datamodel.utils.validation.UniquenessValidationUtils;
 import edu.ie3.util.geo.CoordinateDistance;
 import edu.ie3.util.geo.GeoUtils;
 import java.io.BufferedReader;
@@ -62,14 +65,27 @@ public CsvIdCoordinateSource(IdCoordinateFactory factory, CsvDataSource dataSour
    * @return Mapping from coordinate id to coordinate
    */
   private Map<Integer, Point> setupIdToCoordinateMap() throws SourceException {
-    return buildStreamWithFieldsToAttributesMap()
-        .map(
-            data ->
-                data.map(fieldToValues -> new SimpleFactoryData(fieldToValues, Pair.class))
-                    .map(factory::get))
-        .flatMap(s -> Try.scanStream(s, "Pair<Integer, Point>").transformF(SourceException::new))
-        .map(s -> s.collect(Collectors.toMap(Pair::getKey, Pair::getValue)))
-        .getOrThrow();
+    List<IdCoordinatePair> pairs =
+        buildStreamWithFieldsToAttributesMap()
+            .map(
+                data ->
+                    data.map(
+                            fieldToValues ->
+                                new SimpleFactoryData(fieldToValues, IdCoordinatePair.class))
+                        .map(factory::get))
+            .flatMap(
+                s -> Try.scanStream(s, "Pair<Integer, Point>").transformF(SourceException::new))
+            .getOrThrow()
+            .toList();
+
+    try {
+      // check the uniqueness of the source
+      UniquenessValidationUtils.checkIdCoordinateUniqueness(pairs);
+    } catch (DuplicateEntitiesException de) {
+      throw new SourceException("Due to: ", de);
+    }
+
+    return pairs.stream().collect(Collectors.toMap(Pair::getKey, Pair::getValue));
   }
 
   /**
@@ -171,7 +187,7 @@ public int getCoordinateCount() {
       final String[] headline = dataSource.parseCsvRow(reader.readLine(), dataSource.csvSep);
 
       // validating read file
-      factory.validate(Set.of(headline), Pair.class).getOrThrow();
+      factory.validate(Set.of(headline), IdCoordinatePair.class).getOrThrow();
 
       // by default try-with-resources closes the reader directly when we leave this method (which
       // is wanted to avoid a lock on the file), but this causes a closing of the stream as well.

src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java

@@ -5,6 +5,9 @@
 */
 package edu.ie3.datamodel.io.source.csv;
 
+import static edu.ie3.datamodel.utils.validation.UniquenessValidationUtils.checkWeatherUniqueness;
+
+import edu.ie3.datamodel.exceptions.DuplicateEntitiesException;
 import edu.ie3.datamodel.exceptions.SourceException;
 import edu.ie3.datamodel.exceptions.ValidationException;
 import edu.ie3.datamodel.io.connectors.CsvFileConnector;
@@ -20,6 +23,7 @@
 import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue;
 import edu.ie3.datamodel.models.value.Value;
 import edu.ie3.datamodel.models.value.WeatherValue;
+import edu.ie3.datamodel.utils.ExceptionUtils;
 import edu.ie3.datamodel.utils.TimeSeriesUtils;
 import edu.ie3.datamodel.utils.Try;
 import edu.ie3.datamodel.utils.Try.*;
@@ -188,7 +192,22 @@ private Map<Point, IndividualTimeSeries<WeatherValue>> readWeatherTimeSeries(
         throw new SourceException("Validation failed for file " + data.getFullFilePath() + ".", e);
       }
     }
-    return weatherTimeSeries;
+
+    // checking the uniqueness before returning the time series
+    List<DuplicateEntitiesException> exceptions =
+        Try.getExceptions(
+            weatherTimeSeries.values().stream()
+                .map(
+                    ts ->
+                        Try.ofVoid(
+                            () -> checkWeatherUniqueness(ts.getEntries()),
+                            DuplicateEntitiesException.class)));
+
+    if (exceptions.isEmpty()) {
+      return weatherTimeSeries;
+    } else {
+      throw new SourceException("Due to: " + ExceptionUtils.getMessages(exceptions));
+    }
   }
 
   private Try<Stream<Map<String, String>>, SourceException> buildStreamWithFieldsToAttributesMap(

src/main/java/edu/ie3/datamodel/io/source/sql/SqlIdCoordinateSource.java

@@ -13,6 +13,7 @@
 import edu.ie3.datamodel.io.factory.timeseries.SqlIdCoordinateFactory;
 import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy;
 import edu.ie3.datamodel.io.source.IdCoordinateSource;
+import edu.ie3.datamodel.models.input.IdCoordinatePair;
 import edu.ie3.datamodel.models.value.CoordinateValue;
 import edu.ie3.datamodel.utils.Try;
 import edu.ie3.util.geo.CoordinateDistance;
@@ -21,7 +22,6 @@
 import java.sql.PreparedStatement;
 import java.util.*;
 import javax.measure.quantity.Length;
-import org.apache.commons.lang3.tuple.Pair;
 import org.locationtech.jts.geom.Envelope;
 import org.locationtech.jts.geom.Point;
 import tech.units.indriya.ComparableQuantity;
@@ -65,7 +65,9 @@ public SqlIdCoordinateSource(
                 fieldsOpt
                     .map(
                         fields ->
-                            factory.validate(fields, Pair.class).transformF(SourceException::new))
+                            factory
+                                .validate(fields, IdCoordinatePair.class)
+                                .transformF(SourceException::new))
                     .orElse(Try.Success.empty()))
         .getOrThrow();
 
@@ -198,9 +200,10 @@ public List<CoordinateDistance> getClosestCoordinates(
   private CoordinateValue createCoordinateValue(Map<String, String> fieldToValues) {
     fieldToValues.remove("distance");
 
-    SimpleFactoryData simpleFactoryData = new SimpleFactoryData(fieldToValues, Pair.class);
+    SimpleFactoryData simpleFactoryData =
+        new SimpleFactoryData(fieldToValues, IdCoordinatePair.class);
 
-    Pair<Integer, Point> pair = factory.get(simpleFactoryData).getOrThrow();
+    IdCoordinatePair pair = factory.get(simpleFactoryData).getOrThrow();
     return new CoordinateValue(pair.getKey(), pair.getValue());
   }
 

src/main/java/edu/ie3/datamodel/models/input/IdCoordinatePair.java

@@ -0,0 +1,45 @@
+/*
+ * © 2024. TU Dortmund University,
+ * Institute of Energy Systems, Energy Efficiency and Energy Economics,
+ * Research group Distribution grid planning and operation
+*/
+package edu.ie3.datamodel.models.input;
+
+import edu.ie3.util.geo.GeoUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.locationtech.jts.geom.Point;
+
+public class IdCoordinatePair extends ImmutablePair<Integer, Point> implements InputEntity {
+  /**
+   * Create a new pair instance.
+   *
+   * @param left the left value, may be null
+   * @param right the right value, may be null
+   */
+  private IdCoordinatePair(Integer left, Point right) {
+    super(left, right);
+  }
+
+  /**
+   * Creates an {@link IdCoordinatePair}.
+   *
+   * @param id of the pair
+   * @param lat of the pair
+   * @param lon of the pair
+   * @return a new {@link IdCoordinatePair}
+   */
+  public static IdCoordinatePair of(Integer id, double lat, double lon) {
+    return new IdCoordinatePair(id, GeoUtils.buildPoint(lat, lon));
+  }
+
+  /**
+   * Creates an {@link IdCoordinatePair}.
+   *
+   * @param id of the pair
+   * @param point of the pair
+   * @return a new {@link IdCoordinatePair}
+   */
+  public static IdCoordinatePair of(Integer id, Point point) {
+    return new IdCoordinatePair(id, point);
+  }
+}

src/main/java/edu/ie3/datamodel/utils/validation/UniquenessValidationUtils.java

@@ -10,11 +10,12 @@
 import edu.ie3.datamodel.models.Entity;
 import edu.ie3.datamodel.models.UniqueEntity;
 import edu.ie3.datamodel.models.input.AssetInput;
+import edu.ie3.datamodel.models.input.IdCoordinatePair;
 import edu.ie3.datamodel.models.result.ResultEntity;
 import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue;
 import edu.ie3.datamodel.models.value.WeatherValue;
 import edu.ie3.datamodel.utils.Try;
-import edu.ie3.datamodel.utils.Try.*;
+import edu.ie3.datamodel.utils.Try.Success;
 import java.util.*;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -30,9 +31,10 @@ public class UniquenessValidationUtils extends ValidationUtils {
       entity -> Set.of(entity.getTime(), entity.getInputModel());
   protected static final FieldSetSupplier<MappingEntry> mappingFieldSupplier =
       entity -> Set.of(entity.participant());
-  protected static final FieldSetSupplier<TimeBasedValue<WeatherValue>>
-      timeBasedValueFieldSupplier =
-          entity -> Set.of(entity.getTime(), entity.getValue().getCoordinate());
+  protected static final FieldSetSupplier<IdCoordinatePair> idCoordinateSupplier =
+      pair -> Set.of(pair.getKey(), pair.getValue());
+  protected static final FieldSetSupplier<TimeBasedValue<WeatherValue>> weatherValueFieldSupplier =
+      entity -> Set.of(entity.getTime(), entity.getValue().getCoordinate());
 
   /**
    * Checks the uniqueness of a collection of {@link UniqueEntity}.
@@ -88,6 +90,17 @@ public static void checkMappingEntryUniqueness(Collection<MappingEntry> entities
     checkUniqueness(entities, mappingFieldSupplier).getOrThrow();
   }
 
+  /**
+   * Checks the uniqueness of a collection of
+   *
+   * @param entities to be checked
+   * @throws DuplicateEntitiesException if uniqueness is violated
+   */
+  public static void checkIdCoordinateUniqueness(Collection<IdCoordinatePair> entities)
+      throws DuplicateEntitiesException {
+    checkUniqueness(entities, idCoordinateSupplier).getOrThrow();
+  }
+
   /**
    * Checks the uniqueness of TimeBasedWeatherValues.
    *
@@ -96,7 +109,7 @@ public static void checkMappingEntryUniqueness(Collection<MappingEntry> entities
    */
   public static void checkWeatherUniqueness(Collection<TimeBasedValue<WeatherValue>> entities)
       throws DuplicateEntitiesException {
-    checkUniqueness(entities, timeBasedValueFieldSupplier).getOrThrow();
+    checkUniqueness(entities, weatherValueFieldSupplier).getOrThrow();
   }
 
   /**