Merge branch 'dev' into df/#1131-psdm-update-for-tap-water-demand

Author: danielfeismann

CHANGELOG.md

@@ -35,6 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fix spotless deprecations [#1123](https://github.com/ie3-institute/PowerSystemDataModel/issues/1223)
 - Refactored `CongestionResult`, removed `ModelResultEntity` [#1234](https://github.com/ie3-institute/PowerSystemDataModel/issues/1234)
 - Replaced `LoadProfileInput` with `LoadProfileTimeSeries` [#1228](https://github.com/ie3-institute/PowerSystemDataModel/issues/1228)
+- Enhance `CsvDataSource` [#1246](https://github.com/ie3-institute/PowerSystemDataModel/issues/1246)
 
 ## [5.1.0] - 2024-06-24
 

src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java

@@ -17,6 +17,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.util.*;
+import java.util.function.Function;
 import java.util.stream.Stream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -34,10 +35,17 @@ public class CsvFileConnector implements DataConnector {
   private final Map<Class<? extends Entity>, BufferedCsvWriter> entityWriters = new HashMap<>();
   private final Map<UUID, BufferedCsvWriter> timeSeriesWriters = new HashMap<>();
   private final Path baseDirectory;
+  private final Optional<Function<String, InputStream>> customInputStream;
   private static final String FILE_ENDING = ".csv";
 
   public CsvFileConnector(Path baseDirectory) {
     this.baseDirectory = baseDirectory;
+    this.customInputStream = Optional.empty();
+  }
+
+  public CsvFileConnector(Path baseDirectory, Function<String, InputStream> inputStreamSupplier) {
+    this.baseDirectory = baseDirectory;
+    this.customInputStream = Optional.ofNullable(inputStreamSupplier);
   }
 
   /** Returns the base directory of this connector. */
@@ -159,9 +167,17 @@ public synchronized <C extends Entity> void closeEntityWriter(Class<C> clz) thro
    * @throws FileNotFoundException if no file with the provided file name can be found
    */
   public BufferedReader initReader(Path filePath) throws FileNotFoundException {
-    File fullPath = baseDirectory.resolve(filePath.toString() + FILE_ENDING).toFile();
-    return new BufferedReader(
-        new InputStreamReader(new FileInputStream(fullPath), StandardCharsets.UTF_8), 16384);
+    Path fullPath = baseDirectory.resolve(filePath.toString() + FILE_ENDING);
+
+    InputStream inputStream;
+
+    if (customInputStream.isPresent()) {
+      inputStream = customInputStream.get().apply(fullPath.toString());
+    } else {
+      inputStream = new FileInputStream(fullPath.toFile());
+    }
+
+    return new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8), 16384);
   }
 
   @Override

src/main/java/edu/ie3/datamodel/io/source/EntitySource.java

@@ -9,14 +9,24 @@
 import edu.ie3.datamodel.exceptions.FailedValidationException;
 import edu.ie3.datamodel.exceptions.SourceException;
 import edu.ie3.datamodel.exceptions.ValidationException;
+import edu.ie3.datamodel.io.connectors.CsvFileConnector;
 import edu.ie3.datamodel.io.factory.EntityData;
 import edu.ie3.datamodel.io.factory.EntityFactory;
+import edu.ie3.datamodel.io.naming.FileNamingStrategy;
+import edu.ie3.datamodel.io.source.csv.CsvDataSource;
 import edu.ie3.datamodel.models.Entity;
 import edu.ie3.datamodel.models.UniqueEntity;
 import edu.ie3.datamodel.utils.QuadFunction;
 import edu.ie3.datamodel.utils.TriFunction;
 import edu.ie3.datamodel.utils.Try;
 import edu.ie3.datamodel.utils.Try.Failure;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
 import java.util.*;
 import java.util.function.BiFunction;
 import java.util.function.Function;
@@ -34,6 +44,9 @@
 public abstract class EntitySource {
   protected static final Logger log = LoggerFactory.getLogger(EntitySource.class);
 
+  // file system for build-in entities
+  private static FileSystem jarFileSystem = null;
+
   // convenience collectors
 
   protected static <T extends UniqueEntity> Collector<T, ?, Map<UUID, T>> toMap() {
@@ -94,6 +107,47 @@ protected static <C> Try<Void, ValidationException> validate(
                     .orElse(Try.Success.empty()));
   }
 
+  /**
+   * Method to get a source for the build in entities.
+   *
+   * @param subdirectory from the resource folder
+   * @return a new {@link CsvDataSource}
+   */
+  protected static CsvDataSource getBuildInSource(Class<?> clazz, String subdirectory)
+      throws SourceException {
+    try {
+      URL url = clazz.getResource(subdirectory);
+
+      if (url == null) {
+        throw new SourceException("Resources not found for: " + subdirectory);
+      }
+
+      URI uri = url.toURI();
+      CsvFileConnector connector;
+
+      switch (url.getProtocol()) {
+        case "file" -> connector = new CsvFileConnector(Path.of(uri));
+        case "jar" -> {
+          // handling resources in jar
+          String[] array = uri.toString().split("!");
+
+          if (jarFileSystem == null) {
+            jarFileSystem = FileSystems.newFileSystem(URI.create(array[0]), Collections.emptyMap());
+          }
+
+          connector =
+              new CsvFileConnector(jarFileSystem.getPath(array[1]), clazz::getResourceAsStream);
+        }
+        default -> throw new SourceException(
+            "Protocol " + url.getProtocol() + " is nor supported!");
+      }
+
+      return new CsvDataSource(",", connector, new FileNamingStrategy());
+    } catch (URISyntaxException | IOException e) {
+      throw new SourceException(e);
+    }
+  }
+
   /**
    * Universal method to get a map: uuid to {@link UniqueEntity}.
    *

src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java

@@ -54,6 +54,13 @@ public CsvDataSource(String csvSep, Path directoryPath, FileNamingStrategy fileN
     this.fileNamingStrategy = fileNamingStrategy;
   }
 
+  public CsvDataSource(
+      String csvSep, CsvFileConnector connector, FileNamingStrategy fileNamingStrategy) {
+    this.csvSep = csvSep;
+    this.connector = connector;
+    this.fileNamingStrategy = fileNamingStrategy;
+  }
+
   @Override
   public Optional<Set<String>> getSourceFields(Class<? extends Entity> entityClass)
       throws SourceException {
@@ -85,6 +92,15 @@ public Stream<Map<String, String>> getSourceData(Class<? extends Entity> entityC
     return buildStreamWithFieldsToAttributesMap(entityClass, true).getOrThrow();
   }
 
+  /**
+   * @param filePath to the csv file
+   * @return a stream of maps that represent the rows in the csv file
+   * @throws SourceException on error while reading the source file
+   */
+  public Stream<Map<String, String>> getSourceData(Path filePath) throws SourceException {
+    return buildStreamWithFieldsToAttributesMap(filePath, true).getOrThrow();
+  }
+
   // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 
   /** Returns the set {@link FileNamingStrategy}. */
@@ -231,23 +247,20 @@ protected String[] parseCsvRow(String csvRow, String csvSep) {
   protected Try<Stream<Map<String, String>>, SourceException> buildStreamWithFieldsToAttributesMap(
       Class<? extends Entity> entityClass, boolean allowFileNotExisting) {
     return getFilePath(entityClass)
-        .flatMap(
-            path -> buildStreamWithFieldsToAttributesMap(entityClass, path, allowFileNotExisting));
+        .flatMap(path -> buildStreamWithFieldsToAttributesMap(path, allowFileNotExisting));
   }
 
   /**
    * Reads the first line (considered to be the headline with headline fields) and returns a stream
    * of (fieldName to fieldValue) mapping where each map represents one row of the .csv file. Since
    * the returning stream is a parallel stream, the order of the elements cannot be guaranteed.
    *
-   * @param entityClass the entity class that should be build
    * @param filePath the path of the file to read
    * @return a try containing either a parallel stream of maps, where each map represents one row of
    *     the csv file with the mapping (fieldName to fieldValue) or an exception
    */
-  protected <T extends Entity>
-      Try<Stream<Map<String, String>>, SourceException> buildStreamWithFieldsToAttributesMap(
-          Class<T> entityClass, Path filePath, boolean allowFileNotExisting) {
+  protected Try<Stream<Map<String, String>>, SourceException> buildStreamWithFieldsToAttributesMap(
+      Path filePath, boolean allowFileNotExisting) {
     try (BufferedReader reader = connector.initReader(filePath)) {
       final String[] headline = parseCsvRow(reader.readLine(), csvSep);
 
@@ -264,9 +277,7 @@ Try<Stream<Map<String, String>>, SourceException> buildStreamWithFieldsToAttribu
         return Failure.of(new SourceException("Unable to find file '" + filePath + "'.", e));
       }
     } catch (IOException e) {
-      return Failure.of(
-          new SourceException(
-              "Cannot read file to build entity '" + entityClass.getSimpleName() + "'", e));
+      return Failure.of(new SourceException("Cannot read file '" + filePath + "'.", e));
     }
   }
 

src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java

@@ -163,7 +163,7 @@ protected IndividualTimeSeries<V> buildIndividualTimeSeries(
       throws SourceException {
     Try<Stream<TimeBasedValue<V>>, SourceException> timeBasedValues =
         dataSource
-            .buildStreamWithFieldsToAttributesMap(TimeBasedValue.class, filePath, false)
+            .buildStreamWithFieldsToAttributesMap(filePath, false)
             .flatMap(
                 stream ->
                     Try.scanStream(stream.map(fieldToValueFunction), "TimeBasedValue<V>")

src/main/resources/load/lpts_g0.csv

@@ -0,0 +1,97 @@
+SuSa,SuSu,SuWd,TrSa,TrSu,TrWd,WiSa,WiSu,WiWd,quarterHour
+74.6,68.8,71.5,75.8,68.3,73.0,70.0,63.2,65.5,0
+76.2,67.4,69.0,76.7,66.5,70.1,73.0,61.0,62.6,1
+77.7,65.7,66.3,77.7,64.6,67.1,75.9,58.9,59.6,2
+78.5,63.5,63.5,78.5,62.6,64.5,77.6,57.0,57.0,3
+77.9,60.9,60.9,78.5,60.3,62.3,77.1,55.3,54.8,4
+76.3,58.0,58.6,77.8,57.9,60.6,75.0,53.7,53.1,5
+74.1,55.4,56.6,76.6,55.5,59.2,72.1,52.1,51.7,6
+71.9,53.3,55.1,74.7,53.3,57.9,69.1,50.5,50.5,7
+70.2,52.0,54.2,72.6,51.2,56.7,66.8,48.7,49.4,8
+68.9,51.4,53.7,70.4,49.5,55.6,65.1,46.9,48.5,9
+67.9,51.0,53.4,68.5,48.0,54.7,64.1,45.2,47.9,10
+67.3,50.5,53.3,67.3,46.7,54.2,63.5,43.9,47.7,11
+66.9,49.6,53.1,67.0,45.7,54.1,63.4,43.0,47.9,12
+66.7,48.5,53.2,67.5,44.9,54.6,63.6,42.5,48.7,13
+66.8,47.5,54.1,68.3,44.3,55.8,64.0,42.2,50.2,14
+67.3,46.7,56.1,69.1,43.9,57.9,64.5,42.0,52.3,15
+68.0,46.4,59.4,69.7,43.6,60.9,65.0,41.9,55.1,16
+69.0,46.5,63.3,70.1,43.5,64.4,65.6,41.8,58.2,17
+70.0,46.6,67.2,70.5,43.6,68.0,66.3,41.8,61.2,18
+71.0,46.7,70.1,71.0,43.9,71.0,67.3,42.0,63.5,19
+71.8,46.6,71.4,71.8,44.5,73.2,68.5,42.6,65.0,20
+72.4,46.3,71.6,72.5,45.1,74.7,69.9,43.4,66.0,21
+72.8,46.0,71.2,73.0,45.6,75.7,71.4,44.4,67.1,22
+72.9,45.8,71.0,72.9,45.8,76.6,72.9,45.8,69.1,23
+73.0,45.9,71.5,72.2,45.6,77.7,74.4,47.4,72.5,24
+73.9,46.3,73.1,72.2,45.3,79.3,76.8,49.0,77.1,25
+77.0,47.2,75.9,74.4,45.2,81.9,81.0,50.4,82.9,26
+83.2,48.6,80.4,80.4,45.8,86.0,87.8,51.4,89.7,27
+93.3,50.5,86.6,91.0,47.2,91.9,98.0,51.8,97.6,28
+106.2,52.6,95.1,105.2,49.1,100.4,110.7,51.7,107.3,29
+120.5,54.3,106.3,121.2,51.0,111.9,124.8,51.2,119.9,30
+134.5,55.1,120.5,137.4,52.3,127.1,139.2,50.5,136.4,31
+147.2,54.8,137.8,152.1,52.8,145.9,153.0,49.6,157.1,32
+157.8,53.9,156.3,164.9,52.9,166.2,165.4,49.0,179.5,33
+166.2,53.2,173.4,175.1,53.1,185.2,175.9,48.8,200.5,34
+171.9,53.3,186.9,182.2,54.2,200.0,184.1,49.5,216.8,35
+175.0,54.7,195.0,186.1,56.4,208.5,189.5,51.3,226.2,36
+176.1,57.2,198.7,187.5,59.5,212.1,192.8,53.8,230.0,37
+176.3,60.0,199.8,187.7,62.7,212.6,195.1,56.8,230.4,38
+176.6,62.6,200.0,187.8,65.4,212.1,197.1,59.8,229.9,39
+177.7,64.5,200.6,188.7,67.2,212.2,199.8,62.6,230.0,40
+179.4,65.9,201.9,190.1,68.5,213.1,202.7,65.1,231.2,41
+181.3,67.0,203.6,191.9,69.6,214.6,205.4,67.2,233.0,42
+183.1,68.2,205.6,193.4,71.0,216.8,207.4,69.1,235.5,43
+184.4,69.7,207.6,194.5,73.1,219.2,208.4,70.8,238.1,44
+185.2,71.4,209.1,195.1,75.6,221.2,208.3,72.2,240.0,45
+185.3,73.2,209.6,195.3,78.2,222.0,207.4,73.5,240.4,46
+185.0,74.7,208.4,195.3,80.4,220.5,205.6,74.7,238.3,47
+184.1,76.0,205.1,194.9,81.9,216.3,203.0,76.0,233.0,48
+182.4,77.0,200.0,193.9,82.8,209.8,199.7,77.3,225.1,49
+179.7,77.8,193.8,191.7,83.2,201.9,195.6,78.4,215.7,50
+175.7,78.5,186.9,187.8,83.2,193.4,190.6,79.4,205.6,51
+170.1,79.1,179.9,181.9,82.8,185.0,184.6,80.1,195.7,52
+163.1,79.5,173.4,174.1,82.1,177.4,177.5,80.5,186.7,53
+154.6,79.3,168.1,164.8,80.9,170.9,168.9,80.3,179.2,54
+144.8,78.5,164.4,154.2,79.4,166.3,158.8,79.4,173.8,55
+133.9,76.8,163.0,142.6,77.5,163.9,147.2,77.8,171.0,56
+122.7,74.4,163.5,130.9,75.3,163.6,134.8,75.7,170.7,57
+112.0,71.8,165.4,119.8,73.1,165.1,122.7,73.4,172.7,58
+102.8,69.1,168.2,110.3,71.0,168.2,112.1,71.0,176.6,59
+95.7,66.8,171.5,102.7,69.2,172.5,103.8,68.9,182.1,60
+90.4,64.8,174.6,96.9,67.6,177.3,97.4,67.0,188.2,61
+86.4,63.1,177.2,92.1,66.1,181.4,92.6,65.2,193.9,62
+83.2,61.7,178.5,87.8,64.5,184.1,88.8,63.5,198.1,63
+80.3,60.4,178.2,83.6,62.8,184.5,85.6,62.0,200.1,64
+77.8,59.7,176.8,79.7,61.5,183.6,83.3,61.2,200.7,65
+75.9,59.7,175.1,76.9,61.2,182.1,82.4,61.8,200.9,66
+74.7,60.7,173.8,75.7,62.6,181.3,83.2,64.5,201.8,67
+74.4,63.0,173.2,76.5,65.9,181.5,85.8,69.5,204.0,68
+74.8,66.0,172.4,78.6,70.4,181.7,89.5,76.0,205.8,69
+75.6,69.2,170.2,81.1,75.0,180.0,93.4,82.6,205.5,70
+76.6,71.9,165.4,83.2,78.5,174.7,96.2,87.8,200.9,71
+77.6,73.9,157.1,84.1,80.1,164.8,97.4,90.8,190.7,72
+78.5,75.1,146.1,84.3,80.4,151.5,97.4,92.1,176.6,73
+79.1,76.0,133.8,84.1,80.2,137.0,96.8,92.3,160.5,74
+79.4,76.6,121.5,84.1,80.4,123.3,96.2,92.5,144.8,75
+79.4,77.3,110.1,84.7,81.4,112.1,96.3,93.1,131.3,76
+79.4,78.2,100.5,85.8,83.1,103.4,96.8,94.0,120.1,77
+79.6,79.2,93.2,87.3,85.1,97.3,97.2,94.6,111.3,78
+80.4,80.4,88.8,88.8,86.9,93.4,97.2,94.4,104.6,79
+81.9,81.6,87.4,90.1,88.1,91.6,96.3,93.0,100.0,80
+83.9,82.9,88.1,91.2,88.7,91.1,94.8,90.8,96.7,81
+85.7,83.8,89.6,91.7,88.6,91.1,92.8,88.0,94.1,82
+86.9,84.1,90.6,91.6,87.8,90.6,90.6,85.0,91.6,83
+87.0,83.7,90.2,90.6,86.4,89.2,88.5,82.1,88.5,84
+86.3,82.7,88.5,89.0,84.4,87.2,86.4,79.4,85.2,85
+84.9,81.2,86.3,87.1,82.0,85.0,84.3,76.9,82.1,86
+83.2,79.4,84.1,85.0,79.4,83.2,82.2,74.7,79.4,87
+81.3,77.5,82.4,83.1,76.7,82.1,80.1,72.8,77.6,88
+79.3,75.5,81.2,81.2,74.1,81.7,78.0,71.1,76.4,89
+77.4,73.3,80.3,79.4,71.5,81.6,75.9,69.3,75.6,90
+75.7,71.0,79.4,77.6,69.1,81.3,73.8,67.3,74.7,91
+74.1,68.6,78.4,75.7,67.0,80.6,71.7,65.0,73.7,92
+72.7,66.4,77.2,73.8,65.3,79.5,69.7,62.6,72.3,93
+71.3,64.6,75.7,71.9,64.1,77.9,67.6,60.4,70.5,94
+70.1,63.5,73.8,70.1,63.5,75.7,65.4,58.9,68.2,95