Skip to content

Commit 22bc7da

Browse files
sebastian-petersebastian-peter
authored
Merge pull request #796 from ie3-institute/ms/#795-remove-support-for-old-csv-format
Removing support for old csv format
2 parents 9f95a66 + f871e81 commit 22bc7da

File tree

7 files changed

+343
-507
lines changed

7 files changed

+343
-507
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
3939
- Don't throw exceptions for not yet implemented validations [#879](https://github.com/ie3-institute/PowerSystemDataModel/issues/879)
4040
- `CsvDataSource` throws exceptions on error [#954](https://github.com/ie3-institute/PowerSystemDataModel/issues/954)
4141
- Removing `uuid` as required column from input and result time series [#826](https://github.com/ie3-institute/PowerSystemDataModel/issues/826)
42+
- Removing the support for the old csv format that was marked `deprecated` back in version `1.1.0` [#795](https://github.com/ie3-institute/PowerSystemDataModel/issues/795)
4243

4344
## [4.1.0] - 2023-11-02
4445

‎docs/readthedocs/gettingstarted.md‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
# Getting started
2-
32
Welcome, dear fellow of bottom up power system modelling!
43
This section is meant to give you some help getting hands on our project.
54
If you feel, something is missing, please contact us!
@@ -45,3 +44,32 @@ and add the dependency:
4544
<version>3.0-SNAPSHOT</version>
4645
</dependency>
4746
```
47+
48+
## Important changes
49+
50+
With the release of `PowerSystemDataModel` version `5.0` the support for the old csv file format will be fully removed.
51+
It was already marked as `deprecated` back in version `1.1.0`. For those who are still using a model in the old csv format
52+
the following guide will provide a fast and easy way to convert old format into the new one.
53+
54+
- Since the support is removed in version `5.0`, the `PowerSystemDataModel` version `3.x` or `4.x` must be
55+
used to read the old format. The `PSDM` will automatically write the output model in the new csv format.
56+
57+
58+
``` java
59+
/* Parameterization */
60+
String gridName = "gridWithOldFormat";
61+
String csvSep = ",";
62+
Path folderPath = Path.of("PATH_TO_THE_FOLDER");
63+
boolean isHierarchic = false;
64+
Path output = Path.of("PATH_OF_THE_OUTPUT_FOLDER");
65+
FileNamingStrategy namingStrategy = new FileNamingStrategy();
66+
67+
/* Reading the old format */
68+
JointGridContainer container = CsvJointGridContainerSource.read(gridName, csvSep, folderPath, isHierarchic);
69+
70+
/* Writing in the new format */
71+
CsvFileSink sink = new CsvFileSink(output, namingStrategy, csvSep);
72+
sink.persistJointGrid(container);
73+
```
74+
75+

‎src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java‎

Lines changed: 9 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,14 @@
1111
import edu.ie3.datamodel.io.source.DataSource;
1212
import edu.ie3.datamodel.models.Entity;
1313
import edu.ie3.datamodel.utils.Try;
14-
import edu.ie3.datamodel.utils.Try.*;
14+
import edu.ie3.datamodel.utils.Try.Failure;
15+
import edu.ie3.datamodel.utils.Try.Success;
1516
import edu.ie3.util.StringUtils;
1617
import java.io.BufferedReader;
1718
import java.io.FileNotFoundException;
1819
import java.io.IOException;
1920
import java.nio.file.Path;
2021
import java.util.*;
21-
import java.util.concurrent.atomic.AtomicInteger;
22-
import java.util.regex.Matcher;
23-
import java.util.regex.Pattern;
2422
import java.util.stream.Collectors;
2523
import java.util.stream.IntStream;
2624
import java.util.stream.Stream;
@@ -44,13 +42,6 @@ public class CsvDataSource implements DataSource {
4442

4543
private final FileNamingStrategy fileNamingStrategy;
4644

47-
/**
48-
* @deprecated ensures downward compatibility with old csv data format. Can be removed when
49-
* support for old csv format is removed. *
50-
*/
51-
@Deprecated(since = "1.1.0", forRemoval = true)
52-
private boolean notYetLoggedWarning = true;
53-
5445
public CsvDataSource(String csvSep, Path folderPath, FileNamingStrategy fileNamingStrategy) {
5546
this.csvSep = csvSep;
5647
this.connector = new CsvFileConnector(folderPath, fileNamingStrategy);
@@ -90,6 +81,11 @@ public Stream<Map<String, String>> getSourceData(Class<? extends Entity> entityC
9081

9182
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
9283

84+
/** Returns the set {@link FileNamingStrategy}. */
85+
public FileNamingStrategy getNamingStrategy() {
86+
return fileNamingStrategy;
87+
}
88+
9389
/**
9490
* Takes a row string of a .csv file and a string array of the csv file headline, tries to split
9591
* the csv row string based and zip it together with the headline. This method does not contain
@@ -107,35 +103,14 @@ protected Map<String, String> buildFieldsToAttributes(
107103
TreeMap<String, String> insensitiveFieldsToAttributes =
108104
new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
109105

110-
// todo when replacing deprecated workaround code below add final modifier before parseCsvRow as
111-
// well as remove
112-
// 'finalFieldVals' and notYetLoggedWarning below!
113-
String[] fieldVals = parseCsvRow(csvRow, csvSep);
114-
115-
// start workaround for deprecated data model processing
116-
if (fieldVals.length != headline.length) {
117-
// try to parse old structure
118-
fieldVals = oldFieldVals(csvSep, csvRow);
119-
// if this works log a warning to inform the user that this will not work much longer,
120-
// otherwise parsing will fail regularly as expected below
121-
if (fieldVals.length == headline.length && notYetLoggedWarning) {
122-
notYetLoggedWarning = false;
123-
log.warn(
124-
"You are using an outdated version of the data "
125-
+ "model with invalid formatted csv rows. This is okay for now, but please updated your files, as the "
126-
+ "support for the old model will be removed soon.");
127-
}
128-
}
129-
// end workaround for deprecated data model processing
130-
131106
try {
132-
String[] finalFieldVals = fieldVals;
107+
String[] fieldVals = parseCsvRow(csvRow, csvSep);
133108
insensitiveFieldsToAttributes.putAll(
134109
IntStream.range(0, fieldVals.length)
135110
.boxed()
136111
.collect(
137112
Collectors.toMap(
138-
k -> StringUtils.snakeCaseToCamelCase(headline[k]), v -> finalFieldVals[v])));
113+
k -> StringUtils.snakeCaseToCamelCase(headline[k]), v -> fieldVals[v])));
139114

140115
if (insensitiveFieldsToAttributes.size() != headline.length) {
141116
Set<String> fieldsToAttributesKeySet = insensitiveFieldsToAttributes.keySet();
@@ -178,78 +153,6 @@ protected String[] parseCsvRow(String csvRow, String csvSep) {
178153
.toArray(String[]::new);
179154
}
180155

181-
/**
182-
* Build an array of from the provided csv row string considering special cases where geoJson or
183-
* {@link edu.ie3.datamodel.models.input.system.characteristic.CharacteristicInput} are provided
184-
* in the csv row string.
185-
*
186-
* @param csvSep the column separator of the csv row string
187-
* @param csvRow the csv row string
188-
* @return an array with one entry per column of the provided csv row string
189-
* @deprecated only left for downward compatibility. Will be removed in a major release
190-
*/
191-
@Deprecated(since = "1.1.0", forRemoval = true)
192-
protected String[] oldFieldVals(String csvSep, String csvRow) {
193-
194-
/*geo json support*/
195-
final String geoJsonRegex = "\\{.+?}}}";
196-
final String geoReplacement = "geoJSON";
197-
198-
/*characteristic input support */
199-
final String charInputRegex = "(cP:|olm:|cosPhiFixed:|cosPhiP:|qV:)\\{[^}]++}";
200-
final String charReplacement = "charRepl";
201-
202-
/*removes double double quotes*/
203-
List<String> geoList = extractMatchingStrings(geoJsonRegex, csvRow.replace("\"\"", "\""));
204-
List<String> charList = extractMatchingStrings(charInputRegex, csvRow.replace("\"\"", "\""));
205-
206-
AtomicInteger geoCounter = new AtomicInteger(0);
207-
AtomicInteger charCounter = new AtomicInteger(0);
208-
209-
return Arrays.stream(
210-
csvRow
211-
.replaceAll(charInputRegex, charReplacement)
212-
.replaceAll(geoJsonRegex, geoReplacement)
213-
.replaceAll("\"*", "") // remove all quotes from
214-
.split(csvSep, -1))
215-
.map(
216-
fieldVal -> {
217-
String returningFieldVal = fieldVal;
218-
if (fieldVal.equalsIgnoreCase(geoReplacement)) {
219-
returningFieldVal = geoList.get(geoCounter.getAndIncrement());
220-
}
221-
if (fieldVal.equalsIgnoreCase(charReplacement)) {
222-
returningFieldVal = charList.get(charCounter.getAndIncrement());
223-
}
224-
return returningFieldVal.trim();
225-
})
226-
.toArray(String[]::new);
227-
}
228-
229-
/**
230-
* Extracts all strings from the provided csvRow matching the provided regexString and returns a
231-
* list of strings in the order of their appearance in the csvRow string
232-
*
233-
* @param regexString regex string that should be searched for
234-
* @param csvRow csv row string that should be searched in for the regex string
235-
* @return a list of strings matching the provided regex in the order of their appearance in the
236-
* provided csv row string
237-
*/
238-
private List<String> extractMatchingStrings(String regexString, String csvRow) {
239-
Pattern pattern = Pattern.compile(regexString);
240-
Matcher matcher = pattern.matcher(csvRow);
241-
242-
ArrayList<String> matchingList = new ArrayList<>();
243-
while (matcher.find()) {
244-
matchingList.add(matcher.group());
245-
}
246-
return matchingList;
247-
}
248-
249-
public FileNamingStrategy getNamingStrategy() {
250-
return fileNamingStrategy;
251-
}
252-
253156
/**
254157
* Tries to open a file reader based on the provided entity class and hands it over for further
255158
* processing.

‎src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy‎

Lines changed: 1 addition & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,6 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta {
3333
String csvRow, String csvSep) {
3434
return super.parseCsvRow(csvRow, csvSep)
3535
}
36-
37-
String[] oldFieldVals(
38-
String csvSep, String csvRow) {
39-
return super.oldFieldVals(csvSep, csvRow)
40-
}
4136
}
4237

4338
@Shared
@@ -102,7 +97,7 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta {
10297
"olmcharacteristic",
10398
"cosPhiFixed"
10499
] as String[]
105-
def validCsvRow = "5ebd8f7e-dedb-4017-bb86-6373c4b68eb8,25.0,100.0,0.95,98.0,test_bmTypeInput,50.0,25.0,olm:{(0.0,1.0)},cosPhiFixed:{(0.0,1.0)}"
100+
def validCsvRow = "5ebd8f7e-dedb-4017-bb86-6373c4b68eb8,25.0,100.0,0.95,98.0,test_bmTypeInput,50.0,25.0,\"olm:{(0.0,1.0)}\",\"cosPhiFixed:{(0.0,1.0)}\""
106101

107102
expect:
108103
dummyCsvSource.buildFieldsToAttributes(validCsvRow, validHeadline) == [
@@ -119,97 +114,6 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta {
119114
]
120115
}
121116

122-
def "A CsvDataSource should be able to handle deprecated invalid csvRows correctly"() {
123-
expect:
124-
dummyCsvSource.oldFieldVals(csvSep, csvRow) as List == resultingArray
125-
126-
where:
127-
csvSep | csvRow || resultingArray
128-
"," | "4ca90220-74c2-4369-9afa-a18bf068840d,{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}},node_a,2020-03-25T15:11:31Z[UTC],2020-03-24T15:11:31Z[UTC],8f9682df-0744-4b58-a122-f0dc730f6510,true,1,1.0,Höchstspannung,380.0,olm:{(0.00,1.00)},cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}" || [
129-
"4ca90220-74c2-4369-9afa-a18bf068840d",
130-
"{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
131-
"node_a",
132-
"2020-03-25T15:11:31Z[UTC]",
133-
"2020-03-24T15:11:31Z[UTC]",
134-
"8f9682df-0744-4b58-a122-f0dc730f6510",
135-
"true",
136-
"1",
137-
"1.0",
138-
"Höchstspannung",
139-
"380.0",
140-
"olm:{(0.00,1.00)}",
141-
"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
142-
]
143-
"," | "\"4ca90220-74c2-4369-9afa-a18bf068840d\",\"{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}\",\"node_a\",\"2020-03-25T15:11:31Z[UTC]\",\"2020-03-24T15:11:31Z[UTC]\",\"8f9682df-0744-4b58-a122-f0dc730f6510\",\"true\",\"1\",\"1.0\",\"Höchstspannung\",\"380.0\",\"olm:{(0.00,1.00)}\",\"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}\"" || [
144-
"4ca90220-74c2-4369-9afa-a18bf068840d",
145-
"{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
146-
"node_a",
147-
"2020-03-25T15:11:31Z[UTC]",
148-
"2020-03-24T15:11:31Z[UTC]",
149-
"8f9682df-0744-4b58-a122-f0dc730f6510",
150-
"true",
151-
"1",
152-
"1.0",
153-
"Höchstspannung",
154-
"380.0",
155-
"olm:{(0.00,1.00)}",
156-
"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
157-
]
158-
";" | "4ca90220-74c2-4369-9afa-a18bf068840d;cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)};{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}};node_a;2020-03-25T15:11:31Z[UTC];2020-03-24T15:11:31Z[UTC];8f9682df-0744-4b58-a122-f0dc730f6510;true;1;1.0;Höchstspannung;380.0;olm:{(0.00,1.00)};cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}" || [
159-
"4ca90220-74c2-4369-9afa-a18bf068840d",
160-
"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}",
161-
"{(0.0,1.0),(0.9,1.0),(1.2,-0.3)};{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
162-
"node_a",
163-
"2020-03-25T15:11:31Z[UTC]",
164-
"2020-03-24T15:11:31Z[UTC]",
165-
"8f9682df-0744-4b58-a122-f0dc730f6510",
166-
"true",
167-
"1",
168-
"1.0",
169-
"Höchstspannung",
170-
"380.0",
171-
"olm:{(0.00,1.00)}",
172-
"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
173-
]
174-
";" | "\"4ca90220-74c2-4369-9afa-a18bf068840d\";\"{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}\";\"node_a\";\"2020-03-25T15:11:31Z[UTC]\";\"2020-03-24T15:11:31Z[UTC]\";\"8f9682df-0744-4b58-a122-f0dc730f6510\";\"true\";\"1\";\"1.0\";\"Höchstspannung\";\"380.0\";\"olm:{(0.00,1.00)}\";\"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}\"" || [
175-
"4ca90220-74c2-4369-9afa-a18bf068840d",
176-
"{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
177-
"node_a",
178-
"2020-03-25T15:11:31Z[UTC]",
179-
"2020-03-24T15:11:31Z[UTC]",
180-
"8f9682df-0744-4b58-a122-f0dc730f6510",
181-
"true",
182-
"1",
183-
"1.0",
184-
"Höchstspannung",
185-
"380.0",
186-
"olm:{(0.00,1.00)}",
187-
"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
188-
]
189-
"," | "66275bfd-978b-4974-9f73-f270165a6351,Standard,f18a5a9b-6d45-4843-be12-be6d12de0e6b,{\"type\":\"LineString\",\"coordinates\":[[7.4116482,51.4843281],[7.4116482,51.4843281]],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}},{\"type\":\"Point\",\"coordinates\":[0.25423729,0.75409836],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:0\"}}}\"" || [
190-
"66275bfd-978b-4974-9f73-f270165a6351",
191-
"Standard",
192-
"f18a5a9b-6d45-4843-be12-be6d12de0e6b",
193-
"{\"type\":\"LineString\",\"coordinates\":[[7.4116482,51.4843281],[7.4116482,51.4843281]],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
194-
"{\"type\":\"Point\",\"coordinates\":[0.25423729,0.75409836],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:0\"}}}"
195-
]
196-
"," | "4ca90220-74c2-4369-9afa-a18bf068840d,{\"\"type\"\":\"\"Point\"\",\"\"coordinates\"\":[7.411111,51.492528],\"\"crs\"\":{\"\"type\"\":\"\"name\"\",\"\"properties\"\":{\"\"name\"\":\"\"EPSG:4326\"\"}}},node_a,2020-03-25T15:11:31Z[UTC],2020-03-24T15:11:31Z[UTC],8f9682df-0744-4b58-a122-f0dc730f6510,true,1,1.0,Höchstspannung,380.0,\"olm:{(0.00,1.00)}\",\"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}\"" || [
197-
"4ca90220-74c2-4369-9afa-a18bf068840d",
198-
"{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
199-
"node_a",
200-
"2020-03-25T15:11:31Z[UTC]",
201-
"2020-03-24T15:11:31Z[UTC]",
202-
"8f9682df-0744-4b58-a122-f0dc730f6510",
203-
"true",
204-
"1",
205-
"1.0",
206-
"Höchstspannung",
207-
"380.0",
208-
"olm:{(0.00,1.00)}",
209-
"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
210-
]
211-
}
212-
213117
def "A CsvDataSource should be able to handle a variety of different csvRows correctly"() {
214118
expect:
215119
dummyCsvSource.parseCsvRow(csvRow, csvSep) as List == resultingArray

‎src/test/resources/edu/ie3/datamodel/io/source/csv/_grid/default/line_input.csv‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
uuid,geo_position,id,length,node_a,node_b,olm_characteristic,operates_from,operates_until,operator,parallel_devices,type
2-
92ec3bcf-1777-4d38-af67-0bf7c9fa73c7,"{""type"":""LineString"",""coordinates"":[[7.411111,51.492528],[7.414116,51.484136]],""crs"":{""type"":""name"",""properties"":{""name"":""EPSG:4326""}}}",test_line_AtoB,0.003,4ca90220-74c2-4369-9afa-a18bf068840d,47d29df0-ba2d-4d23-8e75-c82229c5c758,olm:{(0.00,1.00)},2020-03-24T15:11:31Z[UTC],2020-03-25T15:11:31Z[UTC],f15105c4-a2de-4ab8-a621-4bc98e372d92,2,3bed3eb3-9790-4874-89b5-a5434d408088
3-
91ec3bcf-1777-4d38-af67-0bf7c9fa73c7,"{""type"":""LineString"",""coordinates"":[[7.411111,51.492528],[7.414116,51.484136]],""crs"":{""type"":""name"",""properties"":{""name"":""EPSG:4326""}}}",test_line_CtoD,0.003,bd837a25-58f3-44ac-aa90-c6b6e3cd91b2,6e0980e0-10f2-4e18-862b-eb2b7c90509b,olm:{(0.00,1.00)},2020-03-24T15:11:31Z[UTC],2020-03-25T15:11:31Z[UTC],f15105c4-a2de-4ab8-a621-4bc98e372d92,2,3bed3eb3-9790-4874-89b5-a5434d408088
2+
92ec3bcf-1777-4d38-af67-0bf7c9fa73c7,"{""type"":""LineString"",""coordinates"":[[7.411111,51.492528],[7.414116,51.484136]],""crs"":{""type"":""name"",""properties"":{""name"":""EPSG:4326""}}}",test_line_AtoB,0.003,4ca90220-74c2-4369-9afa-a18bf068840d,47d29df0-ba2d-4d23-8e75-c82229c5c758,"olm:{(0.00,1.00)}",2020-03-24T15:11:31Z[UTC],2020-03-25T15:11:31Z[UTC],f15105c4-a2de-4ab8-a621-4bc98e372d92,2,3bed3eb3-9790-4874-89b5-a5434d408088
3+
91ec3bcf-1777-4d38-af67-0bf7c9fa73c7,"{""type"":""LineString"",""coordinates"":[[7.411111,51.492528],[7.414116,51.484136]],""crs"":{""type"":""name"",""properties"":{""name"":""EPSG:4326""}}}",test_line_CtoD,0.003,bd837a25-58f3-44ac-aa90-c6b6e3cd91b2,6e0980e0-10f2-4e18-862b-eb2b7c90509b,"olm:{(0.00,1.00)}",2020-03-24T15:11:31Z[UTC],2020-03-25T15:11:31Z[UTC],f15105c4-a2de-4ab8-a621-4bc98e372d92,2,3bed3eb3-9790-4874-89b5-a5434d408088

0 commit comments

Comments
 (0)