Skip to content

Commit c30ef03

Browse files
staudtMariusstaudtMarius
committed
Fixing some tests.
1 parent dc442ca commit c30ef03

File tree

3 files changed

+81
-95
lines changed

3 files changed

+81
-95
lines changed

‎src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java‎

Lines changed: 79 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.io.IOException;
2626
import java.util.*;
2727
import java.util.concurrent.ConcurrentHashMap;
28+
import java.util.concurrent.atomic.AtomicInteger;
2829
import java.util.concurrent.atomic.LongAdder;
2930
import java.util.function.Function;
3031
import java.util.function.Predicate;
@@ -59,6 +60,13 @@ public abstract class CsvDataSource {
5960
protected static final String TYPE = "type";
6061
protected static final String FIELDS_TO_VALUES_MAP = "fieldsToValuesMap";
6162

63+
/**
64+
* @deprecated ensures downward compatibility with old csv data format. Can be removed when
65+
* support for old csv format is removed. *
66+
*/
67+
@Deprecated(since = "1.1.0", forRemoval = true)
68+
private boolean notYetLoggedWarning = true;
69+
6270
protected CsvDataSource(String csvSep, String folderPath, FileNamingStrategy fileNamingStrategy) {
6371
this.csvSep = csvSep;
6472
this.connector = new CsvFileConnector(folderPath, fileNamingStrategy);
@@ -81,14 +89,35 @@ protected Map<String, String> buildFieldsToAttributes(
8189
TreeMap<String, String> insensitiveFieldsToAttributes =
8290
new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
8391

92+
// todo when replacing deprecated workaround code below add final modifier before parseCsvRow as
93+
// well as remove
94+
// 'finalFieldVals' and notYetLoggedWarning below!
95+
String[] fieldVals = parseCsvRow(csvRow, csvSep);
96+
97+
// start workaround for deprecated data model processing
98+
if (fieldVals.length != headline.length) {
99+
// try to parse old structure
100+
fieldVals = oldFieldVals(csvSep, csvRow);
101+
// if this works log a warning to inform the user that this will not work much longer,
102+
// otherwise parsing will fail regularly as expected below
103+
if (fieldVals.length == headline.length && notYetLoggedWarning) {
104+
notYetLoggedWarning = false;
105+
log.warn(
106+
"You are using an outdated version of the data "
107+
+ "model with invalid formatted csv rows. This is okay for now, but please updated your files, as the "
108+
+ "support for the old model will be removed soon.");
109+
}
110+
}
111+
// end workaround for deprecated data model processing
112+
84113
try {
85-
final String[] fieldVals = parseCsvRow(csvRow, csvSep);
114+
String[] finalFieldVals = fieldVals;
86115
insensitiveFieldsToAttributes.putAll(
87116
IntStream.range(0, fieldVals.length)
88117
.boxed()
89118
.collect(
90119
Collectors.toMap(
91-
k -> StringUtils.snakeCaseToCamelCase(headline[k]), v -> fieldVals[v])));
120+
k -> StringUtils.snakeCaseToCamelCase(headline[k]), v -> finalFieldVals[v])));
92121

93122
if (insensitiveFieldsToAttributes.size() != headline.length) {
94123
Set<String> fieldsToAttributesKeySet = insensitiveFieldsToAttributes.keySet();
@@ -131,6 +160,54 @@ protected String[] parseCsvRow(String csvRow, String csvSep) {
131160
.toArray(String[]::new);
132161
}
133162

163+
/**
164+
* Build an array of from the provided csv row string considering special cases where geoJson or
165+
* {@link edu.ie3.datamodel.models.input.system.characteristic.CharacteristicInput} are provided
166+
* in the csv row string.
167+
*
168+
* @param csvSep the column separator of the csv row string
169+
* @param csvRow the csv row string
170+
* @return an array with one entry per column of the provided csv row string
171+
* @deprecated only left for downward compatibility. Will be removed in a major release
172+
*/
173+
@Deprecated(since = "1.1.0", forRemoval = true)
174+
protected String[] oldFieldVals(String csvSep, String csvRow) {
175+
176+
/*geo json support*/
177+
final String geoJsonRegex = "\\{.+?}}}";
178+
final String geoReplacement = "geoJSON";
179+
180+
/*characteristic input support */
181+
final String charInputRegex = "(cP:|olm:|cosPhiFixed:|cosPhiP:|qV:)\\{[^}]++}";
182+
final String charReplacement = "charRepl";
183+
184+
/*removes double double quotes*/
185+
List<String> geoList = extractMatchingStrings(geoJsonRegex, csvRow.replace("\"\"", "\""));
186+
List<String> charList = extractMatchingStrings(charInputRegex, csvRow.replace("\"\"", "\""));
187+
188+
AtomicInteger geoCounter = new AtomicInteger(0);
189+
AtomicInteger charCounter = new AtomicInteger(0);
190+
191+
return Arrays.stream(
192+
csvRow
193+
.replaceAll(charInputRegex, charReplacement)
194+
.replaceAll(geoJsonRegex, geoReplacement)
195+
.replaceAll("\"*", "") // remove all quotes from
196+
.split(csvSep, -1))
197+
.map(
198+
fieldVal -> {
199+
String returningFieldVal = fieldVal;
200+
if (fieldVal.equalsIgnoreCase(geoReplacement)) {
201+
returningFieldVal = geoList.get(geoCounter.getAndIncrement());
202+
}
203+
if (fieldVal.equalsIgnoreCase(charReplacement)) {
204+
returningFieldVal = charList.get(charCounter.getAndIncrement());
205+
}
206+
return returningFieldVal.trim();
207+
})
208+
.toArray(String[]::new);
209+
}
210+
134211
/**
135212
* Extracts all strings from the provided csvRow matching the provided regexString and returns a
136213
* list of strings in the order of their appearance in the csvRow string

‎src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy‎

Lines changed: 1 addition & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ class CsvDataSourceTest extends Specification {
8686
"olmcharacteristic",
8787
"cosPhiFixed"
8888
] as String[]
89-
def validCsvRow = "5ebd8f7e-dedb-4017-bb86-6373c4b68eb8,25.0,100.0,0.95,98.0,test_bmTypeInput,50.0,25.0,olm:{(0.0,1.0)},cosPhiFixed:{(0.0,1.0)}"
89+
def validCsvRow = "5ebd8f7e-dedb-4017-bb86-6373c4b68eb8,25.0,100.0,0.95,98.0,test_bmTypeInput,50.0,25.0,\"olm:{(0.0,1.0)}\",\"cosPhiFixed:{(0.0,1.0)}\""
9090

9191
expect:
9292
dummyCsvSource.buildFieldsToAttributes(validCsvRow, validHeadline) == [
@@ -103,97 +103,6 @@ class CsvDataSourceTest extends Specification {
103103
]
104104
}
105105

106-
def "A CsvDataSource should be able to handle deprecated invalid csvRows correctly"() {
107-
expect:
108-
dummyCsvSource.oldFieldVals(csvSep, csvRow) as List == resultingArray
109-
110-
where:
111-
csvSep | csvRow || resultingArray
112-
"," | "4ca90220-74c2-4369-9afa-a18bf068840d,{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}},node_a,2020-03-25T15:11:31Z[UTC],2020-03-24T15:11:31Z[UTC],8f9682df-0744-4b58-a122-f0dc730f6510,true,1,1.0,Höchstspannung,380.0,olm:{(0.00,1.00)},cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}" || [
113-
"4ca90220-74c2-4369-9afa-a18bf068840d",
114-
"{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
115-
"node_a",
116-
"2020-03-25T15:11:31Z[UTC]",
117-
"2020-03-24T15:11:31Z[UTC]",
118-
"8f9682df-0744-4b58-a122-f0dc730f6510",
119-
"true",
120-
"1",
121-
"1.0",
122-
"Höchstspannung",
123-
"380.0",
124-
"olm:{(0.00,1.00)}",
125-
"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
126-
]
127-
"," | "\"4ca90220-74c2-4369-9afa-a18bf068840d\",\"{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}\",\"node_a\",\"2020-03-25T15:11:31Z[UTC]\",\"2020-03-24T15:11:31Z[UTC]\",\"8f9682df-0744-4b58-a122-f0dc730f6510\",\"true\",\"1\",\"1.0\",\"Höchstspannung\",\"380.0\",\"olm:{(0.00,1.00)}\",\"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}\"" || [
128-
"4ca90220-74c2-4369-9afa-a18bf068840d",
129-
"{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
130-
"node_a",
131-
"2020-03-25T15:11:31Z[UTC]",
132-
"2020-03-24T15:11:31Z[UTC]",
133-
"8f9682df-0744-4b58-a122-f0dc730f6510",
134-
"true",
135-
"1",
136-
"1.0",
137-
"Höchstspannung",
138-
"380.0",
139-
"olm:{(0.00,1.00)}",
140-
"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
141-
]
142-
";" | "4ca90220-74c2-4369-9afa-a18bf068840d;cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)};{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}};node_a;2020-03-25T15:11:31Z[UTC];2020-03-24T15:11:31Z[UTC];8f9682df-0744-4b58-a122-f0dc730f6510;true;1;1.0;Höchstspannung;380.0;olm:{(0.00,1.00)};cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}" || [
143-
"4ca90220-74c2-4369-9afa-a18bf068840d",
144-
"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}",
145-
"{(0.0,1.0),(0.9,1.0),(1.2,-0.3)};{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
146-
"node_a",
147-
"2020-03-25T15:11:31Z[UTC]",
148-
"2020-03-24T15:11:31Z[UTC]",
149-
"8f9682df-0744-4b58-a122-f0dc730f6510",
150-
"true",
151-
"1",
152-
"1.0",
153-
"Höchstspannung",
154-
"380.0",
155-
"olm:{(0.00,1.00)}",
156-
"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
157-
]
158-
";" | "\"4ca90220-74c2-4369-9afa-a18bf068840d\";\"{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}\";\"node_a\";\"2020-03-25T15:11:31Z[UTC]\";\"2020-03-24T15:11:31Z[UTC]\";\"8f9682df-0744-4b58-a122-f0dc730f6510\";\"true\";\"1\";\"1.0\";\"Höchstspannung\";\"380.0\";\"olm:{(0.00,1.00)}\";\"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}\"" || [
159-
"4ca90220-74c2-4369-9afa-a18bf068840d",
160-
"{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
161-
"node_a",
162-
"2020-03-25T15:11:31Z[UTC]",
163-
"2020-03-24T15:11:31Z[UTC]",
164-
"8f9682df-0744-4b58-a122-f0dc730f6510",
165-
"true",
166-
"1",
167-
"1.0",
168-
"Höchstspannung",
169-
"380.0",
170-
"olm:{(0.00,1.00)}",
171-
"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
172-
]
173-
"," | "66275bfd-978b-4974-9f73-f270165a6351,Standard,f18a5a9b-6d45-4843-be12-be6d12de0e6b,{\"type\":\"LineString\",\"coordinates\":[[7.4116482,51.4843281],[7.4116482,51.4843281]],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}},{\"type\":\"Point\",\"coordinates\":[0.25423729,0.75409836],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:0\"}}}\"" || [
174-
"66275bfd-978b-4974-9f73-f270165a6351",
175-
"Standard",
176-
"f18a5a9b-6d45-4843-be12-be6d12de0e6b",
177-
"{\"type\":\"LineString\",\"coordinates\":[[7.4116482,51.4843281],[7.4116482,51.4843281]],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
178-
"{\"type\":\"Point\",\"coordinates\":[0.25423729,0.75409836],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:0\"}}}"
179-
]
180-
"," | "4ca90220-74c2-4369-9afa-a18bf068840d,{\"\"type\"\":\"\"Point\"\",\"\"coordinates\"\":[7.411111,51.492528],\"\"crs\"\":{\"\"type\"\":\"\"name\"\",\"\"properties\"\":{\"\"name\"\":\"\"EPSG:4326\"\"}}},node_a,2020-03-25T15:11:31Z[UTC],2020-03-24T15:11:31Z[UTC],8f9682df-0744-4b58-a122-f0dc730f6510,true,1,1.0,Höchstspannung,380.0,\"olm:{(0.00,1.00)}\",\"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}\"" || [
181-
"4ca90220-74c2-4369-9afa-a18bf068840d",
182-
"{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
183-
"node_a",
184-
"2020-03-25T15:11:31Z[UTC]",
185-
"2020-03-24T15:11:31Z[UTC]",
186-
"8f9682df-0744-4b58-a122-f0dc730f6510",
187-
"true",
188-
"1",
189-
"1.0",
190-
"Höchstspannung",
191-
"380.0",
192-
"olm:{(0.00,1.00)}",
193-
"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
194-
]
195-
}
196-
197106
def "A CsvDataSource should be able to handle a variety of different csvRows correctly"() {
198107
expect:
199108
dummyCsvSource.parseCsvRow(csvRow, csvSep) as List == resultingArray

‎src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvRawGridSourceTest.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -723,7 +723,7 @@ class CsvRawGridSourceTest extends Specification implements CsvTestDataMeta {
723723
then: "all elements are there"
724724
actual.present
725725
actual.get().with {
726-
/* It's okay, to only test the uuids, because content is tested with the other test mehtods */
726+
/* It's okay, to only test the uuids, because content is tested with the other test methods */
727727
assert nodes.size() == expected.nodes.size()
728728
assert nodes.each {entry -> expected.nodes.contains({it.uuid == entry.uuid})}
729729
assert lines.size() == expected.lines.size()

0 commit comments

Comments
 (0)