V2.9.5

file_checker_exec/pom.xml

@@ -4,7 +4,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>fr.ifremer</groupId>
 	<artifactId>file_checker_exec</artifactId>
-	<version>2.9.4</version>
+	<version>2.9.5</version>
 
 	<name>Argo NetCDF file format checker</name>
 

file_checker_exec/src/main/java/fr/coriolis/checker/specs/ArgoReferenceTable.java

@@ -151,7 +151,7 @@ public static enum DACS {
 	static {
 		DacCenterCodes.put(DACS.AOML, "AO MB NA PM SI UW WH");
 		DacCenterCodes.put(DACS.BODC, "BO");
-		DacCenterCodes.put(DACS.CORIOLIS, "AW GE IO IF LV RU SP VL");
+		DacCenterCodes.put(DACS.CORIOLIS, "AW GE IO IF LV RU SP VL DK EA");
 		DacCenterCodes.put(DACS.CSIO, "HZ");
 		DacCenterCodes.put(DACS.CSIRO, "CS");
 		DacCenterCodes.put(DACS.INCOIS, "IN");

file_checker_exec/src/main/java/fr/coriolis/checker/validators/ArgoMetadataFileValidator.java

@@ -1,12 +1,16 @@
 package fr.coriolis.checker.validators;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -155,7 +159,7 @@ public boolean validateData(boolean ckNulls, boolean... optionalChecks) throws I
 
 	private void validateOptionalParams() {
 		// PROGRAM_NAME - ref table 41
-		checkOptionalParameterValueAgainstRefTable("PROGRAM_NAME", ArgoReferenceTable.PROGRAM_NAME);
+		checkOptionalParameterValueAgainstRefTable("PROGRAM_NAME", ArgoReferenceTable.PROGRAM_NAME, true);
 	}
 
 	/**
@@ -820,6 +824,12 @@ public void validateMandatory_v3(ArgoReferenceTable.DACS dac) throws IOException
 				validationResult.addError(name + "[" + (n + 1) + "]: '" + str + "': Invalid");
 			}
 		}
+		// check unicity in PARAMETER entries :
+		Set<String> duplicateParameters = checkForDuplicate(paramVar);
+		if (duplicateParameters.size() > 0) {
+			validationResult.addWarning(
+					"PARAMETER variable contains duplicate values: [" + String.join(", ", duplicateParameters) + "]");
+		}
 
 		name = "PARAMETER_UNITS"; // ..not empty
 		paramVar = arFile.readStringArr(name);
@@ -886,53 +896,23 @@ public void validateMandatory_v3(ArgoReferenceTable.DACS dac) throws IOException
 			ArgoReferenceTable.ArgoReferenceEntry snsrInfo;
 
 			// ..check SENSOR
-			boolean snsrValid = false;
 			String snsr = sensor[n].trim();
-			log.debug(sensorName + "[{}]: '{}'", n, snsr);
-
-			if ((snsrInfo = ArgoReferenceTable.SENSOR.contains(snsr)).isValid()) {
-				snsrValid = true;
-				if (snsrInfo.isDeprecated) {
-					validationResult
-							.addWarning(sensorName + "[" + (n + 1) + "]: '" + snsr + "' Status: " + snsrInfo.message);
-				}
-
-			} else {
-				validationResult.addError(sensorName + "[" + (n + 1) + "]: '" + snsr + "' Status: " + snsrInfo.message);
-			}
+			snsrInfo = ArgoReferenceTable.SENSOR.contains(snsr);
+			boolean snsrValid = checkParameterValueAgainstRefTable(sensorName + "[" + (n + 1) + "]", snsr,
+					ArgoReferenceTable.SENSOR, false);
 
 			// ..check SENSOR_MAKER
 			String snsrMaker = sensorMaker[n].trim();
-			boolean smkrValid = false;
+			mkrInfo = ArgoReferenceTable.SENSOR_MAKER.contains(snsrMaker);
+			boolean smkrValid = checkParameterValueAgainstRefTable(sensorMakerName + "[" + (n + 1) + "]", snsrMaker,
+					ArgoReferenceTable.SENSOR_MAKER, false);
 			log.debug(sensorMakerName + "[{}]: '{}'", n, snsrMaker);
 
-			if ((mkrInfo = ArgoReferenceTable.SENSOR_MAKER.contains(snsrMaker)).isValid()) {
-				smkrValid = true;
-				if (mkrInfo.isDeprecated) {
-					validationResult.addWarning(
-							sensorMakerName + "[" + (n + 1) + "]: '" + snsrMaker + "' Status: " + mkrInfo.message);
-				}
-
-			} else {
-				validationResult.addError(
-						sensorMakerName + "[" + (n + 1) + "]: '" + snsrMaker + "' Status: " + mkrInfo.message);
-			}
-
 			// ..check SENSOR_MODEL
 			String snsrModel = sensorModel[n].trim();
-			boolean mdlValid = false;
-			log.debug(sensorModelName + "[{}]: '{}'", n, snsrModel);
-
-			if ((mdlInfo = ArgoReferenceTable.SENSOR_MODEL.contains(snsrModel)).isValid()) {
-				mdlValid = true;
-				if (mdlInfo.isDeprecated) {
-					validationResult.addWarning(
-							sensorModelName + "[" + (n + 1) + "]: '" + snsrModel + "' Status: " + mdlInfo.message);
-				}
-			} else {
-				validationResult.addError(
-						sensorModelName + "[" + (n + 1) + "]: '" + snsrModel + "' Status: " + mdlInfo.message);
-			}
+			mdlInfo = ArgoReferenceTable.SENSOR_MODEL.contains(snsrModel);
+			boolean mdlValid = checkParameterValueAgainstRefTable(sensorModelName + "[" + (n + 1) + "]", snsrModel,
+					ArgoReferenceTable.SENSOR_MODEL, false);
 
 			// ..cross-reference SENSOR_MODEL / SENSOR_MAKER
 			if (smkrValid && mdlValid) {
@@ -966,6 +946,12 @@ public void validateMandatory_v3(ArgoReferenceTable.DACS dac) throws IOException
 				}
 			}
 		}
+		// check unicity in SENSOR entries :
+		Set<String> duplicateSensors = checkForDuplicate(sensor);
+		if (duplicateSensors.size() > 0) {
+			validationResult.addWarning(
+					"SENSOR variable contains duplicate values: [" + String.join(", ", duplicateSensors) + "]");
+		}
 
 		// ..........per-positioning_system checks
 		String[] positVar;
@@ -1023,27 +1009,56 @@ public void validateMandatory_v3(ArgoReferenceTable.DACS dac) throws IOException
 		log.debug("....validateMandatory_v3: end.....");
 	} // ..end validateMandatory_v3
 
-	private void checkParameterValueAgainstRefTable(String parameterName, StringTable refTable) {
+	private boolean checkParameterValueAgainstRefTable(String parameterName, String parameterValue,
+			StringTable refTable, boolean warningOnly) {
 		ArgoReferenceTable.ArgoReferenceEntry info;
-		String parameterValue = arFile.readString(parameterName).trim();
 
 		log.debug("{}: '{}'", parameterName, parameterValue);
 		if ((info = refTable.contains(parameterValue)).isValid()) {
 			if (info.isDeprecated) {
 				validationResult.addWarning(parameterName + ": '" + parameterValue + "' Status: " + info.message);
 			}
+			return true;
 
 		} else {
-			validationResult.addError(parameterName + ": '" + parameterValue + "' Status: " + info.message);
+			String resultMessage = parameterName + ": '" + parameterValue + "' Status: " + info.message
+					+ " (not in reference table)";
+			if (warningOnly) {
+				validationResult.addWarning(resultMessage);
+			} else {
+				validationResult.addError(resultMessage);
+			}
+
+			return false;
 		}
 	}
 
-	private void checkOptionalParameterValueAgainstRefTable(String parameterName, StringTable refTable) {
+	private boolean checkOptionalParameterValueAgainstRefTable(String parameterName, StringTable refTable,
+			boolean warningOnly) {
 
 		Variable dataVar = arFile.getNcReader().findVariable(parameterName);
 		if (dataVar != null) {
-			checkParameterValueAgainstRefTable(parameterName, refTable);
+			String parameterValue = arFile.readString(parameterName).trim();
+			return checkParameterValueAgainstRefTable(parameterName, parameterValue, refTable, warningOnly);
 		}
+		return true;
+	}
+
+	/**
+	 * Check if a list of values contains duplicates
+	 * 
+	 * @param paramValuesList (String[]) list of values to check
+	 * @return Set of values which are found multiple time in the list
+	 */
+	private Set<String> checkForDuplicate(String[] paramValuesList) {
+		// for each value of the list, count the number of times it appears
+		Map<String, Long> count = Arrays.stream(paramValuesList).map(String::trim)
+				.collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));
+		// build the list of value which appears more than one time
+		Set<String> doublons = count.entrySet().stream().filter(e -> e.getValue() > 1).map(Map.Entry::getKey)
+				.collect(Collectors.toSet());
+
+		return doublons;
 	}
 
 	/**

file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/TestsUtils.java

@@ -33,42 +33,45 @@ public static void init(Class<?> clazz) {
 		}
 	}
 
-	public static void genericFileCheckerE2ETest(String fileName, String dac, String result, String phase,
-			String testDirName, String options) throws IOException, InterruptedException {
+	private static String executeJarAndGetResult(String fileName, String dac, String testDirName, String options)
+			throws IOException, InterruptedException {
 
-		// ARANGE
-		String inPutDirPath = TestsUtils.TEST_FILES_DIR + "/" + testDirName;
-		File intputDir = new File(inPutDirPath);
-		File testFile = new File(inPutDirPath + "/" + fileName);
-		// before executing jar, verify file and dir exists :
+		// ARRANGE
+		String inputDirPath = TestsUtils.TEST_FILES_DIR + "/" + testDirName;
+		File inputDir = new File(inputDirPath);
+		File testFile = new File(inputDirPath + "/" + fileName);
+
+		// pre-checks
 		assertThat(TestsUtils.jarFile).exists().isFile().as("jar should be created in target folder");
-		;
 		assertThat(testFile).exists().isFile().as("netcdf test file should be in test/netcdf/TEST* resources folder");
 		assertThat(TestsUtils.specDirDir).exists().isDirectory().as("specifications directory should exist");
-		assertThat(intputDir).exists().isDirectory().as("input directory should exist");
+		assertThat(inputDir).exists().isDirectory().as("input directory should exist");
 		assertThat(TestsUtils.outputDir).exists().isDirectory().as("output directory should exist");
 
 		// ACT
 		ProcessBuilder builder = new ProcessBuilder("java", "-jar", TestsUtils.jarPath, options, dac,
-				TestsUtils.SPEC_DIR_PATH, TestsUtils.OUTPUT_DIR_PATH, inPutDirPath, fileName);
+				TestsUtils.SPEC_DIR_PATH, TestsUtils.OUTPUT_DIR_PATH, inputDirPath, fileName);
 		builder.redirectErrorStream(true);
 		Process process = builder.start();
-		process.waitFor();
 
-		// ASSESS
-		// No error
+		// ASSERT - common checks
 		int exitCode = process.waitFor();
 		assertThat(exitCode).isZero().as("execution should complete without errors");
-		// result file created
-		File xmlResultFile = new File(TestsUtils.OUTPUT_DIR_PATH + "\\" + fileName + ".filecheck");
+
+		File xmlResultFile = new File(TestsUtils.OUTPUT_DIR_PATH + "/" + fileName + ".filecheck");
 		assertThat(xmlResultFile).exists().isFile().as("Result file should be created in %s",
 				TestsUtils.OUTPUT_DIR_PATH);
-		// expected status
-		String content = String.join("\n", Files.readAllLines(xmlResultFile.toPath()));
-		assertThat(content).isNotEmpty().contains("<status>" + result);
-		// in the expected
-		assertThat(content).isNotEmpty().contains("<phase>" + phase);
 
+		return String.join("\n", Files.readAllLines(xmlResultFile.toPath()));
+	}
+
+	// ============== CHECK RESULT =================
+	public static void genericFileCheckerE2ETest(String fileName, String dac, String result, String phase,
+			String testDirName, String options) throws IOException, InterruptedException {
+
+		String content = executeJarAndGetResult(fileName, dac, testDirName, options);
+
+		assertThat(content).isNotEmpty().contains("<status>" + result).contains("<phase>" + phase);
 	}
 
 	// We use an overloaded method to provide a default value for the last argument
@@ -80,4 +83,34 @@ public static void genericFileCheckerE2ETest(String fileName, String dac, String
 
 	}
 
+	// ============== CHECK WARNINGS =================
+
+	public static void e2eTestWarningPresence(String fileName, String dac, String warningMessage, String testDirName,
+			String options) throws IOException, InterruptedException {
+
+		String content = executeJarAndGetResult(fileName, dac, testDirName, options);
+
+		assertThat(content).isNotEmpty().contains("<warning>" + warningMessage);
+	}
+
+	public static void e2eTestWarningAbsence(String fileName, String dac, String testDirName, String options)
+			throws IOException, InterruptedException {
+
+		String content = executeJarAndGetResult(fileName, dac, testDirName, options);
+
+		assertThat(content).isNotEmpty().contains("<warnings number=\"0\"/>");
+	}
+
+	public static void e2eTestWarningPresence(String fileName, String dac, String warningMessage, String testDirName)
+			throws IOException, InterruptedException {
+
+		e2eTestWarningPresence(fileName, dac, warningMessage, testDirName, "-no-name-check");
+	}
+
+	public static void e2eTestWarningAbsence(String fileName, String dac, String testDirName)
+			throws IOException, InterruptedException {
+
+		e2eTestWarningAbsence(fileName, dac, testDirName, "-no-name-check");
+	}
+
 }

file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/ValidateMetaDuplicateSensorParameterIT.java

@@ -0,0 +1,42 @@
+package fr.coriolis.checker.e2etests;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+@DisplayName("Check if SENSOR and PARAMETER don't contain duplicate value")
+class ValidateMetaDuplicateSensorParameterIT {
+
+	private final String TEST_DIR_NAME = "TEST_META_0003";
+
+	@BeforeAll
+	public static void init() {
+		TestsUtils.init(ValidateMetaDuplicateSensorParameterIT.class);
+
+	}
+
+	@Tag(TEST_DIR_NAME)
+	@ParameterizedTest(name = "{0} from dac {1} should have status {2} at phase {3}")
+	@CsvSource({ "2903795_meta_duplicate_PARAMETER.nc,coriolis,FILE-ACCEPTED,DATA-VALIDATION" })
+	void fileChecker_shouldAcceptMetaFile_WhenDuplicateInSensorOrParameter(String fileName, String dac, String result,
+			String phase) throws IOException, InterruptedException {
+
+		TestsUtils.genericFileCheckerE2ETest(fileName, dac, result, phase, TEST_DIR_NAME);
+
+	}
+
+	@Tag(TEST_DIR_NAME)
+	@ParameterizedTest(name = "{0} from dac {1} should have warning {2}")
+	@CsvSource(delimiter = '|', value = {
+			"2903795_meta_duplicate_PARAMETER.nc|coriolis|PARAMETER variable contains duplicate values: [PPOX_DOXY, DOXY]",
+			"5902129_meta_duplicate_PARAMETER_SENSOR.nc|coriolis|SENSOR variable contains duplicate values: [CTD_TEMP, CTD_PRES]" })
+	void fileChecker_ShouldRaiseWarning_WhenDuplicateInSensorOrParamete(String fileName, String dac,
+			String warningMessage) throws IOException, InterruptedException {
+		TestsUtils.e2eTestWarningPresence(fileName, dac, warningMessage, TEST_DIR_NAME);
+	}
+
+}

file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/ValidateOptionalParametersValueCheckIT.java

@@ -22,7 +22,7 @@ public static void init() {
 	@Tag(TEST_DIR_NAME)
 	@ParameterizedTest(name = "{0} from dac {1} should have status {2} at phase {3}")
 	@CsvSource({ "6903281_meta_PROGRAM_NAME_in_ref-table-41.nc,coriolis,FILE-ACCEPTED,DATA-VALIDATION",
-			"6903281_meta_PROGRAM_NAME_Not-in_ref-table-41.nc,coriolis,FILE-REJECTED,DATA-VALIDATION" })
+			"6903281_meta_PROGRAM_NAME_Not-in_ref-table-41.nc,coriolis,FILE-ACCEPTED,DATA-VALIDATION" })
 	void fileChecker_shouldAcceptPROGRAMNAME_WhenConformToRefTable41(String fileName, String dac, String result,
 			String phase) throws IOException, InterruptedException {
 

file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/validateProgramNameCheckInMetaFileIT.java

@@ -0,0 +1,51 @@
+package fr.coriolis.checker.e2etests;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+@DisplayName("Check PROGRAM_NAME against reference table")
+class validateProgramNameCheckInMetaFileIT {
+
+	private final String TEST_DIR_NAME = "TEST_META_0004";
+
+	@BeforeAll
+	public static void init() {
+		TestsUtils.init(validateProgramNameCheckInMetaFileIT.class);
+
+	}
+
+	@Tag(TEST_DIR_NAME)
+	@ParameterizedTest(name = "{0} from dac {1} should have status {2} at phase {3}")
+	@CsvSource({ "1902735_meta_bad_programName.nc,coriolis,FILE-ACCEPTED,DATA-VALIDATION",
+			"1902735_meta_empty_programName.nc,coriolis,FILE-ACCEPTED,DATA-VALIDATION" })
+	void fileChecker_shouldAcceptMetaFile_WhenBadProgramNameValue(String fileName, String dac, String result,
+			String phase) throws IOException, InterruptedException {
+
+		TestsUtils.genericFileCheckerE2ETest(fileName, dac, result, phase, TEST_DIR_NAME);
+
+	}
+
+	@Tag(TEST_DIR_NAME)
+	@ParameterizedTest(name = "{0} from dac {1} should have warning {2}")
+	@CsvSource(delimiter = '|', value = {
+			"1902735_meta_bad_programName.nc|coriolis|PROGRAM_NAME: 'test de program name' Status: Invalid (not in reference table)",
+			"1902735_meta_empty_programName.nc|coriolis|PROGRAM_NAME: '' Status: Invalid (not in reference table)" })
+	void fileChecker_ShouldRaiseWarning_WhenBadProgramNameValue(String fileName, String dac, String warningMessage)
+			throws IOException, InterruptedException {
+		TestsUtils.e2eTestWarningPresence(fileName, dac, warningMessage, TEST_DIR_NAME);
+	}
+
+	@Tag(TEST_DIR_NAME)
+	@ParameterizedTest(name = "{0} from dac {1} should not have warning")
+	@CsvSource({ "5907141_meta_good_programName.nc,coriolis" })
+	void fileChecker_ShouldNotRaiseWarning_WhenGoodProgramNameValue(String fileName, String dac)
+			throws IOException, InterruptedException {
+		TestsUtils.e2eTestWarningAbsence(fileName, dac, TEST_DIR_NAME);
+	}
+
+}