diff --git a/file_checker_exec/pom.xml b/file_checker_exec/pom.xml index 8cab763..4a33159 100644 --- a/file_checker_exec/pom.xml +++ b/file_checker_exec/pom.xml @@ -4,7 +4,7 @@ 4.0.0 fr.ifremer file_checker_exec - 2.9.4 + 2.9.5 Argo NetCDF file format checker diff --git a/file_checker_exec/src/main/java/fr/coriolis/checker/specs/ArgoReferenceTable.java b/file_checker_exec/src/main/java/fr/coriolis/checker/specs/ArgoReferenceTable.java index 543eb8b..97dc5df 100644 --- a/file_checker_exec/src/main/java/fr/coriolis/checker/specs/ArgoReferenceTable.java +++ b/file_checker_exec/src/main/java/fr/coriolis/checker/specs/ArgoReferenceTable.java @@ -151,7 +151,7 @@ public static enum DACS { static { DacCenterCodes.put(DACS.AOML, "AO MB NA PM SI UW WH"); DacCenterCodes.put(DACS.BODC, "BO"); - DacCenterCodes.put(DACS.CORIOLIS, "AW GE IO IF LV RU SP VL"); + DacCenterCodes.put(DACS.CORIOLIS, "AW GE IO IF LV RU SP VL DK EA"); DacCenterCodes.put(DACS.CSIO, "HZ"); DacCenterCodes.put(DACS.CSIRO, "CS"); DacCenterCodes.put(DACS.INCOIS, "IN"); diff --git a/file_checker_exec/src/main/java/fr/coriolis/checker/validators/ArgoMetadataFileValidator.java b/file_checker_exec/src/main/java/fr/coriolis/checker/validators/ArgoMetadataFileValidator.java index f32afbd..d9a09f5 100644 --- a/file_checker_exec/src/main/java/fr/coriolis/checker/validators/ArgoMetadataFileValidator.java +++ b/file_checker_exec/src/main/java/fr/coriolis/checker/validators/ArgoMetadataFileValidator.java @@ -1,12 +1,16 @@ package fr.coriolis.checker.validators; import java.io.IOException; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -155,7 +159,7 @@ public boolean validateData(boolean ckNulls, boolean... optionalChecks) throws I private void validateOptionalParams() { // PROGRAM_NAME - ref table 41 - checkOptionalParameterValueAgainstRefTable("PROGRAM_NAME", ArgoReferenceTable.PROGRAM_NAME); + checkOptionalParameterValueAgainstRefTable("PROGRAM_NAME", ArgoReferenceTable.PROGRAM_NAME, true); } /** @@ -820,6 +824,12 @@ public void validateMandatory_v3(ArgoReferenceTable.DACS dac) throws IOException validationResult.addError(name + "[" + (n + 1) + "]: '" + str + "': Invalid"); } } + // check unicity in PARAMETER entries : + Set duplicateParameters = checkForDuplicate(paramVar); + if (duplicateParameters.size() > 0) { + validationResult.addWarning( + "PARAMETER variable contains duplicate values: [" + String.join(", ", duplicateParameters) + "]"); + } name = "PARAMETER_UNITS"; // ..not empty paramVar = arFile.readStringArr(name); @@ -886,53 +896,23 @@ public void validateMandatory_v3(ArgoReferenceTable.DACS dac) throws IOException ArgoReferenceTable.ArgoReferenceEntry snsrInfo; // ..check SENSOR - boolean snsrValid = false; String snsr = sensor[n].trim(); - log.debug(sensorName + "[{}]: '{}'", n, snsr); - - if ((snsrInfo = ArgoReferenceTable.SENSOR.contains(snsr)).isValid()) { - snsrValid = true; - if (snsrInfo.isDeprecated) { - validationResult - .addWarning(sensorName + "[" + (n + 1) + "]: '" + snsr + "' Status: " + snsrInfo.message); - } - - } else { - validationResult.addError(sensorName + "[" + (n + 1) + "]: '" + snsr + "' Status: " + snsrInfo.message); - } + snsrInfo = ArgoReferenceTable.SENSOR.contains(snsr); + boolean snsrValid = checkParameterValueAgainstRefTable(sensorName + "[" + (n + 1) + "]", snsr, + ArgoReferenceTable.SENSOR, false); // ..check SENSOR_MAKER String snsrMaker = sensorMaker[n].trim(); - boolean smkrValid = false; + mkrInfo = ArgoReferenceTable.SENSOR_MAKER.contains(snsrMaker); + boolean smkrValid = checkParameterValueAgainstRefTable(sensorMakerName + "[" + (n + 1) + "]", snsrMaker, + ArgoReferenceTable.SENSOR_MAKER, false); log.debug(sensorMakerName + "[{}]: '{}'", n, snsrMaker); - if ((mkrInfo = ArgoReferenceTable.SENSOR_MAKER.contains(snsrMaker)).isValid()) { - smkrValid = true; - if (mkrInfo.isDeprecated) { - validationResult.addWarning( - sensorMakerName + "[" + (n + 1) + "]: '" + snsrMaker + "' Status: " + mkrInfo.message); - } - - } else { - validationResult.addError( - sensorMakerName + "[" + (n + 1) + "]: '" + snsrMaker + "' Status: " + mkrInfo.message); - } - // ..check SENSOR_MODEL String snsrModel = sensorModel[n].trim(); - boolean mdlValid = false; - log.debug(sensorModelName + "[{}]: '{}'", n, snsrModel); - - if ((mdlInfo = ArgoReferenceTable.SENSOR_MODEL.contains(snsrModel)).isValid()) { - mdlValid = true; - if (mdlInfo.isDeprecated) { - validationResult.addWarning( - sensorModelName + "[" + (n + 1) + "]: '" + snsrModel + "' Status: " + mdlInfo.message); - } - } else { - validationResult.addError( - sensorModelName + "[" + (n + 1) + "]: '" + snsrModel + "' Status: " + mdlInfo.message); - } + mdlInfo = ArgoReferenceTable.SENSOR_MODEL.contains(snsrModel); + boolean mdlValid = checkParameterValueAgainstRefTable(sensorModelName + "[" + (n + 1) + "]", snsrModel, + ArgoReferenceTable.SENSOR_MODEL, false); // ..cross-reference SENSOR_MODEL / SENSOR_MAKER if (smkrValid && mdlValid) { @@ -966,6 +946,12 @@ public void validateMandatory_v3(ArgoReferenceTable.DACS dac) throws IOException } } } + // check unicity in SENSOR entries : + Set duplicateSensors = checkForDuplicate(sensor); + if (duplicateSensors.size() > 0) { + validationResult.addWarning( + "SENSOR variable contains duplicate values: [" + String.join(", ", duplicateSensors) + "]"); + } // ..........per-positioning_system checks String[] positVar; @@ -1023,27 +1009,56 @@ public void validateMandatory_v3(ArgoReferenceTable.DACS dac) throws IOException log.debug("....validateMandatory_v3: end....."); } // ..end validateMandatory_v3 - private void checkParameterValueAgainstRefTable(String parameterName, StringTable refTable) { + private boolean checkParameterValueAgainstRefTable(String parameterName, String parameterValue, + StringTable refTable, boolean warningOnly) { ArgoReferenceTable.ArgoReferenceEntry info; - String parameterValue = arFile.readString(parameterName).trim(); log.debug("{}: '{}'", parameterName, parameterValue); if ((info = refTable.contains(parameterValue)).isValid()) { if (info.isDeprecated) { validationResult.addWarning(parameterName + ": '" + parameterValue + "' Status: " + info.message); } + return true; } else { - validationResult.addError(parameterName + ": '" + parameterValue + "' Status: " + info.message); + String resultMessage = parameterName + ": '" + parameterValue + "' Status: " + info.message + + " (not in reference table)"; + if (warningOnly) { + validationResult.addWarning(resultMessage); + } else { + validationResult.addError(resultMessage); + } + + return false; } } - private void checkOptionalParameterValueAgainstRefTable(String parameterName, StringTable refTable) { + private boolean checkOptionalParameterValueAgainstRefTable(String parameterName, StringTable refTable, + boolean warningOnly) { Variable dataVar = arFile.getNcReader().findVariable(parameterName); if (dataVar != null) { - checkParameterValueAgainstRefTable(parameterName, refTable); + String parameterValue = arFile.readString(parameterName).trim(); + return checkParameterValueAgainstRefTable(parameterName, parameterValue, refTable, warningOnly); } + return true; + } + + /** + * Check if a list of values contains duplicates + * + * @param paramValuesList (String[]) list of values to check + * @return Set of values which are found multiple time in the list + */ + private Set checkForDuplicate(String[] paramValuesList) { + // for each value of the list, count the number of times it appears + Map count = Arrays.stream(paramValuesList).map(String::trim) + .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())); + // build the list of value which appears more than one time + Set doublons = count.entrySet().stream().filter(e -> e.getValue() > 1).map(Map.Entry::getKey) + .collect(Collectors.toSet()); + + return doublons; } /** diff --git a/file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/TestsUtils.java b/file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/TestsUtils.java index 204a6af..2719c8b 100644 --- a/file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/TestsUtils.java +++ b/file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/TestsUtils.java @@ -33,42 +33,45 @@ public static void init(Class clazz) { } } - public static void genericFileCheckerE2ETest(String fileName, String dac, String result, String phase, - String testDirName, String options) throws IOException, InterruptedException { + private static String executeJarAndGetResult(String fileName, String dac, String testDirName, String options) + throws IOException, InterruptedException { - // ARANGE - String inPutDirPath = TestsUtils.TEST_FILES_DIR + "/" + testDirName; - File intputDir = new File(inPutDirPath); - File testFile = new File(inPutDirPath + "/" + fileName); - // before executing jar, verify file and dir exists : + // ARRANGE + String inputDirPath = TestsUtils.TEST_FILES_DIR + "/" + testDirName; + File inputDir = new File(inputDirPath); + File testFile = new File(inputDirPath + "/" + fileName); + + // pre-checks assertThat(TestsUtils.jarFile).exists().isFile().as("jar should be created in target folder"); - ; assertThat(testFile).exists().isFile().as("netcdf test file should be in test/netcdf/TEST* resources folder"); assertThat(TestsUtils.specDirDir).exists().isDirectory().as("specifications directory should exist"); - assertThat(intputDir).exists().isDirectory().as("input directory should exist"); + assertThat(inputDir).exists().isDirectory().as("input directory should exist"); assertThat(TestsUtils.outputDir).exists().isDirectory().as("output directory should exist"); // ACT ProcessBuilder builder = new ProcessBuilder("java", "-jar", TestsUtils.jarPath, options, dac, - TestsUtils.SPEC_DIR_PATH, TestsUtils.OUTPUT_DIR_PATH, inPutDirPath, fileName); + TestsUtils.SPEC_DIR_PATH, TestsUtils.OUTPUT_DIR_PATH, inputDirPath, fileName); builder.redirectErrorStream(true); Process process = builder.start(); - process.waitFor(); - // ASSESS - // No error + // ASSERT - common checks int exitCode = process.waitFor(); assertThat(exitCode).isZero().as("execution should complete without errors"); - // result file created - File xmlResultFile = new File(TestsUtils.OUTPUT_DIR_PATH + "\\" + fileName + ".filecheck"); + + File xmlResultFile = new File(TestsUtils.OUTPUT_DIR_PATH + "/" + fileName + ".filecheck"); assertThat(xmlResultFile).exists().isFile().as("Result file should be created in %s", TestsUtils.OUTPUT_DIR_PATH); - // expected status - String content = String.join("\n", Files.readAllLines(xmlResultFile.toPath())); - assertThat(content).isNotEmpty().contains("" + result); - // in the expected - assertThat(content).isNotEmpty().contains("" + phase); + return String.join("\n", Files.readAllLines(xmlResultFile.toPath())); + } + + // ============== CHECK RESULT ================= + public static void genericFileCheckerE2ETest(String fileName, String dac, String result, String phase, + String testDirName, String options) throws IOException, InterruptedException { + + String content = executeJarAndGetResult(fileName, dac, testDirName, options); + + assertThat(content).isNotEmpty().contains("" + result).contains("" + phase); } // We use an overloaded method to provide a default value for the last argument @@ -80,4 +83,34 @@ public static void genericFileCheckerE2ETest(String fileName, String dac, String } + // ============== CHECK WARNINGS ================= + + public static void e2eTestWarningPresence(String fileName, String dac, String warningMessage, String testDirName, + String options) throws IOException, InterruptedException { + + String content = executeJarAndGetResult(fileName, dac, testDirName, options); + + assertThat(content).isNotEmpty().contains("" + warningMessage); + } + + public static void e2eTestWarningAbsence(String fileName, String dac, String testDirName, String options) + throws IOException, InterruptedException { + + String content = executeJarAndGetResult(fileName, dac, testDirName, options); + + assertThat(content).isNotEmpty().contains(""); + } + + public static void e2eTestWarningPresence(String fileName, String dac, String warningMessage, String testDirName) + throws IOException, InterruptedException { + + e2eTestWarningPresence(fileName, dac, warningMessage, testDirName, "-no-name-check"); + } + + public static void e2eTestWarningAbsence(String fileName, String dac, String testDirName) + throws IOException, InterruptedException { + + e2eTestWarningAbsence(fileName, dac, testDirName, "-no-name-check"); + } + } diff --git a/file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/ValidateMetaDuplicateSensorParameterIT.java b/file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/ValidateMetaDuplicateSensorParameterIT.java new file mode 100644 index 0000000..cd1f21c --- /dev/null +++ b/file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/ValidateMetaDuplicateSensorParameterIT.java @@ -0,0 +1,42 @@ +package fr.coriolis.checker.e2etests; + +import java.io.IOException; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +@DisplayName("Check if SENSOR and PARAMETER don't contain duplicate value") +class ValidateMetaDuplicateSensorParameterIT { + + private final String TEST_DIR_NAME = "TEST_META_0003"; + + @BeforeAll + public static void init() { + TestsUtils.init(ValidateMetaDuplicateSensorParameterIT.class); + + } + + @Tag(TEST_DIR_NAME) + @ParameterizedTest(name = "{0} from dac {1} should have status {2} at phase {3}") + @CsvSource({ "2903795_meta_duplicate_PARAMETER.nc,coriolis,FILE-ACCEPTED,DATA-VALIDATION" }) + void fileChecker_shouldAcceptMetaFile_WhenDuplicateInSensorOrParameter(String fileName, String dac, String result, + String phase) throws IOException, InterruptedException { + + TestsUtils.genericFileCheckerE2ETest(fileName, dac, result, phase, TEST_DIR_NAME); + + } + + @Tag(TEST_DIR_NAME) + @ParameterizedTest(name = "{0} from dac {1} should have warning {2}") + @CsvSource(delimiter = '|', value = { + "2903795_meta_duplicate_PARAMETER.nc|coriolis|PARAMETER variable contains duplicate values: [PPOX_DOXY, DOXY]", + "5902129_meta_duplicate_PARAMETER_SENSOR.nc|coriolis|SENSOR variable contains duplicate values: [CTD_TEMP, CTD_PRES]" }) + void fileChecker_ShouldRaiseWarning_WhenDuplicateInSensorOrParamete(String fileName, String dac, + String warningMessage) throws IOException, InterruptedException { + TestsUtils.e2eTestWarningPresence(fileName, dac, warningMessage, TEST_DIR_NAME); + } + +} diff --git a/file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/ValidateOptionalParametersValueCheckIT.java b/file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/ValidateOptionalParametersValueCheckIT.java index d2c32ea..41421f6 100644 --- a/file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/ValidateOptionalParametersValueCheckIT.java +++ b/file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/ValidateOptionalParametersValueCheckIT.java @@ -22,7 +22,7 @@ public static void init() { @Tag(TEST_DIR_NAME) @ParameterizedTest(name = "{0} from dac {1} should have status {2} at phase {3}") @CsvSource({ "6903281_meta_PROGRAM_NAME_in_ref-table-41.nc,coriolis,FILE-ACCEPTED,DATA-VALIDATION", - "6903281_meta_PROGRAM_NAME_Not-in_ref-table-41.nc,coriolis,FILE-REJECTED,DATA-VALIDATION" }) + "6903281_meta_PROGRAM_NAME_Not-in_ref-table-41.nc,coriolis,FILE-ACCEPTED,DATA-VALIDATION" }) void fileChecker_shouldAcceptPROGRAMNAME_WhenConformToRefTable41(String fileName, String dac, String result, String phase) throws IOException, InterruptedException { diff --git a/file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/validateProgramNameCheckInMetaFileIT.java b/file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/validateProgramNameCheckInMetaFileIT.java new file mode 100644 index 0000000..ac12d96 --- /dev/null +++ b/file_checker_exec/src/test/java/fr/coriolis/checker/e2etests/validateProgramNameCheckInMetaFileIT.java @@ -0,0 +1,51 @@ +package fr.coriolis.checker.e2etests; + +import java.io.IOException; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +@DisplayName("Check PROGRAM_NAME against reference table") +class validateProgramNameCheckInMetaFileIT { + + private final String TEST_DIR_NAME = "TEST_META_0004"; + + @BeforeAll + public static void init() { + TestsUtils.init(validateProgramNameCheckInMetaFileIT.class); + + } + + @Tag(TEST_DIR_NAME) + @ParameterizedTest(name = "{0} from dac {1} should have status {2} at phase {3}") + @CsvSource({ "1902735_meta_bad_programName.nc,coriolis,FILE-ACCEPTED,DATA-VALIDATION", + "1902735_meta_empty_programName.nc,coriolis,FILE-ACCEPTED,DATA-VALIDATION" }) + void fileChecker_shouldAcceptMetaFile_WhenBadProgramNameValue(String fileName, String dac, String result, + String phase) throws IOException, InterruptedException { + + TestsUtils.genericFileCheckerE2ETest(fileName, dac, result, phase, TEST_DIR_NAME); + + } + + @Tag(TEST_DIR_NAME) + @ParameterizedTest(name = "{0} from dac {1} should have warning {2}") + @CsvSource(delimiter = '|', value = { + "1902735_meta_bad_programName.nc|coriolis|PROGRAM_NAME: 'test de program name' Status: Invalid (not in reference table)", + "1902735_meta_empty_programName.nc|coriolis|PROGRAM_NAME: '' Status: Invalid (not in reference table)" }) + void fileChecker_ShouldRaiseWarning_WhenBadProgramNameValue(String fileName, String dac, String warningMessage) + throws IOException, InterruptedException { + TestsUtils.e2eTestWarningPresence(fileName, dac, warningMessage, TEST_DIR_NAME); + } + + @Tag(TEST_DIR_NAME) + @ParameterizedTest(name = "{0} from dac {1} should not have warning") + @CsvSource({ "5907141_meta_good_programName.nc,coriolis" }) + void fileChecker_ShouldNotRaiseWarning_WhenGoodProgramNameValue(String fileName, String dac) + throws IOException, InterruptedException { + TestsUtils.e2eTestWarningAbsence(fileName, dac, TEST_DIR_NAME); + } + +} diff --git a/file_checker_exec/src/test/netcdf-test-files/TEST_META_0003/2903795_meta_duplicate_PARAMETER.nc b/file_checker_exec/src/test/netcdf-test-files/TEST_META_0003/2903795_meta_duplicate_PARAMETER.nc new file mode 100644 index 0000000..4db1bb5 Binary files /dev/null and b/file_checker_exec/src/test/netcdf-test-files/TEST_META_0003/2903795_meta_duplicate_PARAMETER.nc differ diff --git a/file_checker_exec/src/test/netcdf-test-files/TEST_META_0003/5902129_meta_duplicate_PARAMETER_SENSOR.nc b/file_checker_exec/src/test/netcdf-test-files/TEST_META_0003/5902129_meta_duplicate_PARAMETER_SENSOR.nc new file mode 100644 index 0000000..5f8f77b Binary files /dev/null and b/file_checker_exec/src/test/netcdf-test-files/TEST_META_0003/5902129_meta_duplicate_PARAMETER_SENSOR.nc differ diff --git a/file_checker_exec/src/test/netcdf-test-files/TEST_META_0004/1902735_meta_bad_programName.nc b/file_checker_exec/src/test/netcdf-test-files/TEST_META_0004/1902735_meta_bad_programName.nc new file mode 100644 index 0000000..a864e6e Binary files /dev/null and b/file_checker_exec/src/test/netcdf-test-files/TEST_META_0004/1902735_meta_bad_programName.nc differ diff --git a/file_checker_exec/src/test/netcdf-test-files/TEST_META_0004/1902735_meta_empty_programName.nc b/file_checker_exec/src/test/netcdf-test-files/TEST_META_0004/1902735_meta_empty_programName.nc new file mode 100644 index 0000000..8f648b1 Binary files /dev/null and b/file_checker_exec/src/test/netcdf-test-files/TEST_META_0004/1902735_meta_empty_programName.nc differ diff --git a/file_checker_exec/src/test/netcdf-test-files/TEST_META_0004/5907141_meta_good_programName.nc b/file_checker_exec/src/test/netcdf-test-files/TEST_META_0004/5907141_meta_good_programName.nc new file mode 100644 index 0000000..49938b4 Binary files /dev/null and b/file_checker_exec/src/test/netcdf-test-files/TEST_META_0004/5907141_meta_good_programName.nc differ