Add checks for additional invalid metadata

checksit/generic.py

@@ -397,6 +397,7 @@ def check_var(
     variable: Union[str, List[str]],
     defined_attrs: List[str],
     rules_attrs: Optional[Dict[str, str]] = None,
+    additional_attrs_allowed: bool = True,
     skip_spellcheck: bool = False,
 ) -> Tuple[List[str], List[str]]:
     """Check variable exists and attributes defined and/or meet rules.
@@ -416,6 +417,8 @@ def check_var(
           against, and any options needed, as string value (e.g.
           "rule-func:string-of-length:3+"). See documentation on the `check` function
           in the `Rules` class for more information on formatting.
+        additional_attrs_allowed: if False, will return an error if variable has
+          any attributes not defined in `defined_attrs` or `rules_attrs`. Default True.
         skip_spellcheck: skip looking for close misspelling of attribute if not found
           in variable. Default False.
 
@@ -616,6 +619,28 @@ def check_var(
                     errors.extend(rule_errors)
                     warnings.extend(rule_warnings)
 
+    if not additional_attrs_allowed and variable in dct["variables"].keys():
+        all_allowed_attrs = []
+        for attr in defined_attrs:
+            if isinstance(attr, dict) and len(attr.keys()) == 1:
+                for key in attr.keys():
+                    all_allowed_attrs.append(key.split(":")[0])
+            else:
+                all_allowed_attrs.append(attr.split(":")[0])
+        for attr in rules_attrs:
+            if isinstance(attr, dict) and len(attr.keys()) == 1:
+                for key in attr.keys():
+                    all_allowed_attrs.append(key.split(":")[0])
+            else:
+                all_allowed_attrs.append(attr.split(":")[0])
+        if "qc_flag" in variable and "flag_meanings" not in all_allowed_attrs:
+            all_allowed_attrs.append("flag_meanings")
+        for attr in dct["variables"][variable].keys():
+            if attr not in all_allowed_attrs:
+                errors.append(
+                    f"[variable**************:{variable}]: Attribute '{attr}' in variable {variable} is not allowed."
+                )
+
     return errors, warnings
 
 
@@ -981,5 +1006,37 @@ def check_radar_moment_variables(
                 errors.append(
                     f"[variable:**************:{variable}]: Only one of '{attr_options}' should be defined, {matches} found."
                 )
+    return errors, warnings
 
+
+def check_defined_only(
+    dct: Dict[str, Dict[str, Any]],
+    all_global_attrs: List[str],
+    all_dimensions: List[str],
+    all_variables: List[str],
+    skip_spellcheck: bool = False,
+ ):
+    """Checks that only defined global attributes, dimensions and variables are present.
+
+    Args:
+        dct: dictionary of file data, as made by the `to_dict()` function in each
+          reader class, with "variables", "dimensions" and "global_attributes" as keys.
+        all_global_attrs: list of all allowed global attributes.
+        all_dimensions: list of all allowed dimensions.
+        all_variables: list of all allowed variables.
+
+    Returns:
+        A list of errors and a list of warnings
+    """
+    errors = []
+    warnings = []
+    for attr in dct['global_attributes']:
+        if attr not in all_global_attrs:
+            errors.append(f"[global-attributes:**************:{attr}]: Invalid global attribute '{attr}' found in file.")
+    for dim in dct['dimensions']:
+        if dim not in all_dimensions:
+            errors.append(f"[dimension**************:{dim}]: Invalid dimension '{dim}' found in file.")
+    for var in dct['variables']:
+        if var not in all_variables:
+            errors.append(f"[variable**************:{var}]: Invalid variable '{var}' found in file.")
     return errors, warnings

checksit/readers/cdl.py

@@ -55,6 +55,7 @@ def __init__(
         """
         self.inpt = inpt
         self.verbose = verbose
+        self.fmt_errors = []
         self._parse(inpt)
         self._check_format()
 
@@ -98,8 +99,6 @@ def _parse(self, inpt: str) -> None:
         self.variables, self.global_attrs = self._split_vars_globals(sections[1])
 
     def _check_format(self) -> None:
-        self.fmt_errors = []
-
         source = self.global_attrs.get("source", "UNDEFINED")
 
         min_chars = 10
@@ -248,10 +247,20 @@ def _construct_variables(self, content: List[str]) -> Dict[str, Dict[str, str]]:
             else:
                 #                key, value = [x.strip() for x in line.split(":", 1)[1].split("=", 1)]
                 # Send last key and last value (from last iteration of loop) and line to get new value
-                key, value = self._parse_key_value_multiline_safe(
+                key, value, new_key = self._parse_key_value_multiline_safe(
                     line, key, value, variable_attr=True
                 )
-                current[key] = self._safe_parse_value(value)
+                if new_key and key in current.keys():
+                    if current[key] != self._safe_parse_value(value) and self.verbose:
+                        print(
+                            f"[WARNING] Variable attribute '{key}' for variable '{var_id}' already exists,"
+                            f" not overwriting existing value '{current[key]}' with new value '{value}'"
+                        )
+                    self.fmt_errors.append(
+                        f"[DUPLICATE:variable:{var_id}:{key}] Variable attribute '{key}' for variable '{var_id}' defined multiple times"
+                    )
+                else:
+                    current[key] = self._safe_parse_value(value)
 
             last_line = line
         else:
@@ -261,7 +270,7 @@ def _construct_variables(self, content: List[str]) -> Dict[str, Dict[str, str]]:
 
     def _parse_key_value_multiline_safe(
         self, line: str, last_key: str, last_value: str, variable_attr: bool = False
-    ) -> Tuple[str, str]:
+    ) -> Tuple[str, str, bool]:
         """Cater for values over multiple lines in CDL files.
 
         If an attribute value is printed over multiple lines in the CDL file, this
@@ -270,15 +279,17 @@ def _parse_key_value_multiline_safe(
         # Caters for continuation lines for arrays of strings, etc
         if "=" in line:
             # A new (key, value) pair is found
+            new_key = True
             if variable_attr:  # var attr
                 key, value = [x.strip() for x in line.split(":", 1)[1].split("=", 1)]
             else:  # global attr
                 key, value = [x.strip() for x in line.lstrip(":").split("=", 1)]
         else:
             # Assume a continuation of th last value, so set key to None
+            new_key = False
             key, value = last_key, last_value + " " + line.strip().rstrip(";")
 
-        return key, value
+        return key, value, new_key
 
     def _ordered_dict(self, content: List[str]) -> Dict[str, str]:
         """Construct a dictionary from a list of attribute string.
@@ -311,7 +322,7 @@ def _ordered_dict(self, content: List[str]) -> Dict[str, str]:
             # Assume a continuation of th last value
             #                value += " " + line.strip()
             # Send last key and last value (from last iteration of loop) and line to get new value
-            key, value = self._parse_key_value_multiline_safe(line, key, value)
+            key, value, _ = self._parse_key_value_multiline_safe(line, key, value)
 
             # This will overwrite the previous value - which is safe if a continuation happened
             # as the key is the same as last time
@@ -332,7 +343,7 @@ def to_yaml(self) -> str:
             sort_keys=False,
         )
 
-    def to_dict(self) -> Dict[str, Union[Dict[str, str], Dict[str, Dict[str, str]], str]]:
+    def to_dict(self) -> Dict[str, Union[Dict[str, str], Dict[str, Dict[str, str]], str, List[str]]]:
         """Return the parsed CDL content as a dictionary.
 
         Returns: