diff --git a/csv_detective/parsing/columns.py b/csv_detective/parsing/columns.py index 81f4461..edd97b4 100755 --- a/csv_detective/parsing/columns.py +++ b/csv_detective/parsing/columns.py @@ -166,7 +166,10 @@ def build_remaining_tests_per_col(return_table: pd.DataFrame) -> dict[str, list[ for col in table.columns } handle_empty_columns(return_table) - empty_cols = {col for col in table.columns if table[col].dropna().empty} + empty_cols = ( + # if NA values should be considered valid, then we can skip empty columns + {col for col in table.columns if table[col].isna().all()} if skipna else {} + ) remaining_tests_per_col = build_remaining_tests_per_col(return_table) # hashing rows to get nb_duplicates @@ -217,7 +220,7 @@ def build_remaining_tests_per_col(return_table: pd.DataFrame) -> dict[str, list[ fill_value=0, ) for col in list(empty_cols): - if not batch[col].dropna().empty: + if not batch[col].isna().all(): empty_cols.discard(col) remaining_tests_per_col[col] = [ fmt_label