diff --git a/audb/core/config.py b/audb/core/config.py index 737f91f0..5012b1ca 100644 --- a/audb/core/config.py +++ b/audb/core/config.py @@ -35,7 +35,7 @@ def load_configuration_file(config_file: str) -> dict: return {} with open(config_file) as cf: - config = yaml.load(cf, Loader=yaml.BaseLoader) + config = yaml.load(cf, Loader=yaml.SafeLoader) if config is None: return {} diff --git a/audb/core/define.py b/audb/core/define.py index 4dab8d2f..5352cdcb 100644 --- a/audb/core/define.py +++ b/audb/core/define.py @@ -76,6 +76,7 @@ TIMEOUT = 86400 # 24 h CACHED_VERSIONS_TIMEOUT = 10 # Timeout to acquire access to cached versions LOCK_FILE = ".lock" +COMPLETE_FILE = ".complete" TIMEOUT_MSG = "Lock could not be acquired. Timeout exceeded." diff --git a/audb/core/load.py b/audb/core/load.py index b4af0567..97a0a40d 100644 --- a/audb/core/load.py +++ b/audb/core/load.py @@ -182,6 +182,12 @@ def check() -> bool: return complete if check(): + # Create .complete file to mark database as complete + complete_file = os.path.join(db_root, define.COMPLETE_FILE) + open(complete_file, "w").close() + + # Keep updating the header file for backwards compatibility + # with older versions of audb db_root_tmp = database_tmp_root(db_root) db.meta["audb"]["complete"] = True db_original = audformat.Database.load(db_root, load_data=False) @@ -197,6 +203,14 @@ def check() -> bool: def _database_is_complete( db: audformat.Database, ) -> bool: + # First check for .complete file + if "audb" in db.meta and "root" in db.meta["audb"]: + db_root = db.meta["audb"]["root"] + complete_file = os.path.join(db_root, define.COMPLETE_FILE) + if os.path.exists(complete_file): + return True + + # Fall back to checking header for backwards compatibility complete = False if "audb" in db.meta: if "complete" in db.meta["audb"]: diff --git a/tests/test_load.py b/tests/test_load.py index 3a3f6d61..99985016 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -984,3 +984,39 @@ def test_repository(persistent_repository, name, version, error, error_msg): else: repository = audb.repository(name, version) assert repository == persistent_repository + + +def test_database_complete_file(tmpdir, persistent_repository): + """Test database completeness mechanism with .complete file. + + This test checks if: + 1. A .complete file is created when loading a complete database + 2. The _database_is_complete function returns True when the .complete file exists + 3. The completeness check still works via the header for backward compatibility + """ + # Load a database fully to create the .complete file + db = audb.load(DB_NAME, version="1.0.0", cache_root=tmpdir) + + # Get the database root folder path + db_root = audb.core.cache.database_cache_root(DB_NAME, "1.0.0", cache_root=tmpdir) + + # Check if .complete file exists + complete_file = os.path.join(db_root, audb.core.define.COMPLETE_FILE) + assert os.path.exists(complete_file), "The .complete file was not created" + + # Check if the complete flag is also set in the header (for backward compatibility) + from audb.core.load import _database_is_complete + + assert _database_is_complete(db), "Database is not marked as complete" + + # Remove .complete file but keep the header flag to test backward compatibility + os.remove(complete_file) + assert not os.path.exists(complete_file), "The .complete file could not be removed" + assert _database_is_complete( + db + ), "Backward compatibility with header flag not working" + + # Load the database again - this should recreate the .complete file + # as the database detects it's complete from the header flag + db = audb.load(DB_NAME, version="1.0.0", cache_root=tmpdir) + assert os.path.exists(complete_file), "The .complete file was not recreated"