perf: Speed up unit tests via fixture scoping and reduced filesystem I/O

autoconf/conf.py

@@ -239,18 +239,18 @@ def push(
         if not Path(new_path).exists():
             raise ConfigException(f"{new_path} does not exist")
 
-        suffixes = {
-            Path(file).suffix for _, _, files in os.walk(new_path) for file in files
-        }
-
-        CONFIG_SUFFIXES = [
-            ".yml",
-            ".ini",
-            ".json",
-            ".yaml",
-        ]
-
-        if not any((suffix in suffixes for suffix in CONFIG_SUFFIXES)):
+        CONFIG_SUFFIXES = (".yml", ".ini", ".json", ".yaml")
+
+        has_config = False
+        for dirpath, _, files in os.walk(new_path):
+            for file in files:
+                if file.endswith(CONFIG_SUFFIXES):
+                    has_config = True
+                    break
+            if has_config:
+                break
+
+        if not has_config:
             raise ConfigException(
                 f"{new_path} does not contain any files ending with {'/'.join(CONFIG_SUFFIXES)} recursively"
             )

autoconf/directory_config.py

@@ -150,11 +150,24 @@ def _getitem(self, item):
 
 
 class RecursiveConfig(AbstractConfig):
+    def __init__(self, path):
+        self.path = Path(path)
+        self._listing = None
+
+    @property
+    def listing(self):
+        if self._listing is None:
+            try:
+                self._listing = set(os.listdir(self.path))
+            except FileNotFoundError:
+                self._listing = set()
+        return self._listing
+
     def keys(self):
         try:
             return [
                 path.split(".")[0]
-                for path in os.listdir(self.path)
+                for path in self.listing
                 if all(
                     [
                         path != "priors",
@@ -169,9 +182,6 @@ def keys(self):
         except FileNotFoundError as e:
             raise KeyError(f"No configuration found at {self.path}") from e
 
-    def __init__(self, path):
-        self.path = Path(path)
-
     def __eq__(self, other):
         return str(self) == str(other)
 
@@ -182,18 +192,18 @@ def __repr__(self):
         return f"<{self.__class__.__name__} {self.path}>"
 
     def _getitem(self, item):
-        item_path = self.path / f"{item}"
-        file_path = f"{item_path}.ini"
-        if os.path.isfile(file_path):
-            return NamedConfig(file_path)
-        yml_path = item_path.with_suffix(".yml")
-        if yml_path.exists():
-            return YAMLConfig(yml_path)
-        yaml_path = item_path.with_suffix(".yaml")
-        if yaml_path.exists():
-            return YAMLConfig(yaml_path)
-        if os.path.isdir(item_path):
-            return RecursiveConfig(item_path)
+        listing = self.listing
+        ini_name = f"{item}.ini"
+        if ini_name in listing:
+            return NamedConfig(self.path / ini_name)
+        yml_name = f"{item}.yml"
+        if yml_name in listing:
+            return YAMLConfig(self.path / yml_name)
+        yaml_name = f"{item}.yaml"
+        if yaml_name in listing:
+            return YAMLConfig(self.path / yaml_name)
+        if item in listing and os.path.isdir(self.path / item):
+            return RecursiveConfig(self.path / item)
         raise KeyError(f"No configuration found for {item} at path {self.path}")
 
 

test_autoconf/conftest.py

@@ -5,11 +5,19 @@
 from autoconf import conf
 
 
-@pytest.fixture(name="files_directory")
+@pytest.fixture(scope="session", name="files_directory")
 def make_files_directory():
     return pathlib.Path(__file__).parent / "files"
 
 
+@pytest.fixture(scope="session", name="session_config")
+def make_session_config(files_directory):
+    return conf.Config(
+        files_directory / "config",
+        files_directory / "default",
+    )
+
+
 @pytest.fixture(name="config")
 def make_config(files_directory):
     return conf.Config(

test_autoconf/test_default.py

@@ -1,8 +1,8 @@
 from autoconf.mock.mock_real import Redshift
 
 
-def test_override_file(config):
-    hpc = config["general"]["hpc"]
+def test_override_file(session_config):
+    hpc = session_config["general"]["hpc"]
 
     assert hpc["hpc_mode"] is False
     assert hpc["default_field"] == "hello"
@@ -36,52 +36,52 @@ def test_keep_first(config, files_directory):
     assert config["general"]["hpc"]["hpc_mode"] is False
 
 
-def test_override_in_directory(config):
-    superscript = config["text"]["label"]["superscript"]
+def test_override_in_directory(session_config):
+    superscript = session_config["text"]["label"]["superscript"]
 
     assert superscript["Galaxy"] == "g"
     assert superscript["default_field"] == "label default"
 
 
-def test_novel_directory(config):
-    assert config["default"]["other"]["section"]["key"] == "value"
+def test_novel_directory(session_config):
+    assert session_config["default"]["other"]["section"]["key"] == "value"
 
 
-def test_novel_file(config):
-    assert config["default_file"]["section"]["key"] == "file value"
+def test_novel_file(session_config):
+    assert session_config["default_file"]["section"]["key"] == "file value"
 
 
-def test_json(config):
+def test_json(session_config):
     assert (
-        config.prior_config.for_class_and_suffix_path(Redshift, ["redshift"])[
+        session_config.prior_config.for_class_and_suffix_path(Redshift, ["redshift"])[
             "upper_limit"
         ]
         == 3.0
     )
     assert (
-        config.prior_config.for_class_and_suffix_path(Redshift, ["rodshift"])[
+        session_config.prior_config.for_class_and_suffix_path(Redshift, ["rodshift"])[
             "upper_limit"
         ]
         == 4.0
     )
 
 
-def test_embedded_yaml_default(config):
-    embedded_dict = config["embedded"]["first"]["first_a"]
+def test_embedded_yaml_default(session_config):
+    embedded_dict = session_config["embedded"]["first"]["first_a"]
 
     assert embedded_dict["first_a_a"] == "one"
     assert embedded_dict["first_a_b"] == "two"
     assert embedded_dict["first_a_c"] == "three"
 
 
-def test_as_dict(config):
-    embedded_dict = config["embedded"]["first"]["first_a"]
+def test_as_dict(session_config):
+    embedded_dict = session_config["embedded"]["first"]["first_a"]
 
     assert {**embedded_dict}
 
 
-def test_mix_files(config):
-    embedded_dict = config["one"]["two"]["three"]
+def test_mix_files(session_config):
+    embedded_dict = session_config["one"]["two"]["three"]
 
     assert embedded_dict["four"] == "five"
     assert embedded_dict["six"] == "seven"