explicit field persistence

Author: yaugenst-flex

docs/configuration/index.rst

@@ -155,7 +155,7 @@ Command Line Helpers
 --------------------
 
 Use ``tidy3d configure`` to store your API key in ``config.toml``. The command
-creates the directory if it is missing and updates only the ``auth`` section.
+creates the directory if it is missing and updates only the ``web`` section.
 
 If you have older files in ``~/.tidy3d``, run ``tidy3d config migrate`` to move
 them into the new location described above.
@@ -173,4 +173,3 @@ Next Steps
 
 - :doc:`migration`
 - :doc:`../api/configuration`
-

tests/config/conftest.py

@@ -14,6 +14,7 @@
     "TIDY3D_ENV",
     "SIMCLOUD_APIKEY",
     "TIDY3D_AUTH__APIKEY",
+    "TIDY3D_WEB__APIKEY",
     "TIDY3D_BASE_DIR",
 }
 

tests/config/test_loader.py

@@ -20,9 +20,7 @@ def test_save_includes_descriptions(config_manager, mock_config_dir):
     manager.save(include_defaults=True)
 
     content = _config_path(mock_config_dir).read_text(encoding="utf-8")
-    assert "# Logging configuration." in content
     assert "# Web/HTTP configuration." in content
-    assert "# Tidy3D API base URL." in content
 
 
 def test_preserves_user_comments(config_manager, mock_config_dir):
@@ -32,7 +30,7 @@ def test_preserves_user_comments(config_manager, mock_config_dir):
     config_path = _config_path(mock_config_dir)
     text = config_path.read_text(encoding="utf-8")
     text = text.replace(
-        "Tidy3D API base URL.",
+        "Web/HTTP configuration.",
         "user-modified comment",
     )
     config_path.write_text(text, encoding="utf-8")
@@ -43,57 +41,90 @@ def test_preserves_user_comments(config_manager, mock_config_dir):
 
     updated = config_path.read_text(encoding="utf-8")
     assert "user-modified comment" in updated
-    assert "Tidy3D API base URL." not in updated
+    assert "Web/HTTP configuration." not in updated
 
 
 def test_profile_preserves_comments(config_manager, mock_config_dir):
-    manager = config_manager
-    manager.switch_profile("custom")
-    manager.update_section("web", timeout=55)
-    manager.save()
-
-    profile_path = mock_config_dir / "profiles" / "custom.toml"
-    text = profile_path.read_text(encoding="utf-8")
-    assert "HTTP request timeout in seconds." in text
-    text = text.replace("HTTP request timeout in seconds.", "user comment")
-    profile_path.write_text(text, encoding="utf-8")
+    @config_registry.register_plugin("profile_comment")
+    class ProfileComment(ConfigSection):
+        """Profile comment plugin."""
 
-    manager.update_section("web", timeout=56)
-    manager.save()
+        knob: int = pd.Field(
+            1,
+            description="Profile knob description.",
+            json_schema_extra={"persist": True},
+        )
 
-    updated = profile_path.read_text(encoding="utf-8")
-    assert "user comment" in updated
-    assert "HTTP request timeout in seconds." not in updated
+    try:
+        manager = config_manager
+        manager.switch_profile("custom")
+        manager.update_section("plugins.profile_comment", knob=5)
+        manager.save()
+
+        profile_path = mock_config_dir / "profiles" / "custom.toml"
+        text = profile_path.read_text(encoding="utf-8")
+        assert "Profile knob description." in text
+        text = text.replace("Profile knob description.", "user comment")
+        profile_path.write_text(text, encoding="utf-8")
+
+        manager.update_section("plugins.profile_comment", knob=7)
+        manager.save()
+
+        updated = profile_path.read_text(encoding="utf-8")
+        assert "user comment" in updated
+        assert "Profile knob description." not in updated
+    finally:
+        config_registry._SECTIONS.pop("plugins.profile_comment", None)
+        reload_config(profile="default")
 
 
 def test_cli_reset_config(mock_config_dir):
-    reload_config(profile="default")
-    manager = get_manager()
-    manager.update_section("web", api_endpoint="https://example.com")
-    manager.save(include_defaults=True)
-    manager.switch_profile("custom")
-    manager.update_section("web", timeout=70)
-    manager.save()
+    @config_registry.register_plugin("cli_comment")
+    class CLIPlugin(ConfigSection):
+        """CLI plugin configuration."""
 
-    profiles_dir = mock_config_dir / "profiles"
-    assert profiles_dir.exists()
+        knob: int = pd.Field(
+            3,
+            description="CLI knob description.",
+            json_schema_extra={"persist": True},
+        )
 
-    runner = CliRunner()
-    result = runner.invoke(tidy3d_cli, ["config-reset", "--yes"])
-    assert result.exit_code == 0, result.output
-
-    config_text = _config_path(mock_config_dir).read_text(encoding="utf-8")
-    assert "Web/HTTP configuration." in config_text
-    assert "Tidy3D API base URL" in config_text
-    assert not profiles_dir.exists()
+    try:
+        reload_config(profile="default")
+        manager = get_manager()
+        manager.update_section("web", apikey="secret")
+        manager.save(include_defaults=True)
+        manager.switch_profile("custom")
+        manager.update_section("plugins.cli_comment", knob=42)
+        manager.save()
+
+        profiles_dir = mock_config_dir / "profiles"
+        assert profiles_dir.exists()
+
+        runner = CliRunner()
+        result = runner.invoke(tidy3d_cli, ["config-reset", "--yes"])
+        assert result.exit_code == 0, result.output
+
+        config_text = _config_path(mock_config_dir).read_text(encoding="utf-8")
+        assert "Web/HTTP configuration." in config_text
+        assert "[web]" in config_text
+        assert "secret" not in config_text
+        assert not profiles_dir.exists()
+    finally:
+        config_registry._SECTIONS.pop("plugins.cli_comment", None)
+        reload_config(profile="default")
 
 
 def test_plugin_descriptions(mock_config_dir):
     @config_registry.register_plugin("comment_test")
     class CommentPlugin(ConfigSection):
         """Comment plugin configuration."""
 
-        knob: int = pd.Field(3, description="Plugin knob description.")
+        knob: int = pd.Field(
+            3,
+            description="Plugin knob description.",
+            json_schema_extra={"persist": True},
+        )
 
     try:
         reload_config(profile="default")

tests/config/test_manager.py

@@ -3,6 +3,8 @@
 import numpy as np
 import pytest
 
+from tidy3d.config import Env, get_manager, reload_config
+
 
 def test_default_web_settings(config_manager):
     web = config_manager.get_section("web")
@@ -42,6 +44,19 @@ def test_builtin_profiles(profile, config_manager):
     assert web.s3_region is not None
 
 
+def test_uppercase_profile_normalization(monkeypatch):
+    monkeypatch.setenv("TIDY3D_ENV", "DEV")
+    try:
+        reload_config()
+        manager = get_manager()
+        assert manager.profile == "dev"
+        web = manager.get_section("web")
+        assert str(web.api_endpoint) == "https://tidy3d-api.dev-simulation.cloud"
+        assert Env.current.name == "dev"
+    finally:
+        reload_config(profile="default")
+
+
 def test_autograd_defaults(config_manager):
     autograd = config_manager.get_section("autograd")
     assert autograd.min_wvl_fraction == pytest.approx(5e-2)

tests/config/test_plugins.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import pydantic.v1 as pd
 import toml
 
 from tidy3d.config.__init__ import get_manager, reload_config
@@ -13,8 +14,8 @@ def ensure_dummy_plugin():
 
     @register_plugin("dummy")
     class DummyPlugin(ConfigSection):
-        enabled: bool = False
-        precision: int = 1
+        enabled: bool = pd.Field(False, json_schema_extra={"persist": True})
+        precision: int = pd.Field(1, json_schema_extra={"persist": True})
 
 
 def test_plugin_defaults_available(mock_config_dir):

tests/config/test_profiles.py

@@ -4,13 +4,14 @@
 
 
 def test_save_default_profile(config_manager):
+    config_manager.update_section("web", apikey="token")
     config_manager.update_section("web", timeout=30)
     config_manager.save()
 
     config_path = config_manager.config_dir / "config.toml"
     assert config_path.exists()
     data = toml.load(config_path)
-    assert data["web"]["timeout"] == 30
+    assert data["web"]["apikey"] == "token"
 
 
 def test_save_custom_profile(config_manager):
@@ -19,6 +20,4 @@ def test_save_custom_profile(config_manager):
     config_manager.save()
 
     profile_path = config_manager.config_dir / "profiles" / "customer.toml"
-    assert profile_path.exists()
-    profile_data = toml.load(profile_path)
-    assert profile_data == {"logging": {"level": "DEBUG"}}
+    assert not profile_path.exists()

tidy3d/config/legacy.py

@@ -6,7 +6,7 @@
 import ssl
 from typing import Any, Optional
 
-from .manager import ConfigManager
+from .manager import ConfigManager, normalize_profile_name
 from .profiles import BUILTIN_PROFILES
 
 
@@ -116,6 +116,7 @@ def __init__(
     ) -> None:
         if name is None:
             raise ValueError("Environment name is required")
+        name = normalize_profile_name(name)
         self._manager = manager
         self._name = name
         self._environment = environment
@@ -207,7 +208,7 @@ def name(self) -> str:
 
     @name.setter
     def name(self, value: str) -> None:
-        self._name = value
+        self._name = normalize_profile_name(value)
 
     def get_real_url(self, path: str) -> str:
         endpoint = self.web_api_endpoint or ""
@@ -248,14 +249,17 @@ def reset_manager(self, manager: ConfigManager) -> None:
         for name in BUILTIN_PROFILES:
             self.env_map[name] = LegacyEnvironmentConfig(manager, name, environment=self)
 
-        desired = os.getenv("TIDY3D_ENV")
-        desired = desired.lower() if desired else None
-
-        if not desired:
+        desired_env = os.getenv("TIDY3D_ENV")
+        if desired_env:
+            desired = normalize_profile_name(desired_env)
+        else:
             desired = manager.profile
+
         if desired == "default":
             desired = "prod"
 
+        desired = normalize_profile_name(desired)
+
         self._current = self.env_map.setdefault(
             desired, LegacyEnvironmentConfig(manager, desired, environment=self)
         )
@@ -266,13 +270,15 @@ def current(self) -> LegacyEnvironmentConfig:
         return self._current
 
     def set_current(self, env_config: LegacyEnvironmentConfig) -> None:
+        key = normalize_profile_name(env_config.name)
         if env_config.manager is self._manager:
-            if self._manager.profile != env_config.name:
-                self._manager.switch_profile(env_config.name)
-            stored = self.env_map.setdefault(env_config.name, env_config)
+            if self._manager.profile != key:
+                self._manager.switch_profile(key)
+            stored = self.env_map.setdefault(key, env_config)
         else:
             stored = env_config
-            self.env_map[env_config.name] = stored
+            stored.name = key
+            self.env_map[key] = stored
 
         stored._environment = self
         self._current = stored
@@ -289,7 +295,8 @@ def set_ssl_version(self, ssl_version) -> None:
         self._current._overrides["ssl_version"] = ssl_version
 
     def __getattr__(self, name: str) -> LegacyEnvironmentConfig:
-        return self.env_map.setdefault(name, LegacyEnvironmentConfig(self._manager, name))
+        key = normalize_profile_name(name)
+        return self.env_map.setdefault(key, LegacyEnvironmentConfig(self._manager, key))
 
     def _apply_env_vars(self, config: LegacyEnvironmentConfig) -> None:
         self._restore_env_vars()

tidy3d/config/loader.py

@@ -157,7 +157,7 @@ def load_environment_overrides() -> dict[str, Any]:
     overrides: dict[str, Any] = {}
     for key, value in os.environ.items():
         if key == "SIMCLOUD_APIKEY":
-            _assign_path(overrides, ("auth", "apikey"), value)
+            _assign_path(overrides, ("web", "apikey"), value)
             continue
         if not key.startswith("TIDY3D_"):
             continue
@@ -167,6 +167,8 @@ def load_environment_overrides() -> dict[str, Any]:
         segments = tuple(segment.lower() for segment in rest.split("__") if segment)
         if not segments:
             continue
+        if segments[0] == "auth":
+            segments = ("web",) + segments[1:]
         _assign_path(overrides, segments, value)
     return overrides