diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c8a2197b..857a9cf6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -24,6 +24,16 @@ Changed ^^^^^^^ - Removed support for python 3.8 (`#752 `__). +- ``YAML`` comments feature is now implemented in a separate class to allow + better support for custom help formatters without breaking the comments (`#754 + `__). + +Deprecated +^^^^^^^^^^ +- ``DefaultHelpFormatter.*_yaml*_comment*`` methods are deprecated and will be + removed in v5.0.0. This logic has been moved to a new private class + ``YAMLCommentFormatter``. If deemed necessary, this class might be made public + in the future (`#754 `__). v4.40.2 (2025-08-06) diff --git a/jsonargparse/_actions.py b/jsonargparse/_actions.py index 9f0ae313..087b500d 100644 --- a/jsonargparse/_actions.py +++ b/jsonargparse/_actions.py @@ -12,7 +12,7 @@ from ._common import Action, is_subclass, parser_context from ._loaders_dumpers import get_loader_exceptions, load_value from ._namespace import Namespace, NSKeyError, split_key, split_key_root -from ._optionals import _get_config_read_mode +from ._optionals import _get_config_read_mode, ruyaml_support from ._type_checking import ActionsContainer, ArgumentParser from ._util import ( Path, @@ -251,13 +251,16 @@ def __init__( help=( "Print the configuration after applying all other arguments and exit. The optional " "flags customizes the output and are one or more keywords separated by comma. The " - "supported flags are: comments, skip_default, skip_null." - ), + "supported flags are:%s skip_default, skip_null." + ) + % (" comments," if ruyaml_support else ""), ) def __call__(self, parser, namespace, value, option_string=None): kwargs = {"subparser": parser, "key": None, "skip_none": False, "skip_validation": False} - valid_flags = {"": None, "comments": "yaml_comments", "skip_default": "skip_default", "skip_null": "skip_none"} + valid_flags = {"": None, "skip_default": "skip_default", "skip_null": "skip_none"} + if ruyaml_support: + valid_flags["comments"] = "yaml_comments" if value is not None: flags = value[0].split(",") invalid_flags = [f for f in flags if f not in valid_flags] diff --git a/jsonargparse/_deprecated.py b/jsonargparse/_deprecated.py index 17633b70..724cb5ae 100644 --- a/jsonargparse/_deprecated.py +++ b/jsonargparse/_deprecated.py @@ -14,7 +14,7 @@ from ._common import Action, null_logger from ._common import LoggerProperty as InternalLoggerProperty from ._namespace import Namespace -from ._type_checking import ArgumentParser +from ._type_checking import ArgumentParser, ruyamlCommentedMap __all__ = [ "ActionEnum", @@ -701,3 +701,50 @@ class LoggerProperty(InternalLoggerProperty): def namespace_to_dict(namespace: Namespace) -> Dict[str, Any]: """Returns a copy of a nested namespace converted into a nested dictionary.""" return namespace.clone().as_dict() + + +class HelpFormatterDeprecations: + def __init__(self, *args, **kwargs): + from jsonargparse._formatters import YAMLCommentFormatter + + super().__init__(*args, **kwargs) + self._yaml_formatter = YAMLCommentFormatter(self) + + @deprecated("The add_yaml_comments method is deprecated and will be removed in v5.0.0.") + def add_yaml_comments(self, cfg: str) -> str: + """Adds help text as yaml comments.""" + return self._yaml_formatter.add_yaml_comments(cfg) + + @deprecated("The set_yaml_start_comment method is deprecated and will be removed in v5.0.0.") + def set_yaml_start_comment(self, text: str, cfg: ruyamlCommentedMap): + """Sets the start comment to a ruyaml object. + + Args: + text: The content to use for the comment. + cfg: The ruyaml object. + """ + self._yaml_formatter.set_yaml_start_comment(text, cfg) + + @deprecated("The set_yaml_group_comment method is deprecated and will be removed in v5.0.0.") + def set_yaml_group_comment(self, text: str, cfg: ruyamlCommentedMap, key: str, depth: int): + """Sets the comment for a group to a ruyaml object. + + Args: + text: The content to use for the comment. + cfg: The parent ruyaml object. + key: The key of the group. + depth: The nested level of the group. + """ + self._yaml_formatter.set_yaml_group_comment(text, cfg, key, depth) + + @deprecated("The set_yaml_argument_comment method is deprecated and will be removed in v5.0.0.") + def set_yaml_argument_comment(self, text: str, cfg: ruyamlCommentedMap, key: str, depth: int): + """Sets the comment for an argument to a ruyaml object. + + Args: + text: The content to use for the comment. + cfg: The parent ruyaml object. + key: The key of the argument. + depth: The nested level of the argument. + """ + self._yaml_formatter.set_yaml_argument_comment(text, cfg, key, depth) diff --git a/jsonargparse/_formatters.py b/jsonargparse/_formatters.py index 3d662150..14b96dc0 100644 --- a/jsonargparse/_formatters.py +++ b/jsonargparse/_formatters.py @@ -30,6 +30,7 @@ supports_optionals_as_positionals, ) from ._completions import ShtabAction +from ._deprecated import HelpFormatterDeprecations from ._link_arguments import ActionLink from ._namespace import Namespace, NSKeyError from ._optionals import import_ruyaml @@ -54,7 +55,116 @@ class PercentTemplate(Template): """ # type: ignore[assignment] -class DefaultHelpFormatter(HelpFormatter): +class YAMLCommentFormatter: + """Formatter class for adding YAML comments to configuration files.""" + + def __init__(self, help_formatter: HelpFormatter): + self.help_formatter = help_formatter + + def add_yaml_comments(self, cfg: str) -> str: + """Adds help text as yaml comments.""" + ruyaml = import_ruyaml("add_yaml_comments") + yaml = ruyaml.YAML() + cfg = yaml.load(cfg) + + def get_subparsers(parser, prefix=""): + subparsers = {} + if parser._subparsers is not None: + for key, subparser in parser._subparsers._group_actions[0].choices.items(): + full_key = (prefix + "." if prefix else "") + key + subparsers[full_key] = subparser + subparsers.update(get_subparsers(subparser, prefix=full_key)) + return subparsers + + parser = parent_parser.get() + parsers = get_subparsers(parser) + parsers[None] = parser + + group_titles = {} + for parser_key, parser in parsers.items(): + group_titles[parser_key] = parser.description + prefix = "" if parser_key is None else parser_key + "." + for group in parser._action_groups: + actions = filter_default_actions(group._group_actions) + actions = [ + a for a in actions if not isinstance(a, (_ActionConfigLoad, ActionConfigFile, _ActionSubCommands)) + ] + keys = {re.sub(r"\.?[^.]+$", "", a.dest) for a in actions if "." in a.dest} + for key in keys: + group_titles[prefix + key] = group.title + + def set_comments(cfg, prefix="", depth=0): + for key in cfg.keys(): + full_key = (prefix + "." if prefix else "") + key + action = _find_action(parser, full_key) + text = None + if full_key in group_titles and isinstance(cfg[key], dict): + text = group_titles[full_key] + elif action is not None and action.help not in {None, SUPPRESS}: + text = self.help_formatter._expand_help(action) + if isinstance(cfg[key], dict): + if text: + self.set_yaml_group_comment(text, cfg, key, depth) + set_comments(cfg[key], full_key, depth + 1) + elif text: + self.set_yaml_argument_comment(text, cfg, key, depth) + + if parser.description is not None: + self.set_yaml_start_comment(parser.description, cfg) + set_comments(cfg) + out = StringIO() + yaml.dump(cfg, out) + return out.getvalue() + + def set_yaml_start_comment( + self, + text: str, + cfg: ruyamlCommentedMap, + ): + """Sets the start comment to a ruyaml object. + + Args: + text: The content to use for the comment. + cfg: The ruyaml object. + """ + cfg.yaml_set_start_comment(text) + + def set_yaml_group_comment( + self, + text: str, + cfg: ruyamlCommentedMap, + key: str, + depth: int, + ): + """Sets the comment for a group to a ruyaml object. + + Args: + text: The content to use for the comment. + cfg: The parent ruyaml object. + key: The key of the group. + depth: The nested level of the group. + """ + cfg.yaml_set_comment_before_after_key(key, before="\n" + text, indent=2 * depth) + + def set_yaml_argument_comment( + self, + text: str, + cfg: ruyamlCommentedMap, + key: str, + depth: int, + ): + """Sets the comment for an argument to a ruyaml object. + + Args: + text: The content to use for the comment. + cfg: The parent ruyaml object. + key: The key of the argument. + depth: The nested level of the argument. + """ + cfg.yaml_set_comment_before_after_key(key, before="\n" + text, indent=2 * depth) + + +class DefaultHelpFormatter(HelpFormatterDeprecations, HelpFormatter): """Help message formatter that includes types, default values and env var names. This class is an extension of `argparse.HelpFormatter @@ -184,108 +294,6 @@ def add_usage(self, usage: Optional[str], actions: Iterable[Action], *args, **kw actions = [a for a in actions if not isinstance(a, ActionLink)] super().add_usage(usage, actions, *args, **kwargs) - def add_yaml_comments(self, cfg: str) -> str: - """Adds help text as yaml comments.""" - ruyaml = import_ruyaml("add_yaml_comments") - yaml = ruyaml.YAML() - cfg = yaml.load(cfg) - - def get_subparsers(parser, prefix=""): - subparsers = {} - if parser._subparsers is not None: - for key, subparser in parser._subparsers._group_actions[0].choices.items(): - full_key = (prefix + "." if prefix else "") + key - subparsers[full_key] = subparser - subparsers.update(get_subparsers(subparser, prefix=full_key)) - return subparsers - - parser = parent_parser.get() - parsers = get_subparsers(parser) - parsers[None] = parser - - group_titles = {} - for parser_key, parser in parsers.items(): - group_titles[parser_key] = parser.description - prefix = "" if parser_key is None else parser_key + "." - for group in parser._action_groups: - actions = filter_default_actions(group._group_actions) - actions = [ - a for a in actions if not isinstance(a, (_ActionConfigLoad, ActionConfigFile, _ActionSubCommands)) - ] - keys = {re.sub(r"\.?[^.]+$", "", a.dest) for a in actions if "." in a.dest} - for key in keys: - group_titles[prefix + key] = group.title - - def set_comments(cfg, prefix="", depth=0): - for key in cfg.keys(): - full_key = (prefix + "." if prefix else "") + key - action = _find_action(parser, full_key) - text = None - if full_key in group_titles and isinstance(cfg[key], dict): - text = group_titles[full_key] - elif action is not None and action.help not in {None, SUPPRESS}: - text = self._expand_help(action) - if isinstance(cfg[key], dict): - if text: - self.set_yaml_group_comment(text, cfg, key, depth) - set_comments(cfg[key], full_key, depth + 1) - elif text: - self.set_yaml_argument_comment(text, cfg, key, depth) - - if parser.description is not None: - self.set_yaml_start_comment(parser.description, cfg) - set_comments(cfg) - out = StringIO() - yaml.dump(cfg, out) - return out.getvalue() - - def set_yaml_start_comment( - self, - text: str, - cfg: ruyamlCommentedMap, - ): - """Sets the start comment to a ruyaml object. - - Args: - text: The content to use for the comment. - cfg: The ruyaml object. - """ - cfg.yaml_set_start_comment(text) - - def set_yaml_group_comment( - self, - text: str, - cfg: ruyamlCommentedMap, - key: str, - depth: int, - ): - """Sets the comment for a group to a ruyaml object. - - Args: - text: The content to use for the comment. - cfg: The parent ruyaml object. - key: The key of the group. - depth: The nested level of the group. - """ - cfg.yaml_set_comment_before_after_key(key, before="\n" + text, indent=2 * depth) - - def set_yaml_argument_comment( - self, - text: str, - cfg: ruyamlCommentedMap, - key: str, - depth: int, - ): - """Sets the comment for an argument to a ruyaml object. - - Args: - text: The content to use for the comment. - cfg: The parent ruyaml object. - key: The key of the argument. - depth: The nested level of the argument. - """ - cfg.yaml_set_comment_before_after_key(key, before="\n" + text, indent=2 * depth) - def get_env_var( parser_or_formatter: Union[ArgumentParser, DefaultHelpFormatter], diff --git a/jsonargparse/_loaders_dumpers.py b/jsonargparse/_loaders_dumpers.py index db8211c4..f3867446 100644 --- a/jsonargparse/_loaders_dumpers.py +++ b/jsonargparse/_loaders_dumpers.py @@ -2,11 +2,19 @@ import inspect import re +from argparse import HelpFormatter from contextlib import suppress from typing import Any, Callable, Dict, Optional, Set, Tuple, Type from ._common import load_value_mode, parent_parser -from ._optionals import import_jsonnet, import_toml_dumps, import_toml_loads, omegaconf_support, pyyaml_available +from ._optionals import ( + import_jsonnet, + import_toml_dumps, + import_toml_loads, + omegaconf_support, + pyyaml_available, + ruyaml_support, +) from ._type_checking import ArgumentParser __all__ = [ @@ -225,7 +233,8 @@ def yaml_dump(data): def yaml_comments_dump(data, parser): dump = dumpers["yaml"](data) - formatter = parser.formatter_class(parser.prog) + formatter_class = create_help_formatter_with_comments(parser.formatter_class) + formatter = formatter_class(parser.prog) return formatter.add_yaml_comments(dump) @@ -248,13 +257,14 @@ def toml_dump(data): dumpers: Dict[str, Callable] = { "yaml": yaml_dump, - "yaml_comments": yaml_comments_dump, "json": json_compact_dump, "json_compact": json_compact_dump, "json_indented": json_indented_dump, "toml": toml_dump, "jsonnet": json_indented_dump, } +if ruyaml_support: + dumpers["yaml_comments"] = yaml_comments_dump comment_prefix: Dict[str, str] = { "yaml": "# ", @@ -333,3 +343,26 @@ def set_omegaconf_loader(): set_loader("jsonnet", jsonnet_load, get_loader_exceptions("jsonnet")) + + +def create_help_formatter_with_comments(formatter_class: Type[HelpFormatter]) -> Type[HelpFormatter]: + """Creates a dynamic class that combines a formatter with YAML comment functionality. + + Args: + formatter_class: The base formatter class to extend. + + Returns: + A new class that inherits from both the formatter and YAMLCommentFormatter. + """ + from ._formatters import YAMLCommentFormatter + + class DynamicHelpFormatter(formatter_class): # type: ignore[valid-type,misc] + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._yaml_formatter = YAMLCommentFormatter(self) + + def add_yaml_comments(self, cfg: str) -> str: + """Adds help text as yaml comments.""" + return self._yaml_formatter.add_yaml_comments(cfg) + + return DynamicHelpFormatter diff --git a/jsonargparse_tests/test_core.py b/jsonargparse_tests/test_core.py index d3d1e15c..1c9f5099 100644 --- a/jsonargparse_tests/test_core.py +++ b/jsonargparse_tests/test_core.py @@ -761,12 +761,22 @@ def test_print_config_skip_null(print_parser): @pytest.mark.skipif(not ruyaml_support, reason="ruyaml package is required") @skip_if_docstring_parser_unavailable def test_print_config_comments(print_parser): + help_str = get_parser_help(print_parser) + assert "comments," in help_str out = get_parse_args_stdout(print_parser, ["--print_config=comments"]) assert "# cli tool" in out assert "# Option v1. (default: 1)" in out assert "# Option v2. (default: 2)" in out +@pytest.mark.skipif(ruyaml_support, reason="ruyaml package should not be installed") +def test_print_config_comments_unavailable(print_parser): + help_str = get_parser_help(print_parser) + assert "comments," not in help_str + with pytest.raises(ArgumentError, match='Invalid option "comments"'): + get_parse_args_stdout(print_parser, ["--print_config=comments"]) + + def test_print_config_invalid_flag(print_parser): with pytest.raises(ArgumentError) as ctx: print_parser.parse_args(["--print_config=invalid"]) diff --git a/jsonargparse_tests/test_deprecated.py b/jsonargparse_tests/test_deprecated.py index d6996f07..5863f636 100644 --- a/jsonargparse_tests/test_deprecated.py +++ b/jsonargparse_tests/test_deprecated.py @@ -37,11 +37,14 @@ shown_deprecation_warnings, usage_and_exit_error_handler, ) +from jsonargparse._formatters import DefaultHelpFormatter from jsonargparse._optionals import ( docstring_parser_support, get_docstring_parse_options, + import_ruyaml, jsonnet_support, pyyaml_available, + ruyaml_support, url_support, ) from jsonargparse._util import argument_error @@ -726,3 +729,35 @@ def test_namespace_to_dict(): message="namespace_to_dict was deprecated", code="dic1 = namespace_to_dict(ns)", ) + + +@pytest.mark.skipif(not ruyaml_support, reason="ruyaml package is required") +def test_DefaultHelpFormatter_yaml_comments(parser): + parser.add_argument("--arg", type=int, help="Description") + formatter = DefaultHelpFormatter(prog="test") + from jsonargparse._common import parent_parser + + parent_parser.set(parser) + ruyaml = import_ruyaml("test_DefaultHelpFormatter_yaml_comments") + yaml = ruyaml.YAML() + cfg = yaml.load("arg: 1") + + with catch_warnings(record=True) as w: + formatter.add_yaml_comments("arg: 1") + assert "add_yaml_comments method is deprecated and will be removed in v5.0.0" in str(w[-1].message) + assert "formatter.add_yaml_comments(" in source[w[-1].lineno - 1] + + with catch_warnings(record=True) as w: + formatter.set_yaml_start_comment("start", cfg) + assert "set_yaml_start_comment method is deprecated and will be removed in v5.0.0" in str(w[-1].message) + assert "formatter.set_yaml_start_comment(" in source[w[-1].lineno - 1] + + with catch_warnings(record=True) as w: + formatter.set_yaml_group_comment("group", cfg, "arg", 0) + assert "set_yaml_group_comment method is deprecated and will be removed in v5.0.0" in str(w[-1].message) + assert "formatter.set_yaml_group_comment(" in source[w[-1].lineno - 1] + + with catch_warnings(record=True) as w: + formatter.set_yaml_argument_comment("arg", cfg, "arg", 0) + assert "set_yaml_argument_comment method is deprecated and will be removed in v5.0.0" in str(w[-1].message) + assert "formatter.set_yaml_argument_comment(" in source[w[-1].lineno - 1]