Support error codes from plugins in options

Mypy has options for enabling or disabling specific error codes. These
work fine, except that it is not possible to enable or disable error
codes from plugins, only mypy's original error codes.

The crux of the issue is that mypy validates and rejects unknown error
codes passed in the options before it loads plugins and learns about the
any error codes that might get registered.

There are many ways to solve this. This commit tries to find a pragmatic
solution where the relevant options parsing is deferred until after
plugin loading. Error code validation in the config parser, where
plugins are not loaded yet, is also skipped entirely, since the error
code options are re-validated later anyway. This means that this commit
introduces a small observable change in behavior when running with
invalid error codes specified, as shown in the test
test_config_file_error_codes_invalid.

This fixes #12987.

Author: sigvef

mypy/build.py

@@ -504,6 +504,9 @@ def load_plugins(
     """
     custom_plugins, snapshot = load_plugins_from_config(options, errors, stdout)
 
+    if options._on_plugins_loaded is not None:
+        options._on_plugins_loaded()
+
     custom_plugins += extra_plugins
 
     default_plugin: Plugin = DefaultPlugin(options)

mypy/config_parser.py

@@ -8,8 +8,6 @@
 import sys
 from io import StringIO
 
-from mypy.errorcodes import error_codes
-
 if sys.version_info >= (3, 11):
     import tomllib
 else:
@@ -87,15 +85,6 @@ def complain(x: object, additional_info: str = "") -> Never:
         complain(v)
 
 
-def validate_codes(codes: list[str]) -> list[str]:
-    invalid_codes = set(codes) - set(error_codes.keys())
-    if invalid_codes:
-        raise argparse.ArgumentTypeError(
-            f"Invalid error code(s): {', '.join(sorted(invalid_codes))}"
-        )
-    return codes
-
-
 def validate_package_allow_list(allow_list: list[str]) -> list[str]:
     for p in allow_list:
         msg = f"Invalid allow list entry: {p}"
@@ -209,8 +198,8 @@ def split_commas(value: str) -> list[str]:
         [p.strip() for p in split_commas(s)]
     ),
     "enable_incomplete_feature": lambda s: [p.strip() for p in split_commas(s)],
-    "disable_error_code": lambda s: validate_codes([p.strip() for p in split_commas(s)]),
-    "enable_error_code": lambda s: validate_codes([p.strip() for p in split_commas(s)]),
+    "disable_error_code": lambda s: [p.strip() for p in split_commas(s)],
+    "enable_error_code": lambda s: [p.strip() for p in split_commas(s)],
     "package_root": lambda s: [p.strip() for p in split_commas(s)],
     "cache_dir": expand_path,
     "python_executable": expand_path,
@@ -234,8 +223,8 @@ def split_commas(value: str) -> list[str]:
         "always_false": try_split,
         "untyped_calls_exclude": lambda s: validate_package_allow_list(try_split(s)),
         "enable_incomplete_feature": try_split,
-        "disable_error_code": lambda s: validate_codes(try_split(s)),
-        "enable_error_code": lambda s: validate_codes(try_split(s)),
+        "disable_error_code": lambda s: try_split(s),
+        "enable_error_code": lambda s: try_split(s),
         "package_root": try_split,
         "exclude": str_or_array_as_list,
         "packages": try_split,

mypy/main.py

@@ -1353,7 +1353,8 @@ def process_options(
     If a FileSystemCache is passed in, and package_root options are given,
     call fscache.set_package_root() to set the cache's package root.
 
-    Returns a tuple of: a list of source files, an Options collected from flags.
+    Returns a tuple of: a list of source files, an Options collected from
+    flags, and a callback to be called once plugins have loaded.
     """
     stdout = stdout if stdout is not None else sys.stdout
     stderr = stderr if stderr is not None else sys.stderr
@@ -1456,9 +1457,16 @@ def set_strict_flags() -> None:
     validate_package_allow_list(options.untyped_calls_exclude)
     validate_package_allow_list(options.deprecated_calls_exclude)
 
-    options.process_error_codes(error_callback=parser.error)
     options.process_incomplete_features(error_callback=parser.error, warning_callback=print)
 
+    def on_plugins_loaded() -> None:
+        # Processing error codes after plugins have loaded since plugins may
+        # register custom error codes that we don't know about until plugins
+        # have loaded.
+        options.process_error_codes(error_callback=parser.error)
+
+    options._on_plugins_loaded = on_plugins_loaded
+
     # Compute absolute path for custom typeshed (if present).
     if options.custom_typeshed_dir is not None:
         options.abs_custom_typeshed_dir = os.path.abspath(options.custom_typeshed_dir)

mypy/options.py

@@ -416,6 +416,8 @@ def __init__(self) -> None:
         # preserving manual tweaks to generated C file)
         self.mypyc_skip_c_generation = False
 
+        self._on_plugins_loaded: Callable[[], None] | None = None
+
     def use_lowercase_names(self) -> bool:
         warnings.warn(
             "options.use_lowercase_names() is deprecated and will be removed in a future version",

mypy/test/teststubtest.py

@@ -12,6 +12,8 @@
 from collections.abc import Iterator
 from typing import Any, Callable
 
+from pytest import raises
+
 import mypy.stubtest
 from mypy import build, nodes
 from mypy.modulefinder import BuildSource
@@ -171,7 +173,12 @@ def build_helper(source: str) -> build.BuildResult:
 
 
 def run_stubtest_with_stderr(
-    stub: str, runtime: str, options: list[str], config_file: str | None = None
+    stub: str,
+    runtime: str,
+    options: list[str],
+    config_file: str | None = None,
+    output: io.StringIO | None = None,
+    outerr: io.StringIO | None = None,
 ) -> tuple[str, str]:
     with use_tmp_dir(TEST_MODULE_NAME) as tmp_dir:
         with open("builtins.pyi", "w") as f:
@@ -188,8 +195,8 @@ def run_stubtest_with_stderr(
             with open(f"{TEST_MODULE_NAME}_config.ini", "w") as f:
                 f.write(config_file)
             options = options + ["--mypy-config-file", f"{TEST_MODULE_NAME}_config.ini"]
-        output = io.StringIO()
-        outerr = io.StringIO()
+        output = io.StringIO() if output is None else output
+        outerr = io.StringIO() if outerr is None else outerr
         with contextlib.redirect_stdout(output), contextlib.redirect_stderr(outerr):
             test_stubs(parse_options([TEST_MODULE_NAME] + options), use_builtins_fixtures=True)
     filtered_output = remove_color_code(
@@ -2888,14 +2895,20 @@ def test_config_file_error_codes_invalid(self) -> None:
         runtime = "temp = 5\n"
         stub = "temp: int\n"
         config_file = "[mypy]\ndisable_error_code = not-a-valid-name\n"
-        output, outerr = run_stubtest_with_stderr(
-            stub=stub, runtime=runtime, options=[], config_file=config_file
-        )
-        assert output == "Success: no issues found in 1 module\n"
-        assert outerr == (
-            "test_module_config.ini: [mypy]: disable_error_code: "
-            "Invalid error code(s): not-a-valid-name\n"
-        )
+        output = io.StringIO()
+        outerr = io.StringIO()
+        with raises(SystemExit):
+            run_stubtest_with_stderr(
+                stub=stub,
+                runtime=runtime,
+                options=[],
+                config_file=config_file,
+                output=output,
+                outerr=outerr,
+            )
+
+        assert output.getvalue() == "error: Invalid error code(s): not-a-valid-name\n"
+        assert outerr.getvalue() == ""
 
     def test_config_file_wrong_incomplete_feature(self) -> None:
         runtime = "x = 1\n"

test-data/unit/check-plugin-error-codes.test

@@ -0,0 +1,32 @@
+[case testCustomErrorCodeFromPluginIsTargetable]
+# flags: --config-file tmp/mypy.ini --show-error-codes
+
+def main() -> None:
+    return
+main() # E: Custom error  [custom]
+
+[file mypy.ini]
+\[mypy]
+plugins=<ROOT>/test-data/unit/plugins/custom_errorcode.py
+
+[case testCustomErrorCodeCanBeDisabled]
+# flags: --config-file tmp/mypy.ini --show-error-codes --disable-error-code=custom
+
+def main() -> None:
+    return
+main()  # no output expected when disabled
+
+[file mypy.ini]
+\[mypy]
+plugins=<ROOT>/test-data/unit/plugins/custom_errorcode.py
+
+[case testCustomErrorCodeCanBeReenabled]
+# flags: --config-file tmp/mypy.ini --show-error-codes --disable-error-code=custom --enable-error-code=custom
+
+def main() -> None:
+    return
+main() # E: Custom error  [custom]
+
+[file mypy.ini]
+\[mypy]
+plugins=<ROOT>/test-data/unit/plugins/custom_errorcode.py