add ci/cd for conformance testing and option to skip install type checkers

.github/workflows/conformance-test.yml

@@ -0,0 +1,121 @@
+name: Update Typing Conformance Tests Results
+
+on:
+  schedule:
+    - cron: "45 11 * * 1"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  check-versions:
+    runs-on: ubuntu-latest
+    outputs:
+      has_updates: ${{ steps.compare.outputs.has_updates }}
+    steps:
+      - name: Restore previous version state
+        id: restore-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: tool_versions.json
+          # We use a key that will mis  s, forcing it to look at restore-keys
+          key: tool-state-placeholder-${{ github.run_id }}
+          # This picks the most recent cache matching this prefix
+          restore-keys: |
+            tool-state-
+
+      - name: Check and Compare Versions
+        id: compare
+        run: |
+          TOOLS=("zuban" "pyrefly" "pyright" "mypy")
+          STATE_FILE="tool_versions.json"
+          UPDATED=false
+
+          # Load previous state or create empty if first run
+          if [ -f "$STATE_FILE" ]; then
+            echo "Found previous state."
+          else
+            echo "{}" > "$STATE_FILE"
+            echo "No previous state found (First Run). Treating as updated."
+            # We force update to true on first run to populate the cache
+            UPDATED=true
+          fi
+
+          # Temp file for the new state
+          NEW_STATE=$(mktemp)
+          echo "{}" > "$NEW_STATE"
+
+          echo "Fetching versions from PyPI..."
+
+          for tool in "${TOOLS[@]}"; do
+            # Fetch latest version string
+            LATEST=$(curl -sL "https://pypi.org/pypi/$tool/json" | jq -r '.info.version')
+
+            # Get old version from JSON
+            OLD=$(jq -r --arg t "$tool" '.[$t] // ""' "$STATE_FILE")
+
+            echo "Checking $tool: Old=$OLD | New=$LATEST"
+
+            # Write to new state object
+            jq --arg t "$tool" --arg v "$LATEST" '.[$t] = $v' "$NEW_STATE" > "$NEW_STATE.tmp" && mv "$NEW_STATE.tmp" "$NEW_STATE"
+
+            if [ "$LATEST" != "$OLD" ]; then
+              echo ">> UPDATE DETECTED for $tool"
+              UPDATED=true
+            fi
+          done
+
+          if [ "$UPDATED" = "true" ]; then
+            echo "has_updates=true" >> $GITHUB_OUTPUT
+            # Replace the state file with the new one so Cache Save picks it up
+            mv "$NEW_STATE" "$STATE_FILE"
+            cat "$STATE_FILE"
+          else
+            echo "has_updates=false" >> $GITHUB_OUTPUT
+            echo "No updates found."
+          fi
+
+      - name: Save new state to cache
+        if: steps.compare.outputs.has_updates == 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: tool_versions.json
+          # Unique key ensures a new cache entry is created
+          key: tool-state-${{ github.run_id }}
+
+  run-conformance-tests:
+    needs: check-versions
+    if: needs.check-versions.outputs.has_updates == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v6
+        with:
+          sparse-checkout: |
+            conformance
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install Dependencies
+        run: |
+          cd conformance
+          pip install -r requirements.txt
+
+      - name: Run Main Script (Src)
+        run: |
+          cd conformance/src
+          python main.py --skip-install-check
+
+      - name: Commit and Push results
+        run: |
+          # go dir `conformance`
+          cd conformance
+          echo "Typing static type checkers has updated, commit and push new results"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add ./results/*
+          git commit -m "chore: update conformance test results"
+          git push

conformance/results/mypy/callables_annotation.toml

@@ -17,17 +17,17 @@ callables_annotation.py:57: error: Bracketed expression "[...]" is not valid as
 callables_annotation.py:57: note: Did you mean "List[...]"?
 callables_annotation.py:58: error: Please use "Callable[[<parameters>], <return type>]" or "Callable"  [misc]
 callables_annotation.py:59: error: Unexpected "..."  [misc]
-callables_annotation.py:91: error: Incompatible types in assignment (expression has type "Callable[[], str]", variable has type "Callable[[int, VarArg(Any), KwArg(Any)], str]")  [assignment]
-callables_annotation.py:93: error: Incompatible types in assignment (expression has type "Callable[[NamedArg(int, 'a')], str]", variable has type "Callable[[int, VarArg(Any), KwArg(Any)], str]")  [assignment]
+callables_annotation.py:91: error: Incompatible types in assignment (expression has type "Callable[[], str]", variable has type "def (int, /, *Any, **Any) -> str")  [assignment]
+callables_annotation.py:93: error: Incompatible types in assignment (expression has type "def test_cb4(*, a: int) -> str", variable has type "def (int, /, *Any, **Any) -> str")  [assignment]
 callables_annotation.py:157: error: Incompatible types in assignment (expression has type "Proto7", variable has type "Proto6")  [assignment]
 callables_annotation.py:157: note: Following member(s) of "Proto7" have conflicts:
 callables_annotation.py:157: note:     Expected:
 callables_annotation.py:157: note:         def __call__(self, int, /, *args: Any, k: str, **kwargs: Any) -> None
 callables_annotation.py:157: note:     Got:
 callables_annotation.py:157: note:         def __call__(self, float, /, b: int, *, k: str, m: str) -> None
-callables_annotation.py:172: error: Incompatible types in assignment (expression has type "Callable[[], str]", variable has type "Callable[[int, VarArg(Any), KwArg(Any)], str]")  [assignment]
-callables_annotation.py:187: error: Incompatible types in assignment (expression has type "Callable[[int, str], str]", variable has type "Callable[[str, VarArg(Any), KwArg(Any)], str]")  [assignment]
-callables_annotation.py:189: error: Incompatible types in assignment (expression has type "Callable[[int, str], str]", variable has type "Callable[[str, VarArg(Any), KwArg(Any)], str]")  [assignment]
+callables_annotation.py:172: error: Incompatible types in assignment (expression has type "Callable[[], str]", variable has type "def (int, /, *Any, **Any) -> str")  [assignment]
+callables_annotation.py:187: error: Incompatible types in assignment (expression has type "Callable[[int, str], str]", variable has type "def (str, /, *Any, **Any) -> str")  [assignment]
+callables_annotation.py:189: error: Incompatible types in assignment (expression has type "Callable[[int, str], str]", variable has type "def (str, /, *Any, **Any) -> str")  [assignment]
 """
 conformance_automated = "Fail"
 errors_diff = """

conformance/results/mypy/callables_kwargs.toml

@@ -15,12 +15,12 @@ callables_kwargs.py:63: error: "func1" gets multiple values for keyword argument
 callables_kwargs.py:64: error: "func2" gets multiple values for keyword argument "v3"  [misc]
 callables_kwargs.py:64: error: Argument 1 to "func2" has incompatible type "int"; expected "str"  [arg-type]
 callables_kwargs.py:65: error: "func2" gets multiple values for keyword argument "v1"  [misc]
-callables_kwargs.py:101: error: Incompatible types in assignment (expression has type "Callable[[KwArg(TD2)], None]", variable has type "TDProtocol3")  [assignment]
-callables_kwargs.py:101: note: "TDProtocol3.__call__" has type "Callable[[NamedArg(int, 'v1'), NamedArg(int, 'v2'), NamedArg(str, 'v3')], None]"
-callables_kwargs.py:102: error: Incompatible types in assignment (expression has type "Callable[[KwArg(TD2)], None]", variable has type "TDProtocol4")  [assignment]
-callables_kwargs.py:102: note: "TDProtocol4.__call__" has type "Callable[[NamedArg(int, 'v1')], None]"
-callables_kwargs.py:103: error: Incompatible types in assignment (expression has type "Callable[[KwArg(TD2)], None]", variable has type "TDProtocol5")  [assignment]
-callables_kwargs.py:103: note: "TDProtocol5.__call__" has type "Callable[[Arg(int, 'v1'), Arg(str, 'v3')], None]"
+callables_kwargs.py:101: error: Incompatible types in assignment (expression has type "def func1(**kwargs: Unpack[TD2]) -> None", variable has type "TDProtocol3")  [assignment]
+callables_kwargs.py:101: note: "TDProtocol3.__call__" has type "def __call__(self, *, v1: int, v2: int, v3: str) -> None"
+callables_kwargs.py:102: error: Incompatible types in assignment (expression has type "def func1(**kwargs: Unpack[TD2]) -> None", variable has type "TDProtocol4")  [assignment]
+callables_kwargs.py:102: note: "TDProtocol4.__call__" has type "def __call__(self, *, v1: int) -> None"
+callables_kwargs.py:103: error: Incompatible types in assignment (expression has type "def func1(**kwargs: Unpack[TD2]) -> None", variable has type "TDProtocol5")  [assignment]
+callables_kwargs.py:103: note: "TDProtocol5.__call__" has type "def __call__(self, v1: int, v3: str) -> None"
 callables_kwargs.py:111: error: Overlap between argument names and ** TypedDict items: "v1"  [misc]
 callables_kwargs.py:122: error: Unpack item in ** argument must be a TypedDict  [misc]
 """

conformance/results/mypy/callables_protocol.toml

@@ -1,36 +1,36 @@
 conformant = "Pass"
 output = """
-callables_protocol.py:35: error: Incompatible types in assignment (expression has type "Callable[[VarArg(bytes), NamedArg(int | None, 'max_items')], list[bytes]]", variable has type "Proto1")  [assignment]
-callables_protocol.py:35: note: "Proto1.__call__" has type "Callable[[VarArg(bytes), DefaultNamedArg(int | None, 'max_len')], list[bytes]]"
-callables_protocol.py:36: error: Incompatible types in assignment (expression has type "Callable[[VarArg(bytes)], list[bytes]]", variable has type "Proto1")  [assignment]
-callables_protocol.py:36: note: "Proto1.__call__" has type "Callable[[VarArg(bytes), DefaultNamedArg(int | None, 'max_len')], list[bytes]]"
-callables_protocol.py:37: error: Incompatible types in assignment (expression has type "Callable[[VarArg(bytes), NamedArg(str | None, 'max_len')], list[bytes]]", variable has type "Proto1")  [assignment]
-callables_protocol.py:37: note: "Proto1.__call__" has type "Callable[[VarArg(bytes), DefaultNamedArg(int | None, 'max_len')], list[bytes]]"
-callables_protocol.py:67: error: Incompatible types in assignment (expression has type "Callable[[VarArg(bytes)], Any]", variable has type "Proto2")  [assignment]
-callables_protocol.py:67: note: "Proto2.__call__" has type "Callable[[VarArg(bytes), KwArg(str)], None]"
-callables_protocol.py:68: error: Incompatible types in assignment (expression has type "Callable[[VarArg(str), KwArg(str)], Any]", variable has type "Proto2")  [assignment]
-callables_protocol.py:68: note: "Proto2.__call__" has type "Callable[[VarArg(bytes), KwArg(str)], None]"
-callables_protocol.py:69: error: Incompatible types in assignment (expression has type "Callable[[VarArg(bytes), KwArg(bytes)], Any]", variable has type "Proto2")  [assignment]
-callables_protocol.py:69: note: "Proto2.__call__" has type "Callable[[VarArg(bytes), KwArg(str)], None]"
-callables_protocol.py:70: error: Incompatible types in assignment (expression has type "Callable[[KwArg(str)], Any]", variable has type "Proto2")  [assignment]
-callables_protocol.py:70: note: "Proto2.__call__" has type "Callable[[VarArg(bytes), KwArg(str)], None]"
+callables_protocol.py:35: error: Incompatible types in assignment (expression has type "def cb1_bad1(*vals: bytes, max_items: int | None) -> list[bytes]", variable has type "Proto1")  [assignment]
+callables_protocol.py:35: note: "Proto1.__call__" has type "def __call__(self, *vals: bytes, max_len: int | None = ...) -> list[bytes]"
+callables_protocol.py:36: error: Incompatible types in assignment (expression has type "def cb1_bad2(*vals: bytes) -> list[bytes]", variable has type "Proto1")  [assignment]
+callables_protocol.py:36: note: "Proto1.__call__" has type "def __call__(self, *vals: bytes, max_len: int | None = ...) -> list[bytes]"
+callables_protocol.py:37: error: Incompatible types in assignment (expression has type "def cb1_bad3(*vals: bytes, max_len: str | None) -> list[bytes]", variable has type "Proto1")  [assignment]
+callables_protocol.py:37: note: "Proto1.__call__" has type "def __call__(self, *vals: bytes, max_len: int | None = ...) -> list[bytes]"
+callables_protocol.py:67: error: Incompatible types in assignment (expression has type "def cb2_bad1(*a: bytes) -> Any", variable has type "Proto2")  [assignment]
+callables_protocol.py:67: note: "Proto2.__call__" has type "def __call__(self, *vals: bytes, **kwargs: str) -> None"
+callables_protocol.py:68: error: Incompatible types in assignment (expression has type "def cb2_bad2(*a: str, **b: str) -> Any", variable has type "Proto2")  [assignment]
+callables_protocol.py:68: note: "Proto2.__call__" has type "def __call__(self, *vals: bytes, **kwargs: str) -> None"
+callables_protocol.py:69: error: Incompatible types in assignment (expression has type "def cb2_bad3(*a: bytes, **b: bytes) -> Any", variable has type "Proto2")  [assignment]
+callables_protocol.py:69: note: "Proto2.__call__" has type "def __call__(self, *vals: bytes, **kwargs: str) -> None"
+callables_protocol.py:70: error: Incompatible types in assignment (expression has type "def cb2_bad4(**b: str) -> Any", variable has type "Proto2")  [assignment]
+callables_protocol.py:70: note: "Proto2.__call__" has type "def __call__(self, *vals: bytes, **kwargs: str) -> None"
 callables_protocol.py:97: error: Incompatible types in assignment (expression has type "Callable[[int], None]", variable has type "Proto4")  [assignment]
 callables_protocol.py:97: note: "function" is missing following "Proto4" protocol member:
 callables_protocol.py:97: note:     other_attribute
-callables_protocol.py:121: error: Incompatible types in assignment (expression has type "Callable[[VarArg(bytes), DefaultNamedArg(int | None, 'max_len')], list[bytes]]", variable has type "NotProto6")  [assignment]
+callables_protocol.py:121: error: Incompatible types in assignment (expression has type "def cb6_bad1(*vals: bytes, max_len: int | None = ...) -> list[bytes]", variable has type "NotProto6")  [assignment]
 callables_protocol.py:169: error: Incompatible types in assignment (expression has type "Callable[[int], Any]", variable has type "Proto8")  [assignment]
 callables_protocol.py:169: note: "Proto8.__call__" has type overloaded function
 callables_protocol.py:186: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
 callables_protocol.py:187: error: "Proto9[P, R]" has no attribute "xxx"  [attr-defined]
 callables_protocol.py:197: error: "Proto9[[int], str]" has no attribute "other_attribute2"; maybe "other_attribute"?  [attr-defined]
 callables_protocol.py:238: error: Incompatible types in assignment (expression has type "Callable[[int, str], Any]", variable has type "Proto11")  [assignment]
-callables_protocol.py:238: note: "Proto11.__call__" has type "Callable[[int, Arg(str, 'y')], Any]"
-callables_protocol.py:260: error: Incompatible types in assignment (expression has type "Callable[[VarArg(Any), NamedArg(Any, 'kwarg0')], None]", variable has type "Proto12")  [assignment]
-callables_protocol.py:260: note: "Proto12.__call__" has type "Callable[[VarArg(Any), NamedArg(Any, 'kwarg0'), NamedArg(Any, 'kwarg1')], None]"
+callables_protocol.py:238: note: "Proto11.__call__" has type "def __call__(self, int, /, y: str) -> Any"
+callables_protocol.py:260: error: Incompatible types in assignment (expression has type "def cb12_bad1(*args: Any, kwarg0: Any) -> None", variable has type "Proto12")  [assignment]
+callables_protocol.py:260: note: "Proto12.__call__" has type "def __call__(self, *args: Any, kwarg0: Any, kwarg1: Any) -> None"
 callables_protocol.py:284: error: Incompatible types in assignment (expression has type "Callable[[str], str]", variable has type "Proto13_Default")  [assignment]
-callables_protocol.py:284: note: "Proto13_Default.__call__" has type "Callable[[DefaultArg(str, 'path')], str]"
-callables_protocol.py:311: error: Incompatible types in assignment (expression has type "Callable[[NamedArg(str, 'path')], str]", variable has type "Proto14_Default")  [assignment]
-callables_protocol.py:311: note: "Proto14_Default.__call__" has type "Callable[[DefaultNamedArg(str, 'path')], str]"
+callables_protocol.py:284: note: "Proto13_Default.__call__" has type "def __call__(self, path: str = ...) -> str"
+callables_protocol.py:311: error: Incompatible types in assignment (expression has type "def cb14_no_default(*, path: str) -> str", variable has type "Proto14_Default")  [assignment]
+callables_protocol.py:311: note: "Proto14_Default.__call__" has type "def __call__(self, *, path: str = ...) -> str"
 """
 conformance_automated = "Pass"
 errors_diff = """

conformance/results/mypy/dataclasses_transform_converter.toml

@@ -13,7 +13,7 @@ Line 121: Unexpected errors ['dataclasses_transform_converter.py:121: error: Arg
 """
 output = """
 dataclasses_transform_converter.py:48: error: Argument "converter" to "model_field" has incompatible type "Callable[[], int]"; expected "Callable[[Never], int]"  [arg-type]
-dataclasses_transform_converter.py:49: error: Argument "converter" to "model_field" has incompatible type "Callable[[NamedArg(int, 'x')], int]"; expected "Callable[[Never], int]"  [arg-type]
+dataclasses_transform_converter.py:49: error: Argument "converter" to "model_field" has incompatible type "def bad_converter2(*, x: int) -> int"; expected "Callable[[Never], int]"  [arg-type]
 dataclasses_transform_converter.py:107: error: Argument 2 to "DC2" has incompatible type "str"; expected "int"  [arg-type]
 dataclasses_transform_converter.py:107: error: Argument 3 to "DC2" has incompatible type "str"; expected "int"  [arg-type]
 dataclasses_transform_converter.py:107: error: Argument 4 to "DC2" has incompatible type "bytes"; expected "ConverterClass"  [arg-type]

conformance/results/mypy/generics_paramspec_semantics.toml

@@ -3,14 +3,14 @@ output = """
 generics_paramspec_semantics.py:26: error: Unexpected keyword argument "a"  [call-arg]
 generics_paramspec_semantics.py:26: error: Unexpected keyword argument "b"  [call-arg]
 generics_paramspec_semantics.py:27: error: Argument 2 has incompatible type "str"; expected "bool"  [arg-type]
-generics_paramspec_semantics.py:61: error: Argument 2 to "func1" has incompatible type "Callable[[NamedArg(int, 'y')], int]"; expected "Callable[[NamedArg(int, 'x')], int]"  [arg-type]
+generics_paramspec_semantics.py:61: error: Argument 2 to "func1" has incompatible type "def keyword_only_y(*, y: int) -> int"; expected "def (*, x: int) -> int"  [arg-type]
 generics_paramspec_semantics.py:98: error: Argument 1 has incompatible type "int"; expected "str"  [arg-type]
 generics_paramspec_semantics.py:108: error: Argument 1 has incompatible type "int"; expected "bool"  [arg-type]
 generics_paramspec_semantics.py:120: error: Argument 1 has incompatible type "int"; expected "str"  [arg-type]
 generics_paramspec_semantics.py:127: error: Argument 1 to "expects_int_first" has incompatible type "Callable[[str], int]"; expected "Callable[[int], int]"  [arg-type]
 generics_paramspec_semantics.py:127: note: This is likely because "one" has named arguments: "x". Consider marking them positional-only
-generics_paramspec_semantics.py:132: error: Argument 1 to "expects_int_first" has incompatible type "Callable[[NamedArg(int, 'x')], int]"; expected "Callable[[int, NamedArg(int, 'x')], int]"  [arg-type]
-generics_paramspec_semantics.py:137: error: Argument 1 to "expects_int_first" has incompatible type "Callable[[KwArg(int)], int]"; expected "Callable[[int, KwArg(int)], int]"  [arg-type]
+generics_paramspec_semantics.py:132: error: Argument 1 to "expects_int_first" has incompatible type "def two(*, x: int) -> int"; expected "def (int, /, *, x: int) -> int"  [arg-type]
+generics_paramspec_semantics.py:137: error: Argument 1 to "expects_int_first" has incompatible type "def three(**kwargs: int) -> int"; expected "def (int, /, **kwargs: int) -> int"  [arg-type]
 """
 conformance_automated = "Pass"
 errors_diff = """