add ci/cd for conformance testing and option to skip install type checkers

.github/workflows/conformance-test.yml

@@ -0,0 +1,121 @@
+name: Update Typing Conformance Tests Results
+
+on:
+  schedule:
+    - cron: "45 11 * * 1"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  check-versions:
+    runs-on: ubuntu-latest
+    outputs:
+      has_updates: ${{ steps.compare.outputs.has_updates }}
+    steps:
+      - name: Restore previous version state
+        id: restore-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: tool_versions.json
+          # We use a key that will mis  s, forcing it to look at restore-keys
+          key: tool-state-placeholder-${{ github.run_id }}
+          # This picks the most recent cache matching this prefix
+          restore-keys: |
+            tool-state-
+
+      - name: Check and Compare Versions
+        id: compare
+        run: |
+          TOOLS=("zuban" "pyrefly" "pyright" "mypy")
+          STATE_FILE="tool_versions.json"
+          UPDATED=false
+
+          # Load previous state or create empty if first run
+          if [ -f "$STATE_FILE" ]; then
+            echo "Found previous state."
+          else
+            echo "{}" > "$STATE_FILE"
+            echo "No previous state found (First Run). Treating as updated."
+            # We force update to true on first run to populate the cache
+            UPDATED=true
+          fi
+
+          # Temp file for the new state
+          NEW_STATE=$(mktemp)
+          echo "{}" > "$NEW_STATE"
+
+          echo "Fetching versions from PyPI..."
+
+          for tool in "${TOOLS[@]}"; do
+            # Fetch latest version string
+            LATEST=$(curl -sL "https://pypi.org/pypi/$tool/json" | jq -r '.info.version')
+
+            # Get old version from JSON
+            OLD=$(jq -r --arg t "$tool" '.[$t] // ""' "$STATE_FILE")
+
+            echo "Checking $tool: Old=$OLD | New=$LATEST"
+
+            # Write to new state object
+            jq --arg t "$tool" --arg v "$LATEST" '.[$t] = $v' "$NEW_STATE" > "$NEW_STATE.tmp" && mv "$NEW_STATE.tmp" "$NEW_STATE"
+
+            if [ "$LATEST" != "$OLD" ]; then
+              echo ">> UPDATE DETECTED for $tool"
+              UPDATED=true
+            fi
+          done
+
+          if [ "$UPDATED" = "true" ]; then
+            echo "has_updates=true" >> $GITHUB_OUTPUT
+            # Replace the state file with the new one so Cache Save picks it up
+            mv "$NEW_STATE" "$STATE_FILE"
+            cat "$STATE_FILE"
+          else
+            echo "has_updates=false" >> $GITHUB_OUTPUT
+            echo "No updates found."
+          fi
+
+      - name: Save new state to cache
+        if: steps.compare.outputs.has_updates == 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: tool_versions.json
+          # Unique key ensures a new cache entry is created
+          key: tool-state-${{ github.run_id }}
+
+  run-conformance-tests:
+    needs: check-versions
+    if: needs.check-versions.outputs.has_updates == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v6
+        with:
+          sparse-checkout: |
+            conformance
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install Dependencies
+        run: |
+          cd conformance
+          pip install -r requirements.txt
+
+      - name: Run Main Script (Src)
+        run: |
+          cd conformance/src
+          python main.py --skip-install-check
+
+      - name: Commit and Push results
+        run: |
+          # go dir `conformance`
+          cd conformance
+          echo "Typing static type checkers has updated, commit and push new results"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add ./results/*
+          git commit -m "chore: update conformance test results"
+          git push

conformance/results/pyrefly/dataclasses_final.toml

@@ -2,13 +2,13 @@ conformant = "Partial"
 notes = """
 Allows assignment to final attributes that are not initialized on the class
 """
-conformance_automated = "Fail"
+conformance_automated = "Pass"
 errors_diff = """
-Line 35: Expected 1 errors
-Line 37: Expected 1 errors
 """
 output = """
 ERROR dataclasses_final.py:27:1-17: Cannot set field `final_classvar` [read-only]
+ERROR dataclasses_final.py:35:1-19: Cannot set field `final_no_default` [read-only]
 ERROR dataclasses_final.py:36:1-21: Cannot set field `final_with_default` [read-only]
+ERROR dataclasses_final.py:37:1-19: Cannot set field `final_no_default` [read-only]
 ERROR dataclasses_final.py:38:1-21: Cannot set field `final_with_default` [read-only]
 """

conformance/results/pyrefly/dataclasses_transform_converter.toml

@@ -4,9 +4,7 @@ errors_diff = """
 """
 output = """
 ERROR dataclasses_transform_converter.py:48:41-55: Argument `() -> int` is not assignable to parameter `converter` with type `(@_) -> int` in function `model_field` [bad-argument-type]
-ERROR dataclasses_transform_converter.py:48:41-55: Argument `() -> int` is not assignable to parameter `converter` with type `(@_) -> @_` in function `model_field` [bad-argument-type]
 ERROR dataclasses_transform_converter.py:49:41-55: Argument `(*, x: int) -> int` is not assignable to parameter `converter` with type `(@_) -> int` in function `model_field` [bad-argument-type]
-ERROR dataclasses_transform_converter.py:49:41-55: Argument `(*, x: int) -> int` is not assignable to parameter `converter` with type `(@_) -> @_` in function `model_field` [bad-argument-type]
 ERROR dataclasses_transform_converter.py:107:5-6: Argument `Literal[1]` is not assignable to parameter `field0` with type `str` in function `DC2.__init__` [bad-argument-type]
 ERROR dataclasses_transform_converter.py:108:23-24: Argument `Literal[1]` is not assignable to parameter `field3` with type `bytes | str` in function `DC2.__init__` [bad-argument-type]
 ERROR dataclasses_transform_converter.py:109:29-31: Argument `complex` is not assignable to parameter `field4` with type `list[str] | str` in function `DC2.__init__` [bad-argument-type]

conformance/results/pyrefly/generics_defaults.toml

@@ -11,7 +11,7 @@ ERROR generics_defaults.py:24:7-31: Type parameter `T` without a default cannot
 ERROR generics_defaults.py:50:1-20: Expected 5 type arguments for `AllTheDefaults`, got 1 [bad-specialization]
 ERROR generics_defaults.py:107:51-54: Expected default `int` of `Invalid1` to be assignable to the upper bound of `str` [invalid-type-var]
 ERROR generics_defaults.py:114:52-55: Expected default `int` of `Invalid2` to be one of the following constraints: `float`, `str` [invalid-type-var]
-ERROR generics_defaults.py:130:12-27: assert_type(Any, int) failed [assert-type]
+ERROR generics_defaults.py:131:12-27: assert_type(int, Any) failed [assert-type]
 ERROR generics_defaults.py:142:7-11: TypeVar `T5` with a default cannot follow TypeVarTuple `Ts` [invalid-type-var]
 ERROR generics_defaults.py:170:12-57: assert_type([DefaultIntT](Foo7[DefaultIntT]) -> Foo7[DefaultIntT], (Foo7[int]) -> Foo7[int]) failed [assert-type]
 """

conformance/results/pyrefly/generics_syntax_scoping.toml

@@ -2,16 +2,16 @@ conformant = "Partial"
 notes = """
 Does not implement some scoping restrictions for PEP695 generic syntax
 """
-conformance_automated = "Fail"
+conformance_automated = "Pass"
 errors_diff = """
-Line 92: Expected 1 errors
-Line 95: Expected 1 errors
-Line 98: Expected 1 errors
 """
 output = """
 ERROR generics_syntax_scoping.py:14:20-31: Type variable bounds and constraints must be concrete [invalid-annotation]
 ERROR generics_syntax_scoping.py:18:26-27: Expected a type form, got instance of `int` [not-a-type]
 ERROR generics_syntax_scoping.py:35:7-8: `T` is uninitialized [unbound-name]
 ERROR generics_syntax_scoping.py:44:17-18: `T` is uninitialized [unbound-name]
 ERROR generics_syntax_scoping.py:44:17-18: Expected a type form, got instance of `int` [not-a-type]
+ERROR generics_syntax_scoping.py:92:17-18: Type parameter `T` shadows a type parameter of the same name from an enclosing scope [invalid-type-var]
+ERROR generics_syntax_scoping.py:95:17-18: Type parameter `T` shadows a type parameter of the same name from an enclosing scope [invalid-type-var]
+ERROR generics_syntax_scoping.py:98:17-18: Type parameter `T` shadows a type parameter of the same name from an enclosing scope [invalid-type-var]
 """

conformance/results/pyrefly/protocols_class_objects.toml

@@ -4,16 +4,14 @@ Does not require concrete classes to be passed to type[Proto]
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 29: Expected 1 errors
-Line 34: Expected 1 errors
 Line 104: Expected 1 errors
 Line 106: Expected 1 errors
 Line 107: Expected 1 errors
 Line 108: Expected 1 errors
-Line 26: Unexpected errors ['Cannot instantiate `Proto` because it is a protocol [bad-instantiation]']
 """
 output = """
-ERROR protocols_class_objects.py:26:15-17: Cannot instantiate `Proto` because it is a protocol [bad-instantiation]
+ERROR protocols_class_objects.py:29:5-10: Argument `type[Proto]` is not assignable to parameter `cls` with type `type[Proto]` in function `fun` [bad-argument-type]
+ERROR protocols_class_objects.py:34:7-12: `type[Proto]` is not assignable to variable `var` with type `type[Proto]` [bad-assignment]
 ERROR protocols_class_objects.py:58:16-25: `type[ConcreteA]` is not assignable to `ProtoA1` [bad-assignment]
 ERROR protocols_class_objects.py:74:16-25: `type[ConcreteB]` is not assignable to `ProtoB1` [bad-assignment]
 """

conformance/results/pyrefly/protocols_explicit.toml

@@ -5,10 +5,10 @@ Does not detect stub methods inherited from protocols as abstract.
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 27: Expected 1 errors
 Line 89: Expected 1 errors
 """
 output = """
+ERROR protocols_explicit.py:27:16-28: Method `draw` inherited from class `PColor` has no implementation and cannot be accessed via `super()` [missing-attribute]
 ERROR protocols_explicit.py:56:20-36: `tuple[int, int, str]` is not assignable to attribute `rgb` with type `tuple[int, int, int]` [bad-assignment]
 ERROR protocols_explicit.py:60:10-20: Cannot instantiate `Point` because the following members are abstract: `intensity`, `transparency` [bad-instantiation]
 ERROR protocols_explicit.py:134:15-17: Cannot instantiate `Concrete5` because the following members are abstract: `method1` [bad-instantiation]

conformance/results/pyrefly/typeddicts_class_syntax.toml

@@ -3,9 +3,9 @@ conformance_automated = "Pass"
 errors_diff = """
 """
 output = """
-ERROR typeddicts_class_syntax.py:29:9-16: TypedDict item `method1` may not be initialized [bad-class-definition]
-ERROR typeddicts_class_syntax.py:34:9-16: TypedDict item `method2` may not be initialized [bad-class-definition]
-ERROR typeddicts_class_syntax.py:39:9-16: TypedDict item `method3` may not be initialized [bad-class-definition]
+ERROR typeddicts_class_syntax.py:29:9-16: TypedDict members must be declared in the form `field: Annotation` with no assignment [bad-class-definition]
+ERROR typeddicts_class_syntax.py:34:9-16: TypedDict members must be declared in the form `field: Annotation` with no assignment [bad-class-definition]
+ERROR typeddicts_class_syntax.py:39:9-16: TypedDict members must be declared in the form `field: Annotation` with no assignment [bad-class-definition]
 ERROR typeddicts_class_syntax.py:44:7-20: Typed dictionary definitions may not specify a metaclass [invalid-inheritance]
 ERROR typeddicts_class_syntax.py:49:7-20: TypedDict does not support keyword argument `other` [bad-typed-dict]
 """

conformance/results/pyrefly/typeddicts_readonly_inheritance.toml

@@ -2,9 +2,8 @@ conformant = "Partial"
 notes = """
 Does not restrictions around overriding for ReadOnly fields
 """
-conformance_automated = "Fail"
+conformance_automated = "Pass"
 errors_diff = """
-Line 119: Expected 1 errors
 """
 output = """
 ERROR typeddicts_readonly_inheritance.py:36:4-10: Key `name` in TypedDict `Album2` is read-only [read-only]
@@ -16,5 +15,6 @@ ERROR typeddicts_readonly_inheritance.py:84:5-7: Missing required key `ident` fo
 ERROR typeddicts_readonly_inheritance.py:94:5-6: TypedDict field `a` in `F3` cannot be marked read-only; parent TypedDict `F1` defines it as mutable [bad-typed-dict-key]
 ERROR typeddicts_readonly_inheritance.py:98:5-6: TypedDict field `a` in `F4` must remain required because parent TypedDict `F1` defines it as required [bad-typed-dict-key]
 ERROR typeddicts_readonly_inheritance.py:106:5-6: TypedDict field `c` in `F6` cannot be made non-required; parent TypedDict `F1` defines it as required [bad-typed-dict-key]
+ERROR typeddicts_readonly_inheritance.py:119:7-11: Field `x` is declared `float` in ancestor `class TD_A2: ...
 ERROR typeddicts_readonly_inheritance.py:132:7-11: TypedDict field `x` in `TD_B` cannot be made non-required; parent TypedDict `TD_B2` defines it as required [bad-typed-dict-key]
 """

conformance/results/pyrefly/typeddicts_required.toml

@@ -2,15 +2,13 @@ conformant = "Partial"
 notes = """
 Does not handle recursive typed dicts in functional syntax
 """
-conformance_automated = "Fail"
+conformance_automated = "Pass"
 errors_diff = """
-Line 71: Unexpected errors ["Expected a type form, got instance of `Literal['RecursiveMovie']` [not-a-type]"]
 """
 output = """
 ERROR typeddicts_required.py:12:5-6: `Required` may only be used for TypedDict members [invalid-annotation]
 ERROR typeddicts_required.py:16:8-19: `NotRequired` is only allowed inside a class body [invalid-annotation]
 ERROR typeddicts_required.py:16:8-24: `NotRequired` is not allowed in this context [invalid-annotation]
 ERROR typeddicts_required.py:59:8-31: Duplicate qualifier `Required` [invalid-annotation]
 ERROR typeddicts_required.py:60:8-34: Cannot combine `Required` and `NotRequired` for a TypedDict field [invalid-annotation]
-ERROR typeddicts_required.py:71:75-91: Expected a type form, got instance of `Literal['RecursiveMovie']` [not-a-type]
 """

conformance/results/pyrefly/version.toml

@@ -1 +1 @@
-version = "pyrefly 0.39.4"
+version = "pyrefly 0.42.3"

conformance/results/results.html

@@ -162,9 +162,9 @@ <h3>Python Type System Conformance Test Results</h3>
 </th>
 <th class='tc-header'><div class='tc-name'>pyright 1.1.407</div>
 </th>
-<th class='tc-header'><div class='tc-name'>zuban 0.2.1</div>
+<th class='tc-header'><div class='tc-name'>zuban 0.2.3</div>
 </th>
-<th class='tc-header'><div class='tc-name'>pyrefly 0.39.4</div>
+<th class='tc-header'><div class='tc-name'>pyrefly 0.42.3</div>
 </th>
 </tr>
 <tr><th class="column" colspan="5">

conformance/results/zuban/version.toml

@@ -1 +1 @@
-version = "zuban 0.2.1"
+version = "zuban 0.2.3"

conformance/src/main.py

@@ -3,15 +3,14 @@
 """
 
 import os
-from pathlib import Path
 import re
 import sys
+from pathlib import Path
 from time import time
 from typing import Sequence
 
 import tomli
 import tomlkit
-
 from options import parse_options
 from reporting import generate_summary
 from test_groups import get_test_cases, get_test_groups
@@ -262,10 +261,14 @@ def main():
         for type_checker in TYPE_CHECKERS:
             if options.only_run and options.only_run != type_checker.name:
                 continue
-            if not type_checker.install():
-                print(f"Skipping tests for {type_checker.name}")
+            if options.skip_install_check:
+                print("Skipping install check")
             else:
-                run_tests(root_dir, type_checker, test_cases)
+                if not type_checker.install():
+                    print(f"Skipping tests for {type_checker.name}")
+                    continue
+
+            run_tests(root_dir, type_checker, test_cases)
 
     # Generate a summary report.
     generate_summary(root_dir)

conformance/src/options.py

@@ -7,10 +7,12 @@
 
 from type_checker import TYPE_CHECKERS
 
+
 @dataclass
 class _Options:
     report_only: bool | None
     only_run: str | None
+    skip_install_check: bool | None
 
 
 def parse_options(argv: list[str]) -> _Options:
@@ -24,7 +26,13 @@ def parse_options(argv: list[str]) -> _Options:
     reporting_group.add_argument(
         "--only-run",
         help="Only runs the type checker",
-        choices=[tc.name for tc in TYPE_CHECKERS]
+        choices=[tc.name for tc in TYPE_CHECKERS],
     )
+    reporting_group.add_argument(
+        "--skip-install-check",
+        action="store_true",
+        help="Skips the check for whether type checkers are installed",
+    )
+
     ret = _Options(**vars(parser.parse_args(argv)))
     return ret