From 5529ca4673eaf298832ab71e25a13553d98d014e Mon Sep 17 00:00:00 2001 From: Rebecca Chen Date: Mon, 21 Jul 2025 13:41:57 -0700 Subject: [PATCH 1/4] Update conformance results for mypy 1.17.0 and pyre 0.9.25. I ran the conformance tests and noticed that the results have changed a bit for the latest versions of mypy and pyre. --- conformance/results/mypy/aliases_recursive.toml | 2 +- conformance/results/mypy/classes_classvar.toml | 2 +- conformance/results/mypy/dataclasses_kwonly.toml | 14 +------------- .../results/mypy/directives_reveal_type.toml | 2 +- conformance/results/mypy/version.toml | 4 ++-- .../pyre/generics_paramspec_specialization.toml | 4 ---- conformance/results/pyre/version.toml | 4 ++-- conformance/results/pyright/version.toml | 4 ++-- conformance/results/results.html | 16 ++++++++-------- 9 files changed, 18 insertions(+), 34 deletions(-) diff --git a/conformance/results/mypy/aliases_recursive.toml b/conformance/results/mypy/aliases_recursive.toml index 01af644dc..5272c3212 100644 --- a/conformance/results/mypy/aliases_recursive.toml +++ b/conformance/results/mypy/aliases_recursive.toml @@ -6,7 +6,7 @@ aliases_recursive.py:38: error: Incompatible types in assignment (expression has aliases_recursive.py:39: error: Name "t6" already defined on line 38 [no-redef] aliases_recursive.py:50: error: Dict entry 0 has incompatible type "str": "list[int]"; expected "str": "str | int | Mapping[str, RecursiveMapping]" [dict-item] aliases_recursive.py:51: error: Dict entry 2 has incompatible type "str": "list[int]"; expected "str": "str | int | Mapping[str, RecursiveMapping]" [dict-item] -aliases_recursive.py:52: error: Dict entry 2 has incompatible type "str": "list[int]"; expected "str": "str | int | Mapping[str, RecursiveMapping]" [dict-item] +aliases_recursive.py:52: error: Dict entry 2 has incompatible type "str": "dict[str, str | int | list[int]]"; expected "str": "str | int | Mapping[str, RecursiveMapping]" [dict-item] aliases_recursive.py:63: error: List item 0 has incompatible type "float"; expected "GenericTypeAlias1[str] | str" [list-item] aliases_recursive.py:69: error: List item 0 has incompatible type "float"; expected "GenericTypeAlias2[str, int] | str | int" [list-item] aliases_recursive.py:72: error: Invalid recursive alias: a union item of itself [misc] diff --git a/conformance/results/mypy/classes_classvar.toml b/conformance/results/mypy/classes_classvar.toml index 0d9662ecf..0072d63f1 100644 --- a/conformance/results/mypy/classes_classvar.toml +++ b/conformance/results/mypy/classes_classvar.toml @@ -20,6 +20,7 @@ classes_classvar.py:70: error: ClassVar can only be used for assignments in clas classes_classvar.py:71: error: ClassVar can only be used for assignments in class body [misc] classes_classvar.py:73: error: ClassVar can only be used for assignments in class body [misc] classes_classvar.py:77: error: ClassVar can only be used for assignments in class body [misc] +classes_classvar.py:78: error: ClassVar[...] can't be used inside a type alias [valid-type] classes_classvar.py:111: error: Cannot assign to class variable "stats" via instance [misc] classes_classvar.py:140: error: Incompatible types in assignment (expression has type "ProtoAImpl", variable has type "ProtoA") [assignment] classes_classvar.py:140: note: "ProtoAImpl" is missing following "ProtoA" protocol member: @@ -30,6 +31,5 @@ classes_classvar.py:140: note: Protocol member ProtoA.y expected class variable, conformance_automated = "Fail" errors_diff = """ Line 47: Expected 1 errors -Line 78: Expected 1 errors Line 67: Unexpected errors ['classes_classvar.py:67: error: Invalid type: ClassVar nested inside other type [valid-type]'] """ diff --git a/conformance/results/mypy/dataclasses_kwonly.toml b/conformance/results/mypy/dataclasses_kwonly.toml index 118448df3..2656f656e 100644 --- a/conformance/results/mypy/dataclasses_kwonly.toml +++ b/conformance/results/mypy/dataclasses_kwonly.toml @@ -4,21 +4,9 @@ Incorrectly rejects kw_only field with default before positional field. """ output = """ dataclasses_kwonly.py:23: error: Too many positional arguments for "DC1" [misc] -dataclasses_kwonly.py:29: error: Attributes without a default cannot follow attributes with one [misc] -dataclasses_kwonly.py:32: error: Too many positional arguments for "DC2" [misc] -dataclasses_kwonly.py:32: error: Too few arguments for "DC2" [call-arg] -dataclasses_kwonly.py:32: error: Argument 1 to "DC2" has incompatible type "str"; expected "int" [arg-type] -dataclasses_kwonly.py:35: error: "DC2" gets multiple values for keyword argument "b" [misc] -dataclasses_kwonly.py:35: error: Too few arguments for "DC2" [call-arg] -dataclasses_kwonly.py:35: error: Argument 1 to "DC2" has incompatible type "str"; expected "int" [arg-type] dataclasses_kwonly.py:38: error: Too many positional arguments for "DC2" [misc] -dataclasses_kwonly.py:38: error: Argument 1 to "DC2" has incompatible type "str"; expected "int" [arg-type] -dataclasses_kwonly.py:38: error: Argument 2 to "DC2" has incompatible type "int"; expected "str" [arg-type] dataclasses_kwonly.py:53: error: Too many positional arguments for "DC3" [misc] """ -conformance_automated = "Fail" +conformance_automated = "Pass" errors_diff = """ -Line 29: Unexpected errors ['dataclasses_kwonly.py:29: error: Attributes without a default cannot follow attributes with one [misc]'] -Line 32: Unexpected errors ['dataclasses_kwonly.py:32: error: Too many positional arguments for "DC2" [misc]', 'dataclasses_kwonly.py:32: error: Too few arguments for "DC2" [call-arg]', 'dataclasses_kwonly.py:32: error: Argument 1 to "DC2" has incompatible type "str"; expected "int" [arg-type]'] -Line 35: Unexpected errors ['dataclasses_kwonly.py:35: error: "DC2" gets multiple values for keyword argument "b" [misc]', 'dataclasses_kwonly.py:35: error: Too few arguments for "DC2" [call-arg]', 'dataclasses_kwonly.py:35: error: Argument 1 to "DC2" has incompatible type "str"; expected "int" [arg-type]'] """ diff --git a/conformance/results/mypy/directives_reveal_type.toml b/conformance/results/mypy/directives_reveal_type.toml index 8621a0083..c5853ba33 100644 --- a/conformance/results/mypy/directives_reveal_type.toml +++ b/conformance/results/mypy/directives_reveal_type.toml @@ -1,6 +1,6 @@ conformant = "Pass" output = """ -directives_reveal_type.py:14: note: Revealed type is "Union[builtins.int, builtins.str]" +directives_reveal_type.py:14: note: Revealed type is "builtins.int | builtins.str" directives_reveal_type.py:15: note: Revealed type is "builtins.list[builtins.int]" directives_reveal_type.py:16: note: Revealed type is "Any" directives_reveal_type.py:17: note: Revealed type is "directives_reveal_type.ForwardReference" diff --git a/conformance/results/mypy/version.toml b/conformance/results/mypy/version.toml index d3740879d..91c238fce 100644 --- a/conformance/results/mypy/version.toml +++ b/conformance/results/mypy/version.toml @@ -1,2 +1,2 @@ -version = "mypy 1.16.1" -test_duration = 2.2 +version = "mypy 1.17.0" +test_duration = 2.8 diff --git a/conformance/results/pyre/generics_paramspec_specialization.toml b/conformance/results/pyre/generics_paramspec_specialization.toml index 34a215ee6..d4da441aa 100644 --- a/conformance/results/pyre/generics_paramspec_specialization.toml +++ b/conformance/results/pyre/generics_paramspec_specialization.toml @@ -3,8 +3,6 @@ notes = """ Reports error for legitimate use of `...` to specialize ParamSpec """ output = """ -generics_paramspec_specialization.py:32:14 Invalid type parameters [24]: Callable parameters expected for parameter specification `P2`, but a single type `...` was given for generic type ClassB. -generics_paramspec_specialization.py:36:14 Invalid type parameters [24]: Callable parameters expected for parameter specification `P1`, but a single type `...` was given for generic type ClassA. generics_paramspec_specialization.py:44:14 Invalid type parameters [24]: Callable parameters expected for parameter specification `P1`, but a single type `int` was given for generic type ClassA. generics_paramspec_specialization.py:53:4 Too many arguments [19]: PositionalOnly call expects 2 positional arguments, 3 were provided. generics_paramspec_specialization.py:54:4 Too many arguments [19]: PositionalOnly call expects 2 positional arguments, 3 were provided. @@ -15,8 +13,6 @@ generics_paramspec_specialization.py:61:4 Too many arguments [19]: PositionalOnl """ conformance_automated = "Fail" errors_diff = """ -Line 32: Unexpected errors ['generics_paramspec_specialization.py:32:14 Invalid type parameters [24]: Callable parameters expected for parameter specification `P2`, but a single type `...` was given for generic type ClassB.'] -Line 36: Unexpected errors ['generics_paramspec_specialization.py:36:14 Invalid type parameters [24]: Callable parameters expected for parameter specification `P1`, but a single type `...` was given for generic type ClassA.'] Line 53: Unexpected errors ['generics_paramspec_specialization.py:53:4 Too many arguments [19]: PositionalOnly call expects 2 positional arguments, 3 were provided.'] Line 59: Unexpected errors ['generics_paramspec_specialization.py:59:4 Too many arguments [19]: PositionalOnly call expects 2 positional arguments, 3 were provided.'] """ diff --git a/conformance/results/pyre/version.toml b/conformance/results/pyre/version.toml index c759e78d0..f59474fee 100644 --- a/conformance/results/pyre/version.toml +++ b/conformance/results/pyre/version.toml @@ -1,2 +1,2 @@ -version = "pyre 0.9.23" -test_duration = 10.7 +version = "pyre 0.9.25" +test_duration = 4.0 diff --git a/conformance/results/pyright/version.toml b/conformance/results/pyright/version.toml index ca4eb79ca..1e7d5b71e 100644 --- a/conformance/results/pyright/version.toml +++ b/conformance/results/pyright/version.toml @@ -1,2 +1,2 @@ -version = "pyright 1.1.402" -test_duration = 1.8 +version = "pyright 1.1.403" +test_duration = 5.9 diff --git a/conformance/results/results.html b/conformance/results/results.html index 4cc4ea005..508d7d6d6 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -158,14 +158,14 @@

Python Type System Conformance Test Results

- - - - + @@ -423,7 +423,7 @@

Python Type System Conformance Test Results

- + From f7c0544d3f43f0ab56c8db58865148515f7510c1 Mon Sep 17 00:00:00 2001 From: Rebecca Chen Date: Mon, 21 Jul 2025 14:00:49 -0700 Subject: [PATCH 2/4] Update manual conformance result. --- conformance/results/mypy/dataclasses_kwonly.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conformance/results/mypy/dataclasses_kwonly.toml b/conformance/results/mypy/dataclasses_kwonly.toml index 2656f656e..7c4a7985c 100644 --- a/conformance/results/mypy/dataclasses_kwonly.toml +++ b/conformance/results/mypy/dataclasses_kwonly.toml @@ -1,4 +1,4 @@ -conformant = "Partial" +conformant = "Pass" notes = """ Incorrectly rejects kw_only field with default before positional field. """ From bae2e1a9ba31c870f4f9a3af5fc96f773b695a00 Mon Sep 17 00:00:00 2001 From: Rebecca Chen Date: Mon, 21 Jul 2025 14:05:35 -0700 Subject: [PATCH 3/4] Remove stale notes. --- conformance/results/mypy/dataclasses_kwonly.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/conformance/results/mypy/dataclasses_kwonly.toml b/conformance/results/mypy/dataclasses_kwonly.toml index 7c4a7985c..9f83cef04 100644 --- a/conformance/results/mypy/dataclasses_kwonly.toml +++ b/conformance/results/mypy/dataclasses_kwonly.toml @@ -1,7 +1,4 @@ conformant = "Pass" -notes = """ -Incorrectly rejects kw_only field with default before positional field. -""" output = """ dataclasses_kwonly.py:23: error: Too many positional arguments for "DC1" [misc] dataclasses_kwonly.py:38: error: Too many positional arguments for "DC2" [misc] From b72e1c10b515ecd55e958f57bbbf5e2f7b1dd5e7 Mon Sep 17 00:00:00 2001 From: Rebecca Chen Date: Mon, 21 Jul 2025 14:38:19 -0700 Subject: [PATCH 4/4] Rerun 'python main.py' after change to manual grading. --- conformance/results/mypy/version.toml | 2 +- conformance/results/pyre/version.toml | 2 +- conformance/results/pyright/version.toml | 2 +- conformance/results/results.html | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conformance/results/mypy/version.toml b/conformance/results/mypy/version.toml index 91c238fce..a9b73efe7 100644 --- a/conformance/results/mypy/version.toml +++ b/conformance/results/mypy/version.toml @@ -1,2 +1,2 @@ version = "mypy 1.17.0" -test_duration = 2.8 +test_duration = 2.7 diff --git a/conformance/results/pyre/version.toml b/conformance/results/pyre/version.toml index f59474fee..02af2cfe9 100644 --- a/conformance/results/pyre/version.toml +++ b/conformance/results/pyre/version.toml @@ -1,2 +1,2 @@ version = "pyre 0.9.25" -test_duration = 4.0 +test_duration = 3.9 diff --git a/conformance/results/pyright/version.toml b/conformance/results/pyright/version.toml index 1e7d5b71e..a5b181d82 100644 --- a/conformance/results/pyright/version.toml +++ b/conformance/results/pyright/version.toml @@ -1,2 +1,2 @@ version = "pyright 1.1.403" -test_duration = 5.9 +test_duration = 5.6 diff --git a/conformance/results/results.html b/conformance/results/results.html index 508d7d6d6..7b4f244ca 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -159,13 +159,13 @@

Python Type System Conformance Test Results

 
mypy 1.16.1
-
2.2sec
+
mypy 1.17.0
+
2.8sec
pyright 1.1.402
-
1.8sec
+
pyright 1.1.403
+
5.9sec
pyre 0.9.23
-
10.7sec
+
pyre 0.9.25
+
4.0sec
@@ -324,7 +324,7 @@

Python Type System Conformance Test Results

     generics_type_erasure
Partial

Infers Node[Never] instead of Node[Any] when argument is not provided.

False negative on instance attribute access on type(node).

Partial

Missing error regarding `type(instance).generic_attribute`.

Pass
Partial

Does not erase unspecified type variables to `Any` prior to `assert_type` handling.

False negatives on instance attribute access on the type.

Does not infer type of `DefaultDict` with explicit type parameters on constructor.

False negatives on assert_type uses.
     generics_typevartuple_args
     aliases_newtype
Partial

`NewType`s are considered classes, not functions.

Partial

`NewType`s are considered classes, not functions.

Pass
Partial

Does not reject use of NewType in `isinstance` call.

Does not reject use of NewType in class definition statement.

Does not report inconsistency between name of NewType and assigned identifier name.

Does not reject use of NewType with generic class with TypeVar.

Does not reject use of NewType with protocol class.

Does not reject use of NewType with TypedDict class.

Does not reject use of NewType with Any.
     aliases_recursive
- +
 
mypy 1.17.0
-
2.8sec
+
2.7sec
pyright 1.1.403
-
5.9sec
+
5.6sec
pyre 0.9.25
-
4.0sec
+
3.9sec
@@ -648,7 +648,7 @@

Python Type System Conformance Test Results

Partial

Does not reject ClassVar that is overridden by instance variable.

Does not reject instance variable that is overridden by ClassVar.
     dataclasses_kwonly
Partial

Incorrectly rejects kw_only field with default before positional field.

Pass Pass
Partial

Rejects legitimate use of dataclass field with `kw_only=True`.