From 174b9ead578ae1bec563b8369a9a09c872d08f6c Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Mon, 3 Jun 2024 15:45:34 -0700 Subject: [PATCH] Fixed two bugs in the conformance test: 1. The "conformance_automated" was not written to the output in some cases. 2. The output file was not written if it didn't previously exist and the tests passed. This doesn't change the results of any of the tests. --- conformance/results/mypy/constructors_consistency.toml | 5 +++++ conformance/results/mypy/exceptions_context_managers.toml | 1 + conformance/results/pyre/constructors_call_type.toml | 1 + conformance/results/pyre/constructors_consistency.toml | 5 +++++ conformance/results/pyright/constructors_call_new.toml | 1 + conformance/results/pyright/constructors_consistency.toml | 1 + .../results/pyright/exceptions_context_managers.toml | 1 + conformance/results/pytype/constructors_consistency.toml | 1 + conformance/src/main.py | 8 +++++--- 9 files changed, 21 insertions(+), 3 deletions(-) diff --git a/conformance/results/mypy/constructors_consistency.toml b/conformance/results/mypy/constructors_consistency.toml index 225d7847a..654ebaa96 100644 --- a/conformance/results/mypy/constructors_consistency.toml +++ b/conformance/results/mypy/constructors_consistency.toml @@ -2,3 +2,8 @@ conformant = "Pass" notes = """ Does not report inconsistency between __new__ and __init__ (optional). """ +conformance_automated = "Pass" +errors_diff = """ +""" +output = """ +""" diff --git a/conformance/results/mypy/exceptions_context_managers.toml b/conformance/results/mypy/exceptions_context_managers.toml index cd630cbf5..513dbe853 100644 --- a/conformance/results/mypy/exceptions_context_managers.toml +++ b/conformance/results/mypy/exceptions_context_managers.toml @@ -3,3 +3,4 @@ errors_diff = """ """ output = """ """ +conformance_automated = "Pass" diff --git a/conformance/results/pyre/constructors_call_type.toml b/conformance/results/pyre/constructors_call_type.toml index 1e06c340a..b78a1daf4 100644 --- a/conformance/results/pyre/constructors_call_type.toml +++ b/conformance/results/pyre/constructors_call_type.toml @@ -11,3 +11,4 @@ constructors_call_type.py:72:4 Missing argument [20]: Call `Meta1.__call__` expe constructors_call_type.py:81:4 Missing argument [20]: Call `Class2.__new__` expects argument `y`. constructors_call_type.py:82:11 Incompatible parameter type [6]: In call `Class2.__new__`, for 2nd positional argument, expected `str` but got `int`. """ +conformance_automated = "Pass" diff --git a/conformance/results/pyre/constructors_consistency.toml b/conformance/results/pyre/constructors_consistency.toml index 225d7847a..654ebaa96 100644 --- a/conformance/results/pyre/constructors_consistency.toml +++ b/conformance/results/pyre/constructors_consistency.toml @@ -2,3 +2,8 @@ conformant = "Pass" notes = """ Does not report inconsistency between __new__ and __init__ (optional). """ +conformance_automated = "Pass" +errors_diff = """ +""" +output = """ +""" diff --git a/conformance/results/pyright/constructors_call_new.toml b/conformance/results/pyright/constructors_call_new.toml index 8ca005715..5880ad7fc 100644 --- a/conformance/results/pyright/constructors_call_new.toml +++ b/conformance/results/pyright/constructors_call_new.toml @@ -10,3 +10,4 @@ constructors_call_new.py:145:1 - error:     Type "type[Class11[str]]" is incompatible with type "type[Class11[int]]"       Type parameter "T@Class11" is invariant, but "str" is not the same as "int" (reportGeneralTypeIssues) """ +conformance_automated = "Pass" diff --git a/conformance/results/pyright/constructors_consistency.toml b/conformance/results/pyright/constructors_consistency.toml index 8a3840678..e9fee5022 100644 --- a/conformance/results/pyright/constructors_consistency.toml +++ b/conformance/results/pyright/constructors_consistency.toml @@ -6,3 +6,4 @@ constructors_consistency.py:25:9 - error: Mismatch between signature of __new__   Signature of __init__ is "(x: str)"   Signature of __new__ is "()" (reportInconsistentConstructor) """ +conformance_automated = "Pass" diff --git a/conformance/results/pyright/exceptions_context_managers.toml b/conformance/results/pyright/exceptions_context_managers.toml index cd630cbf5..513dbe853 100644 --- a/conformance/results/pyright/exceptions_context_managers.toml +++ b/conformance/results/pyright/exceptions_context_managers.toml @@ -3,3 +3,4 @@ errors_diff = """ """ output = """ """ +conformance_automated = "Pass" diff --git a/conformance/results/pytype/constructors_consistency.toml b/conformance/results/pytype/constructors_consistency.toml index 100481a6b..a5d4de873 100644 --- a/conformance/results/pytype/constructors_consistency.toml +++ b/conformance/results/pytype/constructors_consistency.toml @@ -7,3 +7,4 @@ errors_diff = """ output = """ """ +conformance_automated = "Pass" diff --git a/conformance/src/main.py b/conformance/src/main.py index c1c8d76fa..02453f9f8 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -141,6 +141,7 @@ def update_output_for_test( results_file = results_dir / f"{test_name}.toml" results_file.parent.mkdir(parents=True, exist_ok=True) + should_write = False # Read the existing results file if present. try: @@ -153,7 +154,6 @@ def update_output_for_test( print(f"Error decoding {results_file}") existing_results = {} - should_write = False ignored_errors = existing_results.get("ignore_errors", []) errors_diff = "\n" + diff_expected_errors(type_checker, test_case, output, ignored_errors) old_errors_diff = "\n" + existing_results.get("errors_diff", "") @@ -165,8 +165,10 @@ def update_output_for_test( print(f"New output: {errors_diff}") print("") - existing_results["conformance_automated"] = "Fail" if errors_diff.strip() else "Pass" - + conformance_automated = "Fail" if errors_diff.strip() else "Pass" + if existing_results.get("conformance_automated") != conformance_automated: + should_write = True + existing_results["conformance_automated"] = conformance_automated old_output = existing_results.get("output", "") old_output = f"\n{old_output}"