conformance tests: drop timing information (#2070)

Timing information is nice to have but the current data is not very reliable,
as it is the timing on whatever machine was used by the person who most recently
regenerated the results. It's also a big source of merge conflicts, basically
meaning that every pair of changes to the conformance suite will merge conflict
with each other.

To make it more pleasant to work with the conformance suite, let's just drop the timings.

Author: JelleZijlstra

conformance/results/mypy/version.toml

@@ -1,2 +1 @@
 version = "mypy 1.17.1"
-test_duration = 2.04

conformance/results/pyright/version.toml

@@ -1,2 +1 @@
 version = "pyright 1.1.403"
-test_duration = 1.83

conformance/results/results.html

@@ -159,16 +159,12 @@ <h3>Python Type System Conformance Test Results</h3>
         <div class="table_container"><table><tbody>
 <tr><th class="col1">&nbsp;</th>
 <th class='tc-header'><div class='tc-name'>mypy 1.17.1</div>
-<div class='tc-time'>2.0sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pyright 1.1.403</div>
-<div class='tc-time'>1.3sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pyre 0.9.25</div>
-<div class='tc-time'>10.2sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>zuban 0.0.19</div>
-<div class='tc-time'>0.32sec</div>
 </th>
 </tr>
 <tr><th class="column" colspan="5">

conformance/results/zuban/version.toml

@@ -1,2 +1 @@
 version = "zuban 0.0.19"
-test_duration = 0.32

conformance/src/main.py

@@ -22,14 +22,15 @@ def run_tests(
     root_dir: Path,
     type_checker: TypeChecker,
     test_cases: Sequence[Path],
-    skip_timing: bool = False,
 ):
     print(f"Running tests for {type_checker.name}")
 
     test_start_time = time()
     tests_output = type_checker.run_tests([file.name for file in test_cases])
     test_duration = time() - test_start_time
 
+    print(f"Completed tests for {type_checker.name} in {test_duration:.2f} seconds")
+
     for _, output in tests_output.items():
         type_checker.parse_errors(output.splitlines())
 
@@ -40,7 +41,7 @@ def run_tests(
             type_checker, results_dir, test_case, tests_output.get(test_case.name, "")
         )
 
-    update_type_checker_info(type_checker, root_dir, test_duration, skip_timing=skip_timing)
+    update_type_checker_info(type_checker, root_dir)
 
 
 def get_expected_errors(test_case: Path) -> tuple[
@@ -217,9 +218,7 @@ def update_output_for_test(
             tomlkit.dump(existing_results, f)
 
 
-def update_type_checker_info(
-    type_checker: TypeChecker, root_dir: Path, test_duration: float, skip_timing: bool = False
-):
+def update_type_checker_info(type_checker: TypeChecker, root_dir: Path):
     # Record the version of the type checker used for the latest run.
     version_file = root_dir / "results" / type_checker.name / "version.toml"
 
@@ -234,8 +233,6 @@ def update_type_checker_info(
         existing_info = {}
 
     existing_info["version"] = type_checker.get_version()
-    if not skip_timing:
-        existing_info["test_duration"] = round(test_duration, 2)
 
     version_file.parent.mkdir(parents=True, exist_ok=True)
     with open(version_file, "w") as f:
@@ -266,7 +263,7 @@ def main():
             if not type_checker.install():
                 print(f"Skipping tests for {type_checker.name}")
             else:
-                run_tests(root_dir, type_checker, test_cases, skip_timing=options.skip_timing)
+                run_tests(root_dir, type_checker, test_cases)
 
     # Generate a summary report.
     generate_summary(root_dir)

conformance/src/options.py

@@ -9,7 +9,6 @@
 @dataclass
 class _Options:
     report_only: bool | None
-    skip_timing: bool
 
 
 def parse_options(argv: list[str]) -> _Options:
@@ -20,10 +19,5 @@ def parse_options(argv: list[str]) -> _Options:
         action="store_true",
         help="regenerates the test suite report from past results",
     )
-    reporting_group.add_argument(
-        "--skip-timing",
-        action="store_true",
-        help="do not update timing information in the output files",
-    )
     ret = _Options(**vars(parser.parse_args(argv)))
     return ret

conformance/src/reporting.py

@@ -46,15 +46,8 @@ def generate_summary_html(root_dir: Path) -> str:
             existing_info = {}
 
         version = existing_info["version"] or "Unknown version"
-        test_duration = existing_info.get("test_duration")
 
         summary_html.append(f"<th class='tc-header'><div class='tc-name'>{version}</div>")
-        if test_duration is not None:
-            if test_duration < 1:
-                duration = f"{test_duration:.2f}"
-            else:
-                duration = f"{test_duration:.1f}"
-            summary_html.append(f"<div class='tc-time'>{duration}sec</div>")
         summary_html.append("</th>")
 
     summary_html.append("</tr>")