2240f6c518
Vendor 500 new real Test262 tests (total 550), relax curation filters for now-supported features (arrow functions, classes, destructuring, spread/rest, for-in/of, template literals, let/const, instanceof), add automated triage script, and track pass rate as a CI metric. 190/550 pass (34.5%). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
119 lines
3.6 KiB
Python
Executable File
119 lines
3.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Triage Test262 tests: run the harness, promote passing known_fail tests.
|
|
|
|
Workflow:
|
|
1. Run the JS262 harness test suite
|
|
2. Parse output for tests marked known_fail that now pass
|
|
3. Promote them via wpt_status.py
|
|
4. Print summary
|
|
"""
|
|
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
MANIFEST_PATH = PROJECT_ROOT / "tests" / "external" / "js262" / "js262_manifest.toml"
|
|
WPT_STATUS_SCRIPT = PROJECT_ROOT / "scripts" / "wpt_status.py"
|
|
|
|
# Stable phrase emitted by enforce_case_policy in tests/js262_harness/runner.rs:302
|
|
PROMOTE_PATTERN = re.compile(r"case '([^']+)' is marked known_fail but now passes")
|
|
|
|
|
|
def count_manifest_statuses():
|
|
"""Count pass/known_fail/total for mode=test262 entries in the manifest."""
|
|
content = MANIFEST_PATH.read_text(encoding="utf-8")
|
|
pass_count = 0
|
|
known_fail_count = 0
|
|
total = 0
|
|
|
|
# Simple line-based parsing: look for status lines within test262 mode blocks
|
|
in_case = False
|
|
is_test262 = False
|
|
for line in content.splitlines():
|
|
stripped = line.strip()
|
|
if stripped == "[[case]]":
|
|
in_case = True
|
|
is_test262 = False
|
|
continue
|
|
if in_case:
|
|
if stripped.startswith("mode"):
|
|
if '"test262"' in stripped:
|
|
is_test262 = True
|
|
if stripped.startswith("status") and is_test262:
|
|
total += 1
|
|
if '"pass"' in stripped:
|
|
pass_count += 1
|
|
elif '"known_fail"' in stripped:
|
|
known_fail_count += 1
|
|
|
|
return pass_count, known_fail_count, total
|
|
|
|
|
|
def main():
|
|
print("Running JS262 harness...")
|
|
result = subprocess.run(
|
|
[
|
|
"cargo", "test", "-p", "rust_browser",
|
|
"--test", "js262_harness",
|
|
"js262_suite_matches_manifest_expectations",
|
|
"--", "--nocapture",
|
|
],
|
|
cwd=str(PROJECT_ROOT),
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
# Combine stdout and stderr for parsing (cargo test outputs to stderr)
|
|
output = result.stdout + "\n" + result.stderr
|
|
|
|
# Extract test IDs that should be promoted
|
|
ids_to_promote = []
|
|
for match in PROMOTE_PATTERN.finditer(output):
|
|
test_id = match.group(1)
|
|
if test_id not in ids_to_promote:
|
|
ids_to_promote.append(test_id)
|
|
|
|
if not ids_to_promote:
|
|
print("No tests to promote.")
|
|
pass_count, known_fail_count, total = count_manifest_statuses()
|
|
if total > 0:
|
|
pct = pass_count * 100 // total
|
|
print(f"Test262 pass rate: {pass_count}/{total} ({pct}%)")
|
|
return 0
|
|
|
|
print(f"Found {len(ids_to_promote)} tests to promote:")
|
|
for tid in ids_to_promote:
|
|
print(f" {tid}")
|
|
|
|
# Promote each test
|
|
promoted = 0
|
|
for tid in ids_to_promote:
|
|
cmd = [
|
|
sys.executable, str(WPT_STATUS_SCRIPT),
|
|
"--manifest", str(MANIFEST_PATH),
|
|
"promote", "--id", tid,
|
|
]
|
|
res = subprocess.run(cmd, capture_output=True, text=True)
|
|
if res.returncode == 0:
|
|
promoted += 1
|
|
print(f" promoted: {tid}")
|
|
else:
|
|
print(f" FAILED to promote {tid}: {res.stderr.strip()}", file=sys.stderr)
|
|
|
|
# Print summary
|
|
pass_count, known_fail_count, total = count_manifest_statuses()
|
|
print(f"\nPromoted: {promoted}/{len(ids_to_promote)}")
|
|
if total > 0:
|
|
pct = pass_count * 100 // total
|
|
print(f"Test262 pass rate: {pass_count}/{total} ({pct}%)")
|
|
print(f"Remaining known_fail: {known_fail_count}")
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|