rust_browser/scripts/triage_test262_full.py

#!/usr/bin/env python3
"""
Triage full Test262 suite: run the harness, promote passing known_fail tests.

Workflow:
1. Run the JS262 full-suite harness test
2. Parse output for tests marked known_fail that now pass
3. Update js262_full_status.toml with promoted tests
4. Print summary
"""

import re
import subprocess
import sys
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parent.parent
STATUS_PATH = PROJECT_ROOT / "tests" / "external" / "js262" / "js262_full_status.toml"
MANIFEST_PATH = PROJECT_ROOT / "tests" / "external" / "js262" / "js262_full_manifest.toml"

# Stable phrase emitted by enforce_case_policy in tests/js262_harness/runner.rs
PROMOTE_PATTERN = re.compile(r"case '([^']+)' is marked known_fail but now passes")


def count_manifest_statuses():
    """Count pass/known_fail/total in the full manifest."""
    if not MANIFEST_PATH.exists():
        return 0, 0, 0

    content = MANIFEST_PATH.read_text(encoding="utf-8")
    pass_count = 0
    known_fail_count = 0

    for line in content.splitlines():
        stripped = line.strip()
        if stripped.startswith("status"):
            if '"pass"' in stripped:
                pass_count += 1
            elif '"known_fail"' in stripped:
                known_fail_count += 1

    return pass_count, known_fail_count, pass_count + known_fail_count


def load_existing_status():
    """Load existing status entries from the status file."""
    entries = {}
    if not STATUS_PATH.exists():
        return entries

    content = STATUS_PATH.read_text(encoding="utf-8")
    for line in content.splitlines():
        line = line.strip()
        if not line or line.startswith("#"):
            continue
        m = re.match(r'^([^\s=]+)\s*=\s*"([^"]+)"', line)
        if m:
            entries[m.group(1)] = m.group(2)

    return entries


def save_status(entries):
    """Write the status file with all entries."""
    lines = [
        "# Status overrides for full Test262 scan.",
        "# Tests not listed default to \"known_fail\".",
        "# Managed by: just triage-test262-full",
        "#",
        "# Format:",
        '#   test_id = "pass"',
        '#   test_id = "skip"',
        "",
    ]

    for test_id in sorted(entries.keys()):
        lines.append(f'{test_id} = "{entries[test_id]}"')

    STATUS_PATH.write_text("\n".join(lines) + "\n", encoding="utf-8")


def main():
    print("Running JS262 full-suite harness...")
    result = subprocess.run(
        [
            "cargo", "test", "-p", "rust_browser",
            "--test", "js262_harness",
            "js262_full_suite_matches_manifest_expectations",
            "--", "--nocapture",
        ],
        cwd=str(PROJECT_ROOT),
        capture_output=True,
        text=True,
    )

    output = result.stdout + "\n" + result.stderr

    # Extract test IDs that should be promoted
    ids_to_promote = []
    for match in PROMOTE_PATTERN.finditer(output):
        test_id = match.group(1)
        if test_id not in ids_to_promote:
            ids_to_promote.append(test_id)

    if not ids_to_promote:
        print("No tests to promote.")
        pass_count, known_fail_count, total = count_manifest_statuses()
        if total > 0:
            pct = pass_count * 100 // total
            print(f"Full-suite pass rate: {pass_count}/{total} ({pct}%)")
        return 0

    print(f"Found {len(ids_to_promote)} tests to promote:")
    for tid in ids_to_promote:
        print(f"  {tid}")

    # Update status file
    entries = load_existing_status()
    promoted = 0
    for tid in ids_to_promote:
        entries[tid] = "pass"
        promoted += 1
        print(f"  promoted: {tid}")

    save_status(entries)
    print(f"\nUpdated {STATUS_PATH}")

    # Re-scan to regenerate manifest with new statuses
    print("\nRegenerating full manifest with updated statuses...")
    subprocess.run(
        [sys.executable, str(PROJECT_ROOT / "scripts" / "scan_test262.py")],
        cwd=str(PROJECT_ROOT),
        check=True,
    )

    # Print summary
    pass_count, known_fail_count, total = count_manifest_statuses()
    print(f"\nPromoted: {promoted}/{len(ids_to_promote)}")
    if total > 0:
        pct = pass_count * 100 // total
        print(f"Full-suite pass rate: {pass_count}/{total} ({pct}%)")
        print(f"Remaining known_fail: {known_fail_count}")

    return 0


if __name__ == "__main__":
    raise SystemExit(main())