rust_browser/scripts/curate_test262.py

#!/usr/bin/env python3
"""
Curate real Test262 tests for the rust_browser JS engine.

Clones tc39/test262 (shallow), scans test/language/ for qualifying tests,
copies them into tests/external/js262/test262/language/, and generates
TOML manifest entries for tests/external/js262/js262_manifest.toml.

Usage:
    python3 scripts/curate_test262.py [--max-tests N] [--dry-run]
"""

import argparse
import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path

# --- Configuration ---

# Root of the rust_browser repo
REPO_ROOT = Path(__file__).resolve().parent.parent

# Where vendored test262 tests live
VENDOR_DIR = REPO_ROOT / "tests" / "external" / "js262" / "test262" / "language"

# Manifest file
MANIFEST_PATH = REPO_ROOT / "tests" / "external" / "js262" / "js262_manifest.toml"

# Maximum tests per feature directory to ensure variety
MAX_PER_DIR = 5

# Default total test cap
DEFAULT_MAX_TESTS = 50

# Allowed harness includes (anything else disqualifies the test)
ALLOWED_INCLUDES = {"assert.js", "sta.js"}

# Disqualifying flags
DISQUALIFYING_FLAGS = {"onlyStrict", "async", "module", "raw", "generated"}

# Disqualifying features (if a test requires any of these, skip it)
DISQUALIFYING_FEATURES = {
    "Symbol", "Symbol.iterator", "Symbol.toPrimitive", "Symbol.toStringTag",
    "Symbol.hasInstance", "Symbol.species", "Symbol.match", "Symbol.replace",
    "Symbol.search", "Symbol.split", "Symbol.unscopables", "Symbol.asyncIterator",
    "well-known-symbol",
    "Proxy", "Reflect", "Reflect.construct", "Reflect.apply",
    "WeakMap", "WeakSet", "WeakRef", "Map", "Set",
    "Promise", "async-functions", "async-iteration",
    "arrow-function", "class",
    "computed-property-names",
    "destructuring-binding", "destructuring-assignment",
    "for-of", "for-in",
    "generators", "generator",
    "rest-parameters", "spread",
    "default-parameters",
    "template",  # tagged templates require features we may not support
    "RegExp", "regexp-dotall", "regexp-lookbehind", "regexp-named-groups",
    "regexp-unicode-property-escapes",
    "ArrayBuffer", "TypedArray", "DataView",
    "Atomics", "SharedArrayBuffer",
    "optional-chaining", "nullish-coalescing",
    "dynamic-import", "import.meta",
    "BigInt", "globalThis",
    "Object.fromEntries", "Array.prototype.flat", "String.prototype.matchAll",
    "Array.prototype.flatMap", "Array.prototype.includes",
    "Object.is", "Object.assign", "Object.keys", "Object.values", "Object.entries",
    "String.prototype.trimEnd", "String.prototype.trimStart",
    "Array.from", "Array.of", "Array.prototype.find", "Array.prototype.findIndex",
    "Array.prototype.fill", "Array.prototype.copyWithin", "Array.prototype.entries",
    "Array.prototype.keys", "Array.prototype.values",
    "String.fromCodePoint", "String.prototype.codePointAt",
    "String.prototype.normalize", "String.prototype.repeat",
    "Number.isFinite", "Number.isInteger", "Number.isNaN",
    "Number.parseFloat", "Number.parseInt",
    "Math.trunc", "Math.sign", "Math.cbrt", "Math.log2", "Math.log10",
    "Math.fround", "Math.clz32", "Math.imul", "Math.hypot",
    "let", "const",  # block scoping features that may require complex tests
    "new.target",
    "super",
    "tail-call-optimization",
    "json-superset",
    "numeric-separator-literal",
    "optional-catch-binding",
    "Object.getOwnPropertyDescriptors",
    "String.raw",
    "Intl",
    "FinalizationRegistry",
    "AggregateError",
    "logical-assignment-operators",
    "coalesce-expression",
    "import-assertions", "import-attributes",
    "top-level-await",
    "class-fields-public", "class-fields-private",
    "class-methods-private", "class-static-fields-public",
    "class-static-fields-private", "class-static-methods-private",
    "decorators",
    "hashbang",
    "resizable-arraybuffer",
    "change-array-by-copy",
    "iterator-helpers",
    "set-methods",
    "array-grouping",
    "using-declaration",
    "json-modules",
    "source-phase-imports",
    "Temporal",
    "ShadowRealm",
    "explicit-resource-management",
    "RegExp.escape",
    "Float16Array",
}

# Body patterns that disqualify a test
DISQUALIFYING_BODY_PATTERNS = [
    r"=>",                       # arrow functions
    r"\[",                       # array literals / computed property access
    r"\bclass\b",                # class declarations
    r"\binstanceof\b",           # instanceof operator
    r"\bfor\s*\([^)]*\bin\b",   # for...in
    r"\bfor\s*\([^)]*\bof\b",   # for...of
    r"\.\.\.",                   # spread/rest
    r"\bimport\b",               # import statements
    r"\bexport\b",               # export statements
    r"/[^/*\s][^/\n]*/",         # regex literals (heuristic)
    r"\bassert\s*\(",            # bare assert() calls (not assert.sameValue etc)
    r"\bassert\.throws\b",       # assert.throws (our shim can't verify error type)
    r"\bnew\s+\w",               # new expressions (we want simple tests)
    r"\bdelete\b",               # delete operator
    r"\bin\b\s",                 # in operator (standalone)
    r"\bget\s+\w+\s*\(",        # getter syntax
    r"\bset\s+\w+\s*\(",        # setter syntax
    r"\byield\b",                # generators
    r"\bawait\b",                # async
    r"\basync\b",                # async
    r"`",                        # template literals (some tests require tagged templates)
    r"\bObject\.",               # Object static methods
    r"\bArray\.",                 # Array static methods
    r"\bJSON\.",                  # JSON methods
    r"\bMath\.",                  # Math methods
    r"\bNumber\.",                # Number static methods
    r"\bDate\b",                 # Date
    r"\bRegExp\b",               # RegExp
    r"\bMap\b",                  # Map
    r"\bSet\b",                  # Set
    r"\bSymbol\b",               # Symbol
    r"\bPromise\b",              # Promise
    r"\bProxy\b",                # Proxy
    r"\bReflect\b",              # Reflect
    r"\beval\b",                 # eval
    r"\bwith\b",                 # with statement
    r"\barguments\b",            # arguments object
    r"\b__proto__\b",            # __proto__
    r"\bprototype\b",            # prototype access
    r"\bhasOwnProperty\b",      # hasOwnProperty
    r"\bObject\b",              # any Object usage
    r"\bString\(",              # String constructor
    r"\bNumber\(",              # Number constructor
    r"\bBoolean\(",             # Boolean constructor
]


def parse_frontmatter(content):
    """Parse YAML-like frontmatter between /*--- and ---*/."""
    match = re.search(r"/\*---\s*\n(.*?)---\*/", content, re.DOTALL)
    if not match:
        return {}

    raw = match.group(1)
    result = {}

    # Parse includes
    includes_match = re.search(r"includes:\s*\[(.*?)\]", raw, re.DOTALL)
    if includes_match:
        items = includes_match.group(1)
        result["includes"] = [s.strip().strip("'\"") for s in items.split(",") if s.strip()]

    # Parse flags
    flags_match = re.search(r"flags:\s*\[(.*?)\]", raw, re.DOTALL)
    if flags_match:
        items = flags_match.group(1)
        result["flags"] = [s.strip().strip("'\"") for s in items.split(",") if s.strip()]

    # Parse features
    features_match = re.search(r"features:\s*\[(.*?)\]", raw, re.DOTALL)
    if features_match:
        items = features_match.group(1)
        result["features"] = [s.strip().strip("'\"") for s in items.split(",") if s.strip()]

    # Parse negative
    neg_match = re.search(r"negative:\s*\n\s+phase:\s*(\w+)\s*\n\s+type:\s*(\w+)", raw)
    if neg_match:
        result["negative"] = {
            "phase": neg_match.group(1),
            "type": neg_match.group(2),
        }

    # Parse description
    desc_match = re.search(r"description:\s*[>\|]?\s*\n?\s*(.*?)(?:\n\w|\Z)", raw, re.DOTALL)
    if not desc_match:
        desc_match = re.search(r"description:\s*(.*)", raw)
    if desc_match:
        result["description"] = desc_match.group(1).strip()

    return result


def get_test_body(content):
    """Return the test body (everything after the frontmatter)."""
    match = re.search(r"---\*/\s*\n?", content)
    if match:
        return content[match.end():]
    return content


def qualifies(filepath, content, frontmatter):
    """Check if a test qualifies for inclusion."""
    # Check includes
    includes = frontmatter.get("includes", [])
    for inc in includes:
        if inc not in ALLOWED_INCLUDES:
            return False, f"disqualified include: {inc}"

    # Check flags
    flags = frontmatter.get("flags", [])
    for flag in flags:
        if flag in DISQUALIFYING_FLAGS:
            return False, f"disqualified flag: {flag}"

    # Check features
    features = frontmatter.get("features", [])
    for feat in features:
        if feat in DISQUALIFYING_FEATURES:
            return False, f"disqualified feature: {feat}"

    # Check negative phase
    negative = frontmatter.get("negative")
    if negative:
        phase = negative.get("phase", "")
        if phase not in ("parse", "runtime", "early"):
            return False, f"disqualified negative phase: {phase}"

    # Check body for disqualifying patterns
    body = get_test_body(content)
    for pattern in DISQUALIFYING_BODY_PATTERNS:
        if re.search(pattern, body):
            return False, f"disqualified body pattern: {pattern}"

    return True, "ok"


def error_type_to_manifest(error_type):
    """Map Test262 error type to manifest expected_error value."""
    mapping = {
        "ReferenceError": "reference",
        "TypeError": "type",
        "SyntaxError": "parse",
        "RangeError": "runtime",
        "URIError": "runtime",
        "EvalError": "runtime",
    }
    return mapping.get(error_type, "runtime")


def feature_from_path(rel_path):
    """Derive a feature name from the test's relative path."""
    parts = Path(rel_path).parts
    # test/language/types/typeof/... -> "typeof"
    # test/language/expressions/... -> "expressions"
    if len(parts) >= 3:
        return parts[2]  # third component after test/language/
    if len(parts) >= 2:
        return parts[1]
    return "language"


def make_test_id(rel_path):
    """Generate a manifest test ID from the test's relative path."""
    # test/language/types/typeof/S8.1_A1.js -> t262-types-typeof-S8.1-A1
    stem = Path(rel_path).stem
    parts = list(Path(rel_path).parts[2:])  # skip test/language/
    parts[-1] = stem
    name = "-".join(parts)
    # Clean up for TOML: replace problematic chars
    name = re.sub(r"[_.]", "-", name)
    return f"t262-{name}"


def main():
    parser = argparse.ArgumentParser(description="Curate Test262 tests")
    parser.add_argument("--max-tests", type=int, default=DEFAULT_MAX_TESTS,
                        help=f"Maximum number of tests to vendor (default: {DEFAULT_MAX_TESTS})")
    parser.add_argument("--dry-run", action="store_true",
                        help="Print what would be done without copying files")
    args = parser.parse_args()

    # Clone test262 to temp directory
    with tempfile.TemporaryDirectory() as tmpdir:
        test262_dir = Path(tmpdir) / "test262"
        print("Cloning tc39/test262 (shallow)...")
        subprocess.run(
            ["git", "clone", "--depth", "1", "https://github.com/tc39/test262.git",
             str(test262_dir)],
            check=True,
            capture_output=True,
        )
        print("Clone complete.")

        # Scan test/language/ for .js files
        language_dir = test262_dir / "test" / "language"
        if not language_dir.exists():
            print(f"ERROR: {language_dir} not found", file=sys.stderr)
            sys.exit(1)

        all_tests = sorted(language_dir.rglob("*.js"))
        print(f"Found {len(all_tests)} test files in test/language/")

        qualifying = []
        disqualified_reasons = {}

        for test_path in all_tests:
            rel_path = test_path.relative_to(test262_dir)
            content = test_path.read_text(encoding="utf-8", errors="replace")
            frontmatter = parse_frontmatter(content)
            ok, reason = qualifies(str(rel_path), content, frontmatter)
            if ok:
                qualifying.append((test_path, rel_path, frontmatter))
            else:
                bucket = reason.split(":")[0] if ":" in reason else reason
                disqualified_reasons[bucket] = disqualified_reasons.get(bucket, 0) + 1

        print(f"\nQualifying tests: {len(qualifying)}")
        print(f"Disqualification breakdown:")
        for reason, count in sorted(disqualified_reasons.items(), key=lambda x: -x[1]):
            print(f"  {reason}: {count}")

        # Apply per-directory cap and total cap for variety
        selected = []
        dir_counts = {}

        for test_path, rel_path, frontmatter in qualifying:
            parent = str(rel_path.parent)
            count = dir_counts.get(parent, 0)
            if count >= MAX_PER_DIR:
                continue
            dir_counts[parent] = count + 1
            selected.append((test_path, rel_path, frontmatter))
            if len(selected) >= args.max_tests:
                break

        print(f"\nSelected {len(selected)} tests (max {args.max_tests}, max {MAX_PER_DIR}/dir)")

        if args.dry_run:
            print("\n--- DRY RUN: would vendor these tests ---")
            for _, rel_path, fm in selected:
                neg = fm.get("negative")
                neg_str = f" [negative: {neg['type']}]" if neg else ""
                print(f"  {rel_path}{neg_str}")
            return

        # Copy tests and generate manifest entries
        manifest_entries = []

        for test_path, rel_path, frontmatter in selected:
            # Destination preserving directory structure under language/
            lang_rel = rel_path.relative_to(Path("test") / "language")
            dest = VENDOR_DIR / lang_rel
            dest.parent.mkdir(parents=True, exist_ok=True)
            shutil.copy2(test_path, dest)

            # Generate manifest entry
            test_id = make_test_id(str(rel_path))
            manifest_input = f"test262/language/{lang_rel}"
            feature = feature_from_path(str(rel_path))

            negative = frontmatter.get("negative")

            entry_lines = [
                "[[case]]",
                f'id = "{test_id}"',
                f'input = "{manifest_input}"',
                'mode = "test262"',
            ]

            if negative:
                err_class = error_type_to_manifest(negative["type"])
                entry_lines.append(f'expected_error = "{err_class}"')

            # Default to pass; will be adjusted in step 6
            entry_lines.append('status = "pass"')
            entry_lines.append(f'feature = "{feature}"')
            entry_lines.append('flags = []')

            manifest_entries.append("\n".join(entry_lines))

        # Append to manifest
        manifest_block = "\n\n# --- Real Test262 tests (Phase A) ---\n\n"
        manifest_block += "\n\n".join(manifest_entries) + "\n"

        with open(MANIFEST_PATH, "a") as f:
            f.write(manifest_block)

        print(f"\nVendored {len(selected)} tests to {VENDOR_DIR}")
        print(f"Appended {len(manifest_entries)} entries to {MANIFEST_PATH}")

        # Summary by feature
        feature_counts = {}
        for _, rel_path, _ in selected:
            feat = feature_from_path(str(rel_path))
            feature_counts[feat] = feature_counts.get(feat, 0) + 1
        print("\nPer-feature breakdown:")
        for feat, count in sorted(feature_counts.items()):
            print(f"  {feat}: {count}")


if __name__ == "__main__":
    main()