rust_browser/scripts/curate_test262.py

#!/usr/bin/env python3
"""
Curate real Test262 tests for the rust_browser JS engine.

Clones tc39/test262 (shallow), scans test/language/ for qualifying tests,
copies them into tests/external/js262/test262/language/, and generates
TOML manifest entries for tests/external/js262/js262_manifest.toml.

Usage:
    python3 scripts/curate_test262.py [--max-tests N] [--dry-run]
                                       [--initial-status STATUS]
                                       [--test262-rev REV]
"""

import argparse
import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path

# --- Configuration ---

# Root of the rust_browser repo
REPO_ROOT = Path(__file__).resolve().parent.parent

# Where vendored test262 tests live
VENDOR_DIR = REPO_ROOT / "tests" / "external" / "js262" / "test262" / "language"

# Manifest file
MANIFEST_PATH = REPO_ROOT / "tests" / "external" / "js262" / "js262_manifest.toml"

# Maximum tests per feature directory to ensure variety
MAX_PER_DIR = 20

# Default total test cap
DEFAULT_MAX_TESTS = 500

# Pinned Test262 revision for reproducibility
DEFAULT_TEST262_REV = ""

# Allowed harness includes (anything else disqualifies the test)
ALLOWED_INCLUDES = {"assert.js", "sta.js"}

# Disqualifying flags
DISQUALIFYING_FLAGS = {"onlyStrict", "async", "module", "raw", "generated"}

# Disqualifying features (if a test requires any of these, skip it)
DISQUALIFYING_FEATURES = {
    # Symbol, Symbol.iterator, Symbol.toPrimitive, Symbol.toStringTag,
    # Symbol.hasInstance, Symbol.species, well-known-symbol — now supported
    "Symbol.match", "Symbol.replace",
    "Symbol.search", "Symbol.split", "Symbol.unscopables", "Symbol.asyncIterator",
    # Proxy, WeakMap — now supported
    "Reflect", "Reflect.construct", "Reflect.apply",
    "WeakSet", "WeakRef", "Map", "Set",
    "Promise", "async-functions", "async-iteration",
    # arrow-function, class, computed-property-names — now supported
    # destructuring-binding, destructuring-assignment — now supported
    # for-of, for-in — now supported
    "generators", "generator",
    # rest-parameters, spread — now supported
    "default-parameters",
    # template — now supported
    "RegExp", "regexp-dotall", "regexp-lookbehind", "regexp-named-groups",
    "regexp-unicode-property-escapes",
    "ArrayBuffer", "TypedArray", "DataView",
    "Atomics", "SharedArrayBuffer",
    "optional-chaining", "nullish-coalescing",
    "dynamic-import", "import.meta",
    "BigInt", "globalThis",
    "Object.fromEntries", "Array.prototype.flat", "String.prototype.matchAll",
    "Array.prototype.flatMap", "Array.prototype.includes",
    "Object.is", "Object.assign", "Object.keys", "Object.values", "Object.entries",
    "String.prototype.trimEnd", "String.prototype.trimStart",
    "Array.from", "Array.of", "Array.prototype.find", "Array.prototype.findIndex",
    "Array.prototype.fill", "Array.prototype.copyWithin", "Array.prototype.entries",
    "Array.prototype.keys", "Array.prototype.values",
    "String.fromCodePoint", "String.prototype.codePointAt",
    "String.prototype.normalize", "String.prototype.repeat",
    "Number.isFinite", "Number.isInteger", "Number.isNaN",
    "Number.parseFloat", "Number.parseInt",
    "Math.trunc", "Math.sign", "Math.cbrt", "Math.log2", "Math.log10",
    "Math.fround", "Math.clz32", "Math.imul", "Math.hypot",
    # let, const — now supported
    "new.target",
    # super — now supported
    "tail-call-optimization",
    "json-superset",
    "numeric-separator-literal",
    "optional-catch-binding",
    "Object.getOwnPropertyDescriptors",
    "String.raw",
    "Intl",
    "FinalizationRegistry",
    "AggregateError",
    "logical-assignment-operators",
    "coalesce-expression",
    "import-assertions", "import-attributes",
    "top-level-await",
    "class-fields-public", "class-fields-private",
    "class-methods-private", "class-static-fields-public",
    "class-static-fields-private", "class-static-methods-private",
    "decorators",
    "hashbang",
    "resizable-arraybuffer",
    "change-array-by-copy",
    "iterator-helpers",
    "set-methods",
    "array-grouping",
    "using-declaration",
    "json-modules",
    "source-phase-imports",
    "Temporal",
    "ShadowRealm",
    "explicit-resource-management",
    "RegExp.escape",
    "Float16Array",
}

# Body patterns that disqualify a test
DISQUALIFYING_BODY_PATTERNS = [
    # Still-unsupported syntax/features
    r"\bimport\b",               # import statements
    r"\bexport\b",               # export statements
    r"/[^/*\s][^/\n]*/",         # regex literals (heuristic)
    r"\bdelete\b",               # delete operator
    r"\bin\b\s",                 # in operator (standalone)
    r"\bget\s+\w+\s*\(",        # getter syntax
    r"\bset\s+\w+\s*\(",        # setter syntax
    r"\byield\b",                # generators
    r"\bawait\b",                # async
    r"\basync\b",                # async
    r"\bJSON\.",                  # JSON methods
    r"\bMath\.",                  # Math methods
    r"\bDate\b",                 # Date
    r"\bRegExp\b",               # RegExp
    r"\bMap\b",                  # Map
    r"\bSet\b",                  # Set
    # Symbol, Proxy — now supported
    r"\bPromise\b",              # Promise
    r"\bReflect\b",              # Reflect
    r"\beval\b",                 # eval
    r"\bwith\b",                 # with statement
    r"\barguments\b",            # arguments object
    r"\b__proto__\b",            # __proto__
    r"\bhasOwnProperty\b",      # hasOwnProperty
    r"\bString\(",              # String constructor
    r"\bNumber\(",              # Number constructor
    r"\bBoolean\(",             # Boolean constructor
    r"\bArray\.",                 # Array static methods
    r"\bNumber\.",                # Number static methods
    # Targeted Object method blocks (replaces blanket \bObject\b and \bObject\.)
    r"\bObject\.definePropert",          # Object.defineProperty/defineProperties
    r"\bObject\.create\b",               # Object.create
    r"\bObject\.freeze\b",               # Object.freeze
    r"\bObject\.seal\b",                 # Object.seal
    r"\bObject\.preventExtensions\b",    # Object.preventExtensions
    r"\bObject\.getPrototypeOf\b",       # Object.getPrototypeOf
    r"\bObject\.setPrototypeOf\b",       # Object.setPrototypeOf
    r"\bObject\.getOwnProperty",         # getOwnPropertyDescriptor/Names/Symbols
    r"\bObject\.is\(",                   # Object.is()
    r"\bArray\.isArray\b",               # Array.isArray
    r"\bArray\.from\b",                  # Array.from
    r"\bNumber\.is",                     # Number.isFinite, etc.
    r"\bString\.fromCharCode\b",         # String.fromCharCode
    r"\bFunction\b",                     # Function constructor
]


def parse_frontmatter(content):
    """Parse YAML-like frontmatter between /*--- and ---*/."""
    match = re.search(r"/\*---\s*\n(.*?)---\*/", content, re.DOTALL)
    if not match:
        return {}

    raw = match.group(1)
    result = {}

    # Parse includes
    includes_match = re.search(r"includes:\s*\[(.*?)\]", raw, re.DOTALL)
    if includes_match:
        items = includes_match.group(1)
        result["includes"] = [s.strip().strip("'\"") for s in items.split(",") if s.strip()]

    # Parse flags
    flags_match = re.search(r"flags:\s*\[(.*?)\]", raw, re.DOTALL)
    if flags_match:
        items = flags_match.group(1)
        result["flags"] = [s.strip().strip("'\"") for s in items.split(",") if s.strip()]

    # Parse features
    features_match = re.search(r"features:\s*\[(.*?)\]", raw, re.DOTALL)
    if features_match:
        items = features_match.group(1)
        result["features"] = [s.strip().strip("'\"") for s in items.split(",") if s.strip()]

    # Parse negative
    neg_match = re.search(r"negative:\s*\n\s+phase:\s*(\w+)\s*\n\s+type:\s*(\w+)", raw)
    if neg_match:
        result["negative"] = {
            "phase": neg_match.group(1),
            "type": neg_match.group(2),
        }

    # Parse description
    desc_match = re.search(r"description:\s*[>\|]?\s*\n?\s*(.*?)(?:\n\w|\Z)", raw, re.DOTALL)
    if not desc_match:
        desc_match = re.search(r"description:\s*(.*)", raw)
    if desc_match:
        result["description"] = desc_match.group(1).strip()

    return result


def get_test_body(content):
    """Return the test body (everything after the frontmatter)."""
    match = re.search(r"---\*/\s*\n?", content)
    if match:
        return content[match.end():]
    return content


def qualifies(filepath, content, frontmatter):
    """Check if a test qualifies for inclusion."""
    # Check includes
    includes = frontmatter.get("includes", [])
    for inc in includes:
        if inc not in ALLOWED_INCLUDES:
            return False, f"disqualified include: {inc}"

    # Check flags
    flags = frontmatter.get("flags", [])
    for flag in flags:
        if flag in DISQUALIFYING_FLAGS:
            return False, f"disqualified flag: {flag}"

    # Check features
    features = frontmatter.get("features", [])
    for feat in features:
        if feat in DISQUALIFYING_FEATURES:
            return False, f"disqualified feature: {feat}"

    # Check negative phase
    negative = frontmatter.get("negative")
    if negative:
        phase = negative.get("phase", "")
        if phase not in ("parse", "runtime", "early"):
            return False, f"disqualified negative phase: {phase}"

    # Check body for disqualifying patterns
    body = get_test_body(content)
    for pattern in DISQUALIFYING_BODY_PATTERNS:
        if re.search(pattern, body):
            return False, f"disqualified body pattern: {pattern}"

    return True, "ok"


def error_type_to_manifest(error_type):
    """Map Test262 error type to manifest expected_error value."""
    mapping = {
        "ReferenceError": "reference",
        "TypeError": "type",
        "SyntaxError": "parse",
        "RangeError": "runtime",
        "URIError": "runtime",
        "EvalError": "runtime",
    }
    return mapping.get(error_type, "runtime")


def feature_from_path(rel_path):
    """Derive a feature name from the test's relative path."""
    parts = Path(rel_path).parts
    # test/language/types/typeof/... -> "typeof"
    # test/language/expressions/... -> "expressions"
    if len(parts) >= 3:
        return parts[2]  # third component after test/language/
    if len(parts) >= 2:
        return parts[1]
    return "language"


def make_test_id(rel_path):
    """Generate a manifest test ID from the test's relative path."""
    # test/language/types/typeof/S8.1_A1.js -> t262-types-typeof-S8.1-A1
    stem = Path(rel_path).stem
    parts = list(Path(rel_path).parts[2:])  # skip test/language/
    parts[-1] = stem
    name = "-".join(parts)
    # Clean up for TOML: replace problematic chars
    name = re.sub(r"[_.]", "-", name)
    return f"t262-{name}"


def load_existing_manifest_ids(manifest_path):
    """Load all test IDs already present in the manifest."""
    ids = set()
    if not manifest_path.exists():
        return ids
    content = manifest_path.read_text(encoding="utf-8")
    for match in re.finditer(r'^id\s*=\s*"([^"]+)"', content, re.MULTILINE):
        ids.add(match.group(1))
    return ids


def main():
    parser = argparse.ArgumentParser(description="Curate Test262 tests")
    parser.add_argument("--max-tests", type=int, default=DEFAULT_MAX_TESTS,
                        help=f"Maximum number of tests to vendor (default: {DEFAULT_MAX_TESTS})")
    parser.add_argument("--dry-run", action="store_true",
                        help="Print what would be done without copying files")
    parser.add_argument("--initial-status", default="known_fail",
                        choices=["pass", "known_fail"],
                        help="Status for generated manifest entries (default: known_fail)")
    parser.add_argument("--test262-rev", default=DEFAULT_TEST262_REV,
                        help="Git revision of tc39/test262 to checkout (default: latest)")
    args = parser.parse_args()

    # Load existing manifest IDs for deduplication
    existing_ids = load_existing_manifest_ids(MANIFEST_PATH)
    print(f"Existing manifest IDs: {len(existing_ids)}")

    # Clone test262 to temp directory
    with tempfile.TemporaryDirectory() as tmpdir:
        test262_dir = Path(tmpdir) / "test262"
        print("Cloning tc39/test262 (shallow)...")
        subprocess.run(
            ["git", "clone", "--depth", "1", "https://github.com/tc39/test262.git",
             str(test262_dir)],
            check=True,
            capture_output=True,
        )

        # Checkout specific revision if provided
        if args.test262_rev:
            print(f"Checking out revision: {args.test262_rev}")
            subprocess.run(
                ["git", "-C", str(test262_dir), "fetch", "--depth", "1",
                 "origin", args.test262_rev],
                check=True,
                capture_output=True,
            )
            subprocess.run(
                ["git", "-C", str(test262_dir), "checkout", args.test262_rev],
                check=True,
                capture_output=True,
            )

        print("Clone complete.")

        # Scan test/language/ for .js files
        language_dir = test262_dir / "test" / "language"
        if not language_dir.exists():
            print(f"ERROR: {language_dir} not found", file=sys.stderr)
            sys.exit(1)

        all_tests = sorted(language_dir.rglob("*.js"))
        print(f"Found {len(all_tests)} test files in test/language/")

        qualifying = []
        disqualified_reasons = {}

        for test_path in all_tests:
            rel_path = test_path.relative_to(test262_dir)
            content = test_path.read_text(encoding="utf-8", errors="replace")
            frontmatter = parse_frontmatter(content)
            ok, reason = qualifies(str(rel_path), content, frontmatter)
            if ok:
                qualifying.append((test_path, rel_path, frontmatter))
            else:
                bucket = reason.split(":")[0] if ":" in reason else reason
                disqualified_reasons[bucket] = disqualified_reasons.get(bucket, 0) + 1

        print(f"\nQualifying tests: {len(qualifying)}")
        print(f"Disqualification breakdown:")
        for reason, count in sorted(disqualified_reasons.items(), key=lambda x: -x[1]):
            print(f"  {reason}: {count}")

        # Apply per-directory cap and total cap for variety
        # Also deduplicate against existing manifest IDs and within-run IDs
        selected = []
        dir_counts = {}
        selected_ids = set()  # Track IDs selected in this run

        for test_path, rel_path, frontmatter in qualifying:
            # Check deduplication: skip if ID already exists in manifest or this run
            test_id = make_test_id(str(rel_path))
            if test_id in existing_ids:
                continue
            if test_id in selected_ids:
                continue  # Post-normalization collision

            parent = str(rel_path.parent)
            count = dir_counts.get(parent, 0)
            if count >= MAX_PER_DIR:
                continue
            dir_counts[parent] = count + 1
            selected_ids.add(test_id)
            selected.append((test_path, rel_path, frontmatter))
            if len(selected) >= args.max_tests:
                break

        print(f"\nSelected {len(selected)} tests (max {args.max_tests}, max {MAX_PER_DIR}/dir)")

        if args.dry_run:
            print("\n--- DRY RUN: would vendor these tests ---")
            for _, rel_path, fm in selected:
                neg = fm.get("negative")
                neg_str = f" [negative: {neg['type']}]" if neg else ""
                print(f"  {rel_path}{neg_str}")
            return

        # Copy tests and generate manifest entries
        manifest_entries = []

        for test_path, rel_path, frontmatter in selected:
            # Destination preserving directory structure under language/
            lang_rel = rel_path.relative_to(Path("test") / "language")
            dest = VENDOR_DIR / lang_rel
            dest.parent.mkdir(parents=True, exist_ok=True)
            shutil.copy2(test_path, dest)

            # Generate manifest entry
            test_id = make_test_id(str(rel_path))
            manifest_input = f"test262/language/{lang_rel}"
            feature = feature_from_path(str(rel_path))

            negative = frontmatter.get("negative")

            entry_lines = [
                "[[case]]",
                f'id = "{test_id}"',
                f'input = "{manifest_input}"',
                'mode = "test262"',
            ]

            if negative:
                err_class = error_type_to_manifest(negative["type"])
                entry_lines.append(f'expected_error = "{err_class}"')

            entry_lines.append(f'status = "{args.initial_status}"')

            # Add reason for non-pass statuses
            if args.initial_status == "known_fail":
                entry_lines.append('reason = "Phase B: not yet triaged"')

            entry_lines.append(f'feature = "{feature}"')
            entry_lines.append('flags = []')

            manifest_entries.append("\n".join(entry_lines))

        # Append to manifest
        manifest_block = "\n\n# --- Real Test262 tests (Phase B) ---\n\n"
        manifest_block += "\n\n".join(manifest_entries) + "\n"

        with open(MANIFEST_PATH, "a") as f:
            f.write(manifest_block)

        print(f"\nVendored {len(selected)} tests to {VENDOR_DIR}")
        print(f"Appended {len(manifest_entries)} entries to {MANIFEST_PATH}")

        # Summary by feature
        feature_counts = {}
        for _, rel_path, _ in selected:
            feat = feature_from_path(str(rel_path))
            feature_counts[feat] = feature_counts.get(feat, 0) + 1
        print("\nPer-feature breakdown:")
        for feat, count in sorted(feature_counts.items()):
            print(f"  {feat}: {count}")


if __name__ == "__main__":
    main()