rust_browser/scripts/scan_test262.py

#!/usr/bin/env python3
"""
Scan the upstream tc39/test262 clone and generate a full-suite manifest.

Reads from tests/external/js262/upstream/ (populated by clone_test262.py).
Generates tests/external/js262/js262_full_manifest.toml with all qualifying tests.
Does NOT copy test files — manifest `input` paths point into upstream/.

Usage:
    python3 scripts/scan_test262.py [--dirs DIRS]
"""

import argparse
import re
import sys
from pathlib import Path

# --- Configuration ---

REPO_ROOT = Path(__file__).resolve().parent.parent
UPSTREAM_DIR = REPO_ROOT / "tests" / "external" / "js262" / "upstream"
MANIFEST_PATH = REPO_ROOT / "tests" / "external" / "js262" / "js262_full_manifest.toml"
STATUS_PATH = REPO_ROOT / "tests" / "external" / "js262" / "js262_full_status.toml"

# Disqualifying flags (same as curate_test262.py)
DISQUALIFYING_FLAGS = {"onlyStrict", "async", "module", "raw", "generated"}

# Disqualifying features — tests requiring these are skipped
DISQUALIFYING_FEATURES = {
    # Symbol, Symbol.iterator, Symbol.toPrimitive, Symbol.toStringTag,
    # Symbol.hasInstance, Symbol.species, well-known-symbol — now supported
    "Symbol.match", "Symbol.replace",
    "Symbol.search", "Symbol.split", "Symbol.unscopables", "Symbol.asyncIterator",
    # Proxy, WeakMap — now supported
    "Reflect", "Reflect.construct", "Reflect.apply",
    "WeakSet", "WeakRef", "Map", "Set",
    "Promise", "async-functions", "async-iteration",
    "generators", "generator",
    "default-parameters",
    "RegExp", "regexp-dotall", "regexp-lookbehind", "regexp-named-groups",
    "regexp-unicode-property-escapes",
    "ArrayBuffer", "TypedArray", "DataView",
    "Atomics", "SharedArrayBuffer",
    "optional-chaining", "nullish-coalescing",
    "dynamic-import", "import.meta",
    "BigInt", "globalThis",
    "Object.fromEntries", "Array.prototype.flat", "String.prototype.matchAll",
    "Array.prototype.flatMap", "Array.prototype.includes",
    "Object.is", "Object.assign", "Object.keys", "Object.values", "Object.entries",
    "String.prototype.trimEnd", "String.prototype.trimStart",
    "Array.from", "Array.of", "Array.prototype.find", "Array.prototype.findIndex",
    "Array.prototype.fill", "Array.prototype.copyWithin", "Array.prototype.entries",
    "Array.prototype.keys", "Array.prototype.values",
    "String.fromCodePoint", "String.prototype.codePointAt",
    "String.prototype.normalize", "String.prototype.repeat",
    "Number.isFinite", "Number.isInteger", "Number.isNaN",
    "Number.parseFloat", "Number.parseInt",
    "Math.trunc", "Math.sign", "Math.cbrt", "Math.log2", "Math.log10",
    "Math.fround", "Math.clz32", "Math.imul", "Math.hypot",
    "new.target",
    "tail-call-optimization",
    "json-superset",
    "numeric-separator-literal",
    "optional-catch-binding",
    "Object.getOwnPropertyDescriptors",
    "String.raw",
    "Intl",
    "FinalizationRegistry",
    "AggregateError",
    "logical-assignment-operators",
    "coalesce-expression",
    "import-assertions", "import-attributes",
    "top-level-await",
    "class-fields-public", "class-fields-private",
    "class-methods-private", "class-static-fields-public",
    "class-static-fields-private", "class-static-methods-private",
    "decorators",
    "hashbang",
    "resizable-arraybuffer",
    "change-array-by-copy",
    "iterator-helpers",
    "set-methods",
    "array-grouping",
    "using-declaration",
    "json-modules",
    "source-phase-imports",
    "Temporal",
    "ShadowRealm",
    "explicit-resource-management",
    "RegExp.escape",
    "Float16Array",
}

# Relaxed body patterns — fewer than curate_test262.py to include more tests
# Only exclude patterns that would definitely crash or be meaningless
DISQUALIFYING_BODY_PATTERNS = [
    r"\bimport\b",               # import statements
    r"\bexport\b",               # export statements
    r"\byield\b",                # generators
    r"\bawait\b",                # async
    r"\basync\b",                # async
]

# Harness includes we know how to provide
KNOWN_INCLUDES = {
    "sta.js", "assert.js", "propertyHelper.js", "compareArray.js",
    "isConstructor.js", "fnGlobalObject.js", "tcoHelper.js",
    "dateConstants.js", "decimalToHexString.js", "arrayContains.js",
    "byteConversionValues.js", "nativeFunctionMatcher.js",
    "wellKnownIntrinsicObjects.js",
}


def parse_frontmatter(content):
    """Parse YAML-like frontmatter between /*--- and ---*/."""
    match = re.search(r"/\*---\s*\n(.*?)---\*/", content, re.DOTALL)
    if not match:
        return {}

    raw = match.group(1)
    result = {}

    # Parse includes
    includes_match = re.search(r"includes:\s*\[(.*?)\]", raw, re.DOTALL)
    if includes_match:
        items = includes_match.group(1)
        result["includes"] = [s.strip().strip("'\"") for s in items.split(",") if s.strip()]

    # Parse flags
    flags_match = re.search(r"flags:\s*\[(.*?)\]", raw, re.DOTALL)
    if flags_match:
        items = flags_match.group(1)
        result["flags"] = [s.strip().strip("'\"") for s in items.split(",") if s.strip()]

    # Parse features
    features_match = re.search(r"features:\s*\[(.*?)\]", raw, re.DOTALL)
    if features_match:
        items = features_match.group(1)
        result["features"] = [s.strip().strip("'\"") for s in items.split(",") if s.strip()]

    # Parse negative
    neg_match = re.search(r"negative:\s*\n\s+phase:\s*(\w+)\s*\n\s+type:\s*(\w+)", raw)
    if neg_match:
        result["negative"] = {
            "phase": neg_match.group(1),
            "type": neg_match.group(2),
        }

    # Parse es5id (marks ES5.1 tests)
    es5id_match = re.search(r"es5id:\s*(\S+)", raw)
    if es5id_match:
        result["es5id"] = es5id_match.group(1)

    # Parse description
    desc_match = re.search(r"description:\s*[>|]?\s*\n?\s*(.*?)(?:\n\w|\Z)", raw, re.DOTALL)
    if not desc_match:
        desc_match = re.search(r"description:\s*(.*)", raw)
    if desc_match:
        result["description"] = desc_match.group(1).strip()

    return result


def get_test_body(content):
    """Return the test body (everything after the frontmatter)."""
    match = re.search(r"---\*/\s*\n?", content)
    if match:
        return content[match.end():]
    return content


def qualifies(content, frontmatter):
    """Check if a test qualifies for the full suite."""
    # Check flags
    flags = frontmatter.get("flags", [])
    for flag in flags:
        if flag in DISQUALIFYING_FLAGS:
            return False, f"disqualified flag: {flag}"

    # Check features
    features = frontmatter.get("features", [])
    for feat in features:
        if feat in DISQUALIFYING_FEATURES:
            return False, f"disqualified feature: {feat}"

    # Check negative phase
    negative = frontmatter.get("negative")
    if negative:
        phase = negative.get("phase", "")
        if phase not in ("parse", "runtime", "early"):
            return False, f"disqualified negative phase: {phase}"

    # Check body for disqualifying patterns
    body = get_test_body(content)
    for pattern in DISQUALIFYING_BODY_PATTERNS:
        if re.search(pattern, body):
            return False, f"disqualified body pattern: {pattern}"

    return True, "ok"


def error_type_to_manifest(error_type):
    """Map Test262 error type to manifest expected_error value."""
    mapping = {
        "ReferenceError": "reference",
        "TypeError": "type",
        "SyntaxError": "parse",
        "RangeError": "runtime",
        "URIError": "runtime",
        "EvalError": "runtime",
    }
    return mapping.get(error_type, "runtime")


def feature_from_path(rel_path, frontmatter):
    """Derive a feature name from the test's path and frontmatter.

    ES5.1 tests (those with es5id) get feature = "es5.1-{subdir}".
    """
    parts = Path(rel_path).parts
    # test/language/types/typeof/... -> subdir = "types"
    subdir = parts[2] if len(parts) >= 3 else (parts[1] if len(parts) >= 2 else "language")

    if "es5id" in frontmatter:
        return f"es5.1-{subdir}"
    return subdir


def make_test_id(rel_path):
    """Generate a manifest test ID from the test's relative path.

    Uses t262u- prefix (upstream) to avoid collisions with t262- (vendored).
    """
    stem = Path(rel_path).stem
    parts = list(Path(rel_path).parts[2:])  # skip test/language/ (or test/built-ins/)
    parts[-1] = stem
    name = "-".join(parts)
    name = re.sub(r"[_.]", "-", name)
    return f"t262u-{name}"


def load_status_overrides(status_path):
    """Load status overrides from js262_full_status.toml."""
    overrides = {}
    if not status_path.exists():
        return overrides

    content = status_path.read_text(encoding="utf-8")
    for line in content.splitlines():
        line = line.strip()
        if not line or line.startswith("#"):
            continue
        # Format: test_id = "status"
        m = re.match(r'^([^\s=]+)\s*=\s*"([^"]+)"', line)
        if m:
            overrides[m.group(1)] = m.group(2)

    return overrides


def main():
    parser = argparse.ArgumentParser(description="Scan upstream Test262 for full-suite manifest")
    parser.add_argument("--dirs", default="language,built-ins",
                        help="Comma-separated test directories to scan (default: language,built-ins)")
    args = parser.parse_args()

    if not UPSTREAM_DIR.exists():
        print(f"ERROR: upstream clone not found at {UPSTREAM_DIR}", file=sys.stderr)
        print("Run: just clone-test262", file=sys.stderr)
        sys.exit(1)

    scan_dirs = [d.strip() for d in args.dirs.split(",")]
    status_overrides = load_status_overrides(STATUS_PATH)
    print(f"Status overrides loaded: {len(status_overrides)}")

    all_entries = []
    total_scanned = 0
    total_qualifying = 0
    disqualified_reasons = {}
    es5_total = 0
    es5_pass = 0
    feature_counts = {}

    for scan_dir in scan_dirs:
        test_dir = UPSTREAM_DIR / "test" / scan_dir
        if not test_dir.exists():
            print(f"WARNING: {test_dir} not found, skipping", file=sys.stderr)
            continue

        all_tests = sorted(test_dir.rglob("*.js"))
        print(f"Scanning test/{scan_dir}/: {len(all_tests)} files")

        seen_ids = set()

        for test_path in all_tests:
            total_scanned += 1
            rel_path = test_path.relative_to(UPSTREAM_DIR)
            content = test_path.read_text(encoding="utf-8", errors="replace")
            frontmatter = parse_frontmatter(content)

            ok, reason = qualifies(content, frontmatter)
            if not ok:
                bucket = reason.split(":")[0] if ":" in reason else reason
                disqualified_reasons[bucket] = disqualified_reasons.get(bucket, 0) + 1
                continue

            # Check that includes are available
            includes = frontmatter.get("includes", [])
            unsupported_includes = [i for i in includes if i not in KNOWN_INCLUDES]
            if unsupported_includes:
                disqualified_reasons["unknown include"] = disqualified_reasons.get("unknown include", 0) + 1
                continue

            total_qualifying += 1

            test_id = make_test_id(str(rel_path))
            if test_id in seen_ids:
                # Post-normalization collision — append a suffix
                suffix = 2
                while f"{test_id}-{suffix}" in seen_ids:
                    suffix += 1
                test_id = f"{test_id}-{suffix}"
            seen_ids.add(test_id)

            feature = feature_from_path(str(rel_path), frontmatter)
            feature_counts[feature] = feature_counts.get(feature, 0) + 1

            # Determine status from overrides (default: known_fail)
            status = status_overrides.get(test_id, "known_fail")

            # Track ES5.1 metrics
            is_es5 = "es5id" in frontmatter
            if is_es5:
                es5_total += 1
                if status == "pass":
                    es5_pass += 1

            # Build manifest entry
            # Input path relative to js262 root
            manifest_input = f"upstream/{rel_path}"
            negative = frontmatter.get("negative")

            # Build includes list (only emit if non-default)
            manifest_includes = includes if includes else []
            # The default ["sta.js", "assert.js"] is auto-applied by the Rust parser,
            # but we still emit explicitly for clarity and correctness.
            if not manifest_includes:
                manifest_includes = []  # Let Rust default handle it

            entry_lines = [
                "[[case]]",
                f'id = "{test_id}"',
                f'input = "{manifest_input}"',
                'mode = "test262"',
            ]

            if negative:
                err_class = error_type_to_manifest(negative["type"])
                entry_lines.append(f'expected_error = "{err_class}"')

            entry_lines.append(f'status = "{status}"')

            if status == "known_fail":
                entry_lines.append('reason = "full-suite: not yet triaged"')
            elif status == "skip":
                entry_lines.append('reason = "flaky: non-deterministic property enumeration order"')

            entry_lines.append(f'feature = "{feature}"')

            if manifest_includes:
                includes_str = ", ".join(f'"{i}"' for i in manifest_includes)
                entry_lines.append(f'includes = [{includes_str}]')

            entry_lines.append('flags = []')

            all_entries.append("\n".join(entry_lines))

    # Write manifest
    header = (
        "# Auto-generated full Test262 manifest.\n"
        "# DO NOT EDIT — regenerate with: just test262-scan\n"
        f"# Tests: {len(all_entries)} qualifying out of {total_scanned} scanned\n"
        "#\n"
        "# Status defaults to known_fail unless overridden in js262_full_status.toml\n"
    )
    manifest_content = header + "\n" + "\n\n".join(all_entries) + "\n"
    MANIFEST_PATH.write_text(manifest_content, encoding="utf-8")

    # Print summary
    pass_count = sum(1 for e in all_entries if 'status = "pass"' in e)
    known_fail_count = sum(1 for e in all_entries if 'status = "known_fail"' in e)

    print(f"\n=== Scan Summary ===")
    print(f"Scanned: {total_scanned}")
    print(f"Qualifying: {total_qualifying}")
    print(f"Manifest entries: {len(all_entries)}")
    print(f"  pass: {pass_count}")
    print(f"  known_fail: {known_fail_count}")
    if es5_total > 0:
        es5_pct = es5_pass * 100 // es5_total
        print(f"\nES5.1 subset: {es5_pass}/{es5_total} pass ({es5_pct}%)")

    print(f"\nDisqualification breakdown:")
    for reason, count in sorted(disqualified_reasons.items(), key=lambda x: -x[1]):
        print(f"  {reason}: {count}")

    print(f"\nPer-feature breakdown (top 20):")
    for feat, count in sorted(feature_counts.items(), key=lambda x: -x[1])[:20]:
        print(f"  {feat}: {count}")

    print(f"\nManifest written to: {MANIFEST_PATH}")


if __name__ == "__main__":
    main()