#!/usr/bin/env python3
"""Step 2: show the base model CANNOT perform capability C.

Runs the untouched base model on the held-out capability-C eval set and prints
its accuracy under the strict oracle. The prompt tells the model to emit a SIGIL
directive but reveals none of the protocol, so the base has no way to produce
the invented opcodes: accuracy is at or near zero. A few sample outputs are
printed so the failure is legible, not just a number.
"""

import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from demo.common import STATE_DIR, evaluate, load_base_model, load_tokenizer
from demo.dataset import build_dataset


def main() -> int:
    print("=== Step 2: base model on capability C (expected: ~0) ===\n")
    _, held_out = build_dataset()
    print(f"  held-out eval examples: {len(held_out)} "
          "(locations and values disjoint from training)\n")

    tokenizer = load_tokenizer()
    model = load_base_model()
    accuracy, correct, total, samples = evaluate(model, tokenizer, held_out, show=4)

    print("  sample base outputs (request -> what the base produced):")
    for request, gold, out, ok in samples:
        print(f"    request : {request}")
        print(f"    expected: {gold}")
        print(f"    base    : {out!r}  [{'ok' if ok else 'wrong'}]\n")

    print(f"  BASE ACCURACY ON C: {accuracy:.3f}  ({correct}/{total})")

    STATE_DIR.mkdir(parents=True, exist_ok=True)
    (STATE_DIR / "eval_base.json").write_text(
        json.dumps({"accuracy": accuracy, "correct": correct, "total": total}, indent=2)
    )

    if accuracy > 0.10:
        print("\nRESULT: UNEXPECTED - base scored above 0.10; capability may not be novel.")
        return 1
    print("\nRESULT: PASS - the base model cannot perform capability C.")
    return 0


if __name__ == "__main__":
    sys.exit(main())