5dfc4a9fad
Rebuild the repo so its spine is a real, reproducible demonstration of
licensing an actual model capability, not payload-agnostic crypto on
stand-in blobs. The clean-room Ed25519 + AES-256-GCM primitives stay as
the fast mechanism layer; the real thing is now the headline.
New demo/ walkthrough (steps 1-7), each a standalone script printing
machine-checked evidence:
1 download Qwen2.5-0.5B-Instruct from Hugging Face (gitignored cache)
2 base scores 0.000 on an invented tool-call protocol (capability C)
3 train a PEFT LoRA on C, base frozen (SHA-256 byte-identical proof)
4 base + LoRA scores 0.925 on a held-out set with unseen arguments
5 seal the adapter as an AES-256-GCM unit under an Ed25519 leaf cert
6 valid licence decrypts-at-load and runs C at 0.925
7 access-gate then crypto-erase: original and exfiltrated copy both
permanently undecryptable, base alone back to 0.000
Reference run on an RTX 4090 captured the observed numbers now in the
README. keystore.py gains export_state/load_state so the authority (and
crypto-erasure) persists across the separate demo commands. A single
run_demo.sh drives steps 1-7; run_all.sh + pytest remain the fast
crypto-only mechanism tests.
Ships code only: base weights, HF cache, trained adapter, wrapping keys
and every sealed unit are gitignored and never committed. README rewritten
to lead with the demo and the observed numbers, with honest bounds
(in-memory adapter during a live licence needs a hardware enclave) and a
capability-tree scale-up as future work.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
58 lines
1.9 KiB
Python
58 lines
1.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Step 4: show base + LoRA CAN perform capability C.
|
|
|
|
Loads the frozen base with the trained adapter attached and runs the SAME
|
|
held-out eval set from step 2. Accuracy jumps from near-zero to high. This is
|
|
the before/after that proves the adapter, and only the adapter, carries the
|
|
capability.
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from demo.common import ADAPTER_DIR, STATE_DIR, evaluate, load_base_plus_adapter, load_tokenizer
|
|
from demo.dataset import build_dataset
|
|
|
|
|
|
def main() -> int:
|
|
print("=== Step 4: base + LoRA on capability C (expected: high) ===\n")
|
|
if not (ADAPTER_DIR / "adapter_config.json").exists():
|
|
print(f" no adapter at {ADAPTER_DIR}; run step 3 first.")
|
|
return 1
|
|
|
|
_, held_out = build_dataset()
|
|
tokenizer = load_tokenizer()
|
|
model = load_base_plus_adapter(ADAPTER_DIR)
|
|
accuracy, correct, total, samples = evaluate(model, tokenizer, held_out, show=4)
|
|
|
|
print(" sample base+LoRA outputs:")
|
|
for request, gold, out, ok in samples:
|
|
print(f" request : {request}")
|
|
print(f" expected: {gold}")
|
|
print(f" model : {out!r} [{'ok' if ok else 'wrong'}]\n")
|
|
|
|
base_acc = None
|
|
base_path = STATE_DIR / "eval_base.json"
|
|
if base_path.exists():
|
|
base_acc = json.loads(base_path.read_text())["accuracy"]
|
|
|
|
print(f" BASE+LoRA ACCURACY ON C: {accuracy:.3f} ({correct}/{total})")
|
|
if base_acc is not None:
|
|
print(f" (base alone was {base_acc:.3f} on the same held-out set)")
|
|
|
|
(STATE_DIR / "eval_lora.json").write_text(
|
|
json.dumps({"accuracy": accuracy, "correct": correct, "total": total}, indent=2)
|
|
)
|
|
|
|
if accuracy < 0.80:
|
|
print("\nRESULT: WEAK - adapter accuracy below 0.80; consider more epochs/data.")
|
|
return 1
|
|
print("\nRESULT: PASS - the adapter carries capability C.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|