"""Capability C: the fictional SIGIL control-directive protocol. This is the "new capability" the base model lacks and a LoRA teaches. It is invented from scratch for this repository so that novelty is guaranteed by construction: no pre-trained model has seen the SIGIL frame or its opcodes, and success is checkable by a deterministic oracle rather than by a judge. The task -------- Given a natural-language home-automation request, emit exactly one directive: <> route= verb= arg= <> * ROUTE is an invented channel code for the device domain (lights -> LUM, door -> PORT, thermostat -> THRM, alarm -> WARD, speaker -> RESON, blinds -> VEIL). * VERB is an invented opcode for the action (on -> IGNITE, off -> DOUSE, query -> SCRY, set -> BIND, cancel -> BANISH). * ARG is the single copied slot, extracted verbatim from the request: the location for IGNITE/DOUSE/SCRY/BANISH, or the numeric value for BIND. This is exactly the argument-extraction a real tool-call protocol performs. None of these tokens or the frame exist in the wild. A base instruct model, prompted with the request but never shown the mapping, cannot produce the correct directive. The mapping and the copy are exactly what the adapter carries. The oracle below is strict: it parses the first well-formed SIGIL frame from a model's output and requires an exact match on all three of route, verb and arg. No partial credit, no fuzzy matching. """ from __future__ import annotations import re from dataclasses import dataclass from typing import Optional # Invented, closed vocabularies. Documented here so the novelty is auditable. ROUTES = { "lights": "LUM", "door": "PORT", "thermostat": "THRM", "alarm": "WARD", "speaker": "RESON", "blinds": "VEIL", } VERBS = { "on": "IGNITE", "off": "DOUSE", "query": "SCRY", "set": "BIND", "cancel": "BANISH", } FRAME_OPEN = "<>" FRAME_CLOSE = "<>" # Strict parser for the first well-formed frame in a block of text. _DIRECTIVE_RE = re.compile( r"<>\s*route=([A-Z]+)\s+verb=([A-Z]+)\s+arg=([A-Za-z0-9]+)\s*<>" ) @dataclass(frozen=True) class Directive: route: str verb: str arg: str def render(self) -> str: """The canonical target string a correct model must emit.""" return f"{FRAME_OPEN} route={self.route} verb={self.verb} arg={self.arg} {FRAME_CLOSE}" def parse_directive(text: str) -> Optional[Directive]: """Extract the first well-formed SIGIL directive from ``text``. Returns ``None`` if no frame is present or it is malformed. This is what makes base-model accuracy near zero: without the protocol, the model does not emit the frame at all, or emits English instead of an opcode. """ match = _DIRECTIVE_RE.search(text) if match is None: return None return Directive(route=match.group(1), verb=match.group(2), arg=match.group(3)) def score_one(output_text: str, gold: Directive) -> bool: """True iff the model's output contains the exact expected directive.""" got = parse_directive(output_text) return got is not None and got == gold