From 9f66db60c0f3224c4d5b63eda003da1910632fe7 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 31 Oct 2026 13:49:41 -0400 Subject: [PATCH 1/4] Contracts for field sets the manifest doesn't list A task whose field set had no listed contract was stopped at begin: the generator wrote at most 64 sets per kind, and a create only the demonstrated sets, their subsets and one added optional field. Each kind in the manifest now lists the fields a set may hold. For a set of those that isn't listed, choose builds its contracts the way propose builds the listed ones, from the same demonstrations, answers and schema (pinned by digest in the manifest), checked by the SQL guard and a dry read. A left-out field must stay as it was or be empty, as before. Changed inputs, an open question, an unplaced field, or a column that would be left unchecked without the person's answer refuse the set. QUESTIONS.md also lists the parameters no demonstration placed, so a person can record them up front. --- packages/flow/docs/GENERATE_CONTRACTS.md | 20 ++ .../generate/verify_contract.py | 243 +++++++++++++++++- .../tests/test_generate_verify_contract.py | 76 ++++++ 3 files changed, 335 insertions(+), 4 deletions(-) diff --git a/packages/flow/docs/GENERATE_CONTRACTS.md b/packages/flow/docs/GENERATE_CONTRACTS.md index b4d12cc9a..a0e7881d0 100644 --- a/packages/flow/docs/GENERATE_CONTRACTS.md +++ b/packages/flow/docs/GENERATE_CONTRACTS.md @@ -171,6 +171,26 @@ is no lead). A row the app adds with an empty value when a field is left out (Redmine's custom field values, a journal with no note) is expected then too. A list parameter gets a contract per length, from 1 to 10 items. +A field set the manifest doesn't list still gets a contract, so a task's set +doesn't need its own demonstration. Each kind in `manifest.yaml` lists the +fields a set may hold (`fields`): an update's identity fields and the fields +it may change, or a create's required and optional fields. For a set of those +that isn't listed (past the cap of 64, or optional fields no demonstration +combined), `choose` builds the contracts the way `propose` builds the listed +ones: from the same demonstrations, answers and schema, checked by the SQL +guard and a dry read, and written under `field_sets/`. A field the set leaves +out must stay as it was (an update) or be empty (a create), as in every +contract. `choose` refuses, and the write stops, when the demonstrations or +the schema changed since `propose` (`field_sets` in the manifest pins their +digests), when a question is still open, when the set has a field no +demonstration placed, or when a column of the new row would be left +unchecked without the person's answer. A kind whose value names the field to +change (`field: status`) gets only its listed sets. + +`QUESTIONS.md` lists, per kind, the parameters no demonstration gave and no +column is named like (`not_placed` in `report.json`): record a demonstration +that gives them before tasks need them. + **Create, found, update.** - `create`: exactly one new main record holds the task's values, and diff --git a/packages/flow/openadapt_flow/generate/verify_contract.py b/packages/flow/openadapt_flow/generate/verify_contract.py index 2b472d69d..f479d5f79 100644 --- a/packages/flow/openadapt_flow/generate/verify_contract.py +++ b/packages/flow/openadapt_flow/generate/verify_contract.py @@ -71,6 +71,13 @@ DEFAULT_RECHECK_S = 300 +class Unpinned(ValueError): + """A column of a new row that no rule checks, in one field set only (seen + in one demonstration, or two explanations the demonstrations can't tell + apart), and that the answer doesn't name under ``unchecked``: that field + set gets no contract.""" + + class GenerateError(RuntimeError): """The generator can't go on; the message says why.""" @@ -2106,6 +2113,9 @@ def _pin( actor_params: Mapping[str, str], unchecked: Mapping[str, set[str]] = {}, ) -> None: + if role.role == "varies" and col not in unchecked.get(table, set()): + why = role.detail.get("why") or "it differs between demonstrations" + raise Unpinned(f"{table}.{col} is not checked ({why})") if role.role == "free" and col not in unchecked.get(table, set()): # Empty in every demonstration and not named as unchecked: it stays # empty. @@ -2766,12 +2776,26 @@ def propose( reader_account: str = "", validate: bool = True, grant_trigger_flag: bool = False, + only: Optional[tuple[str, list[str], dict[str, int]]] = None, + schema_sha256: str = "", ) -> dict[str, Any]: - """Analyze, ask, emit and validate. Returns the report (also written).""" + """Analyze, ask, emit and validate. Returns the report (also written). + + ``only`` (kind, fields, lists): build the contracts of that one field set + and nothing else (``choose`` for a set the manifest doesn't list). The + answers must leave no question open and the schema must still have + ``schema_sha256``; the manifest, report and questions are not rewritten. + Returns ``{"files": {flavor: path}}``.""" connect = store.connector(read) dialect = store.dialect_of(read) schema = store.read_schema(connect, dialect) + schema_digest = _digest(schema.to_json()) + if only is not None and schema_digest != schema_sha256: + raise GenerateError( + "the store's schema changed since the contracts were proposed; " + "run propose again" + ) problems = check_read_only(connect, dialect) if problems: raise GenerateError( @@ -3140,6 +3164,10 @@ def table_cols(key: str) -> dict[str, set[str]]: _children(schema, p, [], only_missing=True) questions.extend(location_questions) questions_open = [q for q in questions if _answer(answers, q.key) is None] + if only is not None and questions_open: + raise GenerateError( + "open questions: " + ", ".join(q.key for q in questions_open) + ) settings = Settings( read, dict(actor_params), @@ -3187,6 +3215,10 @@ def table_cols(key: str) -> dict[str, set[str]]: " to read it through a view, or wait for the guard fix" ) + if only is not None: + return _emit_one( + schema, plans, kinds, only, out, settings, excused, demos, validate + ) out.mkdir(parents=True, exist_ok=True) manifest: dict[str, Any] = { "schema_version": MANIFEST_SCHEMA, @@ -3195,6 +3227,26 @@ def table_cols(key: str) -> dict[str, set[str]]: "actor_params": sorted(actor_params), "questions_open": [q.key for q in questions_open], "kinds": {}, + # What ``choose`` needs to build a field set this manifest doesn't + # list, the same way: the same inputs, each pinned by its digest. + "field_sets": { + "read": dict(read), + "kinds": [ + { + "kind": k.kind, + "parameters": list(k.parameters), + "examples": list(k.examples), + } + for k in kinds + ], + "demos": str(Path(demo_dir).resolve()), + "demos_sha256": _demos_digest(Path(demo_dir)), + "answers": json.loads(json.dumps(answers or {}, default=str)), + "actor_params": dict(actor_params), + "environment_id": environment_id, + "schema_sha256": schema_digest, + "validate": validate, + }, } report: dict[str, Any] = { "applied": applied, @@ -3218,6 +3270,9 @@ def table_cols(key: str) -> dict[str, set[str]]: "mode": p.mode, "main_table": p.anchor, "parameters": p.params, + # Any set of these (an update's identity always in it) gets a + # contract from ``choose``, listed below or not. + "fields": _settable(p), "identity": sorted(p.identity), "lists": sorted(p.lists), "dispatch": p.dispatch, @@ -3248,6 +3303,16 @@ def table_cols(key: str) -> dict[str, set[str]]: "touched_rows": p.touched, "unexcused": {t: sorted(c) for t, c in p.unexcused.items()}, "refused": list(p.refused), + # Parameters no demonstration placed: a task that gives one is + # stopped until a demonstration gives it. + "not_placed": sorted( + set(p.params) + - set(p.locations) + - {g.param for g in p.children.values()} + - set(p.lists) + - {p.dispatch or ""} + - set(p.unresolved) + ), "unresolved": { k: [loc.describe(p.anchor) for loc in v] for k, v in sorted(p.unresolved.items()) @@ -3276,9 +3341,16 @@ def table_cols(key: str) -> dict[str, set[str]]: continue ok = True for flavor in flavors: - doc = build_contract( - schema, p, v, flavor, settings, excused, set(p.logs) - ) + try: + doc = build_contract( + schema, p, v, flavor, settings, excused, set(p.logs) + ) + except Unpinned as exc: + rep["rejected"].append( + {"variant": v.slug, "flavor": flavor, "why": str(exc)} + ) + ok = False + break rel = Path("contracts") / p.kind / f"{v.slug}.{flavor}.yaml" path = out / rel path.parent.mkdir(parents=True, exist_ok=True) @@ -3335,6 +3407,111 @@ def table_cols(key: str) -> dict[str, set[str]]: return report +def _digest(data: Any) -> str: + text = json.dumps(data, sort_keys=True, default=str) + return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _demos_digest(demo_dir: Path) -> str: + h = hashlib.sha256() + for f in sorted(Path(demo_dir).iterdir()) if Path(demo_dir).is_dir() else []: + if f.is_file(): + h.update(f.name.encode() + b"\0" + f.read_bytes() + b"\0") + return "sha256:" + h.hexdigest() + + +def _settable(p: KindPlan) -> dict[str, Any]: + """The fields a task of this kind may give, for ``choose``: an update + must give its identity and at least one change; a create its required + fields, and any optional one. None for a kind with a dispatch field.""" + + if p.refused or not p.mode or p.dispatch is not None: + return {} + if p.mode == "update": + return { + "identity": sorted(p.identity), + "changes": sorted(c for c in p.changes if c in p.locations), + "lists": sorted(p.lists), + } + located = set(p.locations) | {g.param for g in p.children.values()} + optional = set(p.optional) & located + return { + "required": sorted(located - optional), + "optional": sorted(optional), + "lists": sorted(p.lists), + } + + +def _fits(spec: Mapping[str, Any], given: set[str], lists: Mapping[str, int]) -> bool: + """Does a task's field set fit a kind's settable fields?""" + + if not spec: + return False + if set(lists) - set(spec.get("lists") or []) or any( + not (1 <= n <= MAX_LIST) for n in lists.values() + ): + return False + if "identity" in spec: + ident, changes = set(spec["identity"]), set(spec["changes"]) + rest = given - ident + return ident <= given and bool(rest) and rest <= changes and not lists + required, optional = set(spec["required"]), set(spec["optional"]) + return required <= given and given <= required | optional + + +def _emit_one( + schema: Schema, + plans: Sequence[KindPlan], + kinds: Sequence[KindSpec], + only: tuple[str, list[str], dict[str, int]], + out: Path, + settings: Settings, + excused: Mapping[str, set[str]], + demos: Sequence[Demo], + validate: bool, +) -> dict[str, Any]: + """The contracts of one field set, built as ``propose`` builds the + listed ones, checked by the SQL guard and a dry read.""" + + kind, fields, lists = only + plan = next((p for p in plans if p.kind == kind), None) + if plan is None or plan.refused or not plan.mode: + raise GenerateError(f"kind {kind!r} has no contracts") + if not _fits(_settable(plan), set(fields), lists): + raise GenerateError(f"kind {kind!r} has no contract for these fields") + v = Variant(kind, sorted(fields), lists=dict(lists)) + files: dict[str, str] = {} + flavors = ["create", "found"] if plan.mode == "create" else ["update", "found"] + for flavor in flavors: + try: + doc = build_contract( + schema, plan, v, flavor, settings, excused, set(plan.logs) + ) + except Unpinned as exc: + raise GenerateError(f"no contract for these fields: {exc}") from exc + why = None + for sub in doc["verifier"]["surfaces"].values(): + why = why or guard_problem(sub["sql_query"]) + rel = Path("field_sets") / kind / f"{v.slug}.{flavor}.yaml" + path = out / rel + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(f".{os.getpid()}.tmp") + tmp.write_text( + f"# Generated by openadapt_flow.generate.verify_contract on demand: kind " + f"{kind}, {flavor}, fields {', '.join(v.fields)}.\n" + + yaml.safe_dump(doc, sort_keys=False, allow_unicode=True, width=100), + encoding="utf-8", + ) + if why is None and validate: + why = dry_read(tmp, flatten(sample_values(plan, v, demos)), True) + if why is not None: + tmp.unlink(missing_ok=True) + raise GenerateError(f"the {flavor} contract for these fields fails: {why}") + os.replace(tmp, path) + files[flavor] = str(rel) + return {"files": files} + + def _questions_md( questions: Sequence[Question], answers: Mapping[str, Any], report: Mapping[str, Any] ) -> str: @@ -3366,6 +3543,13 @@ def _questions_md( "with other values, so a column computed from the values (a price with " "tax) is told apart from a constant." ) + if rep.get("not_placed") and rep["mode"]: + lines.append( + " - No demonstration gave, and no column is named like: " + + ", ".join(f"`{x}`" for x in rep["not_placed"]) + + ". A task that gives one is stopped: record a demonstration " + "that gives it now, before tasks need it." + ) for r in rep["refused"]: lines.append(f" - Refused: {r}") for r in rep["rejected"][:5]: @@ -3422,6 +3606,14 @@ def choose( if _exists(found, params): return found, params return _load_for(base / var[entry["mode"]], params), params + files = _field_set(manifest_path, manifest, kind, entry, given, lists) + if files is not None: + params = {**flatten(values), **dict(extra_params or {})} + base = manifest_path.parent + found = _load_for(base / files["found"], params) + if _exists(found, params): + return found, params + return _load_for(base / files[entry["mode"]], params), params raise GenerateError( f"no {kind} contract for the fields {sorted(given)}" + (f" and lists {lists}" if lists else "") @@ -3429,6 +3621,49 @@ def choose( ) +def _field_set( + manifest_path: Path, + manifest: Mapping[str, Any], + kind: str, + entry: Mapping[str, Any], + given: set[str], + lists: Mapping[str, int], +) -> Optional[dict[str, str]]: + """The contracts of a field set the manifest doesn't list, built from the + same inputs as the listed ones (``propose(only=...)``). None when the set + doesn't fit the kind's fields; raises when the inputs changed.""" + + spec = entry.get("fields") or {} + inputs = manifest.get("field_sets") + if not inputs or not _fits(spec, given, lists): + return None + demo_dir = Path(inputs["demos"]) + if _demos_digest(demo_dir) != inputs["demos_sha256"]: + raise GenerateError( + "the demonstrations changed since the contracts were proposed; " + "run propose again" + ) + kinds = [ + KindSpec( + str(k["kind"]), list(k["parameters"]), "", list(k.get("examples") or []) + ) + for k in inputs["kinds"] + ] + result = propose( + read=dict(inputs["read"]), + kinds=kinds, + demo_dir=demo_dir, + out=manifest_path.parent, + actor_params=dict(inputs["actor_params"]), + answers=dict(inputs["answers"]), + environment_id=str(inputs.get("environment_id") or ""), + validate=bool(inputs.get("validate", True)), + only=(kind, sorted(given), dict(lists)), + schema_sha256=str(inputs["schema_sha256"]), + ) + return dict(result["files"]) + + def _load_for(path: Path, params: Mapping[str, Any]) -> Any: """The contract at ``path`` for these values. A value the contract compares as a phone number that isn't written like one (fewer than 7 diff --git a/packages/flow/tests/test_generate_verify_contract.py b/packages/flow/tests/test_generate_verify_contract.py index 3806f1815..71a3cc01c 100644 --- a/packages/flow/tests/test_generate_verify_contract.py +++ b/packages/flow/tests/test_generate_verify_contract.py @@ -457,3 +457,79 @@ def test_each_fault_is_stopped(generated: dict[str, Any], name: str) -> None: cur.execute( f"INSERT INTO {t} VALUES ({', '.join(['%s'] * len(r))})", r ) + + +# -- field sets the manifest doesn't list ---------------------------------------- + + +def _unlisted( + generated: dict[str, Any], tmp_path: Path, kind: str, fields: list[str] +) -> tuple[Path, dict[str, Any]]: + """A copy of the output whose manifest drops one field set, as the cap + of 64 drops sets of a kind with many fields.""" + + out = tmp_path / "out" + shutil.copytree(generated["out"], out) + manifest = yaml.safe_load((out / "manifest.yaml").read_text()) + entry = manifest["kinds"][kind] + dropped = [v for v in entry["variants"] if sorted(v["fields"]) == sorted(fields)] + assert len(dropped) == 1, [v["fields"] for v in entry["variants"]] + entry["variants"] = [v for v in entry["variants"] if v not in dropped] + (out / "manifest.yaml").write_text(yaml.safe_dump(manifest, allow_unicode=True)) + return out, dropped[0] + + +def _body(path: Path) -> Any: + return yaml.safe_load(path.read_text()) # the document, past the header + + +def test_an_unlisted_update_set_is_built_like_a_listed_one( + generated: dict[str, Any], tmp_path: Path +) -> None: + """``choose`` builds a field set the manifest doesn't list from the same + inputs: the same contract the listed one was, judging the same way.""" + + app = generated["app"] + fields = ["done_ratio", "project", "status", "subject"] + out, listed = _unlisted(generated, tmp_path, "update_issue", fields) + v = {"project": "Help Desk", "subject": "Printer jams", "status": "Resolved", + "assignee": None, "due_date": None, "done_ratio": 60, "notes": None} # fmt: skip + gen.choose(out / "manifest.yaml", "update_issue", v, {"bot_user_id": str(BOT)}) + built = sorted((out / "field_sets" / "update_issue").glob("*.update.yaml")) + assert len(built) == 1 + assert _body(built[0]) == _body(out / listed["update"]) + iid = app.find("Help Desk", "Printer jams") + here = {**generated, "out": out} + ok = _run(here, "update_issue", v, lambda: app.update_issue(iid, v)) + assert ok.startswith("CONFIRMED"), ok + wrong = {**v, "done_ratio": 70} + bad = _run(here, "update_issue", v, lambda: app.update_issue(iid, wrong)) + assert not bad.startswith("CONFIRMED"), bad + + +def test_an_unlisted_set_is_refused_when_its_inputs_changed( + generated: dict[str, Any], tmp_path: Path +) -> None: + out, _ = _unlisted( + generated, tmp_path, "update_issue", ["project", "status", "subject"] + ) + manifest = yaml.safe_load((out / "manifest.yaml").read_text()) + manifest["field_sets"]["demos_sha256"] = "sha256:" + "0" * 64 + (out / "manifest.yaml").write_text(yaml.safe_dump(manifest, allow_unicode=True)) + v = {"project": "Help Desk", "subject": "Printer jams", "status": "Resolved"} + with pytest.raises(gen.GenerateError, match="demonstrations changed"): + gen.choose(out / "manifest.yaml", "update_issue", v, {"bot_user_id": str(BOT)}) + + +def test_a_set_outside_the_kinds_fields_has_no_contract( + generated: dict[str, Any], +) -> None: + manifest = generated["out"] / "manifest.yaml" + extra = {"bot_user_id": str(BOT)} + # No identity: the update's target can't be told apart. + with pytest.raises(gen.GenerateError, match="no update_issue contract"): + gen.choose(manifest, "update_issue", {"status": "Resolved"}, extra) + # A field no demonstration placed. + v = {"project": "Help Desk", "subject": "x", "status": "New", "priority": "High"} + with pytest.raises(gen.GenerateError, match="no create_issue contract"): + gen.choose(manifest, "create_issue", v, extra) From 563eabbeb4fc89b4e1baf7a43c3ade3d5639b9f9 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 31 Oct 2026 13:50:20 -0400 Subject: [PATCH 2/4] CAPABILITIES: contracts for unlisted field sets --- CAPABILITIES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CAPABILITIES.md b/CAPABILITIES.md index e9353b06d..86837e213 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -147,7 +147,7 @@ improvising or retrying when something is off. | Judge an "ensure the record exists" write (found, created once with every value, or needs review) through a FHIR server's read path only: searches, AuditEvent and `_history`, never the database; the coverage claim names that read path. An agent's own correction of the record it created counts as one write when the audit trail and history show only that agent, no other reader, and the same record in every version | On main ([#1640](https://github.com/OpenAdaptAI/OpenAdapt/pull/1640), corrections [#1655](https://github.com/OpenAdaptAI/OpenAdapt/pull/1655)) | Real app (Medplum 5.2.8; three blind fault suites: 17/18, 17/20, 19/20; no fault passed in the last two. Corrections, on 15 blind cases and the 37 public ones: false stops 8 to 2 and 9 to 4 for Claude, no harm passed that the old rule caught) | `openadapt_flow/ensure_record.py`, [ENSURE_RECORD.md](packages/flow/docs/ENSURE_RECORD.md) | | Generate an ensure-record contract from three demonstrations, the server's CapabilityStatement and its SearchParameter definitions: field paths and `[system=…]` filters, comparison kinds, value maps, identity, search parameters, the closed record and the change feed. It refuses when unsure, asks a person to confirm an identity for a type with no reviewed template, and self-checks every demonstration plus a perturbation per field | On main ([#1653](https://github.com/OpenAdaptAI/OpenAdapt/pull/1653)) | Real app (Medplum 5.2.8: all 16 hand-written Patient lines regenerated; one blind write type, Immunization: 78 s of person time, 2 lines edited after the person declined the proposed identity; blind suite 23/24, no fault passed) | `openadapt_flow/generate/ensure_contract.py`, [RESULTS.md](packages/flow/benchmark/contract_autogen/results/RESULTS.md) | | A stop card for a stopped write: the exact record difference and who else wrote, with Undo or Fix that reads back, or Open in the app; cost modelled from published operators and timed machine steps, as pre-registered | On main ([#1653](https://github.com/OpenAdaptAI/OpenAdapt/pull/1653)) | Real app (Medplum 5.2.8, blind suite: 16 stops, 14 one-tap with no failed read-back; modelled 42 s per caught harm against 420; one wrong category on a false stop, gone in the re-measure with `void`: 24/24, 15 stops, no wrong category, no failed read-back) | `benchmark/stop_card/`, [PREREG_REVIEW.md](packages/flow/benchmark/stop_card/PREREG_REVIEW.md) | -| Generate `openadapt.verify/v1` contracts from a MySQL, MariaDB or PostgreSQL schema, the kinds of write with their parameter names, and one demonstration write per example: one contract per field set (an update gets one for each set of fields it may change), list parameters by length, a `found` twin for a record already in place, the whole-database frame on every dialect, rows the app adds on its own declared by count, a recheck window the person confirms, and `frame.versions` (a lost update) from a version column or the app's audit trail of saves. Every undecided point is one numbered question; every contract is loaded and dry-read against the live store before it is written; a needed `TRIGGER` grant or view is applied with a flag or the kind is refused. Not yet: OpenAPI metadata | [#1672](https://github.com/OpenAdaptAI/OpenAdapt/pull/1672) | Real app (first study, instance copies; questions for a person: Dolibarr 4, Redmine 2, EspoCRM 3, against 11 hand decisions each; every in-scope planted fault stopped (22/22, 19/19, 21/21); correct writes confirmed 32/43, 32/38, 27/40; evidence in the private study repository) | `python -m openadapt_flow.generate.verify_contract`, [GENERATE_CONTRACTS.md](packages/flow/docs/GENERATE_CONTRACTS.md) | +| Generate `openadapt.verify/v1` contracts from a MySQL, MariaDB or PostgreSQL schema, the kinds of write with their parameter names, and one demonstration write per example: one contract per field set (an update gets one for each set of fields it may change), and `choose` builds any other set of the kind's fields the same way from the same pinned inputs, list parameters by length, a `found` twin for a record already in place, the whole-database frame on every dialect, rows the app adds on its own declared by count, a recheck window the person confirms, and `frame.versions` (a lost update) from a version column or the app's audit trail of saves. Every undecided point is one numbered question; every contract is loaded and dry-read against the live store before it is written; a needed `TRIGGER` grant or view is applied with a flag or the kind is refused. Not yet: OpenAPI metadata | [#1672](https://github.com/OpenAdaptAI/OpenAdapt/pull/1672) | Real app (first study, instance copies; questions for a person: Dolibarr 4, Redmine 2, EspoCRM 3, against 11 hand decisions each; every in-scope planted fault stopped (22/22, 19/19, 21/21); correct writes confirmed 32/43, 32/38, 27/40; evidence in the private study repository) | `python -m openadapt_flow.generate.verify_contract`, [GENERATE_CONTRACTS.md](packages/flow/docs/GENERATE_CONTRACTS.md) | | `void` in an ensure-record contract: values that make a record assert nothing (`status="entered-in-error"`). A record holding one is never the person, is not a duplicate, and a created one needs review; it names values to leave out, never values to keep, and can't name an identity field | On main ([#1657](https://github.com/OpenAdaptAI/OpenAdapt/pull/1657)) | Real app (Medplum 5.2.8: 8 live cases written in advance, 8/8 with the line, 5/8 and one silent pass without it; frozen blind Immunization suite 24/24); adversarial review recorded | `openadapt_flow/ensure_record.py`, [RESULTS.md](packages/flow/benchmark/fhir_medplum/void/results/RESULTS.md), [REVIEW_VOID.md](packages/flow/benchmark/fhir_medplum/void/REVIEW_VOID.md) | | Check a patient registration on a commercial EHR through its public FHIR sandbox: Epic (open.epic, R4), read through FHIR search and read with SMART backend auth; the frame is the search set, since the sandbox has no AuditEvent or `_history` | On main ([#1656](https://github.com/OpenAdaptAI/OpenAdapt/pull/1656)) | Real app (Epic public sandbox, August 2026 release; blind round 12/13 with none passed as verified, the miss a false refusal fixed in the contract; caught Epic dropping the phone on a 201 create) | `benchmark/commercial_ehr/`, [FINDINGS.md](packages/flow/benchmark/commercial_ehr/FINDINGS.md) | | A public benchmark a vendor reruns with their own agent: `openadapt bench medplum --agent ` runs 37 written-for-it cases (Patient, AllergyIntolerance, Appointment) against a local Medplum through a fault proxy, and scores the agent's claim, a read-back and OpenAdapt's verifier against an end-state oracle | On main ([#1652](https://github.com/OpenAdaptAI/OpenAdapt/pull/1652)) | Real app (Medplum 5.2.8; Claude Sonnet 5.5: OpenAdapt missed 1 of 9 harms with 4 false stops, all corrections that changed the record's identity, against 8 missed by the claim and the read-back; cold run from clone to scorecard 5 min) | [README](packages/flow/benchmark/fhir_medplum/public/README.md), [RESULTS.md](packages/flow/benchmark/fhir_medplum/public/RESULTS.md) | From 6985e68df1e7d3d57b09b1790320c598fae6608b Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 31 Oct 2026 14:46:10 -0400 Subject: [PATCH 3/4] A column the app computes from the row (a date part) is not refused as unpinned --- .../openadapt_flow/generate/verify_contract.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/flow/openadapt_flow/generate/verify_contract.py b/packages/flow/openadapt_flow/generate/verify_contract.py index f479d5f79..bc9c979d8 100644 --- a/packages/flow/openadapt_flow/generate/verify_contract.py +++ b/packages/flow/openadapt_flow/generate/verify_contract.py @@ -78,6 +78,17 @@ class Unpinned(ValueError): set gets no contract.""" +def _computed(role: ColumnRole) -> bool: + """A column the app computes from the row (the day of a date the task + gives, a text holding the row's own key): recognized, so not pinned and + not asked about.""" + + why = str(role.detail.get("why") or "") + return why == "holds the row's own key" or ( + why.startswith("the ") and "of a date the task gives" in why + ) + + class GenerateError(RuntimeError): """The generator can't go on; the message says why.""" @@ -2113,7 +2124,11 @@ def _pin( actor_params: Mapping[str, str], unchecked: Mapping[str, set[str]] = {}, ) -> None: - if role.role == "varies" and col not in unchecked.get(table, set()): + if ( + role.role == "varies" + and col not in unchecked.get(table, set()) + and not _computed(role) + ): why = role.detail.get("why") or "it differs between demonstrations" raise Unpinned(f"{table}.{col} is not checked ({why})") if role.role == "free" and col not in unchecked.get(table, set()): From 7b61099a89a337aa11b94f973141e48346905921 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 2 Nov 2026 19:09:26 -0500 Subject: [PATCH 4/4] Field sets at run time: review round 1 - Pin the generator's own code (generator_sha256) and refuse a run-time build under other rules. - Pin every listed contract file as propose wrote it (listed_sha256); a hand edit refuses run-time builds, so none is looser than an edited one. - Digest exactly the bytes the demonstrations' load read, snapshots outside the folder included, in the same pass. - Write each build through a temporary file of its own (threads). The shared column_roles fix (one-row computed columns) lands in #1682; this branch is rebased onto it after it merges. --- packages/flow/docs/GENERATE_CONTRACTS.md | 15 ++- .../flow/openadapt_flow/generate/sql_store.py | 9 +- .../generate/verify_contract.py | 93 ++++++++++++--- .../tests/test_generate_verify_contract.py | 107 ++++++++++++++++++ 4 files changed, 202 insertions(+), 22 deletions(-) diff --git a/packages/flow/docs/GENERATE_CONTRACTS.md b/packages/flow/docs/GENERATE_CONTRACTS.md index a0e7881d0..84926f38f 100644 --- a/packages/flow/docs/GENERATE_CONTRACTS.md +++ b/packages/flow/docs/GENERATE_CONTRACTS.md @@ -180,12 +180,19 @@ combined), `choose` builds the contracts the way `propose` builds the listed ones: from the same demonstrations, answers and schema, checked by the SQL guard and a dry read, and written under `field_sets/`. A field the set leaves out must stay as it was (an update) or be empty (a create), as in every -contract. `choose` refuses, and the write stops, when the demonstrations or -the schema changed since `propose` (`field_sets` in the manifest pins their -digests), when a question is still open, when the set has a field no +contract. `field_sets` in the manifest pins, by digest, every byte the +demonstrations' load read (snapshots outside the folder too, digested in the +same read), the schema, the generator's own code, and every listed contract +file as `propose` wrote it. `choose` refuses, and the write stops, when any +of these changed: so a listed contract a person edited by hand (a tighter +check) stops every run-time build, since a built contract wouldn't have the +edit; run `propose` again and reapply the edit to build more sets. It also +refuses when a question is still open, when the set has a field no demonstration placed, or when a column of the new row would be left unchecked without the person's answer. A kind whose value names the field to -change (`field: status`) gets only its listed sets. +change (`field: status`) gets only its listed sets. Each build writes +through a temporary file of its own, so threads building one set at once +don't collide. `QUESTIONS.md` lists, per kind, the parameters no demonstration gave and no column is named like (`not_placed` in `report.json`): record a demonstration diff --git a/packages/flow/openadapt_flow/generate/sql_store.py b/packages/flow/openadapt_flow/generate/sql_store.py index db55cab8b..0a16cde94 100644 --- a/packages/flow/openadapt_flow/generate/sql_store.py +++ b/packages/flow/openadapt_flow/generate/sql_store.py @@ -443,7 +443,14 @@ def save_snapshot(path: Path, rows: Mapping[str, TableRows]) -> None: def load_snapshot(path: Path) -> dict[str, TableRows]: - data = json.loads(path.read_text(encoding="utf-8")) + return snapshot_from_bytes(path.read_bytes()) + + +def snapshot_from_bytes(raw: bytes) -> dict[str, TableRows]: + """A snapshot from the bytes of its file (so a caller can digest exactly + what it parsed).""" + + data = json.loads(raw.decode("utf-8")) return {n: TableRows(dict(t["rows"]), t.get("count_only")) for n, t in data.items()} diff --git a/packages/flow/openadapt_flow/generate/verify_contract.py b/packages/flow/openadapt_flow/generate/verify_contract.py index bc9c979d8..3aa845b20 100644 --- a/packages/flow/openadapt_flow/generate/verify_contract.py +++ b/packages/flow/openadapt_flow/generate/verify_contract.py @@ -156,11 +156,23 @@ def load_kinds(path: Path) -> list[KindSpec]: return out -def load_demos(demo_dir: Path) -> list[Demo]: +def load_demos(demo_dir: Path, digest: Any = None) -> list[Demo]: + """The demonstrations in ``demo_dir``. ``digest`` (a ``hashlib`` object): + fed every byte read, demonstration and snapshot alike (a snapshot path + outside the folder too), in one pass, so it covers exactly what was + parsed.""" + demos = [] for f in sorted(Path(demo_dir).glob("*.demo.json")): - d = json.loads(f.read_text(encoding="utf-8")) - after = store.load_snapshot(Path(demo_dir) / d["after_snapshot"]) + raw = f.read_bytes() + d = json.loads(raw.decode("utf-8")) + snap = Path(demo_dir) / d["after_snapshot"] + snap_raw = snap.read_bytes() + if digest is not None: + for part in (f.name, raw, str(d["after_snapshot"]), snap_raw): + b = part.encode("utf-8") if isinstance(part, str) else part + digest.update(len(b).to_bytes(8, "big") + b) + after = store.snapshot_from_bytes(snap_raw) demos.append( Demo( kind=d["kind"], @@ -2793,6 +2805,7 @@ def propose( grant_trigger_flag: bool = False, only: Optional[tuple[str, list[str], dict[str, int]]] = None, schema_sha256: str = "", + demos_sha256: str = "", ) -> dict[str, Any]: """Analyze, ask, emit and validate. Returns the report (also written). @@ -2833,7 +2846,14 @@ def propose( "--grant-trigger --admin-read --reader-account , " "or have the DBA run: GRANT TRIGGER ON .* TO " ) - demos = load_demos(demo_dir) + read_bytes = hashlib.sha256() + demos = load_demos(demo_dir, read_bytes) + demos_digest = "sha256:" + read_bytes.hexdigest() + if only is not None and demos_digest != demos_sha256: + raise GenerateError( + "the demonstrations changed since the contracts were proposed; " + "run propose again" + ) env = environment_id or f"generated-{schema.database}" env = re.sub(r"[^A-Za-z0-9._:-]", "-", env) if len(env) < 8: @@ -3255,7 +3275,10 @@ def table_cols(key: str) -> dict[str, set[str]]: for k in kinds ], "demos": str(Path(demo_dir).resolve()), - "demos_sha256": _demos_digest(Path(demo_dir)), + "demos_sha256": demos_digest, + # The code that builds a contract: a run-time build under other + # rules than the listed contracts is refused. + "generator_sha256": generator_digest(), "answers": json.loads(json.dumps(answers or {}, default=str)), "actor_params": dict(actor_params), "environment_id": environment_id, @@ -3409,6 +3432,16 @@ def table_cols(key: str) -> dict[str, set[str]]: (out / f).unlink(missing_ok=True) manifest["kinds"][p.kind] = entry report["kinds"][p.kind] = rep + # Every listed contract as written. A person may edit them (a hand fix); + # a run-time build never sees the edit, so any change to a listed file + # refuses run-time builds rather than let one be looser. + manifest["field_sets"]["listed_sha256"] = { + f: _file_digest(out / f) + for e in manifest["kinds"].values() + for var in e.get("variants") or [] + for k, f in var.items() + if k in ("create", "update", "found") and isinstance(f, str) + } (out / "manifest.yaml").write_text( yaml.safe_dump(manifest, sort_keys=False, allow_unicode=True), encoding="utf-8" ) @@ -3427,14 +3460,20 @@ def _digest(data: Any) -> str: return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() -def _demos_digest(demo_dir: Path) -> str: +def generator_digest() -> str: + """A digest of the generator's own code (this package's modules).""" + h = hashlib.sha256() - for f in sorted(Path(demo_dir).iterdir()) if Path(demo_dir).is_dir() else []: - if f.is_file(): - h.update(f.name.encode() + b"\0" + f.read_bytes() + b"\0") + for f in sorted(Path(__file__).parent.glob("*.py")): + raw = f.read_bytes() + h.update(f.name.encode() + b"\0" + len(raw).to_bytes(8, "big") + raw) return "sha256:" + h.hexdigest() +def _file_digest(path: Path) -> str: + return "sha256:" + hashlib.sha256(Path(path).read_bytes()).hexdigest() + + def _settable(p: KindPlan) -> dict[str, Any]: """The fields a task of this kind may give, for ``choose``: an update must give its identity and at least one change; a create its required @@ -3510,13 +3549,17 @@ def _emit_one( rel = Path("field_sets") / kind / f"{v.slug}.{flavor}.yaml" path = out / rel path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_suffix(f".{os.getpid()}.tmp") - tmp.write_text( - f"# Generated by openadapt_flow.generate.verify_contract on demand: kind " - f"{kind}, {flavor}, fields {', '.join(v.fields)}.\n" - + yaml.safe_dump(doc, sort_keys=False, allow_unicode=True, width=100), - encoding="utf-8", + # A name of its own per call (threads share a process id). + fd, tmp_name = tempfile.mkstemp( + dir=path.parent, prefix=f".{path.name}.", suffix=".tmp" ) + tmp = Path(tmp_name) + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write( + f"# Generated by openadapt_flow.generate.verify_contract on demand: " + f"kind {kind}, {flavor}, fields {', '.join(v.fields)}.\n" + + yaml.safe_dump(doc, sort_keys=False, allow_unicode=True, width=100) + ) if why is None and validate: why = dry_read(tmp, flatten(sample_values(plan, v, demos)), True) if why is not None: @@ -3653,11 +3696,26 @@ def _field_set( if not inputs or not _fits(spec, given, lists): return None demo_dir = Path(inputs["demos"]) - if _demos_digest(demo_dir) != inputs["demos_sha256"]: + if inputs.get("generator_sha256") != generator_digest(): raise GenerateError( - "the demonstrations changed since the contracts were proposed; " + "the contract generator changed since the contracts were proposed; " "run propose again" ) + listed = inputs.get("listed_sha256") + if not isinstance(listed, dict) or not listed: + raise GenerateError("the manifest doesn't pin its listed contracts") + base = manifest_path.parent + for rel, want in sorted(listed.items()): + try: + have = _file_digest(base / rel) + except OSError: + have = "" + if have != want: + raise GenerateError( + f"the listed contract {rel} changed since it was proposed (a hand " + "edit?): a contract built at run time wouldn't have the edit, so " + "none is built; run propose again" + ) kinds = [ KindSpec( str(k["kind"]), list(k["parameters"]), "", list(k.get("examples") or []) @@ -3675,6 +3733,7 @@ def _field_set( validate=bool(inputs.get("validate", True)), only=(kind, sorted(given), dict(lists)), schema_sha256=str(inputs["schema_sha256"]), + demos_sha256=str(inputs["demos_sha256"]), ) return dict(result["files"]) diff --git a/packages/flow/tests/test_generate_verify_contract.py b/packages/flow/tests/test_generate_verify_contract.py index 71a3cc01c..a4899f6e4 100644 --- a/packages/flow/tests/test_generate_verify_contract.py +++ b/packages/flow/tests/test_generate_verify_contract.py @@ -533,3 +533,110 @@ def test_a_set_outside_the_kinds_fields_has_no_contract( v = {"project": "Help Desk", "subject": "x", "status": "New", "priority": "High"} with pytest.raises(gen.GenerateError, match="no create_issue contract"): gen.choose(manifest, "create_issue", v, extra) + + +# -- review round 1 (#1683) ------------------------------------------------------ + +UNLISTED = ["project", "status", "subject"] +UNLISTED_V = {"project": "Help Desk", "subject": "Printer jams", "status": "Resolved"} + + +def _edit_manifest(out: Path, change: Callable[[dict[str, Any]], None]) -> None: + manifest = yaml.safe_load((out / "manifest.yaml").read_text()) + change(manifest) + (out / "manifest.yaml").write_text(yaml.safe_dump(manifest, allow_unicode=True)) + + +def test_a_hand_edited_listed_contract_refuses_run_time_builds( + generated: dict[str, Any], tmp_path: Path +) -> None: + """A person tightened a listed contract. A set built at run time would + not have the edit, so none is built.""" + + out, listed = _unlisted(generated, tmp_path, "update_issue", UNLISTED) + other = ( + out + / sorted( + f + for f in yaml.safe_load((out / "manifest.yaml").read_text())["field_sets"][ + "listed_sha256" + ] + )[0] + ) + other.write_text(other.read_text() + "\n# hand edit\n") + with pytest.raises(gen.GenerateError, match="changed since it was proposed"): + gen.choose( + out / "manifest.yaml", "update_issue", UNLISTED_V, {"bot_user_id": str(BOT)} + ) + + +def test_another_generator_version_refuses_run_time_builds( + generated: dict[str, Any], tmp_path: Path +) -> None: + out, _ = _unlisted(generated, tmp_path, "update_issue", UNLISTED) + _edit_manifest( + out, lambda m: m["field_sets"].update(generator_sha256="sha256:" + "1" * 64) + ) + with pytest.raises(gen.GenerateError, match="generator changed"): + gen.choose( + out / "manifest.yaml", "update_issue", UNLISTED_V, {"bot_user_id": str(BOT)} + ) + + +def test_a_changed_snapshot_outside_the_folder_refuses( + generated: dict[str, Any], tmp_path: Path +) -> None: + """The digest covers every byte the demonstrations' load read, a + snapshot named by a path outside the folder too.""" + + out, _ = _unlisted(generated, tmp_path, "update_issue", UNLISTED) + demos = tmp_path / "demos" + shutil.copytree( + Path( + yaml.safe_load((out / "manifest.yaml").read_text())["field_sets"]["demos"] + ), + demos, + ) + outside = tmp_path / "elsewhere" + outside.mkdir() + first = sorted(demos.glob("*.demo.json"))[0] + doc = json.loads(first.read_text()) + snap = demos / doc["after_snapshot"] + moved = outside / snap.name + shutil.move(str(snap), moved) + doc["after_snapshot"] = str(Path("..") / "elsewhere" / snap.name) + first.write_text(json.dumps(doc)) + digest = __import__("hashlib").sha256() + gen.load_demos(demos, digest) + _edit_manifest( + out, + lambda m: m["field_sets"].update( + demos=str(demos), demos_sha256="sha256:" + digest.hexdigest() + ), + ) + gen.choose( + out / "manifest.yaml", "update_issue", UNLISTED_V, {"bot_user_id": str(BOT)} + ) + moved.write_text(moved.read_text().replace('"Printer jams"', '"Printer jamz"')) + with pytest.raises(gen.GenerateError, match="demonstrations changed"): + gen.choose( + out / "manifest.yaml", "update_issue", UNLISTED_V, {"bot_user_id": str(BOT)} + ) + + +def test_two_threads_build_one_set_at_once( + generated: dict[str, Any], tmp_path: Path +) -> None: + import concurrent.futures + + out, _ = _unlisted(generated, tmp_path, "update_issue", UNLISTED) + + def build(_: int) -> Any: + return gen.choose( + out / "manifest.yaml", "update_issue", UNLISTED_V, {"bot_user_id": str(BOT)} + ) + + with concurrent.futures.ThreadPoolExecutor(4) as pool: + results = list(pool.map(build, range(4))) + assert all(r[0] is not None for r in results) + assert not list((out / "field_sets").rglob("*.tmp"))