From 3ba4fba4d521446f708b6e2ae46ef4617ef7fc10 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:01:31 +0000 Subject: [PATCH 01/56] refactor(trust): drop the signed trust-root ceremony; the gateway serves the profile Remove trust-sign/trust-verify/keygen, detached signatures, owner key and minimum version pins. The master reads its own TOML; the gateway serves it at GET /v1/trust and validators build their local TrustRoot from it over HTTPS. Bundle bytes and all golden vectors are unchanged. Co-Authored-By: Claude Sonnet 5.5 --- config/challenges.toml.sig | 1 - config/measurements.toml.sig | 1 - config/owner.pubkey | 1 - docs/how-to/trust-root.md | 159 --------------- scripts/run-validator.sh | 59 +++--- src/cortex/cli.py | 72 ------- src/cortex/config.py | 26 +-- src/cortex/gateway/api.py | 7 + src/cortex/gateway/service.py | 8 + src/cortex/gateway/store.py | 26 +-- src/cortex/master.py | 2 +- src/cortex/operator.py | 47 ----- src/cortex/protocol/crypto.py | 1 - src/cortex/protocol/trust.py | 103 ++++------ src/cortex/validator/__main__.py | 79 ++++---- src/cortex/validator/chain.py | 7 +- src/cortex/validator/service.py | 19 +- tests/protocol/test_trust.py | 77 ------- tests/protocol/vectors/trust/challenges.toml | 27 --- .../vectors/trust/challenges.toml.sig | 1 - .../protocol/vectors/trust/measurements.toml | 9 - .../vectors/trust/measurements.toml.sig | 1 - tests/protocol/vectors/trust/owner.pubkey | 1 - tests/test_master.py | 1 - tests/test_operator.py | 191 ------------------ tests/test_run_validator_script.py | 16 +- tests/validator/test_main.py | 55 ++++- 27 files changed, 215 insertions(+), 782 deletions(-) delete mode 100644 config/challenges.toml.sig delete mode 100644 config/measurements.toml.sig delete mode 100644 config/owner.pubkey delete mode 100644 docs/how-to/trust-root.md delete mode 100644 src/cortex/operator.py delete mode 100644 tests/protocol/test_trust.py delete mode 100644 tests/protocol/vectors/trust/challenges.toml delete mode 100644 tests/protocol/vectors/trust/challenges.toml.sig delete mode 100644 tests/protocol/vectors/trust/measurements.toml delete mode 100644 tests/protocol/vectors/trust/measurements.toml.sig delete mode 100644 tests/protocol/vectors/trust/owner.pubkey delete mode 100644 tests/test_operator.py diff --git a/config/challenges.toml.sig b/config/challenges.toml.sig deleted file mode 100644 index a5031065c..000000000 --- a/config/challenges.toml.sig +++ /dev/null @@ -1 +0,0 @@ -c82e38817950e6ad7abcd9efc4af1b198f49a8d8cb11a1b1c6b88ac7f04e4c360bf98432ed352b421a23499d0ed1c3b5bf7a020254d7a20388c249821581b28e diff --git a/config/measurements.toml.sig b/config/measurements.toml.sig deleted file mode 100644 index 11bf83fb2..000000000 --- a/config/measurements.toml.sig +++ /dev/null @@ -1 +0,0 @@ -ea3f1ee532041bdb52ce5a4a2f954283ac8becc5c0d4a5e0f244cee3ccb29618faf219b44c463eb34dae919366a9d58f1b9bd011b9cf96eaeb71045474ab888f diff --git a/config/owner.pubkey b/config/owner.pubkey deleted file mode 100644 index acfde072f..000000000 --- a/config/owner.pubkey +++ /dev/null @@ -1 +0,0 @@ -40ef68c8608ba18eded3b4498230d88314c2e4e298cadd7cb044e5ca8412fa10 diff --git a/docs/how-to/trust-root.md b/docs/how-to/trust-root.md deleted file mode 100644 index 2ac677ea3..000000000 --- a/docs/how-to/trust-root.md +++ /dev/null @@ -1,159 +0,0 @@ -# Sign and verify the trust root - -The owner signs two immutable TOML documents: the challenge allocation and the -measurement allowlist. Cortex verifies both before serving or submitting a -bundle. The committed keys are development fixtures; production ceremonies run -offline with private files that never enter Git. - -## Prepare an offline directory - -```bash -install -d -m 0700 /private/cortex-ceremony -cortex keygen \ - --seed-out /private/cortex-ceremony/owner.seed \ - --public-out /private/cortex-ceremony/owner.pubkey -cortex keygen \ - --seed-out /private/cortex-ceremony/gateway.seed \ - --public-out /private/cortex-ceremony/gateway.pubkey -cortex keygen \ - --seed-out /private/cortex-ceremony/bounty.seed \ - --public-out /private/cortex-ceremony/bounty.pubkey -cortex keygen \ - --seed-out /private/cortex-ceremony/opentype.seed \ - --public-out /private/cortex-ceremony/opentype.pubkey -cortex keygen \ - --seed-out /private/cortex-ceremony/proof.seed \ - --public-out /private/cortex-ceremony/proof.pubkey -``` - -`keygen` creates files exclusively and refuses to overwrite an existing path. -Seeds are raw 32-byte sr25519 seeds with mode 0600. Copy only the public owner -key into `config/owner.pubkey`. Put each challenge public key in its row of the -selected challenges document. The master reads the seed of container challenge -`` from `$BASE_CHALLENGE_KEYS_DIR/.key` and the Proof seed from -`PROOF_SK_FILE`. The preserved development -`config/challenges.toml` is the signed legacy 2000/8000 profile (algorithm 1). -The unsigned `config/challenges-v2.example.toml` is the 3000/7000 activation -template (algorithm 2, challenge-document version >=2). Replace its -`"CHOOSE_ACTIVATION_EPOCH"` placeholder with the coordinated integer epoch; -the unchanged template cannot be signed. The unsigned -`config/challenges-v3.example.toml` is the algorithm 3 template (version >=3): -any 1..64 unique ids summing to 10,000 bps. The gateway public key -is supplied to verification and is never a challenge row. - -## Sign both documents - -Increment `version` before replacing an accepted document and set -`introduced_epoch` deliberately. Then write detached signatures beside the -TOML files: - -```bash -cortex trust-sign \ - --kind challenges \ - --input config/challenges.toml \ - --seed-file /private/cortex-ceremony/owner.seed \ - --signature-out config/challenges.toml.sig - -cortex trust-sign \ - --kind measurements \ - --input config/measurements.toml \ - --seed-file /private/cortex-ceremony/owner.seed \ - --signature-out config/measurements.toml.sig -``` - -The command refuses to overwrite signatures. Move the previous signatures to -operator-controlled backup storage before a planned rotation. Do not work -around that guard with symlinks. - -## Verify the complete set - -```bash -cortex trust-verify \ - --challenges config/challenges.toml \ - --measurements config/measurements.toml \ - --owner-public config/owner.pubkey \ - --gateway-public "$(tr -d '\n' /` and its - leaf seed at `.key` in the master secrets directory. A registered id that - is not yet trusted runs without emission, which is the burn-in period: check - `GET /challenge//version` and its logs before you sign. -3. Drain and pause as for algorithm 2, then sign a `version >= 3` document from - `config/challenges-v3.example.toml` with the agreed `introduced_epoch`. -4. Install the signed documents and minimum version pins on the master and - validators, resume, and check a new `sealed: true` latest response with - algorithm 3 leaves and an independent validator recomputation. - -A trusted id that is missing from the registry, unhealthy or returning invalid -weights emits `NoScore(ChallengeInternal)`, so its share burns and never moves -to another challenge. To retire a challenge, sign a higher version without its -row first, then remove its registry entry. diff --git a/scripts/run-validator.sh b/scripts/run-validator.sh index 4d471be9f..19b06e6b9 100755 --- a/scripts/run-validator.sh +++ b/scripts/run-validator.sh @@ -1,16 +1,17 @@ #!/usr/bin/env bash # Run a Cortex validator: verify the sealed weights served by the Cortex -# gateway and submit them on Bittensor. This script never starts the master, +# gateway and, only if a validator hotkey is given, submit them on Bittensor. This script never starts the master, # the challenge supervisor or a VM host; those are for the master operator only. # # Usage (from anywhere; paths resolve to this repository): +# scripts/run-validator.sh --verify-only # one verification, no hotkey needed # export WALLET_NAME=validator WALLET_HOTKEY=default -# export GATEWAY_PUBLIC= -# scripts/run-validator.sh --verify-only # one verification, never submits # scripts/run-validator.sh # verify and submit every epoch # -# Required env: WALLET_NAME, WALLET_HOTKEY, GATEWAY_PUBLIC. -# Optional env: GATEWAY (https://chain.joinbase.ai), NETUID (100), +# The gateway (HTTPS) is the trust anchor; no signed trust files are read. +# Required env to submit weights (not for --verify-only): WALLET_NAME, WALLET_HOTKEY. +# Optional env: GATEWAY_PUBLIC (extra pin of the gateway sealing hotkey), +# GATEWAY (https://chain.joinbase.ai), NETUID (100), # NETWORK (finney), STATE_DB ($HOME/.cortex/validator.sqlite3), # WALLET_PATH ($HOME/.bittensor/wallets), VERSION_KEY (default: live # subnet weights_version read from the chain). @@ -33,7 +34,7 @@ if [[ "${CORTEX_MASTER:-}" == 1 || -n "${BASE_MASTER_SECRETS_HOST_DIR+x}" || -n exit 2 fi -usage() { sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//'; } +usage() { sed -n '2,19p' "$0" | sed 's/^# \{0,1\}//'; } verify_only=0 once=0 dry_run=0 for arg in "$@"; do @@ -53,14 +54,14 @@ NETWORK="${NETWORK:-finney}" STATE_DB="${STATE_DB:-$HOME/.cortex/validator.sqlite3}" WALLET_PATH="${WALLET_PATH:-$HOME/.bittensor/wallets}" WALLET_PATH="${WALLET_PATH/#\~/$HOME}" # cortex validator expands a leading ~ too -: "${WALLET_NAME:?set WALLET_NAME to your validator wallet name}" -: "${WALLET_HOTKEY:?set WALLET_HOTKEY to your registered validator hotkey}" -[[ -n "${GATEWAY_PUBLIC:-}" ]] || die "GATEWAY_PUBLIC is not set: export the gateway hotkey you pinned from an independent, trusted source (never copy it from the gateway response)" +if ((!verify_only)); then + : "${WALLET_NAME:?set WALLET_NAME to your validator wallet name (only needed to submit weights; use --verify-only otherwise)}" + : "${WALLET_HOTKEY:?set WALLET_HOTKEY to your registered validator hotkey (only needed to submit weights; use --verify-only otherwise)}" +fi [[ "$NETUID" =~ ^[0-9]+$ ]] || die "NETUID must be an integer" ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" -CONFIG="$ROOT/config" UV=(uv run --locked --extra chain) command -v uv >/dev/null || die "uv not found: install it from https://docs.astral.sh/uv/" @@ -71,33 +72,34 @@ status="$(curl -sS -m 20 -o "$body" -w '%{http_code}' "$GATEWAY/v1/weights/lates || die "cannot reach $GATEWAY/v1/weights/latest" [[ "$status" == 200 ]] || die "$GATEWAY/v1/weights/latest answered HTTP $status, expected 200" -read -r challenges_version measurements_version epoch sealed < <( - "${UV[@]}" python - "$body" "$NETUID" "$CONFIG" <<'PY' || echo FAIL -import json, sys, tomllib +read -r epoch sealed < <( + "${UV[@]}" python - "$body" "$NETUID" <<'PY' || echo FAIL +import json, sys from pathlib import Path if sys.version_info < (3, 12): sys.exit(f"python >= 3.12 required, found {sys.version.split()[0]}") -body, netuid, config = sys.argv[1], int(sys.argv[2]), Path(sys.argv[3]) +body, netuid = sys.argv[1], int(sys.argv[2]) try: latest = json.loads(Path(body).read_text()) except ValueError: sys.exit("gateway answered non-JSON for /v1/weights/latest") if not isinstance(latest, dict) or latest.get("netuid") != netuid: sys.exit(f"gateway netuid {latest.get('netuid') if isinstance(latest, dict) else None} != NETUID {netuid}") -versions = [tomllib.loads((config / n).read_text())["version"] for n in ("challenges.toml", "measurements.toml")] -print(*versions, latest.get("epoch"), str(latest.get("sealed")).lower()) +print(latest.get("epoch"), str(latest.get("sealed")).lower()) PY ) -[[ "$challenges_version" != FAIL && -n "$sealed" ]] || die "preflight failed (see message above)" +[[ "$epoch" != FAIL && -n "$sealed" ]] || die "preflight failed (see message above)" echo "run-validator: gateway $GATEWAY netuid=$NETUID epoch=$epoch sealed=$sealed" >&2 -hotkey_file="$WALLET_PATH/$WALLET_NAME/hotkeys/$WALLET_HOTKEY" -if [[ ! -f "$hotkey_file" ]]; then - ((dry_run)) || die "wallet hotkey file not found: $hotkey_file" - echo "run-validator: warning: wallet hotkey file not found (ignored in --dry-run)" >&2 +if [[ -n "${WALLET_NAME:-}" && -n "${WALLET_HOTKEY:-}" ]]; then + hotkey_file="$WALLET_PATH/$WALLET_NAME/hotkeys/$WALLET_HOTKEY" + if [[ ! -f "$hotkey_file" ]]; then + ((dry_run)) || die "wallet hotkey file not found: $hotkey_file" + echo "run-validator: warning: wallet hotkey file not found (ignored in --dry-run)" >&2 + fi fi -if [[ -z "${VERSION_KEY:-}" ]]; then +if [[ -n "${WALLET_NAME:-}" && -z "${VERSION_KEY:-}" ]]; then VERSION_KEY="$("${UV[@]}" python -c ' import sys from bittensor import Subtensor @@ -111,15 +113,12 @@ finally: fi cmd=(cortex validator - --gateway "$GATEWAY" --netuid "$NETUID" --gateway-public "$GATEWAY_PUBLIC" - --network "$NETWORK" - --owner-public "$CONFIG/owner.pubkey" - --challenges "$CONFIG/challenges.toml" --measurements "$CONFIG/measurements.toml" - --minimum-challenges-version "$challenges_version" - --minimum-measurements-version "$measurements_version" - --version-key "$VERSION_KEY" - --wallet-name "$WALLET_NAME" --wallet-hotkey "$WALLET_HOTKEY" --wallet-path "$WALLET_PATH" + --gateway "$GATEWAY" --netuid "$NETUID" --network "$NETWORK" + --version-key "${VERSION_KEY:-0}" --state-db "$STATE_DB") +[[ -z "${GATEWAY_PUBLIC:-}" ]] || cmd+=(--gateway-public "$GATEWAY_PUBLIC") +[[ -z "${WALLET_NAME:-}" || -z "${WALLET_HOTKEY:-}" ]] || + cmd+=(--wallet-name "$WALLET_NAME" --wallet-hotkey "$WALLET_HOTKEY" --wallet-path "$WALLET_PATH") if ((verify_only)); then cmd+=(--verify-only --once); elif ((once)); then cmd+=(--once); fi if ((dry_run)); then diff --git a/src/cortex/cli.py b/src/cortex/cli.py index dc139e049..b15441477 100644 --- a/src/cortex/cli.py +++ b/src/cortex/cli.py @@ -40,26 +40,6 @@ def parser() -> argparse.ArgumentParser: reconcile.add_argument("--attempt-id", required=True) reconcile.add_argument("--result", choices=("submitted", "not_broadcast"), required=True) reconcile.add_argument("--evidence-digest", required=True) - keygen = commands.add_parser("keygen", help="generate an offline sr25519 operator key") - keygen.add_argument("--seed-out", type=Path, required=True) - keygen.add_argument("--public-out", type=Path, required=True) - trust_sign = commands.add_parser( - "trust-sign", help="sign a challenge or measurement trust document" - ) - trust_sign.add_argument("--kind", choices=("challenges", "measurements"), required=True) - trust_sign.add_argument("--input", type=Path, required=True) - trust_sign.add_argument("--seed-file", type=Path, required=True) - trust_sign.add_argument("--signature-out", type=Path, required=True) - trust_verify = commands.add_parser( - "trust-verify", help="verify signed trust documents against local pins" - ) - trust_verify.add_argument("--challenges", type=Path, required=True) - trust_verify.add_argument("--measurements", type=Path, required=True) - trust_verify.add_argument("--owner-public", type=Path, required=True) - trust_verify.add_argument("--gateway-public", required=True) - trust_verify.add_argument("--epoch", type=int, required=True) - trust_verify.add_argument("--minimum-challenges-version", type=int, default=1) - trust_verify.add_argument("--minimum-measurements-version", type=int, default=1) vm = commands.add_parser("vm-host", help="serve the dedicated Firecracker host") vm.add_argument("--config", type=Path, required=True) vm.add_argument( @@ -232,52 +212,6 @@ async def run_master(args: argparse.Namespace) -> None: await asyncio.to_thread(close_subtensor, subtensor) -def run_keygen(args: argparse.Namespace) -> None: - from cortex.operator import generate_key - - public = generate_key(args.seed_out, args.public_out) - print(json.dumps({"public_key": public, "seed_file": str(args.seed_out)})) - - -def run_trust_sign(args: argparse.Namespace) -> None: - from cortex.operator import sign_trust_document - - sign_trust_document( - input_path=args.input, - kind=args.kind, - seed_path=args.seed_file, - signature_path=args.signature_out, - ) - print(json.dumps({"kind": args.kind, "signature_file": str(args.signature_out)})) - - -def run_trust_verify(args: argparse.Namespace) -> None: - from cortex.protocol.trust import load_trust_root - - trust = load_trust_root( - challenges_path=args.challenges, - challenges_signature=Path(str(args.challenges) + ".sig"), - measurements_path=args.measurements, - measurements_signature=Path(str(args.measurements) + ".sig"), - owner_public=decode_hotkey(args.owner_public.read_text().strip()), - gateway_public=decode_hotkey(args.gateway_public), - epoch=args.epoch, - minimum_challenges_version=args.minimum_challenges_version, - minimum_measurements_version=args.minimum_measurements_version, - ) - print( - json.dumps( - { - "challenges": {name.decode(): share for name, share in trust.shares}, - "challenges_version": trust.challenges_version, - "measurements_version": trust.measurements_version, - "measurements_digest": trust.measurements_digest.hex(), - }, - sort_keys=True, - ) - ) - - def run_validator_reconcile(args: argparse.Namespace) -> None: from cortex.validator import SubmissionJournal @@ -323,12 +257,6 @@ def main(argv: list[str] | None = None) -> None: from cortex.vm.server import serve serve(args.config) - elif args.command == "keygen": - run_keygen(args) - elif args.command == "trust-sign": - run_trust_sign(args) - elif args.command == "trust-verify": - run_trust_verify(args) elif args.command == "validator-reconcile": run_validator_reconcile(args) else: diff --git a/src/cortex/config.py b/src/cortex/config.py index ba475c4d7..e96759413 100644 --- a/src/cortex/config.py +++ b/src/cortex/config.py @@ -13,8 +13,8 @@ from cortex.errors import ServiceError from cortex.protocol import TrustRoot -from cortex.protocol.crypto import decode_hotkey, public_key -from cortex.protocol.trust import load_trust_root +from cortex.protocol.crypto import public_key +from cortex.protocol.trust import load_trust from cortex.vm.models import Resources CHAIN_ALIASES = frozenset({"archive", "finney", "local", "test"}) @@ -100,7 +100,6 @@ def _chain_fallback_endpoints(value: object) -> tuple[str, ...]: class MasterConfig: netuid: int state_dir: Path - owner_public_file: Path challenges_file: Path measurements_file: Path gateway_seed_file: Path @@ -114,8 +113,6 @@ class MasterConfig: emit_poll_seconds: float = 120 epoch_refresh_seconds: float = 12 epoch_stale_seconds: float = 60 - minimum_challenges_version: int = 1 - minimum_measurements_version: int = 1 proof_orchestrator_url: str | None = None proof_orchestrator_token_file: Path | None = None proof_orchestrator_ca_file: Path | None = None @@ -137,8 +134,6 @@ def __post_init__(self): or self.emit_poll_seconds <= 0 ): raise ValueError("invalid epoch refresh/emission intervals") - if self.minimum_challenges_version < 1 or self.minimum_measurements_version < 1: - raise ValueError("trust versions must be positive") validate_chain_endpoint(self.chain_endpoint) _chain_fallback_endpoints(self.chain_fallback_endpoints) if self.proof_orchestrator_url: @@ -184,7 +179,6 @@ def optional_path(name: str) -> Path | None: return cls( netuid=int(value("BASE_NETUID")), state_dir=Path(value("BASE_STATE_DIR", "/var/lib/cortex")), - owner_public_file=Path(value("BASE_OWNER_PUBKEY_FILE", "config/owner.pubkey")), challenges_file=Path(value("BASE_CHALLENGES_FILE", "config/challenges.toml")), measurements_file=Path(value("BASE_MEASUREMENTS_FILE", "config/measurements.toml")), gateway_seed_file=required_path("BASE_GATEWAY_SK_FILE"), @@ -202,8 +196,6 @@ def optional_path(name: str) -> Path | None: ), epoch_refresh_seconds=float(value("BASE_EPOCH_REFRESH_SECS", "12")), epoch_stale_seconds=float(value("BASE_EPOCH_STALE_SECS", "60")), - minimum_challenges_version=int(value("BASE_CHALLENGES_MIN_VERSION", "1")), - minimum_measurements_version=int(value("BASE_MEASUREMENTS_MIN_VERSION", "1")), proof_orchestrator_url=value("PROOF_VM_ORCHESTRATOR_URL") or None, proof_orchestrator_token_file=optional_path("PROOF_VM_ORCHESTRATOR_TOKEN_FILE"), proof_orchestrator_ca_file=optional_path("PROOF_VM_ORCHESTRATOR_CA_FILE"), @@ -222,20 +214,14 @@ def challenge_seed_file(self, challenge: bytes) -> Path: def trust_root(self, epoch: int) -> TrustRoot: try: - trust = load_trust_root( + trust = load_trust( challenges_path=self.challenges_file, - challenges_signature=Path(str(self.challenges_file) + ".sig"), measurements_path=self.measurements_file, - measurements_signature=Path(str(self.measurements_file) + ".sig"), - owner_public=decode_hotkey(self.owner_public_file.read_text().strip()), gateway_public=public_key(read_seed(self.gateway_seed_file)), - epoch=epoch, - minimum_challenges_version=self.minimum_challenges_version, - minimum_measurements_version=self.minimum_measurements_version, ) for entry in trust.challenges: if public_key(read_seed(self.challenge_seed_file(entry.id))) != entry.public_key: - raise ServiceError(503, "challenge signing key does not match owner trust") + raise ServiceError(503, "challenge signing key does not match the profile") return trust - except (OSError, ValueError): - raise ServiceError(503, "signed trust root unavailable") from None + except (OSError, ValueError, KeyError): + raise ServiceError(503, "trust profile unavailable") from None diff --git a/src/cortex/gateway/api.py b/src/cortex/gateway/api.py index 4b02027ba..4eb379918 100644 --- a/src/cortex/gateway/api.py +++ b/src/cortex/gateway/api.py @@ -50,6 +50,13 @@ async def seal(request: Request): except ServiceError as error: return _error(error) + @router.get("/v1/trust") + async def trust(): + try: + return service.trust_profile() + except ServiceError as error: + return _error(error) + @router.get("/v1/weights/latest") async def latest(): return service.latest() diff --git a/src/cortex/gateway/service.py b/src/cortex/gateway/service.py index a8161886e..e3f259950 100644 --- a/src/cortex/gateway/service.py +++ b/src/cortex/gateway/service.py @@ -10,6 +10,7 @@ from cortex.protocol.crypto import BUNDLE_DOMAIN, RAW_WEIGHT_DOMAIN, encode_hotkey, verify_raw from cortex.protocol.merkle import merkle_root from cortex.protocol.scale import uint +from cortex.protocol.trust import trust_to_json from cortex.validator import ChainSnapshot from .projection import project, timestamp @@ -54,6 +55,13 @@ def _chain_endpoint(self) -> str: return "" return value if isinstance(value, str) else "" + def trust_profile(self) -> dict: + """Public profile the validators build their local TrustRoot from (TLS is the anchor).""" + latest = self.store.latest() + if latest is not None: + self.refresh_trust(latest.epoch) + return trust_to_json(self.trust) + def refresh_trust(self, epoch: int) -> None: if self.trust_loader is None: return diff --git a/src/cortex/gateway/store.py b/src/cortex/gateway/store.py index c8944b316..e3d82f430 100644 --- a/src/cortex/gateway/store.py +++ b/src/cortex/gateway/store.py @@ -5,7 +5,7 @@ import threading from collections.abc import Callable from contextlib import contextmanager -from dataclasses import asdict, dataclass +from dataclasses import dataclass from hashlib import sha256 from pathlib import Path from uuid import uuid4 @@ -13,15 +13,14 @@ from cortex.errors import ServiceError from cortex.protocol import ( Bundle, - ChallengeEntry, Leaf, NoScore, - ParticipantPolicy, ProtocolError, Score, TrustRoot, ) from cortex.protocol.scale import Reader, uint +from cortex.protocol.trust import trust_from_json, trust_to_json from cortex.state import secure_sqlite_path @@ -133,7 +132,7 @@ def bind_netuid(self, netuid: int) -> None: def accept_trust(self, trust: TrustRoot) -> None: trust.validate() - profile = json.dumps(asdict(trust), default=bytes.hex, sort_keys=True) + profile = json.dumps(trust_to_json(trust), sort_keys=True) with self._transaction() as connection: for name, value in ( ("challenges_version", trust.challenges_version), @@ -143,7 +142,7 @@ def accept_trust(self, trust: TrustRoot) -> None: "SELECT value FROM gateway_metadata WHERE name=?", (name,) ).fetchone() if row is not None and value < int(row[0]): - raise ServiceError(503, "gateway trust version rollback") + raise ServiceError(503, "gateway profile version rollback") connection.execute( "INSERT INTO gateway_metadata VALUES (?,?) " "ON CONFLICT(name) DO UPDATE SET value=excluded.value", @@ -167,22 +166,7 @@ def trust_profiles(self) -> tuple[TrustRoot, ...]: values = json.loads(profile) if not isinstance(values, dict): raise ProtocolError("corrupt stored trust profile") - challenges = [] - for entry in values.pop("challenges"): - policy = entry["policy"] - policy["hotkeys"] = tuple(bytes.fromhex(key) for key in policy["hotkeys"]) - challenges.append( - ChallengeEntry( - bytes.fromhex(entry["id"]), - bytes.fromhex(entry["public_key"]), - entry["emission_share_bps"], - ParticipantPolicy(**policy), - ) - ) - values["measurements_digest"] = bytes.fromhex(values["measurements_digest"]) - values["gateway_hotkey"] = bytes.fromhex(values["gateway_hotkey"]) - trust = TrustRoot(challenges=tuple(challenges), **values) - trust.validate() + trust = trust_from_json(values) profiles.append(trust) except (KeyError, TypeError, ValueError) as error: raise ProtocolError("corrupt stored trust profile") from error diff --git a/src/cortex/master.py b/src/cortex/master.py index dd09c9121..65b19ec97 100644 --- a/src/cortex/master.py +++ b/src/cortex/master.py @@ -394,7 +394,7 @@ async def _emit(self, epoch: int, snapshot: ChainSnapshot) -> None: expected = challenge.policy.expected(rows) seed = self.challenge_seed(challenge.id) if public_key(seed) != challenge.public_key: - raise ServiceError(503, "challenge signing key does not match owner trust") + raise ServiceError(503, "challenge signing key does not match the profile") outcomes = await self._scores(challenge.id, epoch, expected, epoch_at) # No unknown score key can expand E; missing scores are explicit signed absences. leaves = [] diff --git a/src/cortex/operator.py b/src/cortex/operator.py deleted file mode 100644 index e0984de40..000000000 --- a/src/cortex/operator.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Offline operator key generation and detached trust-root signing.""" - -from __future__ import annotations - -import os -import secrets -from pathlib import Path - -from cortex.config import read_seed -from cortex.protocol.crypto import TRUST_ROOT_DOMAIN, public_key, sign_raw -from cortex.protocol.trust import signing_payload - - -def _exclusive_write(path: Path, value: bytes, mode: int) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode) - try: - with os.fdopen(descriptor, "wb", closefd=False) as stream: - stream.write(value) - stream.flush() - os.fsync(stream.fileno()) - finally: - os.close(descriptor) - - -def generate_key(seed_path: Path, public_path: Path) -> str: - """Create a new raw sr25519 seed and its hexadecimal public key without overwrites.""" - seed = secrets.token_bytes(32) - encoded_public = public_key(seed).hex() - _exclusive_write(seed_path, seed, 0o600) - try: - _exclusive_write(public_path, (encoded_public + "\n").encode(), 0o644) - except BaseException: - seed_path.unlink(missing_ok=True) - raise - return encoded_public - - -def sign_trust_document( - *, input_path: Path, kind: str, seed_path: Path, signature_path: Path -) -> str: - """Sign one immutable TOML document and create its detached signature.""" - signature = sign_raw( - read_seed(seed_path), TRUST_ROOT_DOMAIN, signing_payload(input_path, kind) - ).hex() - _exclusive_write(signature_path, (signature + "\n").encode(), 0o644) - return signature diff --git a/src/cortex/protocol/crypto.py b/src/cortex/protocol/crypto.py index 74a1c48d3..234200e0b 100644 --- a/src/cortex/protocol/crypto.py +++ b/src/cortex/protocol/crypto.py @@ -20,7 +20,6 @@ SIGNING_CONTEXT = b"base-sr25519-v1" BUNDLE_DOMAIN = b"base-bundle-v1" RAW_WEIGHT_DOMAIN = b"base-rawweight-v1" -TRUST_ROOT_DOMAIN = b"base-trustroot-v1" _ORDER = 2**252 + 27742317777372353535851937790883648493 _ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" diff --git a/src/cortex/protocol/trust.py b/src/cortex/protocol/trust.py index d8d131092..05f8e6254 100644 --- a/src/cortex/protocol/trust.py +++ b/src/cortex/protocol/trust.py @@ -1,22 +1,14 @@ -"""Read, sign and verify owner-pinned trust roots.""" +"""Read and (de)serialize the gateway trust profile; the gateway is the source of truth.""" +import json import tomllib +from dataclasses import asdict from hashlib import sha256 from pathlib import Path -from .crypto import TRUST_ROOT_DOMAIN, decode_hotkey, verify_raw +from .crypto import decode_hotkey from .models import ChallengeEntry, ParticipantPolicy, TrustRoot -from .scale import ProtocolError, byte_vec, fixed, uint, vector - - -def _signature(path: Path) -> bytes: - raw = path.read_bytes() - if len(raw) == 64: - return raw - try: - return fixed(bytes.fromhex(raw.decode().strip().removeprefix("0x")), 64) - except (ValueError, UnicodeDecodeError) as error: - raise ProtocolError("invalid detached signature") from error +from .scale import ProtocolError, fixed, vector def _policy(value: str | dict) -> ParticipantPolicy: @@ -37,10 +29,6 @@ def _policy(value: str | dict) -> ParticipantPolicy: raise ProtocolError("unknown participant policy") -def trust_payload(version: int, introduced_epoch: int, body: bytes) -> bytes: - return uint(version, 4) + uint(introduced_epoch, 8) + byte_vec(body) - - def challenge_entries(document: dict) -> tuple[ChallengeEntry, ...]: """Decode the challenge body exactly as it is signed on the wire.""" return tuple( @@ -70,57 +58,50 @@ def measurements_body(document: dict) -> bytes: return vector(entries, lambda value: value) -def signing_payload(path: Path, kind: str) -> bytes: - """Return the detached-signature preimage for an operator document.""" - document = tomllib.loads(path.read_text()) - if kind == "challenges": - challenges = challenge_entries(document) - body = vector(challenges, ChallengeEntry.encode) - elif kind == "measurements": - body = measurements_body(document) - else: - raise ProtocolError("unknown trust document kind") - return trust_payload(document["version"], document.get("introduced_epoch", 0), body) - - -def load_trust_root( - *, - challenges_path: Path, - challenges_signature: Path, - measurements_path: Path, - measurements_signature: Path, - owner_public: bytes, - gateway_public: bytes, - epoch: int, - minimum_challenges_version: int = 1, - minimum_measurements_version: int = 1, +def load_trust( + *, challenges_path: Path, measurements_path: Path, gateway_public: bytes ) -> TrustRoot: - """Owner key and minimum versions are local pins, never fetched from the gateway.""" + """Build the profile from the master's own TOML files. No signature: the files are local.""" challenges_doc = tomllib.loads(challenges_path.read_text()) measurements_doc = tomllib.loads(measurements_path.read_text()) - challenges = challenge_entries(challenges_doc) - measurement_bytes = measurements_body(measurements_doc) trust = TrustRoot( - challenges, - sha256(measurement_bytes).digest(), + challenge_entries(challenges_doc), + sha256(measurements_body(measurements_doc)).digest(), gateway_public, challenges_doc["version"], measurements_doc["version"], max(challenges_doc.get("introduced_epoch", 0), measurements_doc.get("introduced_epoch", 0)), ) - for document, body, signature_path, minimum in ( - (challenges_doc, trust.challenges_body(), challenges_signature, minimum_challenges_version), - (measurements_doc, measurement_bytes, measurements_signature, minimum_measurements_version), - ): - version = document["version"] - introduced = document.get("introduced_epoch", 0) - if version < minimum or introduced > epoch: - raise ProtocolError("trust root rollback or future activation") - if not verify_raw( - owner_public, - TRUST_ROOT_DOMAIN, - trust_payload(version, introduced, body), - _signature(signature_path), - ): - raise ProtocolError("invalid trust root owner signature") + trust.validate() return trust + + +def trust_to_json(trust: TrustRoot) -> dict: + """Public profile served at /v1/trust and persisted by the gateway store.""" + return json.loads(json.dumps(asdict(trust), default=bytes.hex)) + + +def trust_from_json(values: object) -> TrustRoot: + try: + if not isinstance(values, dict): + raise TypeError("profile must be an object") + values = dict(values) + challenges = [] + for entry in values.pop("challenges"): + policy = dict(entry["policy"]) + policy["hotkeys"] = tuple(bytes.fromhex(key) for key in policy["hotkeys"]) + challenges.append( + ChallengeEntry( + bytes.fromhex(entry["id"]), + bytes.fromhex(entry["public_key"]), + entry["emission_share_bps"], + ParticipantPolicy(**policy), + ) + ) + values["measurements_digest"] = bytes.fromhex(values["measurements_digest"]) + values["gateway_hotkey"] = bytes.fromhex(values["gateway_hotkey"]) + trust = TrustRoot(challenges=tuple(challenges), **values) + trust.validate() + return trust + except (KeyError, TypeError, ValueError) as error: + raise ProtocolError("invalid trust profile") from error diff --git a/src/cortex/validator/__main__.py b/src/cortex/validator/__main__.py index 5287c656c..ac4d95f31 100644 --- a/src/cortex/validator/__main__.py +++ b/src/cortex/validator/__main__.py @@ -14,7 +14,6 @@ from cortex.config import read_seed, validate_chain_endpoint from cortex.protocol import ProtocolError from cortex.protocol.crypto import decode_hotkey, public_key -from cortex.protocol.trust import load_trust_root from .chain import BittensorChain, close_subtensor from .evidence import peer_app @@ -71,16 +70,19 @@ def parser() -> argparse.ArgumentParser: ) arguments.add_argument("--gateway", required=True) arguments.add_argument("--netuid", required=True, type=int) - arguments.add_argument("--gateway-public", required=True, help="independently pinned hotkey") - arguments.add_argument("--owner-public", type=Path, default=Path("config/owner.pubkey")) - arguments.add_argument("--challenges", type=Path, default=Path("config/challenges.toml")) - arguments.add_argument("--measurements", type=Path, default=Path("config/measurements.toml")) - arguments.add_argument("--minimum-challenges-version", type=int, required=True) - arguments.add_argument("--minimum-measurements-version", type=int, required=True) + arguments.add_argument( + "--gateway-public", + help="optional extra pin: refuse a gateway whose sealing hotkey differs. The gateway " + "itself (HTTPS) is the trust anchor for the emission profile (/v1/trust)", + ) arguments.add_argument("--network", type=primary_endpoint, default="finney") arguments.add_argument("--fallback-endpoints", type=fallback_endpoints, default=[]) - arguments.add_argument("--wallet-name", required=True) - arguments.add_argument("--wallet-hotkey", required=True) + arguments.add_argument( + "--wallet-name", help="validator wallet; required only to submit weights on-chain" + ) + arguments.add_argument( + "--wallet-hotkey", help="validator hotkey; required only to submit weights on-chain" + ) arguments.add_argument("--wallet-path", default="~/.bittensor/wallets") arguments.add_argument("--state-db", type=Path, required=True) arguments.add_argument("--poll-seconds", type=float, default=30) @@ -124,19 +126,6 @@ async def run(arguments: argparse.Namespace, subtensor, wallet) -> TickResult | if epoch is None: raise ProtocolError("cannot read subnet epoch") - def load_trust(epoch): - return load_trust_root( - challenges_path=arguments.challenges, - challenges_signature=Path(str(arguments.challenges) + ".sig"), - measurements_path=arguments.measurements, - measurements_signature=Path(str(arguments.measurements) + ".sig"), - owner_public=decode_hotkey(arguments.owner_public.read_text().strip()), - gateway_public=decode_hotkey(arguments.gateway_public), - epoch=epoch, - minimum_challenges_version=arguments.minimum_challenges_version, - minimum_measurements_version=arguments.minimum_measurements_version, - ) - # The seed signs cross-validator root statements and dissents, which only # exist in a peer-consensus deployment (_crosscheck returns immediately # otherwise, and _dissent does nothing without a seed). Checking it @@ -185,7 +174,6 @@ def consensus_seed(): validator = Validator( gateway_url=arguments.gateway, netuid=arguments.netuid, - trust=load_trust(epoch), chain=BittensorChain(subtensor, wallet), journal=journal, http=http, @@ -198,7 +186,9 @@ def consensus_seed(): peer_consensus=arguments.peer_consensus, min_peer_sample=arguments.min_peer_sample, max_block_lag=arguments.max_block_lag, - trust_loader=load_trust, + gateway_public=decode_hotkey(arguments.gateway_public) + if arguments.gateway_public + else None, verify_only=arguments.verify_only, ) while True: @@ -234,8 +224,14 @@ def main(argv: list[str] | None = None) -> None: arguments = parser().parse_args(argv) if not isfinite(arguments.poll_seconds) or arguments.poll_seconds <= 0: raise SystemExit("--poll-seconds must be positive") - if arguments.minimum_challenges_version < 1 or arguments.minimum_measurements_version < 1: - raise SystemExit("minimum trust versions must be positive") + submits = not arguments.verify_only + if (submits or arguments.peer_consensus) and not ( + arguments.wallet_name and arguments.wallet_hotkey + ): + raise SystemExit( + "--wallet-name and --wallet-hotkey are required to submit weights or run " + "--peer-consensus; use --verify-only to run without a validator hotkey" + ) if not 0 <= arguments.netuid <= 65535: raise SystemExit("--netuid must fit u16") if not 0 <= arguments.version_key <= 2**64 - 1: @@ -261,21 +257,22 @@ def main(argv: list[str] | None = None) -> None: # mnemonic wallet does — //hotkeys/ — so the file # decides how it is read, and the command line stays what the deploy gate # audits. A mnemonic wallet keeps working exactly as before. - wallet_path = Path(arguments.wallet_path).expanduser() - hotkey_file = wallet_path / arguments.wallet_name / "hotkeys" / arguments.wallet_hotkey - wallet: Wallet | PrivateKeyWallet - try: - private = hotkey_file.is_file() and _carries_private_key(hotkey_file) - if private: - wallet = load_private_key_wallet(hotkey_file) - except ProtocolError as error: - raise SystemExit(f"validator wallet refused: {error}") from None - if not private: - wallet = Wallet( - name=arguments.wallet_name, - hotkey=arguments.wallet_hotkey, - path=str(wallet_path), - ) + wallet: Wallet | PrivateKeyWallet | None = None + if arguments.wallet_name and arguments.wallet_hotkey: + wallet_path = Path(arguments.wallet_path).expanduser() + hotkey_file = wallet_path / arguments.wallet_name / "hotkeys" / arguments.wallet_hotkey + try: + private = hotkey_file.is_file() and _carries_private_key(hotkey_file) + if private: + wallet = load_private_key_wallet(hotkey_file) + except ProtocolError as error: + raise SystemExit(f"validator wallet refused: {error}") from None + if not private: + wallet = Wallet( + name=arguments.wallet_name, + hotkey=arguments.wallet_hotkey, + path=str(wallet_path), + ) try: result = asyncio.run(run(arguments, subtensor, wallet)) if ( diff --git a/src/cortex/validator/chain.py b/src/cortex/validator/chain.py index f1554bf33..2577360be 100644 --- a/src/cortex/validator/chain.py +++ b/src/cortex/validator/chain.py @@ -77,7 +77,8 @@ def close_subtensor(subtensor: Any) -> None: class BittensorChain: """Optional ``chain`` dependency; no chain calls are made by constructing this adapter.""" - def __init__(self, subtensor: Any, wallet: Any, *, block_time: float = 12.0): + def __init__(self, subtensor: Any, wallet: Any = None, *, block_time: float = 12.0): + """``wallet`` (the validator hotkey) is needed only to preflight and submit weights.""" self.subtensor = subtensor self.wallet = wallet self.block_time = block_time @@ -149,6 +150,8 @@ async def submit( async def preflight( self, netuid: int, vector: tuple[tuple[int, int], ...], version_key: int ) -> None: + if self.wallet is None: # verify-only without a hotkey: nothing to check on-chain + return try: await asyncio.to_thread(self._preflight, netuid, vector, version_key) except Exception as error: @@ -162,6 +165,8 @@ def _preflight( for uid, value in vector: uint(uid, 2) uint(value, 2) + if self.wallet is None: + raise ProtocolError("submitting weights requires a validator hotkey (wallet)") uint(netuid, 2) uint(version_key, 8) tip = self.subtensor.get_current_block() diff --git a/src/cortex/validator/service.py b/src/cortex/validator/service.py index 5c439d873..0bafc6a5b 100644 --- a/src/cortex/validator/service.py +++ b/src/cortex/validator/service.py @@ -22,6 +22,7 @@ from cortex.protocol.crypto import public_key from cortex.protocol.models import encode_final_vector from cortex.protocol.scale import fixed, uint +from cortex.protocol.trust import trust_from_json from .evidence import EvidenceStore @@ -333,7 +334,7 @@ def __init__( *, gateway_url: str, netuid: int, - trust: TrustRoot, + trust: TrustRoot | None = None, chain: Chain, journal: SubmissionJournal, http: httpx.AsyncClient, @@ -345,6 +346,7 @@ def __init__( max_block_lag: int = 256, trust_loader: Callable[[int], TrustRoot] | None = None, verify_only: bool = False, + gateway_public: bytes | None = None, ): """Verify the gateway seal and submit weights. @@ -358,7 +360,11 @@ def __init__( """ uint(netuid, 2) uint(version_key, 8) - trust.validate() + if trust is not None: + trust.validate() + if gateway_public is not None: + fixed(gateway_public, 32) + self.gateway_public = gateway_public gateway = urlsplit(gateway_url) if ( gateway_url != gateway_url.strip() @@ -373,7 +379,8 @@ def __init__( raise ValueError("gateway URL requires HTTPS without credentials or path") self.gateway_url = gateway_url.rstrip("/") self.netuid = netuid - self.trust = trust + self.trust: TrustRoot = trust # type: ignore[assignment] # fetched before first use + self._fixed_trust = trust is not None self.chain = chain self.journal = journal self.journal.bind_netuid(netuid) @@ -540,6 +547,12 @@ async def _run_once(self) -> TickResult: raise ProtocolError("bundle epoch does not match the pinned chain block") if self.trust_loader: self.trust = self.trust_loader(epoch) + elif not self._fixed_trust: + # The gateway is ours and is the source of truth for the emission profile; + # TLS plus an optional --gateway-public pin are the only anchors. + self.trust = trust_from_json(decode_json(await self._read("/v1/trust", 1024 * 1024))) + if self.gateway_public is not None and self.trust.gateway_hotkey != self.gateway_public: + raise ProtocolError("gateway key differs from the pinned --gateway-public") self.journal.evidence.watermark("challenges_version", self.trust.challenges_version) self.journal.evidence.watermark("measurements_version", self.trust.measurements_version) body, quarantined = verify_inputs( diff --git a/tests/protocol/test_trust.py b/tests/protocol/test_trust.py deleted file mode 100644 index ad25f2a83..000000000 --- a/tests/protocol/test_trust.py +++ /dev/null @@ -1,77 +0,0 @@ -from pathlib import Path - -import pytest - -from cortex.protocol import ProtocolError -from cortex.protocol.crypto import TRUST_ROOT_DOMAIN, decode_hotkey, public_key, sign_raw -from cortex.protocol.trust import load_trust_root, signing_payload - -REFERENCE = Path(__file__).parent / "vectors/trust" - - -def arguments(): - return dict( - challenges_path=REFERENCE / "challenges.toml", - challenges_signature=REFERENCE / "challenges.toml.sig", - measurements_path=REFERENCE / "measurements.toml", - measurements_signature=REFERENCE / "measurements.toml.sig", - owner_public=decode_hotkey((REFERENCE / "owner.pubkey").read_text().strip()), - gateway_public=public_key(bytes([7]) * 32), - epoch=123, - ) - - -def test_verifies_existing_independent_owner_signed_trust_documents(): - root = load_trust_root(**arguments()) - assert root.shares == ((b"bounty", 2000), (b"proof", 8000)) - assert root.measurements_digest.hex() == ( - "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d" - ) - - -def test_cannot_resign_trust_by_substituting_challenge_key(tmp_path): - values = arguments() - path = tmp_path / "tampered.toml" - original = values["challenges_path"].read_text() - path.write_text( - original.replace( - "743688a1e1b2848b309205706b4dcae54bffe4233a5d7018053471e1dce45c21", - public_key(bytes([55]) * 32).hex(), - ) - ) - values["challenges_path"] = path - with pytest.raises(ProtocolError, match="owner signature"): - load_trust_root(**values) - - -def test_local_version_pin_prevents_rollback(): - with pytest.raises(ProtocolError, match="rollback"): - load_trust_root(**arguments(), minimum_challenges_version=2) - - -@pytest.mark.parametrize("version,epoch,valid", [(1, 123, False), (2, 122, False), (2, 123, True)]) -def test_owner_signed_proportional_profile_requires_version_and_activation( - tmp_path, version, epoch, valid -): - values = arguments() - seed = bytes([31]) * 32 - values.update(owner_public=public_key(seed), epoch=epoch) - for kind in ("challenges", "measurements"): - path = tmp_path / f"{kind}.toml" - source = values[f"{kind}_path"].read_text() - if kind == "challenges": - source = ( - source.replace("version = 1", f"version = {version}") - .replace("introduced_epoch = 0", "introduced_epoch = 123") - .replace("2000", "3000") - .replace("8000", "7000") - ) - path.write_text(source) - signature = path.with_suffix(".sig") - signature.write_bytes(sign_raw(seed, TRUST_ROOT_DOMAIN, signing_payload(path, kind))) - values.update({f"{kind}_path": path, f"{kind}_signature": signature}) - if valid: - assert load_trust_root(**values).algorithm_version == 2 - else: - with pytest.raises(ProtocolError, match="version|activation"): - load_trust_root(**values) diff --git a/tests/protocol/vectors/trust/challenges.toml b/tests/protocol/vectors/trust/challenges.toml deleted file mode 100644 index e71f5419e..000000000 --- a/tests/protocol/vectors/trust/challenges.toml +++ /dev/null @@ -1,27 +0,0 @@ -# Owner-signed challenges trust root (D18/D23/D24). -# Signed with throwaway owner key (see owner.pubkey). Production rotation: CEREMONY.md. -# -# Two live challenges. Emission: bounty = 2000, proof = 8000 bps (sum 10000). -# Proof-weighted: 20%/80%, locked regardless of eval digest. Proof still -# answers 503 on submit until a digest is pinned — do not invent a sha256. -# Neither share is a leftover Relearn / Prism / Design inheritance. Operator -# can retune; sum must stay 10000. -# -# `relearn`, `relearn-image`, `relearn-agent`, `relearn-mm`, `design`, and -# `prism` are OFF: they have no row here, so they have no emission and no -# leaf may verify under this root. Their product code is removed from this -# repo (historical miner stubs stay under docs/external-miner/). -version = 1 -introduced_epoch = 0 - -[[challenges]] -id = "bounty" -public_key = "743688a1e1b2848b309205706b4dcae54bffe4233a5d7018053471e1dce45c21" -emission_share_bps = 2000 -policy = "all_metagraph_hotkeys" - -[[challenges]] -id = "proof" -public_key = "3e7f70f09165e265ab89ab04a4fc91dc0531d54a100c538fb14c6f008421c375" -emission_share_bps = 8000 -policy = "all_metagraph_hotkeys" diff --git a/tests/protocol/vectors/trust/challenges.toml.sig b/tests/protocol/vectors/trust/challenges.toml.sig deleted file mode 100644 index a5031065c..000000000 --- a/tests/protocol/vectors/trust/challenges.toml.sig +++ /dev/null @@ -1 +0,0 @@ -c82e38817950e6ad7abcd9efc4af1b198f49a8d8cb11a1b1c6b88ac7f04e4c360bf98432ed352b421a23499d0ed1c3b5bf7a020254d7a20388c249821581b28e diff --git a/tests/protocol/vectors/trust/measurements.toml b/tests/protocol/vectors/trust/measurements.toml deleted file mode 100644 index 2517227b1..000000000 --- a/tests/protocol/vectors/trust/measurements.toml +++ /dev/null @@ -1,9 +0,0 @@ -# Owner-signed measurements allowlist. -# -# base-agent / Phala CVM miner path removed — allowlist cleared (fail-closed). -# Attest crates + BUNDLE_SPEC measurements_digest machinery remain; new design -# runtime rows (if any) land via owner ceremony (CEREMONY.md). -version = 5 -introduced_epoch = 0 - -measurements = [] diff --git a/tests/protocol/vectors/trust/measurements.toml.sig b/tests/protocol/vectors/trust/measurements.toml.sig deleted file mode 100644 index 11bf83fb2..000000000 --- a/tests/protocol/vectors/trust/measurements.toml.sig +++ /dev/null @@ -1 +0,0 @@ -ea3f1ee532041bdb52ce5a4a2f954283ac8becc5c0d4a5e0f244cee3ccb29618faf219b44c463eb34dae919366a9d58f1b9bd011b9cf96eaeb71045474ab888f diff --git a/tests/protocol/vectors/trust/owner.pubkey b/tests/protocol/vectors/trust/owner.pubkey deleted file mode 100644 index acfde072f..000000000 --- a/tests/protocol/vectors/trust/owner.pubkey +++ /dev/null @@ -1 +0,0 @@ -40ef68c8608ba18eded3b4498230d88314c2e4e298cadd7cb044e5ca8412fa10 diff --git a/tests/test_master.py b/tests/test_master.py index 0dc475913..53184a80d 100644 --- a/tests/test_master.py +++ b/tests/test_master.py @@ -85,7 +85,6 @@ def secret(name, value): return MasterConfig( netuid=541, state_dir=tmp_path / "state", - owner_public_file=tmp_path / "owner.pubkey", challenges_file=tmp_path / "challenges.toml", measurements_file=tmp_path / "measurements.toml", gateway_seed_file=secret("gateway.key", bytes([7]) * 32), diff --git a/tests/test_operator.py b/tests/test_operator.py deleted file mode 100644 index 9e7de69e6..000000000 --- a/tests/test_operator.py +++ /dev/null @@ -1,191 +0,0 @@ -from pathlib import Path - -import pytest - -from cortex.cli import main -from cortex.operator import generate_key, sign_trust_document -from cortex.protocol import ProtocolError -from cortex.protocol.crypto import decode_hotkey, public_key -from cortex.protocol.trust import load_trust_root -from cortex.validator import SubmissionJournal - - -def _document(path: Path, body: str) -> Path: - path.write_text(body) - return path - - -def test_generated_key_signs_both_documents_for_runtime_verification(tmp_path): - seed_path, owner_path = tmp_path / "owner.seed", tmp_path / "owner.pubkey" - generated = generate_key(seed_path, owner_path) - challenge_seed = bytes([4]) * 32 - challenges = _document( - tmp_path / "challenges.toml", - f"""version = 1 -introduced_epoch = 0 -[[challenges]] -id = "bounty" -public_key = "{public_key(challenge_seed).hex()}" -emission_share_bps = 2000 -policy = "all_metagraph_hotkeys" -[[challenges]] -id = "proof" -public_key = "{public_key(bytes([5]) * 32).hex()}" -emission_share_bps = 8000 -policy = "all_metagraph_hotkeys" -""", - ) - measurements = _document( - tmp_path / "measurements.toml", "version = 1\nintroduced_epoch = 0\nmeasurements = []\n" - ) - for kind, document in (("challenges", challenges), ("measurements", measurements)): - sign_trust_document( - input_path=document, - kind=kind, - seed_path=seed_path, - signature_path=Path(str(document) + ".sig"), - ) - - trust = load_trust_root( - challenges_path=challenges, - challenges_signature=Path(str(challenges) + ".sig"), - measurements_path=measurements, - measurements_signature=Path(str(measurements) + ".sig"), - owner_public=decode_hotkey(owner_path.read_text().strip()), - gateway_public=public_key(bytes([7]) * 32), - epoch=0, - ) - - assert generated == owner_path.read_text().strip() - assert seed_path.stat().st_mode & 0o777 == 0o600 - assert trust.shares == ((b"bounty", 2000), (b"proof", 8000)) - - -def test_key_generation_refuses_to_overwrite_operator_material(tmp_path): - seed_path, owner_path = tmp_path / "owner.seed", tmp_path / "owner.pubkey" - generate_key(seed_path, owner_path) - - with pytest.raises(FileExistsError): - generate_key(seed_path, tmp_path / "other.pubkey") - - assert len(seed_path.read_bytes()) == 32 - - -def test_proportional_template_requires_an_explicit_activation_epoch_before_signing(tmp_path): - seed_path = tmp_path / "owner.seed" - generate_key(seed_path, tmp_path / "owner.pubkey") - template = Path(__file__).resolve().parents[1] / "config/challenges-v2.example.toml" - signature = tmp_path / "challenges.toml.sig" - - with pytest.raises(ProtocolError): - sign_trust_document( - input_path=template, - kind="challenges", - seed_path=seed_path, - signature_path=signature, - ) - - assert not signature.exists() - selected = _document( - tmp_path / "challenges.toml", - template.read_text().replace('"CHOOSE_ACTIVATION_EPOCH"', "123"), - ) - sign_trust_document( - input_path=selected, - kind="challenges", - seed_path=seed_path, - signature_path=signature, - ) - assert len(bytes.fromhex(signature.read_text())) == 64 - - -def test_cli_verifies_the_signed_files_before_operator_install(tmp_path, capsys): - seed_path, owner_path = tmp_path / "owner.seed", tmp_path / "owner.pubkey" - generate_key(seed_path, owner_path) - challenges = tmp_path / "challenges.toml" - challenges.write_text( - f'''version = 1 -introduced_epoch = 0 -[[challenges]] -id = "bounty" -public_key = "{public_key(bytes([4]) * 32).hex()}" -emission_share_bps = 2000 -policy = "all_metagraph_hotkeys" -[[challenges]] -id = "proof" -public_key = "{public_key(bytes([5]) * 32).hex()}" -emission_share_bps = 8000 -policy = "all_metagraph_hotkeys" -''' - ) - measurements = tmp_path / "measurements.toml" - measurements.write_text("version = 1\nmeasurements = []\n") - for kind, document in (("challenges", challenges), ("measurements", measurements)): - sign_trust_document( - input_path=document, - kind=kind, - seed_path=seed_path, - signature_path=Path(str(document) + ".sig"), - ) - - main( - [ - "trust-verify", - "--challenges", - str(challenges), - "--measurements", - str(measurements), - "--owner-public", - str(owner_path), - "--gateway-public", - public_key(bytes([7]) * 32).hex(), - "--epoch", - "1", - ] - ) - - assert '"challenges": {"bounty": 2000, "proof": 8000}' in capsys.readouterr().out - - -def test_cli_reconciles_only_the_exact_pending_validator_attempt(tmp_path, capsys): - state = tmp_path / "validator.db" - journal = SubmissionJournal(state) - digest = "ab" * 32 - attempt_id = journal.claim(541, 12, digest) - journal.record_uncertain( - 541, - 12, - attempt_id, - extrinsic_hash="0x" + "cd" * 32, - nonce=17, - ) - journal.close() - - main( - [ - "validator-reconcile", - "--state-db", - str(state), - "--netuid", - "541", - "--epoch", - "12", - "--digest", - digest, - "--attempt-id", - attempt_id, - "--result", - "submitted", - "--evidence-digest", - "ef" * 32, - ] - ) - - output = capsys.readouterr().out - assert '"state": "reconciled_submitted"' in output - reopened = SubmissionJournal(state) - assert reopened.pending(541, 12) is None - assert reopened.connection.execute( - "SELECT state FROM weight_submissions WHERE netuid=541 AND epoch=12" - ).fetchone() == ("submitted",) - reopened.close() diff --git a/tests/test_run_validator_script.py b/tests/test_run_validator_script.py index 1d5e7b9a3..7d9750382 100644 --- a/tests/test_run_validator_script.py +++ b/tests/test_run_validator_script.py @@ -59,12 +59,14 @@ def test_master_intent_is_refused(env, args, extra): assert "cortex master" not in result.stdout -def test_missing_gateway_public_fails(env): - del env["GATEWAY_PUBLIC"] - result = run(env, "--dry-run") - assert result.returncode != 0 - assert "GATEWAY_PUBLIC is not set" in result.stderr - +def test_submitting_without_a_hotkey_fails_but_verify_only_does_not(env): + del env["WALLET_NAME"], env["WALLET_HOTKEY"] + refused = run(env, "--dry-run") + assert refused.returncode != 0 + assert "WALLET_NAME" in refused.stderr + verify = run(env, "--dry-run", "--verify-only") + assert verify.returncode == 0, verify.stderr + assert "--wallet" not in verify.stdout def test_netuid_mismatch_fails(env): result = run(env | {"NETUID": "541"}, "--dry-run") @@ -87,6 +89,8 @@ def test_dry_run_prints_masked_validator_command(env): "--verify-only --once", ): assert expected in command + for removed in ("--owner-public", "--challenges", "--measurements", "minimum"): + assert removed not in command assert "epoch=4242 sealed=true" in result.stderr diff --git a/tests/validator/test_main.py b/tests/validator/test_main.py index 3df08e43a..700e4979c 100644 --- a/tests/validator/test_main.py +++ b/tests/validator/test_main.py @@ -15,10 +15,6 @@ def required_arguments(tmp_path): "541", "--gateway-public", "11" * 32, - "--minimum-challenges-version", - "1", - "--minimum-measurements-version", - "1", "--wallet-name", "validator", "--wallet-hotkey", @@ -146,11 +142,54 @@ async def refused(arguments, subtensor, wallet): validator_main.main([*required_arguments(tmp_path), "--verify-only", "--once"]) -@pytest.mark.parametrize("flag", ["--minimum-challenges-version", "--minimum-measurements-version"]) -def test_main_rejects_nonpositive_trust_version_pins(flag, tmp_path): - with pytest.raises(SystemExit, match="minimum trust versions must be positive"): - validator_main.main([*required_arguments(tmp_path), flag, "0"]) +def without(arguments, flag): + """Drop `flag value` from an argument list.""" + index = arguments.index(flag) + return arguments[:index] + arguments[index + 2 :] +def without_hotkey(tmp_path): + return without(without(required_arguments(tmp_path), "--wallet-name"), "--wallet-hotkey") + +def test_weight_setting_validator_without_hotkey_fails_with_a_clear_error(tmp_path): + with pytest.raises(SystemExit, match="required to submit weights"): + validator_main.main(without_hotkey(tmp_path)) + +def test_peer_consensus_without_hotkey_fails_even_when_verify_only(tmp_path): + with pytest.raises(SystemExit, match="required to submit weights or run --peer-consensus"): + validator_main.main([*without_hotkey(tmp_path), "--verify-only", "--peer-consensus"]) + +def test_verify_only_validator_starts_without_hotkey_gateway_pin_or_trust_files( + monkeypatch, tmp_path +): + seen = {} + + class Subtensor: + def __init__(self, **kwargs): + pass + + def close(self): + pass + + bittensor = ModuleType("bittensor") + bittensor.Subtensor = Subtensor + wallet_module = ModuleType("bittensor_wallet") + wallet_module.Wallet = lambda **kwargs: pytest.fail("no wallet must be built") + monkeypatch.setitem(sys.modules, "bittensor", bittensor) + monkeypatch.setitem(sys.modules, "bittensor_wallet", wallet_module) + + async def verified(arguments, subtensor, wallet): + seen["wallet"] = wallet + return TickResult("verified") + + monkeypatch.setattr(validator_main, "run", verified) + arguments = without(without_hotkey(tmp_path), "--gateway-public") + validator_main.main([*arguments, "--verify-only", "--once"]) + assert seen == {"wallet": None} + +def test_removed_trust_signing_options_are_gone(tmp_path): + for flag in ("--owner-public", "--challenges", "--measurements", "--minimum-challenges-version"): + with pytest.raises(SystemExit): + validator_main.parser().parse_args([*required_arguments(tmp_path), flag, "x"]) @pytest.mark.parametrize( ("flag", "value", "message"), From 034a889deb391c1b8de0e02c33c8fbd5f9f67bd2 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:04:33 +0000 Subject: [PATCH 02/56] feat(validator): only the weight-setting role needs a validator hotkey The validator CLI, run-validator.sh and the deploy contract no longer require a wallet unless weights are submitted (or --peer-consensus runs). role-validator.yml becomes verify-only with no wallet; role-weight-setter.yml carries the hotkey. Drop the min-version env pins and BASE_GATEWAY_HOTKEY. Co-Authored-By: Claude Sonnet 5.5 --- .env.example | 1 - deploy/compose/role-master.yml | 3 +- deploy/compose/role-validator.yml | 32 ++------- deploy/compose/role-weight-setter.yml | 84 ++++++++++++++++++++++ deploy/env/master.env.example | 2 - deploy/env/python-validator.env.example | 7 +- scripts/check_deploy.py | 95 +++++++++++-------------- tests/test_deploy_contract.py | 70 ++++++++++-------- tests/test_run_validator_script.py | 1 + tests/validator/test_main.py | 13 +++- 10 files changed, 189 insertions(+), 119 deletions(-) create mode 100644 deploy/compose/role-weight-setter.yml diff --git a/.env.example b/.env.example index a25738a14..2c6c2b6a6 100644 --- a/.env.example +++ b/.env.example @@ -3,7 +3,6 @@ BASE_NETUID=1 BASE_CHAIN_ENDPOINT=finney BASE_CHAIN_FALLBACK_ENDPOINTS=[] BASE_STATE_DIR=.local/state -BASE_OWNER_PUBKEY_FILE=config/owner.pubkey BASE_CHALLENGES_FILE=config/challenges.toml BASE_MEASUREMENTS_FILE=config/measurements.toml BASE_GATEWAY_SK_FILE=.local/secrets/gateway.seed diff --git a/deploy/compose/role-master.yml b/deploy/compose/role-master.yml index 355fdd084..394598ee3 100644 --- a/deploy/compose/role-master.yml +++ b/deploy/compose/role-master.yml @@ -17,7 +17,6 @@ services: required: true environment: BASE_STATE_DIR: /var/lib/base - BASE_OWNER_PUBKEY_FILE: /etc/base/config/owner.pubkey BASE_CHALLENGES_FILE: /etc/base/config/challenges.toml BASE_MEASUREMENTS_FILE: /etc/base/config/measurements.toml BASE_GATEWAY_SK_FILE: /run/secrets/gateway.key @@ -30,7 +29,7 @@ services: - "${BASE_GATEWAY_BIND_ADDRESS:?Set the master VPC address}:8080:8080" volumes: - type: bind - source: ${BASE_TRUST_ROOT_DIR:-config} + source: ${BASE_CONFIG_DIR:-config} target: /etc/base/config read_only: true bind: diff --git a/deploy/compose/role-validator.yml b/deploy/compose/role-validator.yml index 914bbbfa8..9b506c88f 100644 --- a/deploy/compose/role-validator.yml +++ b/deploy/compose/role-validator.yml @@ -1,4 +1,5 @@ -# Standalone validator role. Never start a second submitter with the same wallet. +# Verify-only validator: checks the gateway's sealed weights, never submits, holds no hotkey. +# The weight-setting validator is role-weight-setter.yml. name: cortex-validator services: validator: @@ -9,32 +10,12 @@ services: - ${BASE_GATEWAY_ENDPOINT:?Set the master gateway HTTPS URL} - --netuid - ${BASE_NETUID:?Set the Bittensor netuid} - - --gateway-public - - ${BASE_GATEWAY_HOTKEY:?Set the independently pinned gateway public key} - --network - ${BASE_CHAIN_ENDPOINT:-finney} - --fallback-endpoints - ${BASE_CHAIN_FALLBACK_ENDPOINTS:-[]} - - --owner-public - - /etc/base/config/owner.pubkey - - --challenges - - /etc/base/config/challenges.toml - - --measurements - - /etc/base/config/measurements.toml - - --minimum-challenges-version - - ${BASE_CHALLENGES_MIN_VERSION:?Set the minimum signed challenge version} - - --minimum-measurements-version - - ${BASE_MEASUREMENTS_MIN_VERSION:?Set the minimum signed measurement version} - - --wallet-name - - ${BASE_WALLET_NAME:?Set the validator wallet name} - - --wallet-hotkey - - ${BASE_WALLET_HOTKEY:?Set the validator wallet hotkey name} - - --wallet-path - - /run/wallets - --state-db - /var/lib/base/validator.sqlite3 - - --consensus-seed-file - - /run/validator/consensus.key - --peers - /etc/base/config/peers.json - --peer-bind @@ -53,6 +34,7 @@ services: - ${BASE_MIN_PEER_SAMPLE:-1} - --max-block-lag - ${BASE_MAX_BLOCK_LAG:-256} + - --verify-only restart: unless-stopped init: true read_only: true @@ -64,17 +46,11 @@ services: - "${BASE_VALIDATOR_BIND_ADDRESS:?Set the validator VPC address}:8091:8091" volumes: - type: bind - source: ${BASE_TRUST_ROOT_DIR:-config} + source: ${BASE_CONFIG_DIR:-config} target: /etc/base/config read_only: true bind: create_host_path: false - - type: bind - source: ${BASE_VALIDATOR_WALLETS_DIR:-deploy/secrets/wallets} - target: /run/wallets - read_only: true - bind: - create_host_path: false - type: bind source: ${BASE_VALIDATOR_IDENTITY_DIR:?Set private validator identity directory} target: /run/validator diff --git a/deploy/compose/role-weight-setter.yml b/deploy/compose/role-weight-setter.yml new file mode 100644 index 000000000..72ca9d1ea --- /dev/null +++ b/deploy/compose/role-weight-setter.yml @@ -0,0 +1,84 @@ +# Weight-setting validator: the only role that holds a validator hotkey and submits weights. +# Never start a second submitter with the same wallet. Use role-validator.yml (verify-only) +# for everyone else. +name: cortex-weight-setter +services: + validator: + image: ${CORTEX_IMAGE:?Set the published image repository@sha256:digest} + command: + - validator + - --gateway + - ${BASE_GATEWAY_ENDPOINT:?Set the master gateway HTTPS URL} + - --netuid + - ${BASE_NETUID:?Set the Bittensor netuid} + - --network + - ${BASE_CHAIN_ENDPOINT:-finney} + - --fallback-endpoints + - ${BASE_CHAIN_FALLBACK_ENDPOINTS:-[]} + - --wallet-name + - ${BASE_WALLET_NAME:?Set the validator wallet name} + - --wallet-hotkey + - ${BASE_WALLET_HOTKEY:?Set the validator wallet hotkey name} + - --wallet-path + - /run/wallets + - --state-db + - /var/lib/base/validator.sqlite3 + - --consensus-seed-file + - /run/validator/consensus.key + - --peers + - /etc/base/config/peers.json + - --peer-bind + - 0.0.0.0 + - --peer-port + - "8091" + - --peer-tls-certificate + - /run/validator/tls.crt + - --peer-tls-key + - /run/validator/tls.key + - --poll-seconds + - ${BASE_COORDINATION_INTERVAL_SECS:-30} + - --version-key + - ${BASE_VERSION_KEY:?Set the subnet validator version key} + - --min-peer-sample + - ${BASE_MIN_PEER_SAMPLE:-1} + - --max-block-lag + - ${BASE_MAX_BLOCK_LAG:-256} + restart: unless-stopped + init: true + read_only: true + user: "65532:65532" + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + stop_grace_period: 2m + ports: + - "${BASE_VALIDATOR_BIND_ADDRESS:?Set the validator VPC address}:8091:8091" + volumes: + - type: bind + source: ${BASE_CONFIG_DIR:-config} + target: /etc/base/config + read_only: true + bind: + create_host_path: false + - type: bind + source: ${BASE_VALIDATOR_WALLETS_DIR:-deploy/secrets/wallets} + target: /run/wallets + read_only: true + bind: + create_host_path: false + - type: bind + source: ${BASE_VALIDATOR_IDENTITY_DIR:?Set private validator identity directory} + target: /run/validator + read_only: true + bind: + create_host_path: false + - base-weight-setter-state:/var/lib/base + tmpfs: + - /tmp:size=32m,mode=1777 + healthcheck: + test: [CMD, python, -I, -c, "import ssl,urllib.request; urllib.request.urlopen('https://127.0.0.1:8091/livez', context=ssl._create_unverified_context(), timeout=5).read()"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 90s +volumes: + base-weight-setter-state: diff --git a/deploy/env/master.env.example b/deploy/env/master.env.example index f35aee39b..9a8c21268 100644 --- a/deploy/env/master.env.example +++ b/deploy/env/master.env.example @@ -14,8 +14,6 @@ BASE_CHAIN_FALLBACK_ENDPOINTS=[] BASE_EMIT_POLL_SECS=120 BASE_EPOCH_REFRESH_SECS=12 BASE_EPOCH_STALE_SECS=60 -BASE_CHALLENGES_MIN_VERSION=1 -BASE_MEASUREMENTS_MIN_VERSION=1 # Dedicated KVM host on the VPC. Empty URL keeps Proof unwired. PROOF_VM_ORCHESTRATOR_URL= PROOF_VM_ORCHESTRATOR_TOKEN_FILE=/run/secrets/proof-vm.token diff --git a/deploy/env/python-validator.env.example b/deploy/env/python-validator.env.example index 729918d20..2154ac302 100644 --- a/deploy/env/python-validator.env.example +++ b/deploy/env/python-validator.env.example @@ -4,19 +4,18 @@ BASE_GATEWAY_ENDPOINT= BASE_NETUID=541 BASE_CHAIN_ENDPOINT=test BASE_CHAIN_FALLBACK_ENDPOINTS=[] -BASE_GATEWAY_HOTKEY= +# Validator hotkey: only the weight-setter role (role-weight-setter.yml) uses it. The +# verify-only validator role needs no wallet. BASE_WALLET_NAME= BASE_WALLET_HOTKEY= BASE_COORDINATION_INTERVAL_SECS=30 -BASE_CHALLENGES_MIN_VERSION= -BASE_MEASUREMENTS_MIN_VERSION= # Subnet-specific validator version key. Obtain it from the live subnet; do not guess. BASE_VERSION_KEY= BASE_MIN_PEER_SAMPLE=1 BASE_MAX_BLOCK_LAG=256 # Optional absolute overrides; blank values use config and deploy/secrets/wallets. BASE_VALIDATOR_WALLETS_DIR= -BASE_TRUST_ROOT_DIR= +BASE_CONFIG_DIR= # Same hotkey as the wallet; directory holds consensus.key, tls.crt, tls.key. # Trust directory also contains peers.json: {"": "https://host:8091"}. BASE_VALIDATOR_IDENTITY_DIR= diff --git a/scripts/check_deploy.py b/scripts/check_deploy.py index 8d0587175..33ced6f1e 100644 --- a/scripts/check_deploy.py +++ b/scripts/check_deploy.py @@ -48,7 +48,6 @@ MASTER_ENVIRONMENT = { "BASE_STATE_DIR": "/var/lib/base", - "BASE_OWNER_PUBKEY_FILE": "/etc/base/config/owner.pubkey", "BASE_CHALLENGES_FILE": "/etc/base/config/challenges.toml", "BASE_MEASUREMENTS_FILE": "/etc/base/config/measurements.toml", "BASE_GATEWAY_SK_FILE": "/run/secrets/gateway.key", @@ -64,19 +63,9 @@ VALIDATOR_OPTIONS = { "--gateway", "--netuid", - "--gateway-public", "--network", "--fallback-endpoints", - "--owner-public", - "--challenges", - "--measurements", - "--minimum-challenges-version", - "--minimum-measurements-version", - "--wallet-name", - "--wallet-hotkey", - "--wallet-path", "--state-db", - "--consensus-seed-file", "--peers", "--peer-bind", "--peer-port", @@ -87,6 +76,14 @@ "--min-peer-sample", "--max-block-lag", } +# Only the role that submits weights on-chain holds a validator hotkey (and a consensus seed). +SETTER_OPTIONS = { + "--wallet-name", + "--wallet-hotkey", + "--wallet-path", + "--consensus-seed-file", +} +ROLES = {"master": "gateway", "validator": "validator", "weight-setter": "validator"} def validate_pin(image: str, *, base: bool = False) -> None: @@ -139,18 +136,27 @@ def validate_vm_resource_environment(environment: dict, *, required: bool = True _integer(int(value), minimum, maximum, f"VM resource {name}") -def _compose_options(command) -> dict[str, str]: +def _compose_options(command, role: str = "validator") -> dict[str, str]: if not isinstance(command, list) or not command or command[0] != "validator": raise ValueError("validator must invoke the Python validator entrypoint") - if len(command) % 2 != 1 or not all(isinstance(item, str) for item in command): + if not all(isinstance(item, str) for item in command): + raise ValueError("validator command must contain flag/value pairs") + arguments = command[1:] + # The verify-only role never submits: it is the one place a validator holds no hotkey. + verify_only = arguments[-1:] == ["--verify-only"] + if verify_only: + arguments = arguments[:-1] + if verify_only != (role == "validator"): + raise ValueError("only the validator role is --verify-only; weight-setter submits") + if len(arguments) % 2: raise ValueError("validator command must contain flag/value pairs") options: dict[str, str] = {} - for index in range(1, len(command), 2): - flag, value = command[index], command[index + 1] + for flag, value in zip(arguments[::2], arguments[1::2], strict=True): if not flag.startswith("--") or not value or flag in options: raise ValueError("validator command contains an invalid or duplicate flag") options[flag] = value - if set(options) != VALIDATOR_OPTIONS: + expected = VALIDATOR_OPTIONS | (SETTER_OPTIONS if role == "weight-setter" else set()) + if not expected <= set(options) <= expected | {"--gateway-public"}: raise ValueError("validator command differs from the audited Python runtime contract") return options @@ -340,7 +346,7 @@ def _validate_common_service(service: dict) -> dict[str, dict]: def validate_compose(config: dict, role: str) -> None: - expected = "gateway" if role == "master" else "validator" + expected = ROLES[role] services = _mapping(config.get("services"), "services") allowed = {expected, "challenge-supervisor"} if role == "master" else {expected} if expected not in services or set(services) - allowed: @@ -395,11 +401,15 @@ def validate_compose(config: dict, role: str) -> None: raise ValueError("validator must not require the master profile") if service.get("environment"): raise ValueError("validator configuration must be explicit command arguments") - if not {"/etc/base/config", "/run/wallets", "/run/validator", "/var/lib/base"} <= set(mounts): - raise ValueError("validator trust, wallet, identity and state mounts are required") + setter = role == "weight-setter" + required_mounts = {"/etc/base/config", "/run/validator", "/var/lib/base"} + if not required_mounts | ({"/run/wallets"} if setter else set()) <= set(mounts): + raise ValueError("validator config, identity and state mounts are required") if "/run/secrets" in mounts: raise ValueError("validator must not hold challenge signing keys") - options = _compose_options(command) + if not setter and "/run/wallets" in mounts: + raise ValueError("only the weight-setter role may mount a validator wallet") + options = _compose_options(command, role) _chain_endpoint(options["--network"], "validator chain endpoint") _wss_endpoints(options["--fallback-endpoints"], "validator fallback endpoints") gateway = urlsplit(options["--gateway"]) @@ -416,8 +426,6 @@ def validate_compose(config: dict, role: str) -> None: try: netuid = int(options["--netuid"]) poll = float(options["--poll-seconds"]) - challenge_version = int(options["--minimum-challenges-version"]) - measurement_version = int(options["--minimum-measurements-version"]) version_key = int(options["--version-key"]) min_peer_sample = int(options["--min-peer-sample"]) max_block_lag = int(options["--max-block-lag"]) @@ -427,33 +435,28 @@ def validate_compose(config: dict, role: str) -> None: not 0 <= netuid <= 65535 or not math.isfinite(poll) or poll <= 0 - or challenge_version < 1 - or measurement_version < 1 or not 0 <= version_key <= 2**64 - 1 or not 0 <= min_peer_sample <= 64 or max_block_lag < 1 ): raise ValueError("validator numeric options are invalid") fixed = { - "--owner-public": "/etc/base/config/owner.pubkey", - "--challenges": "/etc/base/config/challenges.toml", - "--measurements": "/etc/base/config/measurements.toml", - "--wallet-path": "/run/wallets", "--state-db": "/var/lib/base/validator.sqlite3", - "--consensus-seed-file": "/run/validator/consensus.key", "--peers": "/etc/base/config/peers.json", "--peer-bind": "0.0.0.0", "--peer-port": "8091", "--peer-tls-certificate": "/run/validator/tls.crt", "--peer-tls-key": "/run/validator/tls.key", } + if setter: + fixed |= { + "--wallet-path": "/run/wallets", + "--consensus-seed-file": "/run/validator/consensus.key", + } if any(options[key] != value for key, value in fixed.items()): raise ValueError("validator paths or peer listener differ from the runtime contract") - if not all( - options[name] - for name in ("--gateway-public", "--network", "--wallet-name", "--wallet-hotkey") - ): - raise ValueError("validator identity and network pins are required") + if setter and not (options["--wallet-name"] and options["--wallet-hotkey"]): + raise ValueError("the weight-setter needs its validator wallet and hotkey names") health = service.get("healthcheck", {}).get("test", []) if health != VALIDATOR_HEALTHCHECK: raise ValueError("validator healthcheck must use its Python liveness endpoint") @@ -855,8 +858,6 @@ def validate_env_examples(master_source: str, validator_source: str) -> None: "BASE_EMIT_POLL_SECS", "BASE_EPOCH_REFRESH_SECS", "BASE_EPOCH_STALE_SECS", - "BASE_CHALLENGES_MIN_VERSION", - "BASE_MEASUREMENTS_MIN_VERSION", "PROOF_VM_ORCHESTRATOR_URL", "PROOF_VM_ORCHESTRATOR_TOKEN_FILE", "PROOF_VM_ORCHESTRATOR_CA_FILE", @@ -888,8 +889,6 @@ def validate_env_examples(master_source: str, validator_source: str) -> None: emit = float(master["BASE_EMIT_POLL_SECS"]) refresh = float(master["BASE_EPOCH_REFRESH_SECS"]) stale = float(master["BASE_EPOCH_STALE_SECS"]) - challenge_version = int(master["BASE_CHALLENGES_MIN_VERSION"]) - measurement_version = int(master["BASE_MEASUREMENTS_MIN_VERSION"]) except ValueError: raise ValueError("master example contains invalid numeric settings") from None if ( @@ -897,8 +896,6 @@ def validate_env_examples(master_source: str, validator_source: str) -> None: or not all(math.isfinite(item) for item in (emit, refresh, stale)) or min(emit, refresh) <= 0 or stale <= refresh - or challenge_version < 1 - or measurement_version < 1 ): raise ValueError("master example contains unsafe timing or netuid settings") _wss_endpoints(master["BASE_CHAIN_FALLBACK_ENDPOINTS"], "master fallback endpoints") @@ -911,17 +908,14 @@ def validate_env_examples(master_source: str, validator_source: str) -> None: "BASE_NETUID", "BASE_CHAIN_ENDPOINT", "BASE_CHAIN_FALLBACK_ENDPOINTS", - "BASE_GATEWAY_HOTKEY", "BASE_WALLET_NAME", "BASE_WALLET_HOTKEY", "BASE_COORDINATION_INTERVAL_SECS", - "BASE_CHALLENGES_MIN_VERSION", - "BASE_MEASUREMENTS_MIN_VERSION", "BASE_VERSION_KEY", "BASE_MIN_PEER_SAMPLE", "BASE_MAX_BLOCK_LAG", "BASE_VALIDATOR_WALLETS_DIR", - "BASE_TRUST_ROOT_DIR", + "BASE_CONFIG_DIR", "BASE_VALIDATOR_IDENTITY_DIR", "BASE_VALIDATOR_BIND_ADDRESS", } @@ -966,7 +960,7 @@ def _bind_sources(role: str, compose_dir: Path = ROOT / "deploy/compose") -> Non def _default_sources(config: dict, role: str) -> None: - service_name = "gateway" if role == "master" else "validator" + service_name = ROLES[role] service = config["services"][service_name] mounts = {item["target"]: Path(item["source"]) for item in service["volumes"]} expected = {"/etc/base/config": ROOT / "config"} @@ -985,7 +979,7 @@ def _default_sources(config: dict, role: str) -> None: ] if resolved not in ([], [(ROOT / "deploy/env/master.env").resolve()]): raise ValueError("master default env file resolves outside the repository deploy tree") - else: + elif role == "weight-setter": expected["/run/wallets"] = ROOT / "deploy/secrets/wallets" for target, source in expected.items(): if mounts.get(target) != source: @@ -1007,7 +1001,7 @@ def check_examples() -> None: removed = { "BASE_MASTER_ENV_FILE", "BASE_MASTER_SECRETS_DIR", - "BASE_TRUST_ROOT_DIR", + "BASE_CONFIG_DIR", "BASE_VALIDATOR_WALLETS_DIR", "BASE_CHALLENGE_REGISTRY_DIR", } @@ -1021,11 +1015,8 @@ def check_examples() -> None: "BASE_GATEWAY_ENDPOINT": "https://master.fixture.invalid", "BASE_NETUID": "541", "BASE_CHAIN_ENDPOINT": "test", - "BASE_GATEWAY_HOTKEY": "e" * 64, "BASE_WALLET_NAME": "fixture", "BASE_WALLET_HOTKEY": "validator", - "BASE_CHALLENGES_MIN_VERSION": "1", - "BASE_MEASUREMENTS_MIN_VERSION": "1", "BASE_VERSION_KEY": "1", "BASE_VALIDATOR_BIND_ADDRESS": "127.0.0.2", "BASE_VALIDATOR_IDENTITY_DIR": "/fixture/identity", @@ -1048,7 +1039,7 @@ def check_examples() -> None: def _render_examples(env: dict[str, str]) -> None: - for role in ("master", "validator"): + for role in ("master", "validator", "weight-setter"): result = subprocess.run( [ "docker", @@ -1082,7 +1073,7 @@ def main() -> None: modes.add_argument("--check-examples", action="store_true") modes.add_argument("--config", type=Path, help="rendered Compose JSON; use - for stdin") modes.add_argument("--vm-host-config", type=Path, help="materialized Proof host TOML") - parser.add_argument("--role", choices=["master", "validator"]) + parser.add_argument("--role", choices=list(ROLES)) args = parser.parse_args() try: if args.base_image: diff --git a/tests/test_deploy_contract.py b/tests/test_deploy_contract.py index 055bad0b8..1d8719b63 100644 --- a/tests/test_deploy_contract.py +++ b/tests/test_deploy_contract.py @@ -28,6 +28,7 @@ def test_deployment_rejects_floating_or_malformed_image_pins(image): def validator_config(): + """The verify-only validator role: no wallet, no hotkey, no consensus seed.""" return { "services": { "validator": { @@ -55,32 +56,12 @@ def validator_config(): "https://master.fixture.invalid", "--netuid", "541", - "--gateway-public", - "e" * 64, "--network", "test", "--fallback-endpoints", "[]", - "--owner-public", - "/etc/base/config/owner.pubkey", - "--challenges", - "/etc/base/config/challenges.toml", - "--measurements", - "/etc/base/config/measurements.toml", - "--minimum-challenges-version", - "1", - "--minimum-measurements-version", - "1", - "--wallet-name", - "fixture", - "--wallet-hotkey", - "validator", - "--wallet-path", - "/run/wallets", "--state-db", "/var/lib/base/validator.sqlite3", - "--consensus-seed-file", - "/run/validator/consensus.key", "--peers", "/etc/base/config/peers.json", "--peer-bind", @@ -99,6 +80,7 @@ def validator_config(): "1", "--max-block-lag", "256", + "--verify-only", ], "ports": [{"target": 8091, "host_ip": "10.0.0.2"}], "tmpfs": ["/tmp:size=32m,mode=1777"], @@ -109,12 +91,6 @@ def validator_config(): "target": "/etc/base/config", "read_only": True, }, - { - "type": "bind", - "source": "/fixture/wallets", - "target": "/run/wallets", - "read_only": True, - }, { "type": "bind", "source": "/fixture/validator", @@ -128,6 +104,45 @@ def validator_config(): } +def weight_setter_config(): + config = deepcopy(validator_config()) + service = config["services"]["validator"] + service["command"].remove("--verify-only") + service["command"] += [ + "--wallet-name", + "fixture", + "--wallet-hotkey", + "validator", + "--wallet-path", + "/run/wallets", + "--consensus-seed-file", + "/run/validator/consensus.key", + ] + service["volumes"].append( + {"type": "bind", "source": "/fixture/wallets", "target": "/run/wallets", "read_only": True} + ) + return config + + +def test_weight_setter_is_the_only_role_with_a_hotkey(): + CHECK["validate_compose"](weight_setter_config(), "weight-setter") + with pytest.raises(ValueError, match="verify-only|wallet"): + CHECK["validate_compose"](weight_setter_config(), "validator") + with pytest.raises(ValueError): + CHECK["validate_compose"](validator_config(), "weight-setter") + leaked = validator_config() + leaked["services"]["validator"]["volumes"].append( + {"type": "bind", "source": "/w", "target": "/run/wallets", "read_only": True} + ) + with pytest.raises(ValueError, match="only the weight-setter"): + CHECK["validate_compose"](leaked, "validator") + nameless = weight_setter_config() + command = nameless["services"]["validator"]["command"] + command[command.index("--wallet-hotkey") + 1] = "" + with pytest.raises(ValueError): + CHECK["validate_compose"](nameless, "weight-setter") + + def master_config(): return { "services": { @@ -143,7 +158,6 @@ def master_config(): "command": ["master", "--bind", "0.0.0.0", "--port", "8080"], "environment": { "BASE_STATE_DIR": "/var/lib/base", - "BASE_OWNER_PUBKEY_FILE": "/etc/base/config/owner.pubkey", "BASE_CHALLENGES_FILE": "/etc/base/config/challenges.toml", "BASE_MEASUREMENTS_FILE": "/etc/base/config/measurements.toml", "BASE_GATEWAY_SK_FILE": "/run/secrets/gateway.key", @@ -343,8 +357,6 @@ def test_docker_control_stays_confined_to_the_secretless_supervisor(mutation): @pytest.mark.parametrize( ("flag", "value"), [ - ("--minimum-challenges-version", "0"), - ("--minimum-measurements-version", "0"), ("--version-key", "-1"), ("--min-peer-sample", "-1"), ("--max-block-lag", "0"), diff --git a/tests/test_run_validator_script.py b/tests/test_run_validator_script.py index 7d9750382..8db6d72a1 100644 --- a/tests/test_run_validator_script.py +++ b/tests/test_run_validator_script.py @@ -68,6 +68,7 @@ def test_submitting_without_a_hotkey_fails_but_verify_only_does_not(env): assert verify.returncode == 0, verify.stderr assert "--wallet" not in verify.stdout + def test_netuid_mismatch_fails(env): result = run(env | {"NETUID": "541"}, "--dry-run") assert result.returncode != 0 diff --git a/tests/validator/test_main.py b/tests/validator/test_main.py index 700e4979c..27a06ae66 100644 --- a/tests/validator/test_main.py +++ b/tests/validator/test_main.py @@ -147,17 +147,21 @@ def without(arguments, flag): index = arguments.index(flag) return arguments[:index] + arguments[index + 2 :] + def without_hotkey(tmp_path): return without(without(required_arguments(tmp_path), "--wallet-name"), "--wallet-hotkey") + def test_weight_setting_validator_without_hotkey_fails_with_a_clear_error(tmp_path): with pytest.raises(SystemExit, match="required to submit weights"): validator_main.main(without_hotkey(tmp_path)) + def test_peer_consensus_without_hotkey_fails_even_when_verify_only(tmp_path): with pytest.raises(SystemExit, match="required to submit weights or run --peer-consensus"): validator_main.main([*without_hotkey(tmp_path), "--verify-only", "--peer-consensus"]) + def test_verify_only_validator_starts_without_hotkey_gateway_pin_or_trust_files( monkeypatch, tmp_path ): @@ -186,11 +190,18 @@ async def verified(arguments, subtensor, wallet): validator_main.main([*arguments, "--verify-only", "--once"]) assert seen == {"wallet": None} + def test_removed_trust_signing_options_are_gone(tmp_path): - for flag in ("--owner-public", "--challenges", "--measurements", "--minimum-challenges-version"): + for flag in ( + "--owner-public", + "--challenges", + "--measurements", + "--minimum-challenges-version", + ): with pytest.raises(SystemExit): validator_main.parser().parse_args([*required_arguments(tmp_path), flag, "x"]) + @pytest.mark.parametrize( ("flag", "value", "message"), [ From 1e46373822e00387fbf04bd44334869cf8e6eb0a Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:06:48 +0000 Subject: [PATCH 03/56] feat(cli): keep keygen for challenge leaf keys The trust ceremony is gone but challenge leaf keys (e.g. hypertrain) are still generated with cortex keygen: raw seed 0600, public key 0644, never overwrites. Co-Authored-By: Claude Sonnet 5.5 --- src/cortex/cli.py | 14 ++++++++++++++ src/cortex/keygen.py | 32 ++++++++++++++++++++++++++++++++ tests/test_keygen.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 src/cortex/keygen.py create mode 100644 tests/test_keygen.py diff --git a/src/cortex/cli.py b/src/cortex/cli.py index b15441477..30673e084 100644 --- a/src/cortex/cli.py +++ b/src/cortex/cli.py @@ -40,6 +40,11 @@ def parser() -> argparse.ArgumentParser: reconcile.add_argument("--attempt-id", required=True) reconcile.add_argument("--result", choices=("submitted", "not_broadcast"), required=True) reconcile.add_argument("--evidence-digest", required=True) + keygen = commands.add_parser( + "keygen", help="generate an sr25519 seed and public key for a challenge leaf key" + ) + keygen.add_argument("--seed-out", type=Path, required=True) + keygen.add_argument("--public-out", type=Path, required=True) vm = commands.add_parser("vm-host", help="serve the dedicated Firecracker host") vm.add_argument("--config", type=Path, required=True) vm.add_argument( @@ -212,6 +217,13 @@ async def run_master(args: argparse.Namespace) -> None: await asyncio.to_thread(close_subtensor, subtensor) +def run_keygen(args: argparse.Namespace) -> None: + from cortex.keygen import generate_key + + public = generate_key(args.seed_out, args.public_out) + print(json.dumps({"public_key": public, "seed_file": str(args.seed_out)})) + + def run_validator_reconcile(args: argparse.Namespace) -> None: from cortex.validator import SubmissionJournal @@ -257,6 +269,8 @@ def main(argv: list[str] | None = None) -> None: from cortex.vm.server import serve serve(args.config) + elif args.command == "keygen": + run_keygen(args) elif args.command == "validator-reconcile": run_validator_reconcile(args) else: diff --git a/src/cortex/keygen.py b/src/cortex/keygen.py new file mode 100644 index 000000000..bb12d1240 --- /dev/null +++ b/src/cortex/keygen.py @@ -0,0 +1,32 @@ +"""Generate a raw sr25519 seed and its public key for a challenge leaf or topic key.""" + +import os +import secrets +from pathlib import Path + +from cortex.protocol.crypto import public_key + + +def _exclusive_write(path: Path, value: bytes, mode: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode) + try: + with os.fdopen(descriptor, "wb", closefd=False) as stream: + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + finally: + os.close(descriptor) + + +def generate_key(seed_path: Path, public_path: Path) -> str: + """Create a new raw seed (0600) and its hexadecimal public key (0644), never overwriting.""" + seed = secrets.token_bytes(32) + encoded_public = public_key(seed).hex() + _exclusive_write(seed_path, seed, 0o600) + try: + _exclusive_write(public_path, (encoded_public + "\n").encode(), 0o644) + except BaseException: + seed_path.unlink(missing_ok=True) + raise + return encoded_public diff --git a/tests/test_keygen.py b/tests/test_keygen.py new file mode 100644 index 000000000..05a1520ef --- /dev/null +++ b/tests/test_keygen.py @@ -0,0 +1,28 @@ +import stat + +import pytest + +from cortex.config import read_seed +from cortex.keygen import generate_key +from cortex.protocol.crypto import public_key + + +def test_generates_private_seed_matching_public_key(tmp_path): + seed, public = tmp_path / "leaf.seed", tmp_path / "leaf.pubkey" + encoded = generate_key(seed, public) + assert stat.S_IMODE(seed.stat().st_mode) == 0o600 + assert public.read_text().strip() == encoded == public_key(read_seed(seed)).hex() + + +def test_never_overwrites_and_leaves_no_orphan_seed(tmp_path): + seed, public = tmp_path / "leaf.seed", tmp_path / "leaf.pubkey" + generate_key(seed, public) + original = seed.read_bytes() + with pytest.raises(FileExistsError): + generate_key(seed, tmp_path / "other.pubkey") + assert seed.read_bytes() == original + public.write_text("x") + other = tmp_path / "second.seed" + with pytest.raises(FileExistsError): + generate_key(other, public) + assert not other.exists() From e2f6cd796e00a463823056aba8a622a3f1364746 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:11:24 +0000 Subject: [PATCH 04/56] refactor(profile): rename trust root to challenge profile in code, checks and config text MasterConfig.trust_root -> challenge_profile, check_trust_roots -> check_challenge_profiles, user-facing error strings and config comments. BUNDLE_SPEC hash pin updated for the provenance amendment. Co-Authored-By: Claude Sonnet 5.5 --- .coderabbit.yaml | 4 ++-- config/challenges-v2.example.toml | 6 ++--- config/challenges-v3.example.toml | 6 ++--- config/challenges.toml | 5 ++-- config/measurements.toml | 5 ++-- deploy/challenges/registry.toml | 2 +- deploy/secrets/README.md | 2 +- scripts/check_deploy.py | 2 +- scripts/check_repo.py | 10 ++++---- src/cortex/challenges/registry.py | 2 +- src/cortex/config.py | 4 ++-- src/cortex/master.py | 4 ++-- src/cortex/protocol/bundle.py | 4 ++-- tests/test_repo_contract.py | 38 ++++++++++++++++--------------- 14 files changed, 49 insertions(+), 45 deletions(-) diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 42a64f916..dfa089371 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -13,7 +13,7 @@ reviews: - path: "**" instructions: | Cortex review rules. - Cortex is a Python Bittensor research subnet. The owner-signed trust root lists + Cortex is a Python Bittensor research subnet. The challenge profile (`config/challenges.toml`, served by the gateway at `GET /v1/trust`) lists the live challenges: legacy bounty/proof 2,000/8,000 (algorithm 1), 3,000/7,000 (algorithm 2, version >=2) or 1..64 unique ids (algorithm 3, version >=3). The sum is always 10,000. Proof is built in; every other challenge is a Docker @@ -27,7 +27,7 @@ reviews: Algorithm 3 pays each challenge share * min(sum(leaves), 10^12) / 10^12 and burns the rest; unpaid mass never moves to another challenge. Never allow an algorithm 1 body under the new profile or activate before - the owner-signed epoch; historical seals and journals stay immutable. + the profile's activation epoch; historical seals and journals stay immutable. - Preserve existing `BASE_*` names and deployed compatibility paths. New master settings may add the matching `CORTEX_*` alias but conflicting values fail. - Missing or unknown image digests, offers, baselines, topics, keys, feeds and diff --git a/config/challenges-v2.example.toml b/config/challenges-v2.example.toml index 1ba940d7a..66d86477e 100644 --- a/config/challenges-v2.example.toml +++ b/config/challenges-v2.example.toml @@ -1,6 +1,6 @@ -# UNSIGNED activation template. Replace development public keys, choose the -# next owner document version and activation epoch, then sign offline using -# docs/how-to/trust-root.md. No production activation occurs from this file. +# Activation template. Replace development public keys, choose the next profile +# version and activation epoch, then install it as config/challenges.toml and +# restart the master. No production activation occurs from this file. version = 2 introduced_epoch = "CHOOSE_ACTIVATION_EPOCH" diff --git a/config/challenges-v3.example.toml b/config/challenges-v3.example.toml index f7b6ccbb2..cf311f7d7 100644 --- a/config/challenges-v3.example.toml +++ b/config/challenges-v3.example.toml @@ -1,6 +1,6 @@ -# UNSIGNED algorithm 3 template (document version >= 3). Choose the shares, the -# next owner document version and the activation epoch, replace development -# public keys, then sign offline using docs/how-to/trust-root.md. +# Algorithm 3 template (profile version >= 3). Choose the shares, the next profile +# version and the activation epoch, replace development public keys, then install +# it as config/challenges.toml and restart the master. # Each challenge pays share * min(sum(leaves), 10^12) / 10^12; the rest burns. # Container ids also need a row in deploy/challenges/registry.toml to run. version = 3 diff --git a/config/challenges.toml b/config/challenges.toml index 0e2d9ca17..f0de33649 100644 --- a/config/challenges.toml +++ b/config/challenges.toml @@ -1,5 +1,6 @@ -# Owner-signed development trust root. Production uses an offline owner ceremony; -# see docs/how-to/trust-root.md. Only semantic fields enter the signed preimage. +# Development challenge profile (emission shares and challenge keys). The master reads +# this unsigned file; the gateway serves it at GET /v1/trust and validators build their +# view from that. To change shares: edit, bump `version`, restart the master. # # Two live challenges. Emission: bounty = 2000, proof = 8000 bps (sum 10000). # Readiness is independent of these shares: a missing feed, topic, baseline, diff --git a/config/measurements.toml b/config/measurements.toml index 7535c0d73..5eb5c2223 100644 --- a/config/measurements.toml +++ b/config/measurements.toml @@ -1,6 +1,7 @@ -# Owner-signed measurement allowlist retained by the frozen bundle contract. +# Measurement allowlist retained by the frozen bundle contract. Served with the +# challenge profile at GET /v1/trust. # The Python Firecracker path does not authorize a legacy CVM measurement, so -# the list is intentionally empty. Additions require the offline owner ceremony. +# the list is intentionally empty. Bump `version` when adding an entry. version = 5 introduced_epoch = 0 diff --git a/deploy/challenges/registry.toml b/deploy/challenges/registry.toml index af2d17a36..3e60cf3e7 100644 --- a/deploy/challenges/registry.toml +++ b/deploy/challenges/registry.toml @@ -1,5 +1,5 @@ # Unsigned operator registry: which challenge containers the master runs. -# Emission is decided only by the owner-signed trust root (config/challenges.toml); +# Emission is decided only by the challenge profile (config/challenges.toml); # a registered id absent from it runs without emission. See docs/CHALLENGES.md. # The supervisor re-reads this file every tick and the master on every change. version = 1 diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index 45f3750bf..db7707299 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -12,7 +12,7 @@ regular files, owned/readable by that identity, with mode 0400 or 0600. | --- | --- | | `gateway.key` | 32-byte sr25519 seed for bundle seals | | `.key` | leaf seed of each trusted container challenge, e.g. `bounty.key` | -| `proof.key` | Proof topic/leaf seed matching the trust root | +| `proof.key` | Proof topic/leaf seed matching the challenge profile | | `operator.token` | bearer for master administrative routes | | `proof-vm.token` | bearer shared with the dedicated VM host | | `proof-vm-ca.pem` | CA used to verify the VM-host TLS certificate | diff --git a/scripts/check_deploy.py b/scripts/check_deploy.py index 33ced6f1e..8777cbda5 100644 --- a/scripts/check_deploy.py +++ b/scripts/check_deploy.py @@ -378,7 +378,7 @@ def validate_compose(config: dict, role: str) -> None: if not {"/etc/base/config", "/run/secrets", "/var/lib/base"} <= set(mounts): raise ValueError("master trust, secret and state mounts are required") if mounts["/etc/base/config"].get("type") != "bind": - raise ValueError("master trust root must be an operator bind mount") + raise ValueError("master challenge profile must be an operator bind mount") if mounts["/run/secrets"].get("type") != "bind": raise ValueError("master credentials must be an operator bind mount") if "/run/wallets" in mounts: diff --git a/scripts/check_repo.py b/scripts/check_repo.py index 7cc7a4d23..5f88c5f73 100644 --- a/scripts/check_repo.py +++ b/scripts/check_repo.py @@ -19,7 +19,7 @@ ROOT = Path(__file__).resolve().parents[1] FROZEN_SPECS = { - "docs/BUNDLE_SPEC.md": "c7a43fca324a2f44e7c40bca5acde1bd5ea30cabde1e8c0a0011aeedbb4330a3", + "docs/BUNDLE_SPEC.md": "6059c4f8f7917a4426c5ef88fb8464ff5c98bd5dc1bc82003eb8c6ce7cb04b7e", } SHARES = {"bounty": 2000, "proof": 8000} PROPORTIONAL_SHARES = {"bounty": 3000, "proof": 7000} @@ -148,7 +148,7 @@ def check_specs(root: Path) -> list[str]: return failures -def check_trust_roots(root: Path) -> list[str]: +def check_challenge_profiles(root: Path) -> list[str]: failures = [] paths = set((root / "config").glob("challenges*.toml")) paths.add(root / "config/challenges.toml") @@ -156,12 +156,12 @@ def check_trust_roots(root: Path) -> list[str]: name = path.relative_to(root) try: if path.is_symlink(): - raise ValueError("symlinked trust root") + raise ValueError("symlinked challenge profile") document = tomllib.loads(path.read_text()) rows = document["challenges"] version = document["version"] if type(version) is not int or version < 1: - raise ValueError("invalid trust document version") + raise ValueError("invalid challenge profile version") if not isinstance(rows, list) or not 1 <= len(rows) <= (64 if version >= 3 else 2): raise ValueError("unsupported challenge count") actual = {} @@ -380,7 +380,7 @@ def check_final(root: Path, files: Iterable[Path]) -> list[str]: def check_repository(root: Path, *, final: bool = False) -> list[str]: files = repository_files(root) - failures = check_specs(root) + check_trust_roots(root) + check_public_contracts(root) + failures = check_specs(root) + check_challenge_profiles(root) + check_public_contracts(root) failures += check_hygiene(root, files) if final: failures += check_final(root, files) diff --git a/src/cortex/challenges/registry.py b/src/cortex/challenges/registry.py index b8447d189..4f9e20ca8 100644 --- a/src/cortex/challenges/registry.py +++ b/src/cortex/challenges/registry.py @@ -1,6 +1,6 @@ """Unsigned operator registry of challenge containers. -The registry decides what runs; only the owner-signed trust root decides emission. +The registry decides what runs; only the challenge profile decides emission. """ from __future__ import annotations diff --git a/src/cortex/config.py b/src/cortex/config.py index e96759413..f64c693d5 100644 --- a/src/cortex/config.py +++ b/src/cortex/config.py @@ -212,7 +212,7 @@ def challenge_seed_file(self, challenge: bytes) -> Path: return self.proof_seed_file return self.challenge_keys_dir / f"{challenge.decode()}.key" - def trust_root(self, epoch: int) -> TrustRoot: + def challenge_profile(self, epoch: int) -> TrustRoot: try: trust = load_trust( challenges_path=self.challenges_file, @@ -224,4 +224,4 @@ def trust_root(self, epoch: int) -> TrustRoot: raise ServiceError(503, "challenge signing key does not match the profile") return trust except (OSError, ValueError, KeyError): - raise ServiceError(503, "trust profile unavailable") from None + raise ServiceError(503, "challenge profile unavailable") from None diff --git a/src/cortex/master.py b/src/cortex/master.py index 65b19ec97..3caf58b63 100644 --- a/src/cortex/master.py +++ b/src/cortex/master.py @@ -564,7 +564,7 @@ async def build_master( prepare_master_state(config.state_dir) clock = EpochClock(epochs, config.netuid, stale_seconds=config.epoch_stale_seconds) state = await clock.refresh() - local_trust = trust or config.trust_root(state.epoch) + local_trust = trust or config.challenge_profile(state.epoch) read_private_file(config.operator_token_file) load_registry(config.challenge_registry_file) # fail fast on an invalid operator registry with ExitStack() as cleanup: @@ -593,7 +593,7 @@ def chain_endpoint() -> str: chain=chain, gateway_seed=lambda: read_seed(config.gateway_seed_file), chain_endpoint=chain_endpoint, - trust_loader=None if trust is not None else config.trust_root, + trust_loader=None if trust is not None else config.challenge_profile, ) proof = _EpochTrackedProof( store=proof_store, diff --git a/src/cortex/protocol/bundle.py b/src/cortex/protocol/bundle.py index 306f4cba2..4c0d68817 100644 --- a/src/cortex/protocol/bundle.py +++ b/src/cortex/protocol/bundle.py @@ -81,7 +81,7 @@ def build_bundle( ) -> Bundle: trust.validate() if epoch < trust.introduced_epoch: - raise ProtocolError("trust root is not active for bundle epoch") + raise ProtocolError("challenge profile is not active for bundle epoch") if public_key(gateway_seed) != trust.gateway_hotkey: raise ProtocolError("gateway key does not match local trust") ordered_rows = canonical_rows(rows) @@ -146,7 +146,7 @@ def verify_inputs( if body.protocol_version != 1 or body.algorithm_version != trust.algorithm_version: raise ProtocolError("unsupported bundle version") if body.epoch < trust.introduced_epoch: - raise ProtocolError("trust root is not active for bundle epoch") + raise ProtocolError("challenge profile is not active for bundle epoch") if body.netuid != netuid or (epoch is not None and body.epoch != epoch): raise ProtocolError("bundle subnet/epoch mismatch") if body.gateway_hotkey != trust.gateway_hotkey: diff --git a/tests/test_repo_contract.py b/tests/test_repo_contract.py index ebfa35386..e5d2d38de 100644 --- a/tests/test_repo_contract.py +++ b/tests/test_repo_contract.py @@ -18,7 +18,7 @@ def put(root, name, content): return target -def trust_root(proof_share=8000, third=""): +def challenge_profile(proof_share=8000, third=""): return ( 'version = 1\n[[challenges]]\nid = "bounty"\n' f'public_key = "{"a" * 64}"\nemission_share_bps = 2000\n' @@ -43,20 +43,22 @@ def test_frozen_spec_single_byte_change_is_detected(tmp_path): @pytest.mark.parametrize("mutation", ["share", "third", "duplicate", "malformed", "staging"]) -def test_trust_root_rejects_emission_drift_or_extra_products(tmp_path, mutation): - path = put(tmp_path, "config/challenges.toml", trust_root()) - assert CHECK["check_trust_roots"](tmp_path) == [] +def test_challenge_profile_rejects_emission_drift_or_extra_products(tmp_path, mutation): + path = put(tmp_path, "config/challenges.toml", challenge_profile()) + assert CHECK["check_challenge_profiles"](tmp_path) == [] if mutation == "share": - path.write_text(trust_root(proof_share=7000)) + path.write_text(challenge_profile(proof_share=7000)) elif mutation == "third": - path.write_text(trust_root(third='[[challenges]]\nid = "prism"\nemission_share_bps = 0\n')) + path.write_text( + challenge_profile(third='[[challenges]]\nid = "prism"\nemission_share_bps = 0\n') + ) elif mutation == "duplicate": - path.write_text(trust_root().replace('id = "proof"', 'id = "bounty"')) + path.write_text(challenge_profile().replace('id = "proof"', 'id = "bounty"')) elif mutation == "malformed": path.write_text('challenges = "not an array"') else: - put(tmp_path, "config/challenges.staging.toml", trust_root(proof_share=7000)) - assert len(CHECK["check_trust_roots"](tmp_path)) == 1 + put(tmp_path, "config/challenges.staging.toml", challenge_profile(proof_share=7000)) + assert len(CHECK["check_challenge_profiles"](tmp_path)) == 1 def test_version_three_accepts_any_unique_challenge_set_summing_to_full_emission(tmp_path): @@ -65,26 +67,26 @@ def test_version_three_accepts_any_unique_challenge_set_summing_to_full_emission for name, share in (("bounty", 3000), ("opentype", 5000), ("proof", 2000)) ) path = put(tmp_path, "config/challenges.toml", "version = 3\n" + rows) - assert CHECK["check_trust_roots"](tmp_path) == [] + assert CHECK["check_challenge_profiles"](tmp_path) == [] path.write_text("version = 3\n" + rows.replace("5000", "4999")) - assert len(CHECK["check_trust_roots"](tmp_path)) == 1 + assert len(CHECK["check_challenge_profiles"](tmp_path)) == 1 path.write_text("version = 3\n" + rows.replace('"opentype"', '"Open_Type"')) - assert len(CHECK["check_trust_roots"](tmp_path)) == 1 + assert len(CHECK["check_challenge_profiles"](tmp_path)) == 1 path.write_text("version = 2\n" + rows) - assert len(CHECK["check_trust_roots"](tmp_path)) == 1 + assert len(CHECK["check_challenge_profiles"](tmp_path)) == 1 @pytest.mark.parametrize("version,accepted", [(1, False), (2, True), (3, True)]) -def test_proportional_trust_template_requires_new_owner_version(tmp_path, version, accepted): - put(tmp_path, "config/challenges.toml", trust_root()) +def test_proportional_profile_template_requires_new_version(tmp_path, version, accepted): + put(tmp_path, "config/challenges.toml", challenge_profile()) put( tmp_path, "config/challenges-v2.example.toml", - trust_root(proof_share=7000) + challenge_profile(proof_share=7000) .replace("2000", "3000") .replace("version = 1", f"version = {version}"), ) - assert (CHECK["check_trust_roots"](tmp_path) == []) is accepted + assert (CHECK["check_challenge_profiles"](tmp_path) == []) is accepted @pytest.mark.parametrize("method", ["get", "post"]) @@ -247,7 +249,7 @@ def test_default_gate_tolerates_rust_until_final_flag_is_enabled(tmp_path): target = tmp_path / name target.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(ROOT / name, target) - put(tmp_path, "config/challenges.toml", trust_root()) + put(tmp_path, "config/challenges.toml", challenge_profile()) put(tmp_path, "Cargo.toml", '[package]\nname="legacy"') assert CHECK["check_repository"](tmp_path) == [] From cccfa5ace586db6731f958cff19716d2b2e5564f Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:11:24 +0000 Subject: [PATCH 05/56] docs: describe the gateway-served challenge profile; drop the trust-root ceremony Add GET /v1/trust, verify-only vs weight-setter roles, Changing emission shares, and the CHANGELOG entry. BUNDLE_SPEC wire format is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- AGENTS.md | 15 +++--- CHANGELOG.md | 17 ++++++ README.md | 6 +-- deploy/README.md | 59 +++++++++++++------- docs/ARCHITECTURE.md | 5 +- docs/BUNDLE_SPEC.md | 36 ++++++++----- docs/BUNDLE_SPEC_CHECKLIST.md | 2 +- docs/CHALLENGES.md | 6 +-- docs/NAMING.md | 5 +- docs/OPERATOR_SECURITY.md | 13 ++--- docs/PROOF.md | 2 +- docs/THREAT_MODEL.md | 6 +-- docs/ecosystem.md | 4 +- docs/external-miner/README.md | 6 +-- docs/external-miner/proof.md | 6 +-- docs/external-miner/validators.md | 86 +++++++++++++++++------------- docs/index.md | 1 - docs/reference/configuration.md | 37 ++++++++----- docs/runbooks/local-testnet-e2e.md | 3 +- docs/validator-quickstart.md | 29 +++++----- 20 files changed, 208 insertions(+), 136 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b9870e514..0ee45615b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,9 +20,9 @@ Shares always sum to 10,000 basis points. Algorithm 1 is the legacy signed bounty/proof 2,000/8,000 profile; algorithm 2 is bounty/proof 3,000/7,000 (document version >=2); algorithm 3 (document version >=3) admits any 1..64 unique ids and pays each challenge `share * min(sum(leaves), 10^12) / 10^12`. -Activation is an offline owner ceremony. Design, Prism and Relearn are retired +Activation is a profile version bump on the master (`config/challenges.toml`). Design, Prism and Relearn are retired products and their documentation has been removed. No active code, service, -trust-root row or leaf may register them. +challenge profile row or leaf may register them. Start with [the architecture](docs/ARCHITECTURE.md), [Proof](docs/PROOF.md), [challenge containers](docs/CHALLENGES.md) and @@ -69,22 +69,21 @@ or permit UID is not a submit path. | Key or token | Role | | --- | --- | -| owner seed | signs challenge and measurement trust documents offline | | gateway seed | signs immutable epoch bundles | -| challenge seed | `.key`; signs that container challenge's leaves and must match the trust root | -| Proof seed | signs Proof topics and leaves and must match the trust root | +| challenge seed | `.key`; signs that container challenge's leaves and must match the challenge profile | +| Proof seed | signs Proof topics and leaves and must match the challenge profile | | operator bearer | protects master administrative routes | | VM orchestrator bearer | authenticates master to the dedicated KVM host; not a wallet | -| validator hotkey | signs root/dissent evidence and Bittensor submissions | +| validator hotkey | weight-setting role only: signs root/dissent evidence and Bittensor submissions | | challenge internal token | master bearer for a container's `get_weights`; never a signing key | | miner hotkey | signs Bounty pairing or Proof submission payloads | Do not conflate gateway sealing, master ownership and validator chain signing. -Follow [the trust-root ceremony](docs/how-to/trust-root.md). +Validators build their view from the gateway profile at `GET /v1/trust`; the gateway is the single source of truth. ## Challenge container contracts -- The owner-signed trust root decides emission; the unsigned registry +- The challenge profile decides emission; the unsigned registry (`deploy/challenges/registry.toml`) only decides what runs. A container never receives a leaf-signing seed. - `challenge-supervisor` is the only process with the Docker socket. It verifies diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f8377a7c..34627d1a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `GET /v1/trust`: the gateway serves the challenge profile (challenge keys, + shares, measurements). Validators build their trust view from it over HTTPS; + `--gateway-public` stays as an optional extra pin. +- `deploy/compose/role-weight-setter.yml`: the only role with the validator + hotkey and consensus seed; `role-validator.yml` is verify-only. - Challenge containers (`docs/CHALLENGES.md`): the master loads any Docker challenge from an operator registry, polls its authenticated `get_weights` once per epoch, signs its leaves and proxies its public routes under @@ -20,8 +25,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Validators need no change beyond the release; they recompute signed leaves. - `GET /v1/metagraph/latest` serves the sealed hotkey map to challenges. +### Changed + +- The validator hotkey (`--wallet-name`/`--wallet-hotkey`) is required only to + submit weights or run `--peer-consensus`; `--verify-only` runs without it. + `scripts/run-validator.sh` needs `WALLET_NAME`/`WALLET_HOTKEY` only to submit. +- The challenge profile is the unsigned `config/challenges.toml` and + `measurements.toml`; changing shares is an edit, a version bump and a master restart. + ### Removed +- The trust-sign system: `cortex trust-sign`, `trust-verify`, owner keygen and + ceremony, `.sig` files, `owner.pubkey`, validator `--owner-public`, + `--challenges`, `--measurements`, `--minimum-*-version` and `BASE_GATEWAY_HOTKEY`, + `BASE_*_MIN_VERSION`, `BASE_OWNER_PUBKEY_FILE`, `docs/how-to/trust-root.md`. - The in-process Bounty service (`src/cortex/bounty`, `docs/BOUNTY.md` and the `BOUNTY_*` master settings). Bounty now runs as the CortexLM/bounty container; its operator guide migrates `bounty.sqlite3` and the session key unchanged. diff --git a/README.md b/README.md index 665a4022c..4065565d2 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Two roles keep that honest: - The **master** runs the gateway, Proof and every challenge container. At the end of each epoch it signs one leaf per miner and seals the epoch into a bundle. - A **validator** runs none of that. It downloads the sealed bundle, checks it against locally held trust files and the chain, and submits the weights. -An owner-signed trust root sets each challenge's emission share. If a challenge earns less than its share, the shortfall burns to UID 0. It never moves to another challenge. +The challenge profile (`config/challenges.toml`, served at `GET /v1/trust`) sets each challenge's emission share. If a challenge earns less than its share, the shortfall burns to UID 0. It never moves to another challenge. Challenges today: @@ -160,7 +160,7 @@ Tests cover signatures and Rust wire vectors, replay protection, challenge conta ``` src/ Python package: gateway, master, validator, miner CLI, Proof, VM host -config/ trust roots and measurements (owner-signed) +config/ challenge profile and measurements (unsigned, served by the gateway) deploy/ master, validator and KVM host deployment docs/ architecture, challenges, guides, specifications scripts/ validator script, repository and deployment checks, smoke tests @@ -184,7 +184,7 @@ What the design relies on: - The master signs one leaf per miner and seals each epoch. Validators recompute weights from the signed leaves and never trust display fields in the gateway response. - Validators pin the gateway key and the owner key themselves. Trust files come from a local copy, not from the gateway. -- Challenge shares come from the owner-signed trust root. Unused share burns to UID 0. +- Challenge shares come from the challenge profile. Unused share burns to UID 0. - A Proof topic opens only after the control plane verifies the host's execution receipts and signs the document. What isn't done, honestly: diff --git a/deploy/README.md b/deploy/README.md index dc138fafe..1f5ea4267 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -96,9 +96,9 @@ Seeds are raw 32 bytes or 64 hexadecimal characters. Bearers are nonempty opaque not hardlinks or symlinks, and mode 0400 or 0600. The Proof VM token and CA are needed only when `PROOF_VM_ORCHESTRATOR_URL` is set. -The trust bind defaults to `config/` and contains `owner.pubkey`, both signed -TOML documents and their adjacent `.sig` files. Verify them before startup with -the [offline ceremony](../docs/how-to/trust-root.md). +The config bind defaults to `config/` and holds the unsigned `challenges.toml` +and `measurements.toml`. The master reads them and the gateway serves them as the +challenge profile at `GET /v1/trust`; validators build their view from that route. ```bash docker compose --project-directory . --env-file deploy/env/master.env \ @@ -138,15 +138,16 @@ chown 65532:65532 "$dir"/*.token ``` Put the matching leaf seed at `deploy/secrets/master/.key` (its public key -is the trust-root row) and any challenge settings in the registry `env` table, +is the challenge profile row) and any challenge settings in the registry `env` table, for example `BOUNTY_BACKEND_PUBLIC_URL`. Challenge-specific secrets, such as Bounty's `session.key`, go next to the tokens; see each challenge's operator guide ([Bounty](https://github.com/CortexLM/bounty/blob/main/docs/operator.md), [OpenType](https://github.com/OpentypeAI/challenge/blob/main/docs/operator.md)). -A registered id that the trust root does not list runs without emission. Use this -burn-in to check `GET /challenge//version` and the logs. Then activate it with -a signed [algorithm 3 profile](../docs/how-to/trust-root.md#activate-container-challenges-algorithm-3). +A registered id that the challenge profile does not list runs without emission. Use +this burn-in to check `GET /challenge//version` and the logs. Then activate it +with an algorithm 3 profile (`config/challenges-v3.example.toml`), see +[Changing emission shares](#changing-emission-shares). A trusted id that is missing, unhealthy or invalid burns its share. Operator switches: @@ -162,13 +163,15 @@ re-reads it every 15 seconds. ## Validator Create `deploy/env/python-validator.env` and set the exact master HTTPS origin, -netuid, independently pinned gateway public key, chain network, wallet name and -hotkey, trust-version minimums, live subnet `version_key`, VPC peer bind address -and private host paths. Optional fallback RPCs are a JSON list of at most eight +netuid, chain network, live subnet `version_key`, VPC peer bind address and private +host paths. `role-validator.yml` is verify-only and needs no wallet. Weight setting +is a separate role, `role-weight-setter.yml`, the only one that also takes the +validator wallet name and hotkey. `--gateway-public` (the gateway sealing key) is an +optional extra pin; the HTTPS gateway is otherwise the source of the challenge profile. Optional fallback RPCs are a JSON list of at most eight credential-free `wss://` URLs. -The wallet directory is mounted read-only at `/run/wallets`. The validator -identity directory contains: +The weight-setter mounts the wallet directory read-only at `/run/wallets`; the +verify-only role mounts none. The validator identity directory contains: ```text consensus.key @@ -176,14 +179,18 @@ tls.crt tls.key ``` -The trust directory additionally contains `peers.json`, a JSON object mapping +The config directory additionally contains `peers.json`, a JSON object mapping each validator public-key hex string to one SAN-valid HTTPS origin. Do not run two validators with the same wallet: their commit/reveal or rate-limit state can conflict. ```bash +# verify-only: no hotkey, no consensus seed docker compose --project-directory . --env-file deploy/env/python-validator.env \ -f deploy/compose/role-validator.yml up -d +# weight setting: the only role with the validator hotkey and consensus seed +docker compose --project-directory . --env-file deploy/env/python-validator.env \ + -f deploy/compose/role-weight-setter.yml up -d ``` Enable chain submission only after the master returns a current `sealed: true` @@ -193,11 +200,27 @@ chain state. The unsealed UID0 fallback is a readiness failure, not a weight to submit. To run a validator without Compose, use `scripts/run-validator.sh` from a -repository clone. It needs `WALLET_NAME`, `WALLET_HOTKEY` and an independently -pinned `GATEWAY_PUBLIC`, takes trust files from `config/`, and refuses to run -master-only commands; see the +repository clone. It needs `WALLET_NAME` and `WALLET_HOTKEY` only to submit weights +(`--verify-only` needs neither), takes an optional `GATEWAY_PUBLIC` pin, reads the +challenge profile from the gateway, and refuses to run master-only commands; see the [quick start](../docs/external-miner/validators.md#quick-start-one-script). +## Changing emission shares + +The challenge profile is the unsigned `config/challenges.toml` (plus +`config/measurements.toml`). The master reads it; the gateway is the single source +of truth and serves it at `GET /v1/trust`. + +1. Edit the `[[challenges]]` shares (basis points, sum 10000) in `config/challenges.toml`. +2. Bump `version` (and set `introduced_epoch` to a future epoch if the change must + not apply to the current one). The gateway store refuses a version rollback. +3. Run `python scripts/check_repo.py`, then restart the master. +4. Validators pick up the new profile from `/v1/trust` at the next epoch; no + validator restart or file copy is needed. Check `GET /v1/trust` shows the new + `challenges_version`. + +A registered container id still needs a row in `deploy/challenges/registry.toml` to run. + ## Proof VM host The host requires Linux, `/dev/kvm`, Firecracker, jailer, nftables/network @@ -322,5 +345,5 @@ and restore both the prior digest and schema-compatible state on failure. Rollback uses the prior reviewed digest and a schema-compatible state backup. Never roll a database backward by deleting rows or replacing the latest seal. -Trust-root rotations are separate signed ceremonies and do not happen as a side -effect of an image rollback. +Challenge profile changes are separate edits of `config/challenges.toml` and do not +happen as a side effect of an image rollback. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5fcb3bd96..3c187795d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,6 +1,7 @@ # Architecture -The owner-signed trust root lists the scoring challenges and their emission +The challenge profile (`config/challenges.toml`, read by the master and served by +the gateway at `GET /v1/trust`) lists the scoring challenges and their emission shares: - algorithm 1: legacy bounty/proof 2,000/8,000; @@ -33,7 +34,7 @@ flowchart LR | Directory | Responsibility | | --- | --- | -| `src/cortex/protocol` | SCALE wire format, signatures, Merkle roots, aggregation, trust roots | +| `src/cortex/protocol` | SCALE wire format, signatures, Merkle roots, aggregation, challenge profile | | `src/cortex/gateway` | Durable raw leaves and immutable sealed bundles | | `src/cortex/validator` | Independent historical chain reads and exact weight submission | | `src/cortex/challenges` | Registry, weights client, public proxy and auto-updating supervisor | diff --git a/docs/BUNDLE_SPEC.md b/docs/BUNDLE_SPEC.md index 89ace1811..80db1e496 100644 --- a/docs/BUNDLE_SPEC.md +++ b/docs/BUNDLE_SPEC.md @@ -13,6 +13,14 @@ the BASE Python master, so that `/v1/weights/latest` can replace the pinned upstream and its frozen vectors win. This supersedes D16, which had made the Python float/JSON vectors characterization only. +**Trust provenance amendment:** bundle bytes, hashes, signatures and every wire-format +section are unchanged. Where this document says "local owner-signed trust root", read +"challenge profile": the unsigned `config/challenges.toml` and `config/measurements.toml` +read by the master and served by the gateway at `GET /v1/trust`. Validators build their +`TrustRoot` from that route over HTTPS (optionally pinning the gateway hotkey) and keep +version watermarks that refuse rollback. The `base-trustroot-v1` domain is no longer +produced; no owner signature over these documents exists. + Checklist map: [`BUNDLE_SPEC_CHECKLIST.md`](./BUNDLE_SPEC_CHECKLIST.md). CI gate: `cargo run -p xtask -- spec-check`. @@ -153,7 +161,7 @@ RawWeightBodyV1 = scale( ) ``` -`challenge_sig` is sr25519 over `msg`, verifiable with the challenge public key from the **local** owner-signed trust root (`config/challenges.toml`), never from gateway HTTP (D18). +`challenge_sig` is sr25519 over `msg`, verifiable with the challenge public key from the challenge profile (`config/challenges.toml`, obtained from the gateway `/v1/trust` over HTTPS, optionally gateway-pinned) (D18). ### 3.5 Sort key before tree build @@ -182,7 +190,7 @@ EpochBundleBodyV1 = scale( metagraph_root: [u8; 32], // §4.3 algorithm_version: u16, // aggregation; 1 = §6 emission_shares: Vec<(Bytes /*challenge_id*/, u16 /*bps*/)>, // sorted by challenge_id - measurements_digest: [u8; 32], // sha256 of local measurements trust root body + measurements_digest: [u8; 32], // sha256 of the measurements profile body uid_map: Vec<([u8; 32] /*hotkey*/, u16 /*uid*/)>, // sorted by hotkey leaves: Vec, // sorted by scale(challenge_id, miner_hotkey) merkle_root: [u8; 32], // recomputed over leaf preimages @@ -223,16 +231,16 @@ metagraph_root = sha256( scale(rows as Vec) ) --- -## 5. Emission shares from owner-signed trust root (f) +## 5. Emission shares from the challenge profile (f) | Rule | Requirement | |------|-------------| -| Source of truth | Owner-signed `config/challenges.toml` (trust root), loaded from **local disk** on every validator and on the gateway (D23, D18) | +| Source of truth | `config/challenges.toml` (challenge profile), read by the master and served by the gateway at `GET /v1/trust`; validators build it from that route (D23, D18) | | Bundle role | Gateway **copies** shares into `emission_shares`; it does not invent them | | Sum | `sum(bps) == 10_000` exactly, or reject | | Sort | `emission_shares` sorted by ascending `scale(challenge_id)` | -| Validator check | Re-read local trust root; require byte-equal challenge_id set and equal bps per id. Mismatch → reject (`EmissionShareMismatch`) | -| Unknown challenge | Leaf `challenge_id` absent from local trust root → reject (D18) | +| Validator check | Refetch the challenge profile; require byte-equal challenge_id set and equal bps per id. Mismatch → reject (`EmissionShareMismatch`) | +| Unknown challenge | Leaf `challenge_id` absent from the challenge profile → reject (D18) | Share values are `u16` basis points. No floats. @@ -433,9 +441,9 @@ duplicate uid_map entry, wrong `algorithm_version`) → class B, dissent Validators **derive** the expected set. They MUST NOT trust a set announced only by the gateway or challenge HTTP API. -### 7.1 Trust-root policy per challenge +### 7.1 Challenge-profile policy per challenge -In owner-signed `challenges.toml`, each challenge carries: +In `challenges.toml`, each challenge carries: ```text ParticipantPolicy (SCALE enum for signed body; TOML maps 1:1): @@ -473,7 +481,7 @@ function expected_participants(challenge_id, policy, block_hash, chain) -> BTree ### 7.3 Completeness rule -Let `E_c = expected_participants(c, ...)` for each challenge `c` in the local trust root with `bps > 0`. +Let `E_c = expected_participants(c, ...)` for each challenge `c` in the challenge profile with `bps > 0`. For every `c` and every `h ∈ E_c`, the bundle `leaves` MUST contain **exactly one** leaf with `(challenge_id=c, miner_hotkey=h)` whose `ScoreOrAbsence` is either `Score` or `NoScore`. @@ -523,7 +531,7 @@ Validators compare: | Validator mirroring | Validators MAY re-serve a bundle they have verified and persisted; peers SHOULD prefer multi-source fetch | | **No last-known-good** | MUST NOT fall back to a previous epoch's bundle, root, or vector when the current epoch fetch/verify fails. Failure → class B / degraded path, not stale success (aligns with D13 spirit for attestation). The unsealed burn response on `/v1/weights/latest` is an operator-safety default, not a last-known-good seal and **not** a submit path: a validator-local copy of a previously verified seal MUST NOT be used for set-weights while latest is unsealed. Mid-epoch tip revision bumps are intentional tip-tracking, not last-known-good | -Gateway signature and leaf signatures are always verified against local trust roots after fetch. +Gateway signature and leaf signatures are always verified against the challenge profile after fetch. --- @@ -579,7 +587,7 @@ DissentV1 = scale( The following paragraph is **normative wording** for docs and threat models. Do not weaken or inflate: -> base guarantees *no equivocation between validators* and *no undetected deviation by the gateway from the owner-signed challenge and measurement artifacts*. It does **not** guarantee (i) that a challenge's scores are honest, (ii) that the owner is honest — the owner signs the trust roots and runs the gateway, so a malicious owner can authorize a dishonest challenge or a backdoored measurement, (iii) completeness beyond what D24 provides, nor (iv) **chain-anchored, third-party-auditable non-equivocation** — per D5 the property is peer-consensus plus local evidence, verifiable by the participating validators and not by an outside observer after the fact. +> base guarantees *no equivocation between validators* and *no undetected deviation by the gateway from the gateway-served challenge and measurement profile*. It does **not** guarantee (i) that a challenge's scores are honest, (ii) that the owner is honest — the owner controls the profile and runs the gateway, so a malicious owner can authorize a dishonest challenge or a backdoored measurement, (iii) completeness beyond what D24 provides, nor (iv) **chain-anchored, third-party-auditable non-equivocation** — per D5 the property is peer-consensus plus local evidence, verifiable by the participating validators and not by an outside observer after the fact. ### 11.2 Mismatch outcomes (D6) @@ -658,7 +666,7 @@ A `verify(bundle, chain, local_trust_root)` implementation MUST check, in order: 2. `gateway_sig` over body 3. `block_hash` matches `chain.block_hash(block_B)` 4. `metagraph_root` and `uid_map` match `metagraph_at(block_hash)` -5. `emission_shares` equal local trust root and sum to `10_000` +5. `emission_shares` equal the challenge profile and sum to `10_000` 6. `measurements_digest` equals local measurements digest 7. Every leaf challenge key known locally; every `challenge_sig` valid 8. Participant-set completeness (§7) @@ -686,7 +694,7 @@ Failure modes map to §10.1 reason codes. | `base-rawweight-v1` | Challenge leaf body | | `base-dissent-v1` | Dissent body | | `base-root-v1` | Peer `(epoch, merkle_root)` | -| `base-trustroot-v1` | Owner trust-root body | +| `base-trustroot-v1` | Retired: formerly the owner-signed profile body; no longer produced | | `base-attest-v1` | Attestation bindings (see AGENT_CHALLENGE / D10; out of scope for leaf math) | Length-prefix rule: SCALE `Bytes` encoding of the tag string UTF-8 bytes (compact length + bytes), then payload. Task 14 owns the exact sign helper; this appendix names the tags only. @@ -705,7 +713,7 @@ Length-prefix rule: SCALE `Bytes` encoding of the tag string UTF-8 bytes (compac | D9 RFC6962 + EMPTY_ROOT | §3 | | D18 local challenge keys | §3.4, §5, §14 | | D19 honest claim | §11.1 | -| D23 emission shares in trust root | §5 | +| D23 emission shares in challenge profile | §5 | | D24 absence + derived set | §3.3, §7 | | D26 peer sample | §11.3 | diff --git a/docs/BUNDLE_SPEC_CHECKLIST.md b/docs/BUNDLE_SPEC_CHECKLIST.md index 20fbfdabd..158bb0c8d 100644 --- a/docs/BUNDLE_SPEC_CHECKLIST.md +++ b/docs/BUNDLE_SPEC_CHECKLIST.md @@ -12,7 +12,7 @@ CI: `cargo run -p xtask -- spec-check` fails if any letter marker is missing fro | (c) | leaf layout, ScoreOrAbsence, sort, EMPTY_ROOT hex | ## 3. Merkle construction (RFC 6962) and leaves (c) | `## 3. Merkle construction (RFC 6962) and leaves (c)` | | (d) | block_B, block_hash, metagraph_root | ## 4. Bundle body and block pin (d) | `## 4. Bundle body and block pin (d)` | | (e) | full integer aggregation formula | ## 6. Aggregation formula, algorithm_version = 1 (e) | `## 6. Aggregation formula, algorithm_version = 1 (e)` | -| (f) | emission shares from owner-signed trust root | ## 5. Emission shares from owner-signed trust root (f) | `## 5. Emission shares from owner-signed trust root (f)` | +| (f) | emission shares from the challenge profile | ## 5. Emission shares from the challenge profile (f) | `## 5. Emission shares from the challenge profile (f)` | | (g) | expected participant set derivation + completeness | ## 7. Expected participant set derivation (g) (D24) | `## 7. Expected participant set derivation (g) (D24)` | | (h) | final vector equality + sha256 of SCALE bytes | ## 8. Final vector comparison (h) | `## 8. Final vector comparison (h)` | | (i) | distribution endpoints; no last-known-good | ## 9. Distribution and caching (i) | `## 9. Distribution and caching (i)` | diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index 578eac655..0dd56e290 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -15,9 +15,9 @@ master polls weights, signs leaves, seals, proxies /challenge/ validator GET /v1/weights/latest + /v1/bundle/, recompute, submit ``` -The owner-signed trust root decides which challenges earn emission and how +The challenge profile decides which challenges earn emission and how much. The unsigned registry only decides what runs. A registered challenge that -is absent from the trust root runs without emission (burn-in). A trusted +is absent from the challenge profile runs without emission (burn-in). A trusted challenge that is not registered or cannot score emits `NoScore(ChallengeInternal)` for every expected hotkey and its share burns. @@ -101,7 +101,7 @@ changes `D`. | Algorithm | Leaf score for `i` in `E` with `w_i > 0` | Challenge payout | | --- | --- | --- | -| 3 (trust root version >= 3) | `floor(10^12 * w_i / D)` | `share * sum(leaves) / 10^12` | +| 3 (profile version >= 3) | `floor(10^12 * w_i / D)` | `share * sum(leaves) / 10^12` | | 2 (bounty/proof 3000/7000) | `w_i`, which must be an integer below 2^64 | Bounty: `share * min(N, 10) / 10`; Proof: `share` | | 1 (legacy bounty/proof 2000/8000) | not scored: every hotkey in `E` gets `NoScore(ChallengeInternal)` | nothing; the share burns | diff --git a/docs/NAMING.md b/docs/NAMING.md index 4e0b531c1..53e67f243 100644 --- a/docs/NAMING.md +++ b/docs/NAMING.md @@ -23,7 +23,6 @@ The preserved signing domains are: | Domain | Purpose | | --- | --- | -| `base-trustroot-v1` | owner-signed challenge and measurement documents | | `base-rawweight-v1` | challenge leaves | | `base-bundle-v1` | sealed epoch bundles | | `base-root-v1` | validator root statements | @@ -39,14 +38,14 @@ preimage. ## Live products -The signed trust root lists the live challenge ids and their shares, which total +The challenge profile lists the live challenge ids and their shares, which total 10,000 basis points: legacy `bounty` 2,000 / `proof` 8,000 (algorithm 1), 3,000 / 7,000 (algorithm 2), or any 1..64 ids matching `[a-z0-9][a-z0-9-]{0,62}` (algorithm 3). `proof` is built in; every other id is a [challenge container](CHALLENGES.md) named `cortex-challenge-`, with state in the `cortex-challenge--data` volume and routes under `/challenge//`. Design, Prism and Relearn are retired products whose documentation has been -removed. They are not services, routes, trust-root rows or emission recipients. +removed. They are not services, routes, challenge profile rows or emission recipients. ## File and command names diff --git a/docs/OPERATOR_SECURITY.md b/docs/OPERATOR_SECURITY.md index 69a023428..d9ebc7c1e 100644 --- a/docs/OPERATOR_SECURITY.md +++ b/docs/OPERATOR_SECURITY.md @@ -1,6 +1,6 @@ # Operator security checklist -Use this checklist before a deployment, trust-root rotation or incident +Use this checklist before a deployment, challenge profile change or incident recovery. It complements the [threat model](THREAT_MODEL.md). ## Credentials @@ -18,14 +18,15 @@ recovery. It complements the [threat model](THREAT_MODEL.md). ## Trust and pins -- [ ] `cortex trust-verify` accepts the installed challenge and measurement - documents at the deployment epoch. +- [ ] `GET /v1/trust` on the gateway returns the intended challenge and measurement + profile at the deployment epoch, over HTTPS (optionally pinned with + `--gateway-public`). - [ ] Every challenge public key matches its mounted `.key` (or `PROOF_SK_FILE`) seed, and the gateway public key differs from all of them. -- [ ] The trust root is legacy bounty/proof 2000/8000 (algorithm 1), 3000/7000 +- [ ] The challenge profile is legacy bounty/proof 2000/8000 (algorithm 1), 3000/7000 (algorithm 2, version >=2), or unique ids summing to 10000 (algorithm 3, - version >=3). Complete the matching - [activation](how-to/trust-root.md#activate-container-challenges-algorithm-3) + version >=3). Bump the profile + version (see [Changing emission shares](../deploy/README.md#changing-emission-shares)) before switching profiles; preserve old journals and sealed bytes. - [ ] Every registry entry names the challenge's own `ghcr.io` repository and GitHub `source`, keeps `attestation = true`, and runs on `stable` or a pin in diff --git a/docs/PROOF.md b/docs/PROOF.md index ada11d29f..11b90d048 100644 --- a/docs/PROOF.md +++ b/docs/PROOF.md @@ -1,7 +1,7 @@ # Proof operator reference Proof is 70% of Cortex emission under algorithm 2 (80% under the legacy -owner-signed profile). An operator supplies a research objective; a +legacy profile). An operator supplies a research objective; a topic-scoped recursive language-model agent installs its environment, proposes measurable rules, creates private evaluation material, measures a baseline and publishes miner documentation. Miners submit code and artifacts against the diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index 42b5fb394..87fa44543 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -7,8 +7,8 @@ scientifically valuable. ## Security claim -An honest validator can detect a bundle that differs from the owner-signed trust -root, challenge leaves, historical metagraph or deterministic aggregation. Peer +An honest validator can detect a bundle that differs from the gateway-served +challenge profile, challenge leaves, historical metagraph or deterministic aggregation. Peer root consensus can expose equivocation among participating validators, and signed dissent can be retained as local evidence. @@ -22,7 +22,7 @@ challenge keys or generated topic rules that validators will faithfully accept. | Boundary | Trusted property | Residual risk | | --- | --- | --- | -| owner trust root | challenge ids, keys, shares, measurement digest | owner can sign a malicious replacement | +| challenge profile (gateway `/v1/trust`) | challenge ids, keys, shares, measurement digest | a compromised gateway or broken TLS can serve a malicious profile; pin `--gateway-public` | | master gateway | durable intake and immutable seals | availability and censorship remain operator risks | | validator | independent chain snapshot, recomputation and dispatch journal | chain RPC eclipse or colluding validators | | challenge container | weights it returns for its own share | can misallocate or zero its own share; never holds a leaf seed, cannot move another challenge's mass | diff --git a/docs/ecosystem.md b/docs/ecosystem.md index 277162717..98ea96d17 100644 --- a/docs/ecosystem.md +++ b/docs/ecosystem.md @@ -38,7 +38,7 @@ flowchart LR | `hypertrain` | [CortexLM/hypertrain](https://github.com/CortexLM/hypertrain) | Models from decentralized, verifiable pretraining | Research stage | | `sentinel` | none yet | Detectors for code review and security | PLANNED, not live | -Each challenge is its own container with its own repository. The owner-signed trust root sets its emission share, and the contract is in [CHALLENGES.md](CHALLENGES.md). +Each challenge is its own container with its own repository. The challenge profile sets its emission share, and the contract is in [CHALLENGES.md](CHALLENGES.md). ## Apps @@ -66,4 +66,4 @@ Sentinel is a future challenge. It is not live and has no repository, container The plan is for miners to improve code review and security detectors, with the result feeding Cortex Security Cloud. Cortex Security Cloud would then compete with products such as CodeRabbit and Greptile, on the claim that miners keep improving it. That claim is untested. Nothing here promises a date, a design or a share. -When Sentinel ships, it would load like any other challenge: its own container, a share in the owner-signed trust root and leaves sealed by the master. +When Sentinel ships, it would load like any other challenge: its own container, a share in the challenge profile and leaves sealed by the master. diff --git a/docs/external-miner/README.md b/docs/external-miner/README.md index 96699ea15..2055f07a0 100644 --- a/docs/external-miner/README.md +++ b/docs/external-miner/README.md @@ -13,8 +13,8 @@ reproduction or on-chain payment. | `bounty` | up to 3000 bps (30%), algorithm 2 | [Pair an account and report bugs](bounty.md) | | `proof` | 7000 bps (70%), algorithm 2 | [Discover topics and submit research](proof.md) | -The legacy owner-signed profile remains 2000/8000 (algorithm 1) until -[activation](../how-to/trust-root.md#activate-proportional-bounty). +The legacy profile remains 2000/8000 (algorithm 1) until the profile version is +bumped ([Changing emission shares](../../deploy/README.md#changing-emission-shares)). A version-3 owner document may add further [challenge containers](../CHALLENGES.md), such as [OpentypeAI/challenge](https://github.com/OpentypeAI/challenge), with their own shares; each documents its miner API in its repository and is served @@ -81,7 +81,7 @@ frozen consensus wire format. The public documentation contract is checked by `uv run python scripts/check_repo.py`. `relearn`, `relearn-image`, `relearn-agent`, `relearn-mm`, `design` and `prism` -are retired products with no trust-root row or emission. Their documentation +are retired products with no challenge profile row or emission. Their documentation has been removed. ## License diff --git a/docs/external-miner/proof.md b/docs/external-miner/proof.md index 38eb1d4b3..e45ccf03a 100644 --- a/docs/external-miner/proof.md +++ b/docs/external-miner/proof.md @@ -241,9 +241,9 @@ pass-floor pool among eligible miners and a novelty pool according to new improvement; duplicates receive no novelty pool. A miner's Proof score is the **sum of per-topic masses**, not a mean of binary passes. The fixed subnet split is Bounty 3000 / Proof 7000 bps under algorithm 2; algorithm 3 takes any -owner-signed split. Legacy owner-signed 2000/8000 deployments retain algorithm 1 -until -[activation](../how-to/trust-root.md#activate-proportional-bounty). +split in the challenge profile. Legacy 2000/8000 deployments retain algorithm 1 +until the profile version is bumped +([Changing emission shares](../../deploy/README.md#changing-emission-shares)). Actual payment additionally needs signed leaves, a valid gateway seal and validator submission on Bittensor. See [validators](validators.md) and diff --git a/docs/external-miner/validators.md b/docs/external-miner/validators.md index 94d913f76..824aa24ec 100644 --- a/docs/external-miner/validators.md +++ b/docs/external-miner/validators.md @@ -7,15 +7,18 @@ From a clone of this repository, after `uv sync --locked --extra chain`: ```bash +scripts/run-validator.sh --verify-only # no wallet or hotkey needed +# To submit weights, a registered validator wallet is required: export WALLET_NAME=validator WALLET_HOTKEY=default +scripts/run-validator.sh +# Optional extra pin of the gateway sealing hotkey: export GATEWAY_PUBLIC= -scripts/run-validator.sh --verify-only # then run without the flag to submit ``` `scripts/run-validator.sh` defaults to `GATEWAY=https://chain.joinbase.ai`, `NETUID=100`, `NETWORK=finney` and `STATE_DB=$HOME/.cortex/validator.sqlite3`. It checks that `GET $GATEWAY/v1/weights/latest` answers with the expected -netuid, takes the trust files and minimum versions from `config/`, reads the +netuid, reads the challenge profile from the gateway, reads the subnet `weights_version` from the chain (override with `VERSION_KEY`) and then runs `cortex validator`. `--dry-run` prints the command without running it; `--once` runs one tick. The script refuses master-only commands and master @@ -24,17 +27,18 @@ environments. The sections below describe the underlying CLI. Cortex runs a centralized authoritative gateway. The master gateway is the authority for weights: it serves `/v1/weights/latest`, `/v1/weights/{epoch}` and `/v1/bundle/{epoch}`. A Python validator consumes those endpoints, -independently verifies the bundle against local owner-signed trust and the -chain, and submits the verified vector on Bittensor. Cross-checking +independently verifies the bundle against the gateway-served challenge profile +and the chain, and submits the verified vector on Bittensor. Cross-checking authenticated peer roots against other independently-operated validators is an opt-in hardening (`--peer-consensus`), off by default. Validators never run challenge containers or Proof evaluation, rent GPUs or receive miner provider credentials; they need no registry, Docker or challenge -token. Challenge shares come from the owner-signed trust root: legacy +token. Challenge shares come from the challenge profile the gateway serves at +`GET /v1/trust` (the master reads `config/challenges.toml`): legacy bounty/proof 2000/8000 (algorithm 1), `bounty` 3000 bps and `proof` 7000 bps (algorithm 2), or any unique challenge set summing to 10000 bps (algorithm 3, -see [activation](../how-to/trust-root.md#activate-container-challenges-algorithm-3)). +see [Changing emission shares](../../deploy/README.md#changing-emission-shares)). Sealed `GET /v1/weights/latest` responses expose `algorithm_version` from the signed bundle. `emission_shares` contains the configured ceilings; each @@ -44,20 +48,23 @@ the ten-report Bounty ramp under algorithm 2 and unmapped hotkeys still burn. Validators always recompute from signed leaves; they never trust these display fields as consensus inputs. -## Prepare local trust and identity +## Prepare identity Install the [Python package with its chain extra](README.md#installation). -Obtain the gateway URL and independently pinned gateway public key, plus the -owner public key and signed `challenges.toml` and `measurements.toml` trust -files. Detached signatures live beside each TOML as `.toml.sig`. Do not replace -local trust files with unverified content received from the gateway. - -Use a registered Bittensor hotkey in a standard wallet. The current validator -CLI also requires `--consensus-seed-file`: a private file containing the -32-byte hexadecimal seed for **that same hotkey**. Startup refuses a mismatch. -This additional seed requirement is a current validator limitation; the miner -CLI's encrypted-wallet adapter and `--dev-seed-file` option do not apply here. -Do not assume that a derived wallet key has an exportable matching seed. +Obtain the gateway HTTPS URL. The validator builds its challenge profile +(challenge keys, shares, measurements) from `GET /v1/trust` on that gateway and +refuses a profile whose version goes backwards. `--gateway-public` is an optional +extra pin: it refuses a gateway whose sealing hotkey differs. Obtain it from an +independent source if you use it. TLS to the gateway is the trust anchor otherwise. + +Verifying needs no wallet: run with `--verify-only` and no `--wallet-name` or +`--wallet-hotkey`. Submitting weights needs a registered Bittensor hotkey in a +standard wallet (`--wallet-name`, `--wallet-hotkey`, optional `--wallet-path`); +the CLI refuses to submit without them. `--consensus-seed-file`, a private file +with the 32-byte hexadecimal seed for **that same hotkey**, is required only with +`--peer-consensus`; startup refuses a mismatch. The miner CLI's encrypted-wallet +adapter and `--dev-seed-file` option do not apply here. Do not assume that a +derived wallet key has an exportable matching seed. The Bittensor wallet signs on-chain extrinsics. The consensus seed signs peer roots and dissent using the frozen Cortex context. No gateway wallet or @@ -102,23 +109,27 @@ not authentication; the expected hotkey signature is required. ## Run the validator -This command can submit on-chain weights. Set its network, netuid, peer address -and local paths to the intended deployment before running it: +Verify-only (no hotkey, never submits): + +```bash +uv run cortex validator --verify-only --once \ + --gateway "$GATEWAY" --network "$CHAIN_NETWORK" --netuid "$NETUID" \ + --version-key "$VALIDATOR_VERSION_KEY" --state-db ./state/validator.sqlite3 +``` + +The command below can submit on-chain weights and is the only form that needs a +wallet. Set its network, netuid, peer address and local paths to the intended +deployment before running it: ```bash uv run cortex validator \ - --gateway "$GATEWAY" --gateway-public "$GATEWAY_PUBLIC_KEY" \ + --gateway "$GATEWAY" [--gateway-public "$GATEWAY_PUBLIC_KEY"] \ --network "$CHAIN_NETWORK" --netuid "$NETUID" \ --fallback-endpoints "$CHAIN_FALLBACK_ENDPOINTS_JSON" \ - --owner-public ./trust/owner.pubkey \ - --challenges ./trust/challenges.toml \ - --measurements ./trust/measurements.toml \ - --minimum-challenges-version "$CHALLENGES_MIN_VERSION" \ - --minimum-measurements-version "$MEASUREMENTS_MIN_VERSION" \ --version-key "$VALIDATOR_VERSION_KEY" \ --wallet-name validator --wallet-hotkey default \ --consensus-seed-file /private/consensus.key \ - --peers ./trust/peers.json \ + --peers ./config/peers.json \ --state-db ./state/validator.sqlite3 \ --peer-bind "$PEER_BIND" --peer-port 8091 \ --peer-tls-certificate /private/tls.crt \ @@ -126,9 +137,8 @@ uv run cortex validator \ ``` `--wallet-path` defaults to `~/.bittensor/wallets`; polling defaults to -30 seconds. Trust minimums and the subnet validator version key are required; -obtain them from the signed production ceremony and live subnet instead of -guessing. `--fallback-endpoints` is a JSON list of at most eight unique `wss://` +30 seconds. The subnet validator version key is required; obtain it from the live +subnet instead of guessing. `--fallback-endpoints` is a JSON list of at most eight unique `wss://` RPC origins without credentials, queries or fragments. The Bittensor SDK uses them when the primary endpoint fails. Fallbacks are origins only: URL paths are also rejected so provider bearer material cannot appear in the process argv or @@ -149,21 +159,23 @@ or peer-unavailable states return a nonzero process status. For Compose use the Python [validator environment example](../../deploy/env/python-validator.env.example) -and [validator role](../../deploy/compose/role-validator.yml). Keep the SQLite -state persistent and wallet, identity and trust directories read-only. The +with the verify-only [validator role](../../deploy/compose/role-validator.yml) (no +hotkey) or the [weight-setter role](../../deploy/compose/role-weight-setter.yml) +(the only role with the validator hotkey and consensus seed). Keep the SQLite +state persistent and wallet, identity and config directories read-only. The role points to the master gateway and has no gateway or challenge service. ## Verify before dispatch The validator reads `GET /v1/weights/latest`, obtains the corresponding binary -bundle, then verifies the independently pinned gateway signature and local -owner-signed trust root. It checks protocol version, epoch and block hash, +bundle, then verifies the gateway signature (against `/v1/trust`, plus +`--gateway-public` when given) and the gateway-served challenge profile. It checks protocol version, epoch and block hash, metagraph stakes and UID mapping, challenge keys and emission shares, participant completeness, measurements and Merkle root. Frozen wire details are in the [bundle specification](../BUNDLE_SPEC.md). -Trust files are reloaded, their activation windows and minimum versions are -checked, and durable watermarks reject rollback. The sealed block must not be +The profile is refetched each tick, its activation window is checked, and +durable version watermarks reject rollback. The sealed block must not be in the future or more than `--max-block-lag` blocks old (default `256`). Before dispatch the validator rechecks both chain snapshot and latest gateway state; a reorg or a changed/unsealed latest response prevents submission. @@ -238,7 +250,7 @@ The peer listener exposes: Consensus uses sr25519 signing context `base-sr25519-v1` and SCALE-encoded domain/payload vectors. Exact domains are `base-root-v1`, `base-dissent-v1`, -`base-bundle-v1`, `base-rawweight-v1` and `base-trustroot-v1`. Generic Substrate +`base-bundle-v1`, `base-rawweight-v1` (`base-trustroot-v1` is retired). Generic Substrate `wallet.sign()` signatures cannot replace these signatures. The Python metagraph projection uses chain-provided integer stakes. Validate diff --git a/docs/index.md b/docs/index.md index 7d15ba518..6d417df05 100644 --- a/docs/index.md +++ b/docs/index.md @@ -30,7 +30,6 @@ Start here: [Cortex ecosystem](ecosystem.md) and [Validator quickstart](validato ## Operate - [Configuration](reference/configuration.md): Python services and credential files. -- [Trust-root ceremony](how-to/trust-root.md): generate keys, sign and verify operator documents. - [Build a Proof guest rootfs](how-to/build-guest-rootfs.md): convert the pinned Python guest stage into measured ext4 bytes. - [Deployment](../deploy/README.md): master, validator and dedicated KVM host. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 0b1912a63..7660136f3 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -12,9 +12,8 @@ conflicting values are an error. Existing names and crypto domains are preserved | `BASE_CHAIN_ENDPOINT` | Official Bittensor SDK alias (`finney`, `test`, `archive`, `local`) or credential-free `wss://` RPC origin | | `BASE_CHAIN_FALLBACK_ENDPOINTS` | JSON list of up to eight credential-free `wss://` fallback RPC endpoints | | `BASE_STATE_DIR` | Durable private state directory; default `/var/lib/cortex` | -| `BASE_OWNER_PUBKEY_FILE` | Trust-root signing public key | -| `BASE_CHALLENGES_FILE` | Signed challenge configuration; adjacent `.sig` required | -| `BASE_MEASUREMENTS_FILE` | Signed measurement configuration; adjacent `.sig` required | +| `BASE_CHALLENGES_FILE` | Unsigned challenge profile (challenge keys and emission shares); served by the gateway at `GET /v1/trust` | +| `BASE_MEASUREMENTS_FILE` | Unsigned measurement profile; served with the challenge profile | | `BASE_GATEWAY_SK_FILE` | Gateway seal seed file | | `PROOF_SK_FILE` | Proof topic and leaf seed file | | `BASE_GATEWAY_ADMIN_TOKEN_FILE` | Required operator bearer file | @@ -42,8 +41,17 @@ Container challenges such as Bounty take their settings from the registry A trusted challenge that is not registered, unhealthy or returns invalid weights burns its share. The Proof seed remains required because every completed epoch still needs signed Proof absence leaves. The unavailable Proof share burns and is -never reassigned. Trust-root profiles and their activation are described in -[the trust-root ceremony](../how-to/trust-root.md). +never reassigned. Changing the challenge profile is described in +[Changing emission shares](../../deploy/README.md#changing-emission-shares). + +### Gateway routes + +The gateway serves the public `/v1` routes `GET /v1/weights/latest`, +`GET /v1/weights/{epoch}`, `GET /v1/bundle/{epoch}` and `GET /v1/trust`. +`GET /v1/trust` returns the current challenge profile (challenges with keys, +shares and policy, measurements digest, gateway hotkey, versions and +`introduced_epoch`) as JSON. It is public, unsigned and read-only; validators +build their trust view from it over HTTPS. One resource shape applies to the persistent topic VM and each fresh experiment VM. The examples explicitly request 1 vCPU, 1024 MiB RAM and 16384 MiB disk; @@ -73,7 +81,9 @@ The Compose role also needs `BASE_CHALLENGE_SECRETS_HOST_DIR` and ## Validator -The Compose validator maps these host settings to required CLI arguments: +The Compose validator maps these host settings to CLI arguments. The verify-only +`role-validator.yml` needs no wallet; `role-weight-setter.yml` is the only role +with a validator hotkey and consensus seed: | Variable | Meaning | | --- | --- | @@ -81,17 +91,18 @@ The Compose validator maps these host settings to required CLI arguments: | `BASE_NETUID` | Exact deployed subnet UID | | `BASE_CHAIN_ENDPOINT` | Official Bittensor SDK alias (`finney`, `test`, `archive`, `local`) or credential-free `wss://` RPC origin | | `BASE_CHAIN_FALLBACK_ENDPOINTS` | JSON list of up to eight credential-free `wss://` fallback RPC endpoints | -| `BASE_GATEWAY_HOTKEY` | Independently pinned gateway signing public key | -| `BASE_CHALLENGES_MIN_VERSION` | Minimum accepted owner-signed challenge document version | -| `BASE_MEASUREMENTS_MIN_VERSION` | Minimum accepted owner-signed measurement document version | +| `BASE_WALLET_NAME`, `BASE_WALLET_HOTKEY` | Validator wallet and hotkey; weight-setter role only | | `BASE_VERSION_KEY` | Exact live subnet validator version key | | `BASE_MIN_PEER_SAMPLE` | Required independent permitted-validator peer sample, 0-64 | | `BASE_MAX_BLOCK_LAG` | Maximum age of a sealed metagraph block, at least 1 | -The trust minimums and version key are intentionally blank in the example. -Populate them from reviewed signed documents and the live subnet; do not infer -them from tests. `--verify-only --once` is the non-dispatch preflight. It still -requires wallet identity, peer consensus and full historical-chain access. +The version key is intentionally blank in the example. Populate it from the live +subnet; do not infer it from tests. The CLI flags `--wallet-name` and +`--wallet-hotkey` are required only to submit weights (or with `--peer-consensus`); +`--gateway-public` is an optional extra pin of the gateway sealing key. The +challenge profile always comes from `GET /v1/trust`. `--verify-only --once` is the +non-dispatch preflight; it needs no wallet but still requires full historical-chain +access. ## VM host diff --git a/docs/runbooks/local-testnet-e2e.md b/docs/runbooks/local-testnet-e2e.md index 72917379d..37de5348a 100644 --- a/docs/runbooks/local-testnet-e2e.md +++ b/docs/runbooks/local-testnet-e2e.md @@ -5,8 +5,7 @@ services, deployment overlays and local testnet scripts described here are retired and cannot run the Python subnet. Use the [Python deployment guide](../../deploy/README.md) for master, validator -and dedicated KVM-host setup, and the [trust-root ceremony](../how-to/trust-root.md) -for signing keys and trust documents. The [validator guide](../external-miner/validators.md) +and dedicated KVM-host setup. The [validator guide](../external-miner/validators.md) describes independent sealed-bundle verification and chain submission. Offline reward-path tests, a live model test and a KVM lifecycle test establish diff --git a/docs/validator-quickstart.md b/docs/validator-quickstart.md index a46c287d1..3747858bf 100644 --- a/docs/validator-quickstart.md +++ b/docs/validator-quickstart.md @@ -2,14 +2,14 @@ A Cortex validator downloads the sealed epoch bundle from the gateway, verifies it and submits the weights on Bittensor subnet 100. `scripts/run-validator.sh` does the setup. The master (gateway, Proof, challenge containers) runs only on the master operator's machine, and this script never starts it. -For the underlying CLI and trust files, see the [validator guide](external-miner/validators.md). +For the underlying CLI and the challenge profile served by the gateway, see the [validator guide](external-miner/validators.md). ## Prerequisites - Linux, Python 3.12 or 3.13, [uv](https://docs.astral.sh/uv/), `curl` and libsodium 1.0.18 or newer. - A clone of this repository. -- A Bittensor wallet with a hotkey registered on netuid 100. -- The gateway's public hotkey, pinned from an independent, trusted source. Don't copy it from the gateway response. +- To submit weights only: a Bittensor wallet with a hotkey registered on netuid 100. Verifying needs no wallet. +- Optional: the gateway's public hotkey, pinned from an independent, trusted source, as an extra check. Don't copy it from the gateway response. ```bash sudo apt-get install libsodium23 @@ -18,17 +18,17 @@ uv sync --locked --extra chain ## Three steps -1. Set the wallet and the pinned gateway key. +1. Run one verification. It never submits and needs no wallet. ```bash - export WALLET_NAME=validator WALLET_HOTKEY=default - export GATEWAY_PUBLIC= + scripts/run-validator.sh --verify-only ``` -2. Run one verification. It never submits. +2. To submit, set the wallet. Optionally add the independently pinned gateway key. ```bash - scripts/run-validator.sh --verify-only + export WALLET_NAME=validator WALLET_HOTKEY=default + export GATEWAY_PUBLIC= # optional ``` 3. Run for real. It verifies and submits every epoch. @@ -43,9 +43,9 @@ Environment variables: | Variable | Default | Meaning | | --- | --- | --- | -| `WALLET_NAME` | required | Validator wallet name | -| `WALLET_HOTKEY` | required | Registered validator hotkey | -| `GATEWAY_PUBLIC` | required | Independently pinned gateway hotkey | +| `WALLET_NAME` | required to submit | Validator wallet name | +| `WALLET_HOTKEY` | required to submit | Registered validator hotkey | +| `GATEWAY_PUBLIC` | optional | Extra pin of the gateway sealing hotkey | | `GATEWAY` | `https://chain.joinbase.ai` | Gateway URL | | `NETUID` | `100` | Subnet | | `NETWORK` | `finney` | Bittensor network | @@ -102,8 +102,11 @@ The gateway answered for a different subnet than `NETUID`. Check `GATEWAY` and ` **It can't read `weights_version`.** Set `VERSION_KEY` yourself, or check that `NETWORK` is reachable. -**Where do the trust files come from?** -From `config/` in your clone. See the [trust-root ceremony](how-to/trust-root.md). +**Where does the challenge profile come from?** +From the gateway at `GET /v1/trust` over HTTPS; no local files are needed. `--gateway-public` (`GATEWAY_PUBLIC` in the script) is an optional extra pin of the gateway sealing key. + +**Do I need a hotkey?** +Only to submit weights. `scripts/run-validator.sh --verify-only` and the CLI `--verify-only` need no wallet; submitting needs `WALLET_NAME`/`WALLET_HOTKEY` (`--wallet-name`/`--wallet-hotkey`). **Does the validator CLI need anything else?** The [validator guide](external-miner/validators.md) lists the current requirements for the underlying CLI, such as the consensus seed file. From 27d6b33e55a767c1910627c56f7e9a85f166f192 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:15:44 +0000 Subject: [PATCH 06/56] test(gateway): validator without local trust uses the gateway-served profile; pin mismatch refuses Co-Authored-By: Claude Sonnet 5.5 --- tests/gateway/test_gateway.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/gateway/test_gateway.py b/tests/gateway/test_gateway.py index 1facf3a00..7d63d9f97 100644 --- a/tests/gateway/test_gateway.py +++ b/tests/gateway/test_gateway.py @@ -25,6 +25,7 @@ sign_leaf, ) from cortex.protocol.crypto import encode_hotkey, public_key +from cortex.protocol.scale import ProtocolError from cortex.validator import ChainSnapshot, SubmissionJournal, Validator @@ -162,6 +163,39 @@ async def test_signed_intake_seal_latest_validator_and_chain_dispatch(network): assert network.chain.submissions == [(541, ((1, 65535),), 1)] +async def test_validator_without_local_trust_uses_the_gateway_served_profile(network): + await intake(network) + assert (await seal(network)).status_code == 200 + served = (await network.client.get("/v1/trust")).json() + assert served == network.service.trust_profile() + validator = Validator( + gateway_url="https://master.invalid", + netuid=541, + chain=network.chain, + journal=network.journal, + http=network.client, + ) + assert (await validator.run_once()).outcome == "submitted" + assert validator.trust == network.trust + assert network.chain.submissions == [(541, ((1, 65535),), 1)] + + +async def test_validator_pinned_to_another_gateway_key_refuses_the_served_profile(network): + await intake(network) + assert (await seal(network)).status_code == 200 + validator = Validator( + gateway_url="https://master.invalid", + netuid=541, + chain=network.chain, + journal=network.journal, + http=network.client, + gateway_public=bytes(32), + ) + with pytest.raises(ProtocolError, match="pinned --gateway-public"): + await validator.run_once() + assert network.chain.submissions == [] + + async def test_unsealed_burn_is_available_without_owner_wallet_or_chain(network): response = await network.client.get("/v1/weights/latest") assert response.status_code == 200 From 4c24c76a7432d09013f5b1c73353a3316a58834b Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:06:39 +0000 Subject: [PATCH 07/56] docs,deploy: drop gateway/leaf keys, keygen and signature docs for wire v2 BUNDLE_SPEC protocol_version 2 (no leaf/bundle signatures), deploy env and secrets without gateway.key / challenge leaf keys, run-validator without GATEWAY_PUBLIC, check_repo pins refreshed. Co-Authored-By: Claude Sonnet 5.5 --- .coderabbit.yaml | 2 +- .env.example | 3 - CHANGELOG.md | 18 ++++- README.md | 19 +++--- config/challenges-v2.example.toml | 2 - config/challenges-v3.example.toml | 3 - config/challenges.toml | 4 +- deploy/README.md | 18 ++--- deploy/compose/role-master.yml | 2 - deploy/secrets/README.md | 6 +- docs/ARCHITECTURE.md | 10 +-- docs/BUNDLE_SPEC.md | 103 ++++++++++++----------------- docs/CHALLENGES.md | 8 +-- docs/NAMING.md | 2 - docs/OPERATOR_SECURITY.md | 6 +- docs/THREAT_MODEL.md | 6 +- docs/external-miner/validators.md | 24 +++---- docs/reference/configuration.md | 8 +-- docs/validator-quickstart.md | 10 +-- scripts/check_deploy.py | 5 +- scripts/check_repo.py | 4 +- scripts/run-validator.sh | 6 +- tests/test_deploy_contract.py | 2 - tests/test_run_validator_script.py | 10 ++- 24 files changed, 120 insertions(+), 161 deletions(-) diff --git a/.coderabbit.yaml b/.coderabbit.yaml index dfa089371..09543dd47 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -43,7 +43,7 @@ reviews: - Shared observations are untrusted and private until owner-signed approval. - A trusted challenge container that is unregistered, failing or returns invalid weights yields explicit `ChallengeInternal` leaves; never a stale or guessed - score. Containers never receive a leaf seed; only `challenge-supervisor` + score. Containers never receive a signing key; only `challenge-supervisor` mounts the Docker socket; the proxy never forwards `internal/` paths. - Validators independently fetch historical chain state, recompute exact u16 weights and refuse the unsealed UID0 fallback. A sealed UID0 burn is valid. diff --git a/.env.example b/.env.example index 2c6c2b6a6..03f7aa355 100644 --- a/.env.example +++ b/.env.example @@ -5,11 +5,8 @@ BASE_CHAIN_FALLBACK_ENDPOINTS=[] BASE_STATE_DIR=.local/state BASE_CHALLENGES_FILE=config/challenges.toml BASE_MEASUREMENTS_FILE=config/measurements.toml -BASE_GATEWAY_SK_FILE=.local/secrets/gateway.seed BASE_GATEWAY_ADMIN_TOKEN_FILE=.local/secrets/operator.token PROOF_SK_FILE=.local/secrets/proof.seed -# Leaf seeds of container challenges: /.key for every trusted id but proof. -BASE_CHALLENGE_KEYS_DIR=.local/secrets # Optional operator registry; see docs/CHALLENGES.md. Unset runs no container challenge. # BASE_CHALLENGE_REGISTRY_FILE=deploy/challenges/registry.toml BASE_CHALLENGE_SECRETS_DIR=.local/challenge-secrets diff --git a/CHANGELOG.md b/CHANGELOG.md index 34627d1a1..e1c04746a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `GET /v1/trust`: the gateway serves the challenge profile (challenge keys, - shares, measurements). Validators build their trust view from it over HTTPS; - `--gateway-public` stays as an optional extra pin. +- `GET /v1/trust`: the gateway serves the challenge profile (ids, shares, + measurements). Validators build their trust view from it over HTTPS. - `deploy/compose/role-weight-setter.yml`: the only role with the validator hotkey and consensus seed; `role-validator.yml` is verify-only. - Challenge containers (`docs/CHALLENGES.md`): the master loads any Docker @@ -27,6 +26,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Wire format v2 (`protocol_version` 2, BREAKING): leaves and bundles carry no + signatures and no gateway key. `Leaf` ends at its score, `Bundle` is the body, + `ChallengeEntry` has no public key. The old v1 bundles and a gateway `state_dir` + created under v1 are refused: start the gateway on a new `state_dir`. Hash + integrity (Merkle root, metagraph root, block hash, recomputed final vector) is + unchanged, and so is the validator hotkey signature on the chain submission. +- `POST /v1/weights/raw` requires the operator bearer (it was protected only by the + challenge leaf signature). - The validator hotkey (`--wallet-name`/`--wallet-hotkey`) is required only to submit weights or run `--peer-consensus`; `--verify-only` runs without it. `scripts/run-validator.sh` needs `WALLET_NAME`/`WALLET_HOTKEY` only to submit. @@ -35,6 +42,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed +- Leaf and bundle signatures: `sign_leaf`, `challenge_sig`, `gateway_sig`, + `gateway_hotkey`, challenge leaf keys (`.key`, `BASE_CHALLENGE_KEYS_DIR`), + the gateway sealing key (`gateway.key`, `BASE_GATEWAY_SK_FILE`), validator + `--gateway-public`/`GATEWAY_PUBLIC` and `cortex keygen`. The gateway and TLS are + the only anchor for what validators and miners read. - The trust-sign system: `cortex trust-sign`, `trust-verify`, owner keygen and ceremony, `.sig` files, `owner.pubkey`, validator `--owner-public`, `--challenges`, `--measurements`, `--minimum-*-version` and `BASE_GATEWAY_HOTKEY`, diff --git a/README.md b/README.md index 4065565d2..034f53464 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

- An autonomous research network on Bittensor subnet 100. Miners compete in challenges, a master seals each epoch into one signed bundle, and validators verify that bundle and submit the weights on-chain. + An autonomous research network on Bittensor subnet 100. Miners compete in challenges, a master seals each epoch into one bundle, and validators verify that bundle and submit the weights on-chain.

@@ -38,8 +38,8 @@ Cortex is the subnet that runs research competitions. Each competition is a **ch Two roles keep that honest: -- The **master** runs the gateway, Proof and every challenge container. At the end of each epoch it signs one leaf per miner and seals the epoch into a bundle. -- A **validator** runs none of that. It downloads the sealed bundle, checks it against locally held trust files and the chain, and submits the weights. +- The **master** runs the gateway, Proof and every challenge container. At the end of each epoch it collects one score leaf per miner and seals the epoch into an unsigned bundle. +- A **validator** runs none of that. It downloads the sealed bundle, checks its hashes, Merkle root and the gateway-served challenge profile against the chain, and submits the weights. The challenge profile (`config/challenges.toml`, served at `GET /v1/trust`) sets each challenge's emission share. If a challenge earns less than its share, the shortfall burns to UID 0. It never moves to another challenge. @@ -76,10 +76,10 @@ Sentinel is planned as a miner-improved competitor to CodeRabbit and Greptile, p ```mermaid flowchart LR - M["Miner"] --> C["Challenge container"] --> L["Signed leaf"] --> G["Gateway seals epoch"] --> V["Validators verify"] --> N["Weights on chain"] + M["Miner"] --> C["Challenge container"] --> L["Score leaf"] --> G["Gateway seals epoch"] --> V["Validators verify"] --> N["Weights on chain"] ``` -A miner submits work to a challenge. The challenge scores it and the master signs a leaf for that miner. When the epoch ends, the gateway seals all leaves into one bundle. Validators fetch the bundle from `GET https://chain.joinbase.ai/v1/weights/latest`, recompute the weights from the signed leaves and submit them to Bittensor. +A miner submits work to a challenge. The challenge scores it and the master records a leaf for that miner. When the epoch ends, the gateway seals all leaves into one bundle. Validators fetch the bundle from `GET https://chain.joinbase.ai/v1/weights/latest`, recompute the weights from the leaves and submit them to Bittensor. ### 2. Who runs what @@ -114,7 +114,6 @@ sudo apt-get install libsodium23 uv sync --locked --extra chain export WALLET_NAME=validator WALLET_HOTKEY=default -export GATEWAY_PUBLIC= scripts/run-validator.sh --verify-only # one verification, never submits scripts/run-validator.sh # verify and submit every epoch @@ -154,7 +153,7 @@ uv run python scripts/check_deploy.py --check-examples uv build --no-build-isolation ``` -Tests cover signatures and Rust wire vectors, replay protection, challenge container outages, auto-update rollback, artifact validation, topic setup, rejected submissions, VM lifecycle failures, RLM recursion and compaction, reward allocation and sealed-weight submission. CI runs offline and never rents a GPU or boots Firecracker. +Tests cover hashes, Merkle proofs and Rust wire vectors, replay protection, challenge container outages, auto-update rollback, artifact validation, topic setup, rejected submissions, VM lifecycle failures, RLM recursion and compaction, reward allocation and sealed-weight submission. CI runs offline and never rents a GPU or boots Firecracker. ## Repository layout @@ -182,14 +181,14 @@ tests/ offline test suite What the design relies on: -- The master signs one leaf per miner and seals each epoch. Validators recompute weights from the signed leaves and never trust display fields in the gateway response. -- Validators pin the gateway key and the owner key themselves. Trust files come from a local copy, not from the gateway. +- The master records one leaf per miner and seals each epoch. Validators recompute weights from the leaves and never trust display fields in the gateway response. +- The gateway is the trust anchor (TLS plus its own tokens). Bundles and leaves carry no signatures: integrity rests on content hashes, the Merkle root and the chain snapshot. A compromised gateway can serve a dishonest bundle. - Challenge shares come from the challenge profile. Unused share burns to UID 0. - A Proof topic opens only after the control plane verifies the host's execution receipts and signs the document. What isn't done, honestly: -- **The gateway is a central authority.** Validators verify what the master signed. They don't reproduce Proof or challenge scoring themselves. Peer cross-checking between validators is opt-in and off by default. +- **The gateway is a central authority.** Validators verify the bundle hashes and chain snapshot; they trust the gateway for which leaves exist. They don't reproduce Proof or challenge scoring themselves. Peer cross-checking between validators is opt-in and off by default. - **Live status is unproven.** Offline tests don't show live KVM execution, a live model run or confirmed on-chain payment. - **Hypertrain is research.** Quality parity with centralized training is not claimed. - **Sentinel is not built.** The same goes for monetization in the apps. diff --git a/config/challenges-v2.example.toml b/config/challenges-v2.example.toml index 66d86477e..40270a5b9 100644 --- a/config/challenges-v2.example.toml +++ b/config/challenges-v2.example.toml @@ -6,12 +6,10 @@ introduced_epoch = "CHOOSE_ACTIVATION_EPOCH" [[challenges]] id = "bounty" -public_key = "743688a1e1b2848b309205706b4dcae54bffe4233a5d7018053471e1dce45c21" emission_share_bps = 3000 policy = "all_metagraph_hotkeys" [[challenges]] id = "proof" -public_key = "3e7f70f09165e265ab89ab04a4fc91dc0531d54a100c538fb14c6f008421c375" emission_share_bps = 7000 policy = "all_metagraph_hotkeys" diff --git a/config/challenges-v3.example.toml b/config/challenges-v3.example.toml index cf311f7d7..eaa732e7a 100644 --- a/config/challenges-v3.example.toml +++ b/config/challenges-v3.example.toml @@ -8,18 +8,15 @@ introduced_epoch = "CHOOSE_ACTIVATION_EPOCH" [[challenges]] id = "bounty" -public_key = "743688a1e1b2848b309205706b4dcae54bffe4233a5d7018053471e1dce45c21" emission_share_bps = 3000 policy = "all_metagraph_hotkeys" [[challenges]] id = "opentype" -public_key = "ac6cad384e46122a7000a6efa71d084a9a2d6407ab22e86aa2b2e5e9dcb2e43f" emission_share_bps = 3500 policy = "all_metagraph_hotkeys" [[challenges]] id = "proof" -public_key = "3e7f70f09165e265ab89ab04a4fc91dc0531d54a100c538fb14c6f008421c375" emission_share_bps = 3500 policy = "all_metagraph_hotkeys" diff --git a/config/challenges.toml b/config/challenges.toml index f0de33649..10dd0d700 100644 --- a/config/challenges.toml +++ b/config/challenges.toml @@ -1,4 +1,4 @@ -# Development challenge profile (emission shares and challenge keys). The master reads +# Development challenge profile (emission shares). The master reads # this unsigned file; the gateway serves it at GET /v1/trust and validators build their # view from that. To change shares: edit, bump `version`, restart the master. # @@ -15,12 +15,10 @@ introduced_epoch = 0 [[challenges]] id = "bounty" -public_key = "743688a1e1b2848b309205706b4dcae54bffe4233a5d7018053471e1dce45c21" emission_share_bps = 2000 policy = "all_metagraph_hotkeys" [[challenges]] id = "proof" -public_key = "3e7f70f09165e265ab89ab04a4fc91dc0531d54a100c538fb14c6f008421c375" emission_share_bps = 8000 policy = "all_metagraph_hotkeys" diff --git a/deploy/README.md b/deploy/README.md index 1f5ea4267..5f538b27c 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -84,15 +84,13 @@ Prepare a private directory, owned so container UID 65532 can read it: ```text deploy/secrets/master/ - gateway.key - proof.key - bounty.key # one .key per trusted container challenge + proof.key # signs Proof topics (miners pin its public key) operator.token proof-vm.token proof-vm-ca.pem ``` -Seeds are raw 32 bytes or 64 hexadecimal characters. Bearers are nonempty opaque values. Files are regular, +The seed is raw 32 bytes or 64 hexadecimal characters. Nothing else on the master signs: the gateway seals unsigned wire-v2 bundles, so there is no gateway key and no per-challenge leaf key. Bearers are nonempty opaque values. Files are regular, not hardlinks or symlinks, and mode 0400 or 0600. The Proof VM token and CA are needed only when `PROOF_VM_ORCHESTRATOR_URL` is set. @@ -137,8 +135,7 @@ install -d -m 0700 -o 65532 -g 65532 "$dir" chown 65532:65532 "$dir"/*.token ``` -Put the matching leaf seed at `deploy/secrets/master/.key` (its public key -is the challenge profile row) and any challenge settings in the registry `env` table, +Put any challenge settings in the registry `env` table, for example `BOUNTY_BACKEND_PUBLIC_URL`. Challenge-specific secrets, such as Bounty's `session.key`, go next to the tokens; see each challenge's operator guide ([Bounty](https://github.com/CortexLM/bounty/blob/main/docs/operator.md), @@ -166,8 +163,7 @@ Create `deploy/env/python-validator.env` and set the exact master HTTPS origin, netuid, chain network, live subnet `version_key`, VPC peer bind address and private host paths. `role-validator.yml` is verify-only and needs no wallet. Weight setting is a separate role, `role-weight-setter.yml`, the only one that also takes the -validator wallet name and hotkey. `--gateway-public` (the gateway sealing key) is an -optional extra pin; the HTTPS gateway is otherwise the source of the challenge profile. Optional fallback RPCs are a JSON list of at most eight +validator wallet name and hotkey. The HTTPS gateway is the only anchor for the challenge profile and the bundle; nothing it serves is signed. Optional fallback RPCs are a JSON list of at most eight credential-free `wss://` URLs. The weight-setter mounts the wallet directory read-only at `/run/wallets`; the @@ -201,7 +197,7 @@ submit. To run a validator without Compose, use `scripts/run-validator.sh` from a repository clone. It needs `WALLET_NAME` and `WALLET_HOTKEY` only to submit weights -(`--verify-only` needs neither), takes an optional `GATEWAY_PUBLIC` pin, reads the +(`--verify-only` needs neither), reads the challenge profile from the gateway, and refuses to run master-only commands; see the [quick start](../docs/external-miner/validators.md#quick-start-one-script). @@ -338,8 +334,8 @@ backing up state, pulling that digest and recreating one role. The Cortex image itself has deliberately no `git pull`, Docker-socket watcher, CI SSH deployment or unattended host updater; only challenge containers -auto-update, through the supervisor gates above, and they never hold a leaf -seed. Safe automation of the Cortex image must preserve the same operator gate, +auto-update, through the supervisor gates above, and they never hold a signing +key. Safe automation of the Cortex image must preserve the same operator gate, take a consistent SQLite backup, verify role health plus a fresh sealed bundle, and restore both the prior digest and schema-compatible state on failure. diff --git a/deploy/compose/role-master.yml b/deploy/compose/role-master.yml index 394598ee3..896d52945 100644 --- a/deploy/compose/role-master.yml +++ b/deploy/compose/role-master.yml @@ -19,10 +19,8 @@ services: BASE_STATE_DIR: /var/lib/base BASE_CHALLENGES_FILE: /etc/base/config/challenges.toml BASE_MEASUREMENTS_FILE: /etc/base/config/measurements.toml - BASE_GATEWAY_SK_FILE: /run/secrets/gateway.key PROOF_SK_FILE: /run/secrets/proof.key BASE_GATEWAY_ADMIN_TOKEN_FILE: /run/secrets/operator.token - BASE_CHALLENGE_KEYS_DIR: /run/secrets BASE_CHALLENGE_REGISTRY_FILE: /etc/base/challenges/registry.toml BASE_CHALLENGE_SECRETS_DIR: /run/challenge-secrets ports: diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index db7707299..80804ae89 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -10,9 +10,7 @@ regular files, owned/readable by that identity, with mode 0400 or 0600. | File | Purpose | | --- | --- | -| `gateway.key` | 32-byte sr25519 seed for bundle seals | -| `.key` | leaf seed of each trusted container challenge, e.g. `bounty.key` | -| `proof.key` | Proof topic/leaf seed matching the challenge profile | +| `proof.key` | 32-byte sr25519 seed that signs Proof topics; miners pin its public key (`--proof-public`) | | `operator.token` | bearer for master administrative routes | | `proof-vm.token` | bearer shared with the dedicated VM host | | `proof-vm-ca.pem` | CA used to verify the VM-host TLS certificate | @@ -41,6 +39,8 @@ identity directory is mounted at `/run/validator` and contains `consensus.key`, `tls.crt` and `tls.key`. The validator never receives gateway, challenge, Proof, operator or provider credentials. +The master holds no gateway or challenge-leaf signing key: bundles and leaves are unsigned (wire v2). + ## VM-host files Private host files live under `/etc/proof-vm`, outside this repository: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 3c187795d..9f3cb9ab4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -34,7 +34,7 @@ flowchart LR | Directory | Responsibility | | --- | --- | -| `src/cortex/protocol` | SCALE wire format, signatures, Merkle roots, aggregation, challenge profile | +| `src/cortex/protocol` | SCALE wire format, hashes, Merkle roots, aggregation, challenge profile | | `src/cortex/gateway` | Durable raw leaves and immutable sealed bundles | | `src/cortex/validator` | Independent historical chain reads and exact weight submission | | `src/cortex/challenges` | Registry, weights client, public proxy and auto-updating supervisor | @@ -50,13 +50,13 @@ flowchart LR 1. At the end of each epoch, the master calls `GET /internal/v1/get_weights?epoch=` on each trusted, registered container, with its private bearer. -2. It converts the weights of the sealed participant set into signed leaves: +2. It converts the weights of the sealed participant set into leaves: `floor(10^12 * w / max(W, full_share_mass))` under algorithm 3. 3. A missing, failing or invalid answer becomes `NoScore(ChallengeInternal)`, and that challenge's share burns. 4. The gateway seals all leaves. 5. Validators recompute `share * min(sum(leaves), 10^12) / 10^12` per challenge - from the signed leaves alone. + from the leaves alone. The supervisor does the following: @@ -116,7 +116,7 @@ published rules without a separately signed revision. The master records chain epochs and snapshot blocks durably. It emits exact participant coverage for both challenges and seals completed epochs only after -accepted jobs and in-flight intake have finished. A failed challenge emits signed +accepted jobs and in-flight intake have finished. A failed challenge emits `NoScore(ChallengeInternal)` rather than leaving the epoch uncovered. Validators recompute the bundle against independent chain state. Missing or @@ -126,7 +126,7 @@ remain pending in the journal to prevent duplicate submission after restart. ## Compatibility and limits -The bundle/SCALE/signature contracts preserve cross-language vectors. Topic and +The bundle/SCALE/hash contracts preserve cross-language vectors. Topic and VM JSON protocols are explicitly version 2; old JSON topic signatures are not silently reinterpreted. The custom Schnorrkel signing context uses Python Merlin transcripts and native libsodium Ristretto operations; keep its vectors and diff --git a/docs/BUNDLE_SPEC.md b/docs/BUNDLE_SPEC.md index 80db1e496..cdbdd6ab7 100644 --- a/docs/BUNDLE_SPEC.md +++ b/docs/BUNDLE_SPEC.md @@ -3,9 +3,9 @@ **Status:** FROZEN (task 8 wave gate) **Normative for:** `bundle`, `aggregate`, gateway seal, validator verify/recompute **Encoding:** parity SCALE (`parity-scale-codec`), little-endian multi-byte integers -**protocol_version of this document:** `1` +**protocol_version of this document:** `2` -This file is the single source of truth for hashed and signed epoch-bundle bytes. +This file is the single source of truth for hashed epoch-bundle bytes. Where this document and any other source disagree, **this document wins**, with one scoped exception: the aggregation formula in §6 is defined by behavioural parity with the BASE Python master, so that `/v1/weights/latest` can replace @@ -13,13 +13,21 @@ the BASE Python master, so that `/v1/weights/latest` can replace the pinned upstream and its frozen vectors win. This supersedes D16, which had made the Python float/JSON vectors characterization only. -**Trust provenance amendment:** bundle bytes, hashes, signatures and every wire-format -section are unchanged. Where this document says "local owner-signed trust root", read -"challenge profile": the unsigned `config/challenges.toml` and `config/measurements.toml` +**Wire v2 amendment (founder decision 2026-10-19):** protocol_version `2` removes every +signature from the master/bundle path. There is no `challenge_sig` on leaves, no `gateway_hotkey` +and no `gateway_sig` on the bundle, and no per-challenge public key in the challenge profile. +Integrity rests on content hashes (leaf hash, Merkle root, `measurements_digest`), the chain +snapshot (`block_hash`, `metagraph_root`, `uid_map`) and the gateway itself: validators trust the +gateway over TLS, and gateway intake uses internal/admin/worker bearer tokens. Version `1` +bundles are rejected (`ProtocolVersionUnsupported`). Signatures that remain are outside the +bundle bytes: the validator hotkey signature on the on-chain weight extrinsic, peer root +statements and dissents (`base-root-v1`, `base-dissent-v1`), Proof topics and miner submissions. + +The challenge profile is the unsigned `config/challenges.toml` and `config/measurements.toml` read by the master and served by the gateway at `GET /v1/trust`. Validators build their -`TrustRoot` from that route over HTTPS (optionally pinning the gateway hotkey) and keep -version watermarks that refuse rollback. The `base-trustroot-v1` domain is no longer -produced; no owner signature over these documents exists. +`TrustRoot` from that route over HTTPS and keep version watermarks that refuse rollback. The +`base-trustroot-v1` domain is no longer produced; no owner signature over these documents exists. +Where this document below says "signed" for the bundle, leaves or `trust root`, read "hashed". Checklist map: [`BUNDLE_SPEC_CHECKLIST.md`](./BUNDLE_SPEC_CHECKLIST.md). CI gate: `cargo run -p xtask -- spec-check`. @@ -59,7 +67,7 @@ JSON MAY appear only on human HTTP error bodies and operator logs. It MUST NOT a - leaf preimages - merkle inputs -- bundle body bytes under signature +- bundle body bytes - dissent payloads under signature - peer root statements under signature - on-chain `WeightsTlockPayload` construction inputs beyond the four fields in §12 @@ -76,7 +84,7 @@ protocol_version: u16 | Rule | Requirement | |------|-------------| -| Current | `1` | +| Current | `2` | | Compatibility | A validator that does not implement the received `protocol_version` MUST reject the bundle (`DissentReasonCode::ProtocolVersionUnsupported`) and MUST NOT submit weights derived from it | | Schema change | Any change to field set, field order, enum discriminants, hash domain tags, aggregation semantics, or merkle construction is a **major** bump of `protocol_version` | | Patch docs | Editorial doc fixes that do not change bytes MAY keep the same version; the frozen byte contract does not move | @@ -119,8 +127,7 @@ LeafV1 = scale( challenge_id: Bytes, // Vec, UTF-8 challenge id miner_hotkey: [u8; 32], epoch: u64, - score_or_absence: ScoreOrAbsence, - challenge_sig: [u8; 64] // sr25519 signature bytes + score_or_absence: ScoreOrAbsence ) ``` @@ -144,24 +151,14 @@ ScoreOrAbsence (SCALE enum, u8 discriminant): | 7 | `PolicySkip` | Reserved; MUST NOT be used to shrink the expected set (D24). Validators reject bundles that use this to omit coverage | | 8–255 | _reserved_ | Reject unknown codes on verify | -Absence is **signed by the challenge key** over the same domain as scores (see §3.4). Silence is not absence. - -### 3.4 Challenge signature - -Domain-separated message (plan task 14 tags): - -```text -msg = "base-rawweight-v1" as length-prefixed UTF-8 tag ‖ scale(RawWeightBodyV1) +Absence is recorded as an explicit leaf, exactly like a score. Silence is not absence. -RawWeightBodyV1 = scale( - challenge_id: Bytes, - miner_hotkey: [u8; 32], - epoch: u64, - score_or_absence: ScoreOrAbsence -) -``` +### 3.4 Leaf hash -`challenge_sig` is sr25519 over `msg`, verifiable with the challenge public key from the challenge profile (`config/challenges.toml`, obtained from the gateway `/v1/trust` over HTTPS, optionally gateway-pinned) (D18). +There is no leaf signature (removed in protocol_version 2). A leaf is identified by +`sha256(LeafV1)`; the gateway dedupes on it and the Merkle tree commits to it. Which +challenge may emit a leaf is decided by the challenge profile (D18) and by gateway +authentication of the submitting challenge/master. ### 3.5 Sort key before tree build @@ -182,7 +179,7 @@ Field order is normative. SCALE encode in this order only: ```text EpochBundleBodyV1 = scale( - protocol_version: u16, // must be 1 for this doc + protocol_version: u16, // must be 2 for this doc epoch: u64, netuid: u16, block_B: u64, // epoch_end_block (inclusive end of epoch window) @@ -194,19 +191,11 @@ EpochBundleBodyV1 = scale( uid_map: Vec<([u8; 32] /*hotkey*/, u16 /*uid*/)>, // sorted by hotkey leaves: Vec, // sorted by scale(challenge_id, miner_hotkey) merkle_root: [u8; 32], // recomputed over leaf preimages - final_vector: Vec<(u16 /*uid*/, u16 /*weight*/)>, // sorted by uid - gateway_hotkey: [u8; 32] + final_vector: Vec<(u16 /*uid*/, u16 /*weight*/)> // sorted by uid ) ``` -Signed envelope: - -```text -EpochBundleV1 = scale( - body: EpochBundleBodyV1, - gateway_sig: [u8; 64] // sr25519 over tag "base-bundle-v1" ‖ scale(body) -) -``` +The sealed bundle is exactly the SCALE bytes of this body: no envelope, no signature. (Type names keep the `V1` suffix for continuity; the field set is the v2 one.) ### 4.2 `block_B` and `block_hash` @@ -279,7 +268,7 @@ uid_map: Vec<(hotkey, uid: u16)> algorithm_version: u16 // must be 1 ``` -Only leaves that already passed signature and participant-set checks enter aggregation. +Only leaves that already passed participant-set checks enter aggregation. ### 6.2 Constants @@ -296,7 +285,7 @@ algorithm_version: u16 = 1 `min_allowed_weights` and `max_weight_limit` are pinned constants, not chain reads. The deployed master constructs `AggregationService(session_factory, freshness_seconds=…)` without passing either, so it runs on the `aggregate_challenge_weights` defaults. -Pinning them keeps recompute a pure function of the signed body. +Pinning them keeps recompute a pure function of the body. ### 6.3 Float determinism (normative) @@ -446,7 +435,7 @@ Validators **derive** the expected set. They MUST NOT trust a set announced only In `challenges.toml`, each challenge carries: ```text -ParticipantPolicy (SCALE enum for signed body; TOML maps 1:1): +ParticipantPolicy (SCALE enum for the body; TOML maps 1:1): 0 = AllMetagraphHotkeys // every hotkey in metagraph_at(block_hash) 1 = StakeAtLeast { min_stake: u64 } @@ -531,7 +520,7 @@ Validators compare: | Validator mirroring | Validators MAY re-serve a bundle they have verified and persisted; peers SHOULD prefer multi-source fetch | | **No last-known-good** | MUST NOT fall back to a previous epoch's bundle, root, or vector when the current epoch fetch/verify fails. Failure → class B / degraded path, not stale success (aligns with D13 spirit for attestation). The unsealed burn response on `/v1/weights/latest` is an operator-safety default, not a last-known-good seal and **not** a submit path: a validator-local copy of a previously verified seal MUST NOT be used for set-weights while latest is unsealed. Mid-epoch tip revision bumps are intentional tip-tracking, not last-known-good | -Gateway signature and leaf signatures are always verified against the challenge profile after fetch. +Hashes, Merkle root and the challenge profile are always verified after fetch; the gateway is trusted over TLS. --- @@ -559,7 +548,7 @@ DissentV1 = scale( | Code | Name | Typical class | |------|------|----------------| | 0 | `VectorMismatch` | A — inputs ok, peer roots ok, gateway vector ≠ recomputation | -| 1 | `LeafSignatureInvalid` | B | +| 1 | _retired_ (`LeafSignatureInvalid`; number reserved) | — | | 2 | `LeafChallengeKeyUnknown` | B (D18) | | 3 | `IncompleteParticipantSet` | B (D24) | | 4 | `MerkleRootMismatch` | B | @@ -570,7 +559,7 @@ DissentV1 = scale( | 9 | `PeerRootConflict` | B | | 10 | `PeerSampleInsufficient` | Degraded / no submit (D26) | | 11 | `ShareMassBelowThreshold` | B after quarantine | -| 12 | `BundleSignatureInvalid` | B | +| 12 | _retired_ (`BundleSignatureInvalid`; number reserved) | — | | 13 | `AggregationOverflow` | B | | 14 | `EmptyScoreVectorNoSubmit` | no-submit | | 15 | `UidMapMismatch` | B | @@ -640,21 +629,19 @@ There is **no** field for a 256-bit merkle root. `version_key` is 64 bits and MU Non-equivocation rests on: 1. In-epoch signed peer root exchange (hotkey-authenticated HTTPS). -2. Durable local persistence of the signed bundle and peer root statements. +2. Durable local persistence of the bundle and peer root statements. 3. Optional commitments-pallet announcement **only if** metadata snapshot proves the pallet exists — not required by this spec. Any code or doc that claims the merkle root is committed inside `WeightsTlockPayload` is wrong and MUST be corrected. --- -## 13. Gateway bundle signature - -```text -msg = tag "base-bundle-v1" ‖ scale(EpochBundleBodyV1) -gateway_sig = sr25519_sign(gateway_hotkey_sk, msg) -``` +## 13. Gateway authentication (no bundle signature) -`gateway_hotkey` MUST equal on-chain `SubnetOwnerHotkey` for master-only gateway operation (D3); validators still verify the signature cryptographically and MAY additionally check owner equality. +Protocol version 2 has no gateway signature. A validator trusts the gateway it is configured +for (HTTPS) and verifies everything it can recompute: hashes, Merkle root, chain snapshot, +participant set and the final vector. Gateway intake (`POST /v1/weights/raw` and admin routes) +requires bearer tokens. --- @@ -663,12 +650,12 @@ gateway_sig = sr25519_sign(gateway_hotkey_sk, msg) A `verify(bundle, chain, local_trust_root)` implementation MUST check, in order: 1. `protocol_version` supported -2. `gateway_sig` over body +2. _(retired: gateway signature)_ 3. `block_hash` matches `chain.block_hash(block_B)` 4. `metagraph_root` and `uid_map` match `metagraph_at(block_hash)` 5. `emission_shares` equal the challenge profile and sum to `10_000` 6. `measurements_digest` equals local measurements digest -7. Every leaf challenge key known locally; every `challenge_sig` valid +7. Every leaf `challenge_id` is in the challenge profile 8. Participant-set completeness (§7) 9. Leaf sort order canonical; `merkle_root` recomputes 10. `algorithm_version == 1` and `final_vector` equals `aggregate(...)` under §8 dual equality @@ -690,8 +677,6 @@ Failure modes map to §10.1 reason codes. | Tag string | Used for | |------------|----------| -| `base-bundle-v1` | Epoch bundle body | -| `base-rawweight-v1` | Challenge leaf body | | `base-dissent-v1` | Dissent body | | `base-root-v1` | Peer `(epoch, merkle_root)` | | `base-trustroot-v1` | Retired: formerly the owner-signed profile body; no longer produced | @@ -711,7 +696,7 @@ Length-prefix rule: SCALE `Bytes` encoding of the tag string UTF-8 bytes (compac | D7 SCALE + metagraph_root | §1, §4.3 | | D8 integer / Hamilton / checked | §6 | | D9 RFC6962 + EMPTY_ROOT | §3 | -| D18 local challenge keys | §3.4, §5, §14 | +| D18 challenge profile membership | §3.4, §5, §14 | | D19 honest claim | §11.1 | | D23 emission shares in challenge profile | §5 | | D24 absence + derived set | §3.3, §7 | @@ -719,4 +704,4 @@ Length-prefix rule: SCALE `Bytes` encoding of the tag string UTF-8 bytes (compac --- -**End of frozen BUNDLE_SPEC protocol_version=1.** +**End of frozen BUNDLE_SPEC protocol_version=2.** diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index 0dd56e290..77ebd3ff7 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -3,15 +3,15 @@ # Challenge containers A Cortex challenge is a Docker image that scores miners. The master runs it, -reads its weights once per completed epoch, signs one leaf per expected hotkey +reads its weights once per completed epoch, records one leaf per expected hotkey and seals the epoch. Validators only verify the sealed bundle and submit it. ```text -owner (offline) challenges.toml v>=3: id, leaf key, emission bps, policy +owner (offline) challenges.toml v>=3: id, emission bps, policy master operator challenge-registry.toml: image, channel, resources, env challenge-supervisor pulls, verifies, canaries, runs and updates containers challenge container public routes + GET /internal/v1/get_weights -master polls weights, signs leaves, seals, proxies /challenge// +master polls weights, records leaves, seals, proxies /challenge// validator GET /v1/weights/latest + /v1/bundle/, recompute, submit ``` @@ -42,7 +42,7 @@ writable named volume at `/data`. The image must create `/data` owned by Every file under `/run/secrets/` comes from the host directory `//`, mounted read-only. A container never -receives a leaf-signing seed. +receives a signing key. ### Routes diff --git a/docs/NAMING.md b/docs/NAMING.md index 53e67f243..e3362a1c0 100644 --- a/docs/NAMING.md +++ b/docs/NAMING.md @@ -23,8 +23,6 @@ The preserved signing domains are: | Domain | Purpose | | --- | --- | -| `base-rawweight-v1` | challenge leaves | -| `base-bundle-v1` | sealed epoch bundles | | `base-root-v1` | validator root statements | | `base-dissent-v1` | validator dissent evidence | | `base-proof-topic-v1` | published Proof topics | diff --git a/docs/OPERATOR_SECURITY.md b/docs/OPERATOR_SECURITY.md index d9ebc7c1e..3f781cfc6 100644 --- a/docs/OPERATOR_SECURITY.md +++ b/docs/OPERATOR_SECURITY.md @@ -19,10 +19,8 @@ recovery. It complements the [threat model](THREAT_MODEL.md). ## Trust and pins - [ ] `GET /v1/trust` on the gateway returns the intended challenge and measurement - profile at the deployment epoch, over HTTPS (optionally pinned with - `--gateway-public`). -- [ ] Every challenge public key matches its mounted `.key` (or - `PROOF_SK_FILE`) seed, and the gateway public key differs from all of them. + profile at the deployment epoch, over HTTPS. +- [ ] The Proof topic key matches `PROOF_SK_FILE`; no gateway or leaf key exists. - [ ] The challenge profile is legacy bounty/proof 2000/8000 (algorithm 1), 3000/7000 (algorithm 2, version >=2), or unique ids summing to 10000 (algorithm 3, version >=3). Bump the profile diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index 87fa44543..67e8dc166 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -16,16 +16,16 @@ The Merkle root is not part of the Bittensor weight extrinsic. Peer consensus is therefore not a public chain transparency log. A colluding validator set can agree on one bad root, and an operator can delete local evidence. The owner signs the roots and runs the master, so a malicious owner can authorize dishonest -challenge keys or generated topic rules that validators will faithfully accept. +challenge shares or generated topic rules that validators will faithfully accept. ## Trust boundaries | Boundary | Trusted property | Residual risk | | --- | --- | --- | -| challenge profile (gateway `/v1/trust`) | challenge ids, keys, shares, measurement digest | a compromised gateway or broken TLS can serve a malicious profile; pin `--gateway-public` | +| challenge profile (gateway `/v1/trust`) | challenge ids, shares, measurement digest | a compromised gateway or broken TLS can serve a malicious profile; TLS is the only anchor | | master gateway | durable intake and immutable seals | availability and censorship remain operator risks | | validator | independent chain snapshot, recomputation and dispatch journal | chain RPC eclipse or colluding validators | -| challenge container | weights it returns for its own share | can misallocate or zero its own share; never holds a leaf seed, cannot move another challenge's mass | +| challenge container | weights it returns for its own share | can misallocate or zero its own share; holds no signing key, cannot move another challenge's mass | | challenge-supervisor | Docker control, image provenance and label checks | holds the Docker socket (host-root equivalent); a compromised GitHub repository with valid provenance ships code | | CortexLM/backend feed | Bounty scoring publication | backend controls the underlying adjudication truth | | topic RLM guest | topic-scoped setup state | model output is untrusted until checked and signed | diff --git a/docs/external-miner/validators.md b/docs/external-miner/validators.md index 824aa24ec..b7eb72abf 100644 --- a/docs/external-miner/validators.md +++ b/docs/external-miner/validators.md @@ -11,8 +11,6 @@ scripts/run-validator.sh --verify-only # no wallet or hotkey needed # To submit weights, a registered validator wallet is required: export WALLET_NAME=validator WALLET_HOTKEY=default scripts/run-validator.sh -# Optional extra pin of the gateway sealing hotkey: -export GATEWAY_PUBLIC= ``` `scripts/run-validator.sh` defaults to `GATEWAY=https://chain.joinbase.ai`, @@ -41,21 +39,20 @@ bounty/proof 2000/8000 (algorithm 1), `bounty` 3000 bps and `proof` 7000 bps see [Changing emission shares](../../deploy/README.md#changing-emission-shares)). Sealed `GET /v1/weights/latest` responses expose `algorithm_version` from the -signed bundle. `emission_shares` contains the configured ceilings; each +bundle. `emission_shares` contains the configured ceilings; each `source_challenges[].emission_percent` reflects its allocated share, including the ten-report Bounty ramp under algorithm 2 and `min(sum(leaves), 10^12) / 10^12` of the share under algorithm 3. Author allocations to UID0 or -unmapped hotkeys still burn. Validators always recompute from signed leaves; +unmapped hotkeys still burn. Validators always recompute from the leaves; they never trust these display fields as consensus inputs. ## Prepare identity Install the [Python package with its chain extra](README.md#installation). Obtain the gateway HTTPS URL. The validator builds its challenge profile -(challenge keys, shares, measurements) from `GET /v1/trust` on that gateway and -refuses a profile whose version goes backwards. `--gateway-public` is an optional -extra pin: it refuses a gateway whose sealing hotkey differs. Obtain it from an -independent source if you use it. TLS to the gateway is the trust anchor otherwise. +(challenge ids, shares, measurements) from `GET /v1/trust` on that gateway and +refuses a profile whose version goes backwards. TLS to the gateway is the trust anchor; +bundles are unsigned and checked by content hashes and the chain. Verifying needs no wallet: run with `--verify-only` and no `--wallet-name` or `--wallet-hotkey`. Submitting weights needs a registered Bittensor hotkey in a @@ -123,7 +120,7 @@ deployment before running it: ```bash uv run cortex validator \ - --gateway "$GATEWAY" [--gateway-public "$GATEWAY_PUBLIC_KEY"] \ + --gateway "$GATEWAY" \ --network "$CHAIN_NETWORK" --netuid "$NETUID" \ --fallback-endpoints "$CHAIN_FALLBACK_ENDPOINTS_JSON" \ --version-key "$VALIDATOR_VERSION_KEY" \ @@ -168,9 +165,8 @@ role points to the master gateway and has no gateway or challenge service. ## Verify before dispatch The validator reads `GET /v1/weights/latest`, obtains the corresponding binary -bundle, then verifies the gateway signature (against `/v1/trust`, plus -`--gateway-public` when given) and the gateway-served challenge profile. It checks protocol version, epoch and block hash, -metagraph stakes and UID mapping, challenge keys and emission shares, +bundle, then verifies the gateway-served challenge profile. It checks protocol version, epoch and block hash, +metagraph stakes and UID mapping, emission shares, participant completeness, measurements and Merkle root. Frozen wire details are in the [bundle specification](../BUNDLE_SPEC.md). @@ -186,7 +182,7 @@ a reorg or a changed/unsealed latest response prevents submission. | Verified sealed Match with `burn_outcome: true`, `uids: [0]`, `weights: [1.0]` | Submit the sealed burn to UID 0 | | Verified sealed vector allocating 100% to a nonzero owner or validator-permit UID | Refuse; this is not a burn | | Valid inputs (and agreeing peers where required), but gateway's final vector differs | Class A: submit independently recomputed weights and persist signed dissent | -| A challenge has invalid leaf signatures, wrong leaf epoch or incomplete participants | Quarantine that challenge; submit only if surviving signed mass is at least 5000 bps | +| A challenge has a wrong leaf epoch or a leaf hash that breaks the Merkle root or incomplete participants | Quarantine that challenge; submit only if surviving mass is at least 5000 bps | | Structural failure, unknown challenge, invalid Merkle root, own prior root for the epoch disagrees, or peer disagreement with `--peer-consensus` | Refuse | Under algorithm 2, quarantining Bounty retains Proof's absolute 7000 bps @@ -250,7 +246,7 @@ The peer listener exposes: Consensus uses sr25519 signing context `base-sr25519-v1` and SCALE-encoded domain/payload vectors. Exact domains are `base-root-v1`, `base-dissent-v1`, -`base-bundle-v1`, `base-rawweight-v1` (`base-trustroot-v1` is retired). Generic Substrate +(`base-bundle-v1`, `base-rawweight-v1` and `base-trustroot-v1` are retired). Generic Substrate `wallet.sign()` signatures cannot replace these signatures. The Python metagraph projection uses chain-provided integer stakes. Validate diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 7660136f3..c75eb53cf 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -12,12 +12,10 @@ conflicting values are an error. Existing names and crypto domains are preserved | `BASE_CHAIN_ENDPOINT` | Official Bittensor SDK alias (`finney`, `test`, `archive`, `local`) or credential-free `wss://` RPC origin | | `BASE_CHAIN_FALLBACK_ENDPOINTS` | JSON list of up to eight credential-free `wss://` fallback RPC endpoints | | `BASE_STATE_DIR` | Durable private state directory; default `/var/lib/cortex` | -| `BASE_CHALLENGES_FILE` | Unsigned challenge profile (challenge keys and emission shares); served by the gateway at `GET /v1/trust` | +| `BASE_CHALLENGES_FILE` | Unsigned challenge profile (emission shares); served by the gateway at `GET /v1/trust` | | `BASE_MEASUREMENTS_FILE` | Unsigned measurement profile; served with the challenge profile | -| `BASE_GATEWAY_SK_FILE` | Gateway seal seed file | -| `PROOF_SK_FILE` | Proof topic and leaf seed file | +| `PROOF_SK_FILE` | Proof topic seed file | | `BASE_GATEWAY_ADMIN_TOKEN_FILE` | Required operator bearer file | -| `BASE_CHALLENGE_KEYS_DIR` | Directory of container challenge leaf seeds, `.key`; default `/run/secrets` | | `BASE_CHALLENGE_REGISTRY_FILE` | Optional [challenge registry](../CHALLENGES.md); unset runs no container challenge | | `BASE_CHALLENGE_SECRETS_DIR` | Per-challenge `/internal.token` bearers; default `/run/challenge-secrets` | | `PROOF_VM_ORCHESTRATOR_URL` | Dedicated VM host HTTPS origin | @@ -99,7 +97,7 @@ with a validator hotkey and consensus seed: The version key is intentionally blank in the example. Populate it from the live subnet; do not infer it from tests. The CLI flags `--wallet-name` and `--wallet-hotkey` are required only to submit weights (or with `--peer-consensus`); -`--gateway-public` is an optional extra pin of the gateway sealing key. The +The challenge profile always comes from `GET /v1/trust`. `--verify-only --once` is the non-dispatch preflight; it needs no wallet but still requires full historical-chain access. diff --git a/docs/validator-quickstart.md b/docs/validator-quickstart.md index 3747858bf..65e16db3c 100644 --- a/docs/validator-quickstart.md +++ b/docs/validator-quickstart.md @@ -24,11 +24,10 @@ uv sync --locked --extra chain scripts/run-validator.sh --verify-only ``` -2. To submit, set the wallet. Optionally add the independently pinned gateway key. +2. To submit, set the wallet. ```bash export WALLET_NAME=validator WALLET_HOTKEY=default - export GATEWAY_PUBLIC= # optional ``` 3. Run for real. It verifies and submits every epoch. @@ -45,7 +44,6 @@ Environment variables: | --- | --- | --- | | `WALLET_NAME` | required to submit | Validator wallet name | | `WALLET_HOTKEY` | required to submit | Registered validator hotkey | -| `GATEWAY_PUBLIC` | optional | Extra pin of the gateway sealing hotkey | | `GATEWAY` | `https://chain.joinbase.ai` | Gateway URL | | `NETUID` | `100` | Subnet | | `NETWORK` | `finney` | Bittensor network | @@ -81,7 +79,6 @@ Then `cortex validator` takes over. It uses the owner public key and the signed - The script refuses `master`, `challenge-supervisor` and `vm-host` as arguments, and exits with code 2. - It also refuses to run if `CORTEX_MASTER=1`, `BASE_MASTER_SECRETS_HOST_DIR` or `BASE_CHALLENGE_SECRETS_HOST_DIR` is set. Unset them first. -- Pin `GATEWAY_PUBLIC` yourself. A key taken from the gateway proves nothing. - Keep wallet files private. `--dry-run` masks the wallet name and hotkey in its output. - Run `--verify-only` before the first real run. @@ -93,9 +90,6 @@ No. Validators never run challenge containers or Proof evaluation, and they need **Do I need a GPU or a KVM host?** No. Those are for the master operator. -**The script says `GATEWAY_PUBLIC is not set`.** -Export the gateway hotkey you pinned from an independent source. - **The script says the netuid doesn't match.** The gateway answered for a different subnet than `NETUID`. Check `GATEWAY` and `NETUID`. @@ -103,7 +97,7 @@ The gateway answered for a different subnet than `NETUID`. Check `GATEWAY` and ` Set `VERSION_KEY` yourself, or check that `NETWORK` is reachable. **Where does the challenge profile come from?** -From the gateway at `GET /v1/trust` over HTTPS; no local files are needed. `--gateway-public` (`GATEWAY_PUBLIC` in the script) is an optional extra pin of the gateway sealing key. +From the gateway at `GET /v1/trust` over HTTPS; no local files are needed. **Do I need a hotkey?** Only to submit weights. `scripts/run-validator.sh --verify-only` and the CLI `--verify-only` need no wallet; submitting needs `WALLET_NAME`/`WALLET_HOTKEY` (`--wallet-name`/`--wallet-hotkey`). diff --git a/scripts/check_deploy.py b/scripts/check_deploy.py index 8777cbda5..30e23b117 100644 --- a/scripts/check_deploy.py +++ b/scripts/check_deploy.py @@ -50,10 +50,8 @@ "BASE_STATE_DIR": "/var/lib/base", "BASE_CHALLENGES_FILE": "/etc/base/config/challenges.toml", "BASE_MEASUREMENTS_FILE": "/etc/base/config/measurements.toml", - "BASE_GATEWAY_SK_FILE": "/run/secrets/gateway.key", "PROOF_SK_FILE": "/run/secrets/proof.key", "BASE_GATEWAY_ADMIN_TOKEN_FILE": "/run/secrets/operator.token", - "BASE_CHALLENGE_KEYS_DIR": "/run/secrets", "BASE_CHALLENGE_REGISTRY_FILE": "/etc/base/challenges/registry.toml", "BASE_CHALLENGE_SECRETS_DIR": "/run/challenge-secrets", } @@ -156,7 +154,7 @@ def _compose_options(command, role: str = "validator") -> dict[str, str]: raise ValueError("validator command contains an invalid or duplicate flag") options[flag] = value expected = VALIDATOR_OPTIONS | (SETTER_OPTIONS if role == "weight-setter" else set()) - if not expected <= set(options) <= expected | {"--gateway-public"}: + if set(options) != expected: raise ValueError("validator command differs from the audited Python runtime contract") return options @@ -368,7 +366,6 @@ def validate_compose(config: dict, role: str) -> None: raise ValueError("master file credentials must use the audited container paths") forbidden_values = { "OPENROUTER_API_KEY", - "BASE_GATEWAY_SK", "PROOF_SK", "BASE_GATEWAY_ADMIN_TOKEN", "PROOF_VM_ORCHESTRATOR_TOKEN", diff --git a/scripts/check_repo.py b/scripts/check_repo.py index 5f88c5f73..b4164f84c 100644 --- a/scripts/check_repo.py +++ b/scripts/check_repo.py @@ -19,7 +19,7 @@ ROOT = Path(__file__).resolve().parents[1] FROZEN_SPECS = { - "docs/BUNDLE_SPEC.md": "6059c4f8f7917a4426c5ef88fb8464ff5c98bd5dc1bc82003eb8c6ce7cb04b7e", + "docs/BUNDLE_SPEC.md": "45f3b43f95c64678b2074bbd2a984f9743137ffbed71a8dfe14071602f98c12d", } SHARES = {"bounty": 2000, "proof": 8000} PROPORTIONAL_SHARES = {"bounty": 3000, "proof": 7000} @@ -171,8 +171,6 @@ def check_challenge_profiles(root: Path) -> list[str]: raise ValueError("duplicate id or invalid share") if not CHALLENGE_ID.fullmatch(identifier): raise ValueError("invalid challenge id") - if not re.fullmatch(r"[0-9a-f]{64}", row["public_key"]): - raise ValueError("invalid challenge public key") actual[identifier] = share if version >= 3: if sum(actual.values()) != 10000: diff --git a/scripts/run-validator.sh b/scripts/run-validator.sh index 19b06e6b9..b2a2d435d 100755 --- a/scripts/run-validator.sh +++ b/scripts/run-validator.sh @@ -8,10 +8,9 @@ # export WALLET_NAME=validator WALLET_HOTKEY=default # scripts/run-validator.sh # verify and submit every epoch # -# The gateway (HTTPS) is the trust anchor; no signed trust files are read. +# The gateway (HTTPS) is the trust anchor; nothing it serves is signed and no keys are pinned. # Required env to submit weights (not for --verify-only): WALLET_NAME, WALLET_HOTKEY. -# Optional env: GATEWAY_PUBLIC (extra pin of the gateway sealing hotkey), -# GATEWAY (https://chain.joinbase.ai), NETUID (100), +# Optional env: GATEWAY (https://chain.joinbase.ai), NETUID (100), # NETWORK (finney), STATE_DB ($HOME/.cortex/validator.sqlite3), # WALLET_PATH ($HOME/.bittensor/wallets), VERSION_KEY (default: live # subnet weights_version read from the chain). @@ -116,7 +115,6 @@ cmd=(cortex validator --gateway "$GATEWAY" --netuid "$NETUID" --network "$NETWORK" --version-key "${VERSION_KEY:-0}" --state-db "$STATE_DB") -[[ -z "${GATEWAY_PUBLIC:-}" ]] || cmd+=(--gateway-public "$GATEWAY_PUBLIC") [[ -z "${WALLET_NAME:-}" || -z "${WALLET_HOTKEY:-}" ]] || cmd+=(--wallet-name "$WALLET_NAME" --wallet-hotkey "$WALLET_HOTKEY" --wallet-path "$WALLET_PATH") if ((verify_only)); then cmd+=(--verify-only --once); elif ((once)); then cmd+=(--once); fi diff --git a/tests/test_deploy_contract.py b/tests/test_deploy_contract.py index 1d8719b63..4226e8506 100644 --- a/tests/test_deploy_contract.py +++ b/tests/test_deploy_contract.py @@ -160,10 +160,8 @@ def master_config(): "BASE_STATE_DIR": "/var/lib/base", "BASE_CHALLENGES_FILE": "/etc/base/config/challenges.toml", "BASE_MEASUREMENTS_FILE": "/etc/base/config/measurements.toml", - "BASE_GATEWAY_SK_FILE": "/run/secrets/gateway.key", "PROOF_SK_FILE": "/run/secrets/proof.key", "BASE_GATEWAY_ADMIN_TOKEN_FILE": "/run/secrets/operator.token", - "BASE_CHALLENGE_KEYS_DIR": "/run/secrets", "BASE_CHALLENGE_REGISTRY_FILE": "/etc/base/challenges/registry.toml", "BASE_CHALLENGE_SECRETS_DIR": "/run/challenge-secrets", }, diff --git a/tests/test_run_validator_script.py b/tests/test_run_validator_script.py index 8db6d72a1..851cf73ce 100644 --- a/tests/test_run_validator_script.py +++ b/tests/test_run_validator_script.py @@ -31,7 +31,6 @@ def env(tmp_path): "LATEST_JSON": str(tmp_path / "latest.json"), "WALLET_NAME": "coldwallet-secret", "WALLET_HOTKEY": "hotkey-secret", - "GATEWAY_PUBLIC": "5GatewayPinnedKey", "VERSION_KEY": "7", } @@ -85,12 +84,17 @@ def test_dry_run_prints_masked_validator_command(env): assert "--wallet-name \\*\\*\\* --wallet-hotkey \\*\\*\\*" in command for expected in ( "--netuid 100", - "--gateway-public 5GatewayPinnedKey", "--version-key 7", "--verify-only --once", ): assert expected in command - for removed in ("--owner-public", "--challenges", "--measurements", "minimum"): + for removed in ( + "--owner-public", + "--challenges", + "--measurements", + "minimum", + "--gateway-public", + ): assert removed not in command assert "epoch=4242 sealed=true" in result.stderr From b463ce4acf17ec3d3eb332cdee390ee6134c3628 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:08:09 +0000 Subject: [PATCH 08/56] Remove signatures from leaves, bundle and gateway; wire format v2 Leaf.challenge_sig, Bundle.gateway_sig, gateway_hotkey, ChallengeEntry.public_key, gateway seed/key files and --gateway-public are gone. Integrity is the Merkle root, metagraph root, block hash and recomputed final vector; POST /v1/weights/raw needs the operator bearer. protocol_version 2 (v1 refused); gateway store refuses a v1 database. Consensus, topic/offer/miner and on-chain hotkey signatures are kept. rust_wire.json keeps only the signature vectors; wire_v2.json is generated by scripts/gen_wire_vector.py. Co-Authored-By: Claude Sonnet 5.5 --- scripts/gen_wire_vector.py | 49 +++++++ src/cortex/config.py | 21 +-- src/cortex/gateway/api.py | 3 +- src/cortex/gateway/models.py | 3 - src/cortex/gateway/service.py | 45 +++---- src/cortex/gateway/store.py | 27 ++++ src/cortex/master.py | 23 +--- src/cortex/protocol/__init__.py | 4 +- src/cortex/protocol/bundle.py | 40 ++---- src/cortex/protocol/consensus.py | 3 +- src/cortex/protocol/crypto.py | 2 - src/cortex/protocol/models.py | 27 ++-- src/cortex/protocol/trust.py | 12 +- src/cortex/validator/__main__.py | 8 -- src/cortex/validator/service.py | 14 +- tests/gateway/test_gateway.py | 177 ++++++++++++-------------- tests/protocol/test_aggregate.py | 19 ++- tests/protocol/test_bundle.py | 88 +++++++------ tests/protocol/test_crypto.py | 24 ++-- tests/protocol/vectors/rust_wire.json | 4 +- tests/protocol/vectors/wire_v2.json | 12 ++ tests/test_keygen_removed.py | 11 ++ tests/test_master.py | 58 ++++----- tests/test_master_config.py | 1 - tests/test_network_e2e.py | 19 ++- tests/validator/test_main.py | 5 +- tests/validator/test_validator.py | 34 ++--- 27 files changed, 349 insertions(+), 384 deletions(-) create mode 100644 scripts/gen_wire_vector.py create mode 100644 tests/protocol/vectors/wire_v2.json create mode 100644 tests/test_keygen_removed.py diff --git a/scripts/gen_wire_vector.py b/scripts/gen_wire_vector.py new file mode 100644 index 000000000..9bb69edf9 --- /dev/null +++ b/scripts/gen_wire_vector.py @@ -0,0 +1,49 @@ +"""Print the frozen v2 bundle wire vector (deterministic). + +scripts/gen_wire_vector.py > tests/protocol/vectors/wire_v2.json +""" + +import json +from hashlib import sha256 + +from cortex.protocol import ( + ChallengeEntry, + Leaf, + MetagraphRow, + NoScore, + Score, + TrustRoot, + build_bundle, +) + + +def vector() -> dict[str, object]: + rows = [MetagraphRow(bytes([tag]) * 32, tag) for tag in range(3)] + trust = TrustRoot( + (ChallengeEntry(b"bounty", 2000), ChallengeEntry(b"proof", 8000)), sha256(b"\x00").digest() + ) + leaves = [ + Leaf(name, row.hotkey, 123, Score(50) if row.uid == 1 else NoScore()) + for name in (b"bounty", b"proof") + for row in rows + ] + bundle = build_bundle( + epoch=123, + netuid=541, + block_b=99, + block_hash=bytes([9]) * 32, + rows=rows, + leaves=leaves, + trust=trust, + ) + return { + "authority": "Python reference encoder, protocol_version 2 (no signatures); fixed inputs", + "bundle_scale": bundle.encode().hex(), + "leaf_scale": bundle.body.leaves[0].encode().hex(), + "merkle_root": bundle.body.merkle_root.hex(), + "final_vector": [list(pair) for pair in bundle.body.final_vector], + } + + +if __name__ == "__main__": + print(json.dumps(vector(), indent=2)) diff --git a/src/cortex/config.py b/src/cortex/config.py index f64c693d5..c6c2085a8 100644 --- a/src/cortex/config.py +++ b/src/cortex/config.py @@ -13,7 +13,6 @@ from cortex.errors import ServiceError from cortex.protocol import TrustRoot -from cortex.protocol.crypto import public_key from cortex.protocol.trust import load_trust from cortex.vm.models import Resources @@ -102,10 +101,8 @@ class MasterConfig: state_dir: Path challenges_file: Path measurements_file: Path - gateway_seed_file: Path proof_seed_file: Path operator_token_file: Path - challenge_keys_dir: Path = Path("/run/secrets") challenge_registry_file: Path | None = None challenge_secrets_dir: Path = Path("/run/challenge-secrets") chain_endpoint: str = "finney" @@ -181,10 +178,8 @@ def optional_path(name: str) -> Path | None: state_dir=Path(value("BASE_STATE_DIR", "/var/lib/cortex")), challenges_file=Path(value("BASE_CHALLENGES_FILE", "config/challenges.toml")), measurements_file=Path(value("BASE_MEASUREMENTS_FILE", "config/measurements.toml")), - gateway_seed_file=required_path("BASE_GATEWAY_SK_FILE"), proof_seed_file=required_path("PROOF_SK_FILE"), operator_token_file=required_path("BASE_GATEWAY_ADMIN_TOKEN_FILE"), - challenge_keys_dir=Path(value("BASE_CHALLENGE_KEYS_DIR", "/run/secrets")), challenge_registry_file=optional_path("BASE_CHALLENGE_REGISTRY_FILE"), challenge_secrets_dir=Path( value("BASE_CHALLENGE_SECRETS_DIR", "/run/challenge-secrets") @@ -206,22 +201,10 @@ def optional_path(name: str) -> Path | None: ), ) - def challenge_seed_file(self, challenge: bytes) -> Path: - """Proof keeps its topic key; every other challenge id uses /.key.""" - if challenge == b"proof": - return self.proof_seed_file - return self.challenge_keys_dir / f"{challenge.decode()}.key" - def challenge_profile(self, epoch: int) -> TrustRoot: try: - trust = load_trust( - challenges_path=self.challenges_file, - measurements_path=self.measurements_file, - gateway_public=public_key(read_seed(self.gateway_seed_file)), + return load_trust( + challenges_path=self.challenges_file, measurements_path=self.measurements_file ) - for entry in trust.challenges: - if public_key(read_seed(self.challenge_seed_file(entry.id))) != entry.public_key: - raise ServiceError(503, "challenge signing key does not match the profile") - return trust except (OSError, ValueError, KeyError): raise ServiceError(503, "challenge profile unavailable") from None diff --git a/src/cortex/gateway/api.py b/src/cortex/gateway/api.py index 4eb379918..539898eef 100644 --- a/src/cortex/gateway/api.py +++ b/src/cortex/gateway/api.py @@ -1,4 +1,4 @@ -"""Master gateway routes; only sealing requires the rotating operator bearer.""" +"""Master gateway routes; writes (raw weights, sealing) require the rotating operator bearer.""" from fastapi import APIRouter, Request from fastapi.responses import JSONResponse, Response @@ -27,6 +27,7 @@ def create_router(service: GatewayService, operator_auth: OperatorAuth) -> APIRo @router.post("/v1/weights/raw", status_code=202) async def raw_weight(request: Request): try: + operator_auth.require(request) body = RawWeightRequest.model_validate(decode_json(await bounded_body(request, 4096))) return service.accept_leaf(body.to_leaf()) except ValidationError: diff --git a/src/cortex/gateway/models.py b/src/cortex/gateway/models.py index 3e0a1522c..bfab094d1 100644 --- a/src/cortex/gateway/models.py +++ b/src/cortex/gateway/models.py @@ -50,7 +50,6 @@ class RawWeightRequest(RequestModel): miner_hotkey: str = Field(max_length=132) epoch: U64 score_or_absence: ScoreWire | AbsenceWire - challenge_sig: str = Field(max_length=132) def to_leaf(self) -> Leaf: value = self.score_or_absence @@ -64,7 +63,6 @@ def to_leaf(self) -> Leaf: _hex(self.miner_hotkey, 32), self.epoch, score, - _hex(self.challenge_sig, 64), ) @@ -86,5 +84,4 @@ def leaf_request(leaf: Leaf) -> dict: miner_hotkey=leaf.miner_hotkey.hex(), epoch=leaf.epoch, score_or_absence=score, - challenge_sig=leaf.challenge_sig.hex(), ) diff --git a/src/cortex/gateway/service.py b/src/cortex/gateway/service.py index e3f259950..cf6405570 100644 --- a/src/cortex/gateway/service.py +++ b/src/cortex/gateway/service.py @@ -1,4 +1,4 @@ -"""Gateway sealing boundary: local keys and trust, pinned chain snapshot, durable store.""" +"""Gateway sealing boundary: local trust, pinned chain snapshot, durable store.""" import sqlite3 from collections.abc import Callable @@ -6,8 +6,16 @@ from typing import Protocol from cortex.errors import ServiceError -from cortex.protocol import Bundle, Leaf, ProtocolError, TrustRoot, aggregate_leaves, build_bundle -from cortex.protocol.crypto import BUNDLE_DOMAIN, RAW_WEIGHT_DOMAIN, encode_hotkey, verify_raw +from cortex.protocol import ( + BUNDLE_PROTOCOL_VERSION, + Bundle, + Leaf, + ProtocolError, + TrustRoot, + aggregate_leaves, + build_bundle, +) +from cortex.protocol.crypto import encode_hotkey from cortex.protocol.merkle import merkle_root from cortex.protocol.scale import uint from cortex.protocol.trust import trust_to_json @@ -31,7 +39,6 @@ def __init__( trust: TrustRoot, netuid: int, chain: SnapshotProvider, - gateway_seed: Callable[[], bytes], clock: Callable[[], datetime] | None = None, chain_endpoint: str | Callable[[], str] = "", trust_loader: Callable[[int], TrustRoot] | None = None, @@ -42,7 +49,6 @@ def __init__( self.trust = trust self.netuid = netuid self.chain = chain - self.gateway_seed = gateway_seed self.clock = clock or (lambda: datetime.now(UTC)) self.chain_endpoint = chain_endpoint self.trust_loader = trust_loader @@ -81,8 +87,6 @@ def accept_leaf(self, leaf: Leaf) -> dict: leaf.encode() except (ValueError, TypeError): raise ServiceError(400, "invalid raw weight") from None - if not verify_raw(entry.public_key, RAW_WEIGHT_DOMAIN, leaf.payload(), leaf.challenge_sig): - raise ServiceError(401, "invalid challenge signature") try: return self.store.put_leaf(leaf) except (sqlite3.Error, ProtocolError): @@ -111,10 +115,6 @@ def replace_challenge_leaves( raise ServiceError(400, "invalid challenge snapshot") from None if leaf.challenge_id != challenge_id or leaf.epoch != epoch: raise ServiceError(400, "challenge snapshot identity mismatch") - if not verify_raw( - entry.public_key, RAW_WEIGHT_DOMAIN, leaf.payload(), leaf.challenge_sig - ): - raise ServiceError(401, "invalid challenge signature") try: return self.store.replace_challenge_leaves(challenge_id, epoch, leaves) except ServiceError: @@ -138,33 +138,24 @@ def _decode_stored(self, stored: StoredBundle, trust: TrustRoot | None = None) - bundle = Bundle.decode(stored.encoded) if bundle.body.epoch != stored.epoch: raise ProtocolError("stored epoch mismatch") - # The signature protects the chain root and participant coverage checked at - # sealing. Do not reconstruct chain stakes from a uid_map that omits them. + # Chain root and participant coverage were checked at sealing. Do not reconstruct + # chain stakes from a uid_map that omits them. body = bundle.body if ( - body.protocol_version != 1 + body.protocol_version != BUNDLE_PROTOCOL_VERSION or body.algorithm_version != trust.algorithm_version or body.epoch < trust.introduced_epoch or body.netuid != self.netuid - or body.gateway_hotkey != self.trust.gateway_hotkey - or body.gateway_hotkey != trust.gateway_hotkey or body.emission_shares != trust.shares or body.measurements_digest != trust.measurements_digest - or not verify_raw(body.gateway_hotkey, BUNDLE_DOMAIN, body.encode(), bundle.gateway_sig) ): raise ProtocolError("invalid stored seal") if merkle_root(leaf.encode() for leaf in body.leaves) != body.merkle_root: raise ProtocolError("invalid stored leaf root") - keys = {entry.id: entry.public_key for entry in trust.challenges} + known = {entry.id for entry in trust.challenges} for leaf in body.leaves: - if ( - leaf.epoch != body.epoch - or leaf.challenge_id not in keys - or not verify_raw( - keys[leaf.challenge_id], RAW_WEIGHT_DOMAIN, leaf.payload(), leaf.challenge_sig - ) - ): - raise ProtocolError("invalid stored challenge signature") + if leaf.epoch != body.epoch or leaf.challenge_id not in known: + raise ProtocolError("invalid stored leaf") final = aggregate_leaves( body.leaves, body.emission_shares, @@ -259,11 +250,9 @@ async def seal( snapshot = await self.chain.snapshot(chosen_block, self.netuid) if snapshot.block != chosen_block: raise ServiceError(503, "chain snapshot block mismatch") - seed = self.gateway_seed() def build(leaves: tuple[Leaf, ...]) -> Bundle: return build_bundle( - gateway_seed=seed, epoch=epoch, netuid=self.netuid, block_b=chosen_block, diff --git a/src/cortex/gateway/store.py b/src/cortex/gateway/store.py index e3d82f430..6c22bdc7b 100644 --- a/src/cortex/gateway/store.py +++ b/src/cortex/gateway/store.py @@ -12,6 +12,7 @@ from cortex.errors import ServiceError from cortex.protocol import ( + BUNDLE_PROTOCOL_VERSION, Bundle, Leaf, NoScore, @@ -94,6 +95,7 @@ def __init__(self, path: Path | str): digest BLOB PRIMARY KEY, profile TEXT NOT NULL ); """) + self._require_wire_format() for epoch, encoded in self._connection.execute( "SELECT epoch,bundle_scale FROM gateway_bundles " "WHERE epoch NOT IN (SELECT epoch FROM gateway_bundle_roots)" @@ -106,6 +108,31 @@ def __init__(self, path: Path | str): "INSERT OR IGNORE INTO gateway_bundle_roots VALUES (?,?)", (root, epoch) ) + def _require_wire_format(self) -> None: + """v1 rows carry leaf/gateway signatures and cannot be read as v2: refuse, never convert.""" + with self._transaction() as connection: + row = connection.execute( + "SELECT value FROM gateway_metadata WHERE name='wire_format'" + ).fetchone() + if row is None: + used = connection.execute( + "SELECT 1 FROM gateway_raw_leaves UNION ALL " + "SELECT 1 FROM gateway_bundles LIMIT 1" + ).fetchone() + if used: + raise ServiceError( + 503, + "gateway database uses the retired signed wire format: use a new state dir", + ) + connection.execute( + "INSERT INTO gateway_metadata VALUES ('wire_format', ?)", + (str(BUNDLE_PROTOCOL_VERSION),), + ) + elif row[0] != str(BUNDLE_PROTOCOL_VERSION): + raise ServiceError( + 503, "gateway database wire format mismatch: use a new state dir" + ) + @contextmanager def _transaction(self): with self._lock: diff --git a/src/cortex/master.py b/src/cortex/master.py index 3caf58b63..4724fbfaf 100644 --- a/src/cortex/master.py +++ b/src/cortex/master.py @@ -31,7 +31,7 @@ from cortex.proof.scoring import SCORE_MAX from cortex.proof.service import EvaluationBackend, ProofService, UnwiredBackend from cortex.proof.store import ProofStore -from cortex.protocol import NoScore, NoScoreReason, Score, TrustRoot, sign_leaf +from cortex.protocol import Leaf, NoScore, NoScoreReason, Score, TrustRoot from cortex.protocol.crypto import decode_hotkey, public_key from cortex.protocol.merkle import canonical_rows from cortex.protocol.models import FULL_SHARE_SCORE @@ -325,12 +325,11 @@ def __init__( proof: _EpochTrackedProof, clock: EpochClock, journal: EmissionJournal, - challenge_seed: Callable[[bytes], bytes], registry: Callable[[], dict[str, RegistryEntry]], challenges: ChallengeClient, ): self.gateway, self.proof, self.clock, self.journal = gateway, proof, clock, journal - self.challenge_seed, self.registry, self.challenges = challenge_seed, registry, challenges + self.registry, self.challenges = registry, challenges self._lock = asyncio.Lock() async def _scores( @@ -385,18 +384,11 @@ async def _emit(self, epoch: int, snapshot: ChainSnapshot) -> None: and epoch_at is None ): raise ServiceError(503, "completed epoch timestamp unavailable") - self.proof.topic_public_key = next( - (entry.public_key for entry in self.gateway.trust.challenges if entry.id == b"proof"), - self.proof.topic_public_key, - ) rows = canonical_rows(snapshot.rows) for challenge in self.gateway.trust.challenges: expected = challenge.policy.expected(rows) - seed = self.challenge_seed(challenge.id) - if public_key(seed) != challenge.public_key: - raise ServiceError(503, "challenge signing key does not match the profile") outcomes = await self._scores(challenge.id, epoch, expected, epoch_at) - # No unknown score key can expand E; missing scores are explicit signed absences. + # No unknown score key can expand E; missing scores are explicit absences. leaves = [] for hotkey in sorted(expected): score = outcomes.get(hotkey, NoScore(NoScoreReason.NOT_ATTEMPTED)) @@ -404,7 +396,7 @@ async def _emit(self, epoch: int, snapshot: ChainSnapshot) -> None: score.encode() except ValueError: score = NoScore() - leaves.append(sign_leaf(seed, challenge.id, hotkey, epoch, score)) + leaves.append(Leaf(challenge.id, hotkey, epoch, score)) self.gateway.replace_challenge_leaves(challenge.id, epoch, expected, tuple(leaves)) async def tick(self) -> list[int]: @@ -591,16 +583,12 @@ def chain_endpoint() -> str: trust=local_trust, netuid=config.netuid, chain=chain, - gateway_seed=lambda: read_seed(config.gateway_seed_file), chain_endpoint=chain_endpoint, trust_loader=None if trust is not None else config.challenge_profile, ) proof = _EpochTrackedProof( store=proof_store, - topic_public_key=next( - (c.public_key for c in local_trust.challenges if c.id == b"proof"), - public_key(read_seed(config.proof_seed_file)), - ), + topic_public_key=public_key(read_seed(config.proof_seed_file)), vault=FileVault(config.state_dir / "miner-byok"), artifact_dir=config.state_dir / "artifacts", backend=proof_backend or UnwiredBackend(), @@ -612,7 +600,6 @@ def chain_endpoint() -> str: proof=proof, clock=clock, journal=journal, - challenge_seed=lambda challenge: read_seed(config.challenge_seed_file(challenge)), registry=ChallengeRegistry(config.challenge_registry_file), challenges=ChallengeClient(http, config.challenge_secrets_dir), ) diff --git a/src/cortex/protocol/__init__.py b/src/cortex/protocol/__init__.py index ab1f9c75e..fdb432d8e 100644 --- a/src/cortex/protocol/__init__.py +++ b/src/cortex/protocol/__init__.py @@ -1,7 +1,7 @@ """Consensus-compatible protocol for the Bounty and Proof research subnet.""" from .aggregate import FinalWeights, aggregate_leaves -from .bundle import build_bundle, bundle_digest, sign_leaf, verify_bundle +from .bundle import BUNDLE_PROTOCOL_VERSION, build_bundle, bundle_digest, verify_bundle from .models import ( LIVE_SHARES, Bundle, @@ -18,6 +18,7 @@ from .scale import ProtocolError __all__ = [ + "BUNDLE_PROTOCOL_VERSION", "LIVE_SHARES", "Bundle", "BundleBody", @@ -34,6 +35,5 @@ "aggregate_leaves", "build_bundle", "bundle_digest", - "sign_leaf", "verify_bundle", ] diff --git a/src/cortex/protocol/bundle.py b/src/cortex/protocol/bundle.py index 4c0d68817..5e82b636f 100644 --- a/src/cortex/protocol/bundle.py +++ b/src/cortex/protocol/bundle.py @@ -1,11 +1,9 @@ -"""Seal and verify complete, independently recomputed epoch bundles.""" +"""Build and verify complete, independently recomputed epoch bundles (hash integrity only).""" from collections.abc import Iterable -from dataclasses import replace from hashlib import sha256 from .aggregate import aggregate_leaves -from .crypto import BUNDLE_DOMAIN, RAW_WEIGHT_DOMAIN, public_key, sign_raw, verify_raw from .merkle import canonical_rows, merkle_root, metagraph_root from .models import ( PROPORTIONAL_SHARES, @@ -13,22 +11,11 @@ BundleBody, Leaf, MetagraphRow, - NoScore, - Score, TrustRoot, ) from .scale import ProtocolError, byte_vec - -def sign_leaf( - seed: bytes, - challenge_id: bytes, - miner_hotkey: bytes, - epoch: int, - score: Score | NoScore, -) -> Leaf: - unsigned = Leaf(challenge_id, miner_hotkey, epoch, score, bytes(64)) - return replace(unsigned, challenge_sig=sign_raw(seed, RAW_WEIGHT_DOMAIN, unsigned.payload())) +BUNDLE_PROTOCOL_VERSION = 2 # v1 carried leaf and gateway signatures; never confused with v2 def _verify_leaves( @@ -49,12 +36,6 @@ def _verify_leaves( raise ProtocolError("unknown challenge") if leaf.epoch != body.epoch: bad.add(challenge.id) - if not verify_raw( - challenge.public_key, RAW_WEIGHT_DOMAIN, leaf.payload(), leaf.challenge_sig - ): - if not allow_quarantine: - raise ProtocolError("invalid challenge signature") - bad.add(challenge.id) for challenge in trust.challenges: present = {leaf.miner_hotkey for leaf in body.leaves if leaf.challenge_id == challenge.id} if present != challenge.policy.expected(rows): @@ -70,7 +51,6 @@ def _verify_leaves( def build_bundle( *, - gateway_seed: bytes, epoch: int, netuid: int, block_b: int, @@ -82,8 +62,6 @@ def build_bundle( trust.validate() if epoch < trust.introduced_epoch: raise ProtocolError("challenge profile is not active for bundle epoch") - if public_key(gateway_seed) != trust.gateway_hotkey: - raise ProtocolError("gateway key does not match local trust") ordered_rows = canonical_rows(rows) ordered_leaves = tuple(sorted(leaves, key=lambda leaf: leaf.sort_key)) uid_map = tuple((row.hotkey, row.uid) for row in ordered_rows) @@ -91,7 +69,7 @@ def build_bundle( ordered_leaves, trust.shares, uid_map, algorithm_version=trust.algorithm_version ) body = BundleBody( - 1, + BUNDLE_PROTOCOL_VERSION, epoch, netuid, block_b, @@ -104,10 +82,9 @@ def build_bundle( ordered_leaves, merkle_root(leaf.encode() for leaf in ordered_leaves), final.final_vector, - trust.gateway_hotkey, ) _verify_leaves(body, ordered_rows, trust) - return Bundle(body, sign_raw(gateway_seed, BUNDLE_DOMAIN, body.encode())) + return Bundle(body) def verify_bundle( @@ -143,16 +120,15 @@ def verify_inputs( """Rows/hash must come from the chain at body.block_b, never from the gateway.""" trust.validate() body = bundle.body - if body.protocol_version != 1 or body.algorithm_version != trust.algorithm_version: + if ( + body.protocol_version != BUNDLE_PROTOCOL_VERSION + or body.algorithm_version != trust.algorithm_version + ): raise ProtocolError("unsupported bundle version") if body.epoch < trust.introduced_epoch: raise ProtocolError("challenge profile is not active for bundle epoch") if body.netuid != netuid or (epoch is not None and body.epoch != epoch): raise ProtocolError("bundle subnet/epoch mismatch") - if body.gateway_hotkey != trust.gateway_hotkey: - raise ProtocolError("untrusted gateway signer") - if not verify_raw(body.gateway_hotkey, BUNDLE_DOMAIN, body.encode(), bundle.gateway_sig): - raise ProtocolError("invalid gateway signature") if body.block_hash != block_hash: raise ProtocolError("chain block hash mismatch") ordered_rows = canonical_rows(rows) diff --git a/src/cortex/protocol/consensus.py b/src/cortex/protocol/consensus.py index 28a01421e..226a662c4 100644 --- a/src/cortex/protocol/consensus.py +++ b/src/cortex/protocol/consensus.py @@ -12,7 +12,7 @@ class DissentReason(IntEnum): VECTOR_MISMATCH = 0 - LEAF_SIGNATURE_INVALID = 1 + # 1 and 12 were LEAF_/BUNDLE_SIGNATURE_INVALID (signatures removed); numbers stay reserved. LEAF_CHALLENGE_KEY_UNKNOWN = 2 INCOMPLETE_PARTICIPANT_SET = 3 MERKLE_ROOT_MISMATCH = 4 @@ -23,7 +23,6 @@ class DissentReason(IntEnum): PEER_ROOT_CONFLICT = 9 PEER_SAMPLE_INSUFFICIENT = 10 SHARE_MASS_BELOW_THRESHOLD = 11 - BUNDLE_SIGNATURE_INVALID = 12 AGGREGATION_OVERFLOW = 13 EMPTY_SCORE_VECTOR_NO_SUBMIT = 14 UID_MAP_MISMATCH = 15 diff --git a/src/cortex/protocol/crypto.py b/src/cortex/protocol/crypto.py index 234200e0b..667877574 100644 --- a/src/cortex/protocol/crypto.py +++ b/src/cortex/protocol/crypto.py @@ -18,8 +18,6 @@ from .scale import ProtocolError, byte_vec, fixed SIGNING_CONTEXT = b"base-sr25519-v1" -BUNDLE_DOMAIN = b"base-bundle-v1" -RAW_WEIGHT_DOMAIN = b"base-rawweight-v1" _ORDER = 2**252 + 27742317777372353535851937790883648493 _ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" diff --git a/src/cortex/protocol/models.py b/src/cortex/protocol/models.py index f0e6026a1..0d835f902 100644 --- a/src/cortex/protocol/models.py +++ b/src/cortex/protocol/models.py @@ -51,7 +51,6 @@ class Leaf: miner_hotkey: bytes epoch: int score: Score | NoScore - challenge_sig: bytes @property def sort_key(self) -> bytes: @@ -67,7 +66,7 @@ def payload(self) -> bytes: return self.sort_key + uint(self.epoch, 8) + self.score.encode() def encode(self) -> bytes: - return self.payload() + fixed(self.challenge_sig, 64) + return self.payload() @classmethod def read(cls, reader: Reader) -> "Leaf": @@ -84,7 +83,7 @@ def read(cls, reader: Reader) -> "Leaf": raise ProtocolError("unknown NoScore reason") from error else: raise ProtocolError("unknown score discriminant") - return cls(challenge, hotkey, epoch, score, reader.take(64)) + return cls(challenge, hotkey, epoch, score) @dataclass(frozen=True) @@ -131,28 +130,21 @@ def expected(self, rows: tuple[MetagraphRow, ...]) -> set[bytes]: @dataclass(frozen=True) class ChallengeEntry: id: bytes - public_key: bytes emission_share_bps: int policy: ParticipantPolicy = ParticipantPolicy() def encode(self) -> bytes: if not 1 <= len(self.id) <= 64: raise ProtocolError("challenge id must contain 1..64 bytes") - return ( - byte_vec(self.id) - + fixed(self.public_key, 32) - + uint(self.emission_share_bps, 2) - + self.policy.encode() - ) + return byte_vec(self.id) + uint(self.emission_share_bps, 2) + self.policy.encode() @dataclass(frozen=True) class TrustRoot: - """Local trust, constructed only from independently verified owner documents.""" + """Emission profile: challenge ids, shares, policies, and the measurement allowlist digest.""" challenges: tuple[ChallengeEntry, ...] measurements_digest: bytes - gateway_hotkey: bytes challenges_version: int = 1 measurements_version: int = 1 introduced_epoch: int = 0 @@ -189,7 +181,6 @@ def validate(self) -> None: if self.shares == PROPORTIONAL_SHARES and self.challenges_version < 2: raise ProtocolError("proportional shares require challenges version >= 2") fixed(self.measurements_digest, 32) - fixed(self.gateway_hotkey, 32) uint(self.challenges_version, 4) uint(self.measurements_version, 4) uint(self.introduced_epoch, 8) @@ -216,7 +207,6 @@ class BundleBody: leaves: tuple[Leaf, ...] merkle_root: bytes final_vector: tuple[tuple[int, int], ...] - gateway_hotkey: bytes def encode(self) -> bytes: return b"".join( @@ -234,7 +224,6 @@ def encode(self) -> bytes: vector(self.leaves, Leaf.encode), fixed(self.merkle_root, 32), encode_final_vector(self.final_vector), - fixed(self.gateway_hotkey, 32), ) ) @@ -254,22 +243,22 @@ def read(cls, reader: Reader) -> "BundleBody": reader.vector(lambda: Leaf.read(reader)), reader.take(32), reader.vector(lambda: (reader.uint(2), reader.uint(2)), limit=65536), - reader.take(32), ) @dataclass(frozen=True) class Bundle: + """An epoch bundle is its body: integrity comes from hashes, not from a signature.""" + body: BundleBody - gateway_sig: bytes def encode(self) -> bytes: - return self.body.encode() + fixed(self.gateway_sig, 64) + return self.body.encode() @classmethod def decode(cls, data: bytes) -> "Bundle": reader = Reader(data) - result = cls(BundleBody.read(reader), reader.take(64)) + result = cls(BundleBody.read(reader)) reader.finish() if result.encode() != data: raise ProtocolError("noncanonical bundle encoding") diff --git a/src/cortex/protocol/trust.py b/src/cortex/protocol/trust.py index 05f8e6254..a4b5c68f7 100644 --- a/src/cortex/protocol/trust.py +++ b/src/cortex/protocol/trust.py @@ -30,11 +30,10 @@ def _policy(value: str | dict) -> ParticipantPolicy: def challenge_entries(document: dict) -> tuple[ChallengeEntry, ...]: - """Decode the challenge body exactly as it is signed on the wire.""" + """Decode the challenge rows (id, share, participant policy).""" return tuple( ChallengeEntry( row["id"].encode(), - decode_hotkey(row["public_key"]), row["emission_share_bps"], _policy(row["policy"]), ) @@ -43,7 +42,7 @@ def challenge_entries(document: dict) -> tuple[ChallengeEntry, ...]: def measurements_body(document: dict) -> bytes: - """Encode the measurement allowlist exactly as it is signed on the wire.""" + """Encode the measurement allowlist; its sha256 is the profile's measurements digest.""" entries = [] for row in document.get("measurements", []): fields = [] @@ -58,16 +57,13 @@ def measurements_body(document: dict) -> bytes: return vector(entries, lambda value: value) -def load_trust( - *, challenges_path: Path, measurements_path: Path, gateway_public: bytes -) -> TrustRoot: +def load_trust(*, challenges_path: Path, measurements_path: Path) -> TrustRoot: """Build the profile from the master's own TOML files. No signature: the files are local.""" challenges_doc = tomllib.loads(challenges_path.read_text()) measurements_doc = tomllib.loads(measurements_path.read_text()) trust = TrustRoot( challenge_entries(challenges_doc), sha256(measurements_body(measurements_doc)).digest(), - gateway_public, challenges_doc["version"], measurements_doc["version"], max(challenges_doc.get("introduced_epoch", 0), measurements_doc.get("introduced_epoch", 0)), @@ -93,13 +89,11 @@ def trust_from_json(values: object) -> TrustRoot: challenges.append( ChallengeEntry( bytes.fromhex(entry["id"]), - bytes.fromhex(entry["public_key"]), entry["emission_share_bps"], ParticipantPolicy(**policy), ) ) values["measurements_digest"] = bytes.fromhex(values["measurements_digest"]) - values["gateway_hotkey"] = bytes.fromhex(values["gateway_hotkey"]) trust = TrustRoot(challenges=tuple(challenges), **values) trust.validate() return trust diff --git a/src/cortex/validator/__main__.py b/src/cortex/validator/__main__.py index ac4d95f31..d175ab211 100644 --- a/src/cortex/validator/__main__.py +++ b/src/cortex/validator/__main__.py @@ -70,11 +70,6 @@ def parser() -> argparse.ArgumentParser: ) arguments.add_argument("--gateway", required=True) arguments.add_argument("--netuid", required=True, type=int) - arguments.add_argument( - "--gateway-public", - help="optional extra pin: refuse a gateway whose sealing hotkey differs. The gateway " - "itself (HTTPS) is the trust anchor for the emission profile (/v1/trust)", - ) arguments.add_argument("--network", type=primary_endpoint, default="finney") arguments.add_argument("--fallback-endpoints", type=fallback_endpoints, default=[]) arguments.add_argument( @@ -186,9 +181,6 @@ def consensus_seed(): peer_consensus=arguments.peer_consensus, min_peer_sample=arguments.min_peer_sample, max_block_lag=arguments.max_block_lag, - gateway_public=decode_hotkey(arguments.gateway_public) - if arguments.gateway_public - else None, verify_only=arguments.verify_only, ) while True: diff --git a/src/cortex/validator/service.py b/src/cortex/validator/service.py index 0bafc6a5b..16a7b1299 100644 --- a/src/cortex/validator/service.py +++ b/src/cortex/validator/service.py @@ -346,7 +346,6 @@ def __init__( max_block_lag: int = 256, trust_loader: Callable[[int], TrustRoot] | None = None, verify_only: bool = False, - gateway_public: bytes | None = None, ): """Verify the gateway seal and submit weights. @@ -362,9 +361,6 @@ def __init__( uint(version_key, 8) if trust is not None: trust.validate() - if gateway_public is not None: - fixed(gateway_public, 32) - self.gateway_public = gateway_public gateway = urlsplit(gateway_url) if ( gateway_url != gateway_url.strip() @@ -435,8 +431,6 @@ async def run_once(self) -> TickResult: return await self._run_once() except ProtocolError as error: reasons = { - "signature": DissentReason.BUNDLE_SIGNATURE_INVALID, - "challenge signature": DissentReason.LEAF_SIGNATURE_INVALID, "unknown challenge": DissentReason.LEAF_CHALLENGE_KEY_UNKNOWN, "participant": DissentReason.INCOMPLETE_PARTICIPANT_SET, "Merkle": DissentReason.MERKLE_ROOT_MISMATCH, @@ -548,11 +542,9 @@ async def _run_once(self) -> TickResult: if self.trust_loader: self.trust = self.trust_loader(epoch) elif not self._fixed_trust: - # The gateway is ours and is the source of truth for the emission profile; - # TLS plus an optional --gateway-public pin are the only anchors. + # The gateway is ours and is the source of truth for the emission profile and the + # bundle; TLS is the anchor. Integrity of the bundle is recomputed, not signed. self.trust = trust_from_json(decode_json(await self._read("/v1/trust", 1024 * 1024))) - if self.gateway_public is not None and self.trust.gateway_hotkey != self.gateway_public: - raise ProtocolError("gateway key differs from the pinned --gateway-public") self.journal.evidence.watermark("challenges_version", self.trust.challenges_version) self.journal.evidence.watermark("measurements_version", self.trust.measurements_version) body, quarantined = verify_inputs( @@ -603,7 +595,7 @@ async def _run_once(self) -> TickResult: if quarantined: if self.consensus_seed is None: raise ProtocolError("quarantine requires signed dissent") - self._dissent(DissentReason.LEAF_SIGNATURE_INVALID, local_vector) + self._dissent(DissentReason.INCOMPLETE_PARTICIPANT_SET, local_vector) preflight_outcome = ( "dissent_vector_mismatch_and_quarantine" if vector_mismatch diff --git a/tests/gateway/test_gateway.py b/tests/gateway/test_gateway.py index 7d63d9f97..0c562ee77 100644 --- a/tests/gateway/test_gateway.py +++ b/tests/gateway/test_gateway.py @@ -17,15 +17,14 @@ from cortex.protocol import ( Bundle, ChallengeEntry, + Leaf, MetagraphRow, NoScore, NoScoreReason, Score, TrustRoot, - sign_leaf, ) -from cortex.protocol.crypto import encode_hotkey, public_key -from cortex.protocol.scale import ProtocolError +from cortex.protocol.crypto import encode_hotkey from cortex.validator import ChainSnapshot, SubmissionJournal, Validator @@ -73,18 +72,16 @@ async def network(tmp_path): rows = tuple(MetagraphRow(bytes([tag]) * 32, tag) for tag in range(3)) trust = TrustRoot( ( - ChallengeEntry(b"bounty", public_key(bytes([1]) * 32), 2000), - ChallengeEntry(b"proof", public_key(bytes([2]) * 32), 8000), + ChallengeEntry(b"bounty", 2000), + ChallengeEntry(b"proof", 8000), ), sha256(b"\x00").digest(), - public_key(bytes([7]) * 32), ) chain = FakeChain(rows) settings = dict( trust=trust, netuid=541, chain=chain, - gateway_seed=lambda: bytes([7]) * 32, clock=lambda: datetime(2026, 9, 16, 12, tzinfo=UTC), ) service = GatewayService(store=store, **settings) @@ -113,21 +110,22 @@ async def network(tmp_path): def leaves(network, epoch=12, winner=1, reason=NoScoreReason.NOT_ATTEMPTED): return [ - sign_leaf( - bytes([seed]) * 32, + Leaf( challenge, row.hotkey, epoch, Score(50) if row.uid == winner else NoScore(reason), ) - for challenge, seed in ((b"bounty", 1), (b"proof", 2)) + for challenge in (b"bounty", b"proof") for row in network.rows ] async def intake(network, epoch=12, winner=1, reason=NoScoreReason.NOT_ATTEMPTED): for leaf in leaves(network, epoch, winner, reason): - response = await network.client.post("/v1/weights/raw", json=leaf_request(leaf)) + response = await network.client.post( + "/v1/weights/raw", json=leaf_request(leaf), headers=network.auth + ) assert response.status_code == 202, response.text @@ -137,7 +135,7 @@ async def seal(network, epoch=12, **extra): ) -async def test_signed_intake_seal_latest_validator_and_chain_dispatch(network): +async def test_intake_seal_latest_validator_and_chain_dispatch(network): await intake(network) response = await seal(network) assert response.status_code == 200, response.text @@ -180,22 +178,6 @@ async def test_validator_without_local_trust_uses_the_gateway_served_profile(net assert network.chain.submissions == [(541, ((1, 65535),), 1)] -async def test_validator_pinned_to_another_gateway_key_refuses_the_served_profile(network): - await intake(network) - assert (await seal(network)).status_code == 200 - validator = Validator( - gateway_url="https://master.invalid", - netuid=541, - chain=network.chain, - journal=network.journal, - http=network.client, - gateway_public=bytes(32), - ) - with pytest.raises(ProtocolError, match="pinned --gateway-public"): - await validator.run_once() - assert network.chain.submissions == [] - - async def test_unsealed_burn_is_available_without_owner_wallet_or_chain(network): response = await network.client.get("/v1/weights/latest") assert response.status_code == 200 @@ -329,7 +311,9 @@ async def test_sealed_burn_exposes_the_public_weights_contract(network): async def test_seal_requires_exact_participant_coverage(network): for leaf in leaves(network)[:-1]: assert ( - await network.client.post("/v1/weights/raw", json=leaf_request(leaf)) + await network.client.post( + "/v1/weights/raw", json=leaf_request(leaf), headers=network.auth + ) ).status_code == 202 response = await seal(network) assert response.status_code == 409 @@ -338,11 +322,11 @@ async def test_seal_requires_exact_participant_coverage(network): assert network.service.latest()["sealed"] is False -async def test_validly_signed_extra_participant_cannot_enter_the_seal(network): +async def test_extra_participant_cannot_enter_the_seal(network): await intake(network) - extra = sign_leaf(bytes([1]) * 32, b"bounty", bytes([99]) * 32, 12, Score(1)) + extra = Leaf(b"bounty", bytes([99]) * 32, 12, Score(1)) assert ( - await network.client.post("/v1/weights/raw", json=leaf_request(extra)) + await network.client.post("/v1/weights/raw", json=leaf_request(extra), headers=network.auth) ).status_code == 202 assert (await seal(network)).status_code == 409 @@ -350,7 +334,7 @@ async def test_validly_signed_extra_participant_cannot_enter_the_seal(network): @pytest.mark.parametrize( "mutation,status", [ - ("signature", 401), + ("signature", 400), ("retired", 404), ("bool", 400), ("unknown_field", 400), @@ -372,28 +356,25 @@ async def test_invalid_intake_leaves_no_row(network, mutation, status): body["score_or_absence"] = {"no_score": {"reason": 8}} else: del body["miner_hotkey"] - response = await network.client.post("/v1/weights/raw", json=body) + response = await network.client.post("/v1/weights/raw", json=body, headers=network.auth) assert response.status_code == status assert network.store.leaves(12) == () async def test_duplicate_and_nonpositive_leaves_cannot_revoke_positive_score(network): positive = leaves(network)[1] - first = await network.client.post("/v1/weights/raw", json=leaf_request(positive)) - repeated = sign_leaf(bytes([1]) * 32, b"bounty", positive.miner_hotkey, 12, Score(50)) + first = await network.client.post( + "/v1/weights/raw", json=leaf_request(positive), headers=network.auth + ) + repeated = Leaf(b"bounty", positive.miner_hotkey, 12, Score(50)) burns = [ - sign_leaf( - bytes([1]) * 32, - b"bounty", - positive.miner_hotkey, - 12, - NoScore(reason), - ) - for reason in NoScoreReason + Leaf(b"bounty", positive.miner_hotkey, 12, NoScore(reason)) for reason in NoScoreReason ] - zero = sign_leaf(bytes([1]) * 32, b"bounty", positive.miner_hotkey, 12, Score(0)) + zero = Leaf(b"bounty", positive.miner_hotkey, 12, Score(0)) for incoming in (repeated, *burns, zero): - response = await network.client.post("/v1/weights/raw", json=leaf_request(incoming)) + response = await network.client.post( + "/v1/weights/raw", json=leaf_request(incoming), headers=network.auth + ) assert response.status_code == 409 assert response.json()["original"]["id"] == first.json()["id"] assert response.json()["original"]["score"] == 50 @@ -402,13 +383,7 @@ async def test_duplicate_and_nonpositive_leaves_cannot_revoke_positive_score(net def bounty_leaves(network, *, winner, reason=NoScoreReason.CHALLENGE_INTERNAL): return tuple( - sign_leaf( - bytes([1]) * 32, - b"bounty", - row.hotkey, - 12, - Score(50) if row.uid == winner else NoScore(reason), - ) + Leaf(b"bounty", row.hotkey, 12, Score(50) if row.uid == winner else NoScore(reason)) for row in network.rows ) @@ -447,7 +422,7 @@ def test_complete_challenge_snapshot_moves_the_winner_without_leaving_stale_scor assert scores[network.rows[2].hotkey] == Score(50) -@pytest.mark.parametrize("mutation", ["missing", "duplicate", "wrong_challenge", "wrong_sig"]) +@pytest.mark.parametrize("mutation", ["missing", "duplicate", "wrong_challenge", "wrong_epoch"]) def test_invalid_complete_challenge_snapshot_never_mutates_existing_rows(network, mutation): expected = {row.hotkey for row in network.rows} original = bounty_leaves(network, winner=1) @@ -458,15 +433,9 @@ def test_invalid_complete_challenge_snapshot_never_mutates_existing_rows(network elif mutation == "duplicate": incoming[-1] = incoming[0] elif mutation == "wrong_challenge": - incoming[-1] = sign_leaf( - bytes([2]) * 32, - b"proof", - network.rows[-1].hotkey, - 12, - Score(50), - ) + incoming[-1] = Leaf(b"proof", network.rows[-1].hotkey, 12, Score(50)) else: - incoming[-1] = replace(incoming[-1], challenge_sig=b"\x00" * 64) + incoming[-1] = replace(incoming[-1], epoch=13) with pytest.raises(ServiceError): network.service.replace_challenge_leaves(b"bounty", 12, expected, tuple(incoming)) @@ -533,8 +502,10 @@ async def test_changed_payload_supersedes_and_sealed_bytes_stay_immutable(networ await intake(network) assert (await seal(network)).status_code == 200 original = (await network.client.get("/v1/bundle/12")).content - changed = sign_leaf(bytes([1]) * 32, b"bounty", network.rows[1].hotkey, 12, Score(100)) - response = await network.client.post("/v1/weights/raw", json=leaf_request(changed)) + changed = Leaf(b"bounty", network.rows[1].hotkey, 12, Score(100)) + response = await network.client.post( + "/v1/weights/raw", json=leaf_request(changed), headers=network.auth + ) assert response.status_code == 202 and response.json()["superseded"] is True assert (await seal(network)).status_code == 200 assert (await network.client.get("/v1/bundle/12")).content == original @@ -639,12 +610,9 @@ async def test_simultaneous_seals_converge_on_the_same_immutable_bundle(network) assert first.json() == second.json() -async def test_wrong_subnet_and_unavailable_signing_key_never_persist_a_seal(network): +async def test_wrong_subnet_never_persists_a_seal(network): await intake(network) assert (await seal(network, netuid=1)).status_code == 400 - network.service.gateway_seed = lambda: bytes([88]) * 32 - response = await seal(network) - assert response.status_code == 503 assert network.store.bundle(12) is None @@ -680,35 +648,13 @@ async def test_historical_root_refuses_corrupt_or_missing_accepted_profile(netwo assert response.status_code == 503 -async def test_historical_root_uses_accepted_challenge_keys_after_key_rotation(network): - await intake(network) - assert (await seal(network)).status_code == 200 - original = network.service.bundle_bytes(12) - root = Bundle.decode(original).body.merkle_root.hex() - trust = replace( - network.trust, - challenges=tuple( - replace(entry, public_key=public_key(bytes([70 + index]) * 32)) - for index, entry in enumerate(network.trust.challenges) - ), - challenges_version=2, - introduced_epoch=13, - ) - network.service.trust_loader = lambda epoch: trust - network.service.refresh_trust(13) - second = GatewayStore(network.db_path) - try: - restarted = GatewayService(store=second, **{**network.settings, "trust": trust}) - assert restarted.latest()["sealed"] is False - assert restarted.bundle_by_root(root) == original - assert restarted.trust == trust - finally: - second.close() - - async def test_duplicate_json_and_oversized_request_are_rejected_before_store(network): - duplicate = await network.client.post("/v1/weights/raw", content='{"epoch": 1, "epoch": 2}') - oversized = await network.client.post("/v1/weights/raw", content=b"x" * 4097) + duplicate = await network.client.post( + "/v1/weights/raw", content='{"epoch": 1, "epoch": 2}', headers=network.auth + ) + oversized = await network.client.post( + "/v1/weights/raw", content=b"x" * 4097, headers=network.auth + ) assert duplicate.status_code == 400 assert oversized.status_code == 413 assert network.store.leaves(12) == () @@ -762,3 +708,42 @@ async def test_content_addressed_bundle_lookup_returns_identical_signed_bytes(ne assert response.content == network.service.bundle_bytes(12) assert (await network.client.get("/v1/bundle/root/" + "00" * 32)).status_code == 404 assert (await network.client.get("/v1/bundle/root/invalid")).status_code == 400 + + +async def test_raw_weight_intake_requires_the_operator_bearer(network): + body = leaf_request(leaves(network)[0]) + assert (await network.client.post("/v1/weights/raw", json=body)).status_code == 401 + wrong = {"authorization": "Bearer wrong"} + assert ( + await network.client.post("/v1/weights/raw", json=body, headers=wrong) + ).status_code == 401 + assert network.store.leaves(12) == () + assert ( + await network.client.post("/v1/weights/raw", json=body, headers=network.auth) + ).status_code == 202 + + +def test_new_store_is_marked_with_the_wire_format(tmp_path): + store = GatewayStore(tmp_path / "fresh.db") + store.close() + with sqlite3.connect(tmp_path / "fresh.db") as connection: + marker = connection.execute( + "SELECT value FROM gateway_metadata WHERE name='wire_format'" + ).fetchone() + assert marker is not None and int(marker[0]) == 2 + GatewayStore(tmp_path / "fresh.db").close() + + +async def test_store_holding_signed_v1_data_is_refused(network): + await intake(network) + with sqlite3.connect(network.db_path) as connection: + connection.execute("DELETE FROM gateway_metadata WHERE name='wire_format'") + with pytest.raises(ServiceError, match="new state dir"): + GatewayStore(network.db_path) + + +async def test_store_with_another_wire_format_marker_is_refused(network): + with sqlite3.connect(network.db_path) as connection: + connection.execute("UPDATE gateway_metadata SET value='1' WHERE name='wire_format'") + with pytest.raises(ServiceError, match="wire format"): + GatewayStore(network.db_path) diff --git a/tests/protocol/test_aggregate.py b/tests/protocol/test_aggregate.py index e3ac517d9..7b737d979 100644 --- a/tests/protocol/test_aggregate.py +++ b/tests/protocol/test_aggregate.py @@ -15,7 +15,7 @@ VECTORS = sorted( path for path in (Path(__file__).parent / "vectors").glob("*.json") - if path.name != "rust_wire.json" + if path.name not in ("rust_wire.json", "wire_v2.json") ) @@ -52,8 +52,8 @@ def test_upstream_served_vector_parity_bit_for_bit(path): def test_unavailable_proof_burns_its_eighty_percent_without_reallocating_to_bounty(): miner = bytes([1]) * 32 leaves = ( - Leaf(b"bounty", miner, 1, Score(20), bytes(64)), - Leaf(b"proof", miner, 1, NoScore(), bytes(64)), + Leaf(b"bounty", miner, 1, Score(20)), + Leaf(b"proof", miner, 1, NoScore()), ) result = aggregate_leaves(leaves, LIVE_SHARES, ((miner, 1),)) assert result.weights == (0.8, 0.2) @@ -63,8 +63,8 @@ def test_unavailable_proof_burns_its_eighty_percent_without_reallocating_to_boun def test_no_score_does_not_erase_other_miner_scores(): miners = (bytes([1]) * 32, bytes([2]) * 32) leaves = ( - Leaf(b"proof", miners[0], 1, Score(100), bytes(64)), - Leaf(b"proof", miners[1], 1, NoScore(), bytes(64)), + Leaf(b"proof", miners[0], 1, Score(100)), + Leaf(b"proof", miners[1], 1, NoScore()), ) result = aggregate_leaves(leaves, LIVE_SHARES, ((miners[0], 1), (miners[1], 2))) assert result.final_vector == ((0, 13107), (1, 52428)) @@ -75,9 +75,8 @@ def test_v2_bounty_pays_proportionally_up_to_thirty_percent(reports): miners = (bytes([1]) * 32, bytes([2]) * 32, bytes([3]) * 32) counts = (reports // 2, reports - reports // 2) leaves = tuple( - Leaf(b"bounty", key, 1, Score(count), bytes(64)) - for key, count in zip(miners[:2], counts, strict=True) - ) + (Leaf(b"proof", miners[2], 1, Score(100), bytes(64)),) + Leaf(b"bounty", key, 1, Score(count)) for key, count in zip(miners[:2], counts, strict=True) + ) + (Leaf(b"proof", miners[2], 1, Score(100)),) result = aggregate_leaves( leaves, ((b"bounty", 3000), (b"proof", 7000)), @@ -94,7 +93,7 @@ def test_v2_bounty_pays_proportionally_up_to_thirty_percent(reports): def test_v2_burns_uid0_and_unmapped_authors_without_transferring_their_mass(): miners = tuple(bytes([index]) * 32 for index in range(3)) result = aggregate_leaves( - tuple(Leaf(b"bounty", key, 1, Score(5), bytes(64)) for key in miners), + tuple(Leaf(b"bounty", key, 1, Score(5)) for key in miners), ((b"bounty", 3000), (b"proof", 7000)), ((miners[0], 0), (miners[1], 1)), algorithm_version=2, @@ -143,7 +142,7 @@ def test_proportional_profile_cannot_silently_use_default_legacy_algorithm(): miner = bytes([1]) * 32 with pytest.raises(ProtocolError, match="version 2"): aggregate_leaves( - (Leaf(b"bounty", miner, 1, Score(1), bytes(64)),), + (Leaf(b"bounty", miner, 1, Score(1)),), ((b"bounty", 3000), (b"proof", 7000)), ((miner, 1),), ) diff --git a/tests/protocol/test_bundle.py b/tests/protocol/test_bundle.py index 92044b661..88f38b4ca 100644 --- a/tests/protocol/test_bundle.py +++ b/tests/protocol/test_bundle.py @@ -1,24 +1,21 @@ -import json from dataclasses import replace from hashlib import sha256 -from pathlib import Path import pytest from cortex.protocol import ( Bundle, ChallengeEntry, + Leaf, MetagraphRow, NoScore, ProtocolError, Score, TrustRoot, build_bundle, - sign_leaf, verify_bundle, ) from cortex.protocol.bundle import recompute -from cortex.protocol.crypto import BUNDLE_DOMAIN, public_key, sign_raw, verify_raw from cortex.protocol.merkle import merkle_root from cortex.protocol.scale import Reader @@ -28,19 +25,17 @@ def network(): rows = tuple(MetagraphRow(bytes([tag]) * 32, tag) for tag in range(3)) trust = TrustRoot( ( - ChallengeEntry(b"bounty", public_key(bytes([1]) * 32), 2000), - ChallengeEntry(b"proof", public_key(bytes([2]) * 32), 8000), + ChallengeEntry(b"bounty", 2000), + ChallengeEntry(b"proof", 8000), ), sha256(b"\x00").digest(), - public_key(bytes([7]) * 32), ) leaves = tuple( - sign_leaf(bytes([seed]) * 32, name, row.hotkey, 12, Score(50) if row.uid else NoScore()) - for name, seed in ((b"bounty", 1), (b"proof", 2)) + Leaf(name, row.hotkey, 12, Score(50) if row.uid else NoScore()) + for name in (b"bounty", b"proof") for row in rows ) bundle = build_bundle( - gateway_seed=bytes([7]) * 32, epoch=12, netuid=541, block_b=99, @@ -52,7 +47,7 @@ def network(): return bundle, dict(rows=rows, block_hash=bytes([9]) * 32, trust=trust, netuid=541, epoch=12) -def test_complete_signed_seal_roundtrip_and_independent_recompute(network): +def test_complete_seal_roundtrip_and_independent_recompute(network): bundle, arguments = network decoded = Bundle.decode(bundle.encode()) verified = verify_bundle(decoded, **arguments) @@ -74,7 +69,6 @@ def proportional_network(network): introduced_epoch=12, ) bundle = build_bundle( - gateway_seed=bytes([7]) * 32, epoch=12, netuid=541, block_b=99, @@ -90,7 +84,7 @@ def test_v2_owner_profile_selects_algorithm_and_cannot_downgrade(proportional_ne bundle, arguments = proportional_network assert verify_bundle(Bundle.decode(bundle.encode()), **arguments).algorithm_version == 2 body = replace(bundle.body, algorithm_version=1) - downgraded = Bundle(body, sign_raw(bytes([7]) * 32, BUNDLE_DOMAIN, body.encode())) + downgraded = Bundle(body) with pytest.raises(ProtocolError, match="version"): verify_bundle(downgraded, **arguments) with pytest.raises(ProtocolError, match="version"): @@ -115,7 +109,6 @@ def test_v2_profile_cannot_activate_before_its_owner_signed_epoch(proportional_n verify_bundle(bundle, **{**arguments, "trust": future_trust}) with pytest.raises(ProtocolError, match="not active"): build_bundle( - gateway_seed=bytes([7]) * 32, epoch=12, netuid=541, block_b=99, @@ -126,19 +119,11 @@ def test_v2_profile_cannot_activate_before_its_owner_signed_epoch(proportional_n ) -def test_frozen_rust_wire_fixture_matches_exact_fields_and_signature(): - reference = json.loads((Path(__file__).parent / "vectors/rust_wire.json").read_text()) - raw = bytes.fromhex(reference["bundle_scale"]) - bundle = Bundle.decode(raw) - assert bundle.encode() == raw - assert bundle.body.leaves[0].encode().hex() == reference["leaf_scale"] - assert bundle.body.netuid == 541 - assert bundle.body.epoch == 123 - assert bundle.body.emission_shares == ((b"bounty", 2000), (b"proof", 8000)) - assert bundle.body.merkle_root == merkle_root([bundle.body.leaves[0].encode()]) - assert verify_raw( - bundle.body.gateway_hotkey, BUNDLE_DOMAIN, bundle.body.encode(), bundle.gateway_sig - ) +def test_v1_bundles_are_refused(network): + bundle, arguments = network + old = Bundle(replace(bundle.body, protocol_version=1)) + with pytest.raises(ProtocolError, match="unsupported bundle version"): + verify_bundle(old, **arguments) @pytest.mark.parametrize( @@ -153,10 +138,10 @@ def test_frozen_rust_wire_fixture_matches_exact_fields_and_signature(): ("epoch", 13, "epoch"), ], ) -def test_rejects_validly_signed_but_dishonest_gateway_inputs(network, field, value, reason): +def test_rejects_tampered_bundle_fields(network, field, value, reason): bundle, arguments = network body = replace(bundle.body, **{field: value}) - forged = Bundle(body, sign_raw(bytes([7]) * 32, BUNDLE_DOMAIN, body.encode())) + forged = Bundle(body) with pytest.raises(ProtocolError, match=reason): verify_bundle(forged, **arguments) @@ -167,11 +152,10 @@ def test_rejects_validly_signed_but_dishonest_gateway_inputs(network, field, val ("missing", "incomplete"), ("duplicate", "unique"), ("reorder", "sorted"), - ("signature", "signature"), ("retired", "unknown challenge"), ], ) -def test_rejects_incomplete_duplicate_forged_or_retired_challenge_leaves(network, change, reason): +def test_rejects_incomplete_duplicate_or_retired_challenge_leaves(network, change, reason): bundle, arguments = network leaves = list(bundle.body.leaves) if change == "missing": @@ -180,26 +164,29 @@ def test_rejects_incomplete_duplicate_forged_or_retired_challenge_leaves(network leaves.append(leaves[-1]) elif change == "reorder": leaves.reverse() - elif change == "signature": - leaves[0] = replace(leaves[0], challenge_sig=bytes(64)) else: leaves[0] = replace(leaves[0], challenge_id=b"prism") leaves.sort(key=lambda leaf: leaf.sort_key) body = replace( bundle.body, leaves=tuple(leaves), merkle_root=merkle_root(leaf.encode() for leaf in leaves) ) - forged = Bundle(body, sign_raw(bytes([7]) * 32, BUNDLE_DOMAIN, body.encode())) + forged = Bundle(body) with pytest.raises(ProtocolError, match=reason): verify_bundle(forged, **arguments) -def test_unknown_gateway_signer_cannot_replace_owner_trust(network): +def test_tampered_leaf_is_detected_by_the_merkle_root(network): bundle, arguments = network - seed = bytes([8]) * 32 - body = replace(bundle.body, gateway_hotkey=public_key(seed)) - forged = Bundle(body, sign_raw(seed, BUNDLE_DOMAIN, body.encode())) - with pytest.raises(ProtocolError, match="untrusted gateway"): - verify_bundle(forged, **arguments) + leaves = list(bundle.body.leaves) + leaves[1] = replace(leaves[1], score=Score(51)) + with pytest.raises(ProtocolError, match="Merkle root"): + verify_bundle(Bundle(replace(bundle.body, leaves=tuple(leaves))), **arguments) + + +def test_tampered_merkle_root_is_detected(network): + bundle, arguments = network + with pytest.raises(ProtocolError, match="Merkle root"): + verify_bundle(Bundle(replace(bundle.body, merkle_root=bytes(32))), **arguments) def test_decode_rejects_trailing_bytes_and_compact_malleability(network): @@ -213,3 +200,24 @@ def test_rfc6962_odd_leaf_promotion_matches_external_known_answer(): assert merkle_root([b"", b"\x00", b"\x10"]).hex() == ( "aeb6bcfe274b70a14fb067a5e5578264db0fa9b51af5e0ba159158f329e06e77" ) + + +def test_frozen_v2_wire_vector_is_exact_and_roundtrips(): + import importlib.util + import json + from pathlib import Path + + root = Path(__file__).parents[2] + frozen = json.loads((root / "tests/protocol/vectors/wire_v2.json").read_text()) + spec = importlib.util.spec_from_file_location( + "gen_wire_vector", root / "scripts/gen_wire_vector.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + assert module.vector() == frozen # exact bytes, deterministic + raw = bytes.fromhex(frozen["bundle_scale"]) + bundle = Bundle.decode(raw) + assert bundle.encode() == raw + assert bundle.body.protocol_version == 2 + assert bundle.body.leaves[0].encode().hex() == frozen["leaf_scale"] + assert bundle.body.merkle_root == merkle_root(leaf.encode() for leaf in bundle.body.leaves) diff --git a/tests/protocol/test_crypto.py b/tests/protocol/test_crypto.py index b147e2f06..c9284b946 100644 --- a/tests/protocol/test_crypto.py +++ b/tests/protocol/test_crypto.py @@ -4,8 +4,6 @@ import pytest from cortex.protocol.crypto import ( - BUNDLE_DOMAIN, - RAW_WEIGHT_DOMAIN, decode_hotkey, encode_hotkey, public_key, @@ -16,6 +14,10 @@ ) from cortex.protocol.scale import ProtocolError +# Context bytes the frozen Rust vectors were generated with; any consensus context behaves alike. +DOMAIN = b"base-bundle-v1" +OTHER_DOMAIN = b"base-rawweight-v1" + REFERENCE = json.loads((Path(__file__).parent / "vectors/rust_wire.json").read_text()) @@ -29,9 +31,9 @@ def test_verifies_independent_rust_schnorrkel_vectors(reference): signature = bytes.fromhex(reference["signature"]) payload = bytes.fromhex(reference["payload"]) assert public_key(bytes.fromhex(reference["seed"])) == key - assert verify_raw(key, BUNDLE_DOMAIN, payload, signature) - assert not verify_raw(key, RAW_WEIGHT_DOMAIN, payload, signature) - assert not verify_raw(key, BUNDLE_DOMAIN, payload + b"!", signature) + assert verify_raw(key, DOMAIN, payload, signature) + assert not verify_raw(key, OTHER_DOMAIN, payload, signature) + assert not verify_raw(key, DOMAIN, payload + b"!", signature) @pytest.mark.parametrize("mutation", ["marker", "scalar", "point", "public", "short"]) @@ -52,27 +54,25 @@ def test_rejects_signature_malleability_and_invalid_group_elements(mutation): key = bytes(32) elif mutation == "short": signature.pop() - assert not verify_raw(key, BUNDLE_DOMAIN, b"", bytes(signature)) + assert not verify_raw(key, DOMAIN, b"", bytes(signature)) def test_contexts_cannot_be_silently_substituted(): seed = bytes([7]) * 32 message = b"signed payload" substrate_signature = sign_substrate(seed, message) - consensus_signature = sign_raw(seed, BUNDLE_DOMAIN, message) + consensus_signature = sign_raw(seed, DOMAIN, message) assert verify_substrate(public_key(seed), message, substrate_signature) - assert verify_raw(public_key(seed), BUNDLE_DOMAIN, message, consensus_signature) + assert verify_raw(public_key(seed), DOMAIN, message, consensus_signature) assert not verify_substrate(public_key(seed), message, consensus_signature) - assert not verify_raw(public_key(seed), BUNDLE_DOMAIN, message, substrate_signature) + assert not verify_raw(public_key(seed), DOMAIN, message, substrate_signature) def test_python_signature_matches_vector_independently_verified_by_rust(monkeypatch): # Fixed RNG is test-only. Rust crypto::verify_raw/schnorrkel 0.11.5 verified # these exact signature bytes, with seed [7;32], on 2026-09-16. monkeypatch.setattr("cortex.protocol.crypto.os.urandom", lambda size: b"\xab" * size) - signature = sign_raw( - bytes([7]) * 32, BUNDLE_DOMAIN, b"independent Rust schnorrkel signature vector" - ) + signature = sign_raw(bytes([7]) * 32, DOMAIN, b"independent Rust schnorrkel signature vector") assert signature.hex() == ( "22d93e0325fba78d275d92beccc854feecf37cd9233a3a9d501124d3309a23322" "68dc94d9141adfcb0ae0919519b5c6bcc207152ed34d0423037538a430a298d" diff --git a/tests/protocol/vectors/rust_wire.json b/tests/protocol/vectors/rust_wire.json index 7976b50cd..80a96f4fe 100644 --- a/tests/protocol/vectors/rust_wire.json +++ b/tests/protocol/vectors/rust_wire.json @@ -1,7 +1,5 @@ { - "authority": "Rust crypto (schnorrkel 0.11.5), bundle and parity-scale-codec 3.7; independently generated from source, 2026-09-16", - "bundle_scale": "01007b000000000000001d0263000000000000000404040404040404040404040404040404040404040404040404040404040404050505050505050505050505050505050505050505050505050505050505050501000818626f756e7479d0071470726f6f66401f06060606060606060606060606060606060606060606060606060606060606060403030303030303030303030303030303030303030303030303030303030303030000041470726f6f6603030303030303030303030303030303030303030303030303030303030303037b0000000000000001064825f4e7fab57f2e71eeeca3bd71193067b1e816d47d9035d9027a2fc62bc03a6e7ae93c5a7a37e357c6b5fe4abcbc4e21de252286e3dd8040342224a0c3278e4601aa25a1695704e722d8316e09fa639b1249ae222d98295de121c1c0835f53040000ffff7c0f469d3bd340bae718203fa30ca071a5e37c751e891dbded837b213d45d91d2c8d0f4c5ceea5f0cc2efaa9f553d2633c12329b3024fcb3216717f1efa8ff434e337a0fb00a4ae0b375dcfa4a4e8468ab5c699605155bc1be55e4987c85c484", - "leaf_scale": "1470726f6f6603030303030303030303030303030303030303030303030303030303030303037b0000000000000001064825f4e7fab57f2e71eeeca3bd71193067b1e816d47d9035d9027a2fc62bc03a6e7ae93c5a7a37e357c6b5fe4abcbc4e21de252286e3dd8040342224a0c3278e", + "authority": "Rust crypto (schnorrkel 0.11.5), independently generated from source, 2026-09-16; signatures only (bundle/leaf wire moved to wire_v2.json)", "signatures": [ { "payload": "", diff --git a/tests/protocol/vectors/wire_v2.json b/tests/protocol/vectors/wire_v2.json new file mode 100644 index 000000000..b3e377cd2 --- /dev/null +++ b/tests/protocol/vectors/wire_v2.json @@ -0,0 +1,12 @@ +{ + "authority": "Python reference encoder, protocol_version 2 (no signatures); fixed inputs", + "bundle_scale": "02007b000000000000001d026300000000000000090909090909090909090909090909090909090909090909090909090909090946888dc74fe6d901a7068137d9f50fe7e71ee2e2f5494bf0a9a0842e3d9a74b901000818626f756e7479d0071470726f6f66401f6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d0c000000000000000000000000000000000000000000000000000000000000000000000101010101010101010101010101010101010101010101010101010101010101010002020202020202020202020202020202020202020202020202020202020202020200181470726f6f6600000000000000000000000000000000000000000000000000000000000000007b0000000000000001061470726f6f6601010101010101010101010101010101010101010101010101010101010101017b000000000000000032000000000000001470726f6f6602020202020202020202020202020202020202020202020202020202020202027b00000000000000010618626f756e747900000000000000000000000000000000000000000000000000000000000000007b00000000000000010618626f756e747901010101010101010101010101010101010101010101010101010101010101017b0000000000000000320000000000000018626f756e747902020202020202020202020202020202020202020202020202020202020202027b000000000000000106d712299c422590a430aa17176b7e127205ed560f2714f4b46da80f60a1ade934040100ffff", + "leaf_scale": "1470726f6f6600000000000000000000000000000000000000000000000000000000000000007b000000000000000106", + "merkle_root": "d712299c422590a430aa17176b7e127205ed560f2714f4b46da80f60a1ade934", + "final_vector": [ + [ + 1, + 65535 + ] + ] +} diff --git a/tests/test_keygen_removed.py b/tests/test_keygen_removed.py new file mode 100644 index 000000000..5e8d6364a --- /dev/null +++ b/tests/test_keygen_removed.py @@ -0,0 +1,11 @@ +import importlib.util + +import pytest + +from cortex.cli import parser + + +def test_keygen_command_and_module_are_gone(): + assert importlib.util.find_spec("cortex.keygen") is None + with pytest.raises(SystemExit): + parser().parse_args(["keygen", "--seed-out", "a", "--public-out", "b"]) diff --git a/tests/test_master.py b/tests/test_master.py index 53184a80d..b82efe1d1 100644 --- a/tests/test_master.py +++ b/tests/test_master.py @@ -13,7 +13,15 @@ from cortex.master import BittensorEpochProvider, EpochClock, EpochState, build_master from cortex.proof.models import Baseline, Endpoint, EvaluationReport, Metric, Rule, Topic, digest from cortex.proof.service import Readiness, sign_submission, sign_topic -from cortex.protocol import ChallengeEntry, MetagraphRow, NoScore, NoScoreReason, Score, TrustRoot +from cortex.protocol import ( + ChallengeEntry, + Leaf, + MetagraphRow, + NoScore, + NoScoreReason, + Score, + TrustRoot, +) from cortex.protocol.crypto import public_key from cortex.validator import ChainSnapshot, SubmissionJournal, Validator @@ -81,16 +89,13 @@ def secret(name, value): path.chmod(0o600) return path - secret("bounty.key", bytes([1]) * 32) return MasterConfig( netuid=541, state_dir=tmp_path / "state", challenges_file=tmp_path / "challenges.toml", measurements_file=tmp_path / "measurements.toml", - gateway_seed_file=secret("gateway.key", bytes([7]) * 32), proof_seed_file=secret("proof.key", bytes([2]) * 32), operator_token_file=secret("operator.token", b"fixture-operator-token"), - challenge_keys_dir=tmp_path, challenge_secrets_dir=tmp_path / "challenge-secrets", ) @@ -101,11 +106,10 @@ async def master(tmp_path): chain = FakeEpochChain() trust = TrustRoot( ( - ChallengeEntry(b"bounty", public_key(bytes([1]) * 32), 2000), - ChallengeEntry(b"proof", public_key(bytes([2]) * 32), 8000), + ChallengeEntry(b"bounty", 2000), + ChallengeEntry(b"proof", 8000), ), sha256(b"\x00").digest(), - public_key(bytes([7]) * 32), ) runtime = await build_master(config, chain=chain, epochs=chain, trust=trust) yield SimpleNamespace(runtime=runtime, config=config, chain=chain, trust=trust) @@ -191,11 +195,9 @@ async def test_latest_projection_tracks_the_sdk_active_fallback_endpoint(master, async def test_restart_recovers_epoch_pin_and_replaces_stale_positive_with_internal_burn(master): - from cortex.protocol import sign_leaf - runtime = master.runtime await runtime.emitter.tick() - scored = sign_leaf(bytes([2]) * 32, b"proof", master.chain.rows[1].hotkey, 12, Score(100)) + scored = Leaf(b"proof", master.chain.rows[1].hotkey, 12, Score(100)) runtime.gateway.accept_leaf(scored) restarted = await build_master( master.config, chain=master.chain, epochs=master.chain, trust=master.trust @@ -403,33 +405,31 @@ async def test_private_routes_require_rotating_file_and_master_routes_are_compos def test_private_seed_rejects_symlinks_permissions_and_accepts_raw_or_hex(tmp_path): config = master_config(tmp_path) - assert read_seed(config.gateway_seed_file) == bytes([7]) * 32 - config.gateway_seed_file.write_text("07" * 32) - assert read_seed(config.gateway_seed_file) == bytes([7]) * 32 + assert read_seed(config.proof_seed_file) == bytes([2]) * 32 + config.proof_seed_file.write_text("02" * 32) + assert read_seed(config.proof_seed_file) == bytes([2]) * 32 link = tmp_path / "link" - link.symlink_to(config.gateway_seed_file) + link.symlink_to(config.proof_seed_file) with pytest.raises(ServiceError): read_seed(link) - config.gateway_seed_file.chmod(0o644) + config.proof_seed_file.chmod(0o644) with pytest.raises(ServiceError): - read_seed(config.gateway_seed_file) + read_seed(config.proof_seed_file) def test_config_base_aliases_fail_closed_and_do_not_accept_inline_secret_values(): env = { "BASE_NETUID": "541", - "BASE_GATEWAY_SK_FILE": "/private/gateway", "PROOF_SK_FILE": "/private/proof", "BASE_GATEWAY_ADMIN_TOKEN_FILE": "/private/token", - "BASE_CHALLENGE_KEYS_DIR": "/private", } alias = {key.replace("BASE_", "CORTEX_"): value for key, value in env.items()} assert MasterConfig.from_env(env) == MasterConfig.from_env(alias) with pytest.raises(ValueError, match="conflicting"): MasterConfig.from_env({**env, "CORTEX_NETUID": "1"}) - del env["BASE_GATEWAY_SK_FILE"] - env["BASE_GATEWAY_SK"] = "secret-inline-material" - with pytest.raises(ValueError, match="BASE_GATEWAY_SK_FILE"): + del env["PROOF_SK_FILE"] + env["PROOF_SK"] = "secret-inline-material" + with pytest.raises(ValueError, match="PROOF_SK_FILE"): MasterConfig.from_env(env) @@ -454,9 +454,6 @@ async def decay_master(tmp_path): from cortex.challenges.registry import parse_registry config = master_config(tmp_path) - key = tmp_path / "opentype.key" - key.write_bytes(bytes([3]) * 32) - key.chmod(0o600) entries = [] for slug in ("bounty", "opentype"): token = config.challenge_secrets_dir / slug / "internal.token" @@ -473,11 +470,10 @@ async def decay_master(tmp_path): chain = FakeEpochChain() trust = TrustRoot( ( - ChallengeEntry(b"bounty", public_key(bytes([1]) * 32), 3000), - ChallengeEntry(b"opentype", public_key(bytes([3]) * 32), 7000), + ChallengeEntry(b"bounty", 3000), + ChallengeEntry(b"opentype", 7000), ), sha256(b"\x00").digest(), - public_key(bytes([7]) * 32), challenges_version=3, ) seen = [] @@ -514,12 +510,10 @@ def upstream(request): async def test_missing_or_invalid_epoch_time_postpones_before_any_leaves_or_seal( decay_master, timestamp ): - from cortex.protocol import sign_leaf - master = decay_master runtime = master.runtime await runtime.emitter.tick() - original = sign_leaf(bytes([1]) * 32, b"bounty", master.chain.rows[1].hotkey, 12, Score(123)) + original = Leaf(b"bounty", master.chain.rows[1].hotkey, 12, Score(123)) runtime.gateway.accept_leaf(original) before = runtime.gateway.store.leaves(12) master.chain.timestamp_seconds = timestamp @@ -583,9 +577,7 @@ async def test_historical_epoch_time_reaches_http_and_decayed_mass_seals_without async def test_bounty_only_algorithm_three_still_seals_without_epoch_time(decay_master): master = decay_master runtime = master.runtime - runtime.gateway.trust = replace( - master.trust, challenges=(ChallengeEntry(b"bounty", public_key(bytes([1]) * 32), 10000),) - ) + runtime.gateway.trust = replace(master.trust, challenges=(ChallengeEntry(b"bounty", 10000),)) master.chain.timestamp_seconds = None await runtime.emitter.tick() master.chain.state = EpochState(13, 100, 105) diff --git a/tests/test_master_config.py b/tests/test_master_config.py index 5ddeea55a..f4286eee8 100644 --- a/tests/test_master_config.py +++ b/tests/test_master_config.py @@ -11,7 +11,6 @@ def configured_environment(): return { "BASE_NETUID": "541", - "BASE_GATEWAY_SK_FILE": "/run/secrets/gateway", "PROOF_SK_FILE": "/run/secrets/proof", "BASE_GATEWAY_ADMIN_TOKEN_FILE": "/run/secrets/operator", "PROOF_VM_ORCHESTRATOR_URL": "https://vm.example", diff --git a/tests/test_network_e2e.py b/tests/test_network_e2e.py index 6c4074284..30fc4be2f 100644 --- a/tests/test_network_e2e.py +++ b/tests/test_network_e2e.py @@ -218,11 +218,10 @@ async def test_bounty_container_weights_seal_and_validator_dispatch( bounty_share = 2000 if version == 1 else 3000 trust = TrustRoot( ( - ChallengeEntry(b"bounty", public_key(bytes([1]) * 32), bounty_share), - ChallengeEntry(b"proof", public_key(bytes([2]) * 32), 10000 - bounty_share), + ChallengeEntry(b"bounty", bounty_share), + ChallengeEntry(b"proof", 10000 - bounty_share), ), hashlib.sha256(b"\x00").digest(), - public_key(bytes([7]) * 32), challenges_version=version, ) fake = FakeChallenges() @@ -350,13 +349,12 @@ async def test_three_container_challenges_burn_failures_and_unpaid_mass( chain.rows += (MetagraphRow(public_key(bytes([22]) * 32), 2),) trust = TrustRoot( ( - ChallengeEntry(b"audit", public_key(bytes([4]) * 32), 1000), - ChallengeEntry(b"bounty", public_key(bytes([1]) * 32), 3000), - ChallengeEntry(b"opentype", public_key(bytes([3]) * 32), 4000), - ChallengeEntry(b"proof", public_key(bytes([2]) * 32), 2000), + ChallengeEntry(b"audit", 1000), + ChallengeEntry(b"bounty", 3000), + ChallengeEntry(b"opentype", 4000), + ChallengeEntry(b"proof", 2000), ), hashlib.sha256(b"\x00").digest(), - public_key(bytes([7]) * 32), challenges_version=3, ) fake = FakeChallenges() @@ -431,11 +429,10 @@ async def test_owner_setup_miner_submission_seal_and_validator_dispatch(tmp_path chain = FakeEpochChain() trust = TrustRoot( ( - ChallengeEntry(b"bounty", public_key(bytes([1]) * 32), 2000), - ChallengeEntry(b"proof", public_key(bytes([2]) * 32), 8000), + ChallengeEntry(b"bounty", 2000), + ChallengeEntry(b"proof", 8000), ), hashlib.sha256(b"\x00").digest(), - public_key(bytes([7]) * 32), ) hypervisor = ExecutingHypervisor(tmp_path / "guests") hypervisor.config = SimpleNamespace(images={"ab" * 32: None}) diff --git a/tests/validator/test_main.py b/tests/validator/test_main.py index 27a06ae66..92b082cd7 100644 --- a/tests/validator/test_main.py +++ b/tests/validator/test_main.py @@ -13,8 +13,6 @@ def required_arguments(tmp_path): "https://master.invalid", "--netuid", "541", - "--gateway-public", - "11" * 32, "--wallet-name", "validator", "--wallet-hotkey", @@ -186,13 +184,14 @@ async def verified(arguments, subtensor, wallet): return TickResult("verified") monkeypatch.setattr(validator_main, "run", verified) - arguments = without(without_hotkey(tmp_path), "--gateway-public") + arguments = without_hotkey(tmp_path) validator_main.main([*arguments, "--verify-only", "--once"]) assert seen == {"wallet": None} def test_removed_trust_signing_options_are_gone(tmp_path): for flag in ( + "--gateway-public", "--owner-public", "--challenges", "--measurements", diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py index c1a8a85cb..e7cdc75d1 100644 --- a/tests/validator/test_validator.py +++ b/tests/validator/test_validator.py @@ -8,6 +8,7 @@ from cortex.protocol import ( Bundle, ChallengeEntry, + Leaf, MetagraphRow, NoScore, ProtocolError, @@ -16,9 +17,8 @@ aggregate_leaves, build_bundle, bundle_digest, - sign_leaf, ) -from cortex.protocol.crypto import BUNDLE_DOMAIN, public_key, sign_raw +from cortex.protocol.crypto import public_key from cortex.validator import ChainSnapshot, SubmissionJournal, Validator from cortex.validator.service import DispatchNotBroadcast, DispatchUncertain @@ -58,27 +58,19 @@ def network(tmp_path): rows = tuple(MetagraphRow(public_key(bytes([20 + tag]) * 32), tag) for tag in range(3)) trust = TrustRoot( ( - ChallengeEntry(b"bounty", public_key(bytes([1]) * 32), 2000), - ChallengeEntry(b"proof", public_key(bytes([2]) * 32), 8000), + ChallengeEntry(b"bounty", 2000), + ChallengeEntry(b"proof", 8000), ), sha256(b"\x00").digest(), - public_key(bytes([7]) * 32), ) def seal(winner=None): leaves = tuple( - sign_leaf( - bytes([seed]) * 32, - name, - row.hotkey, - 12, - Score(50) if row.uid == winner else NoScore(), - ) - for name, seed in ((b"bounty", 1), (b"proof", 2)) + Leaf(name, row.hotkey, 12, Score(50) if row.uid == winner else NoScore()) + for name in (b"bounty", b"proof") for row in rows ) return build_bundle( - gateway_seed=bytes([7]) * 32, epoch=12, netuid=541, block_b=99, @@ -228,9 +220,9 @@ async def test_latest_projection_tampering_refuses_dispatch(network, changes, re assert network.chain.submissions == [] -async def test_dishonest_signed_final_vector_is_never_dispatched(network): +async def test_dishonest_final_vector_is_never_dispatched(network): body = replace(network.bundle.body, final_vector=((2, 65535),)) - network.bundle = Bundle(body, sign_raw(bytes([7]) * 32, BUNDLE_DOMAIN, body.encode())) + network.bundle = Bundle(body) with pytest.raises(ProtocolError, match="final vector"): await network.validator.run_once() assert network.chain.submissions == [] @@ -467,7 +459,7 @@ async def test_class_a_submits_independent_vector_and_signed_dissent_after_peer_ from cortex.protocol.models import encode_final_vector body = replace(network.bundle.body, final_vector=((2, 65535),)) - network.bundle = Bundle(body, sign_raw(bytes([7]) * 32, BUNDLE_DOMAIN, body.encode())) + network.bundle = Bundle(body) signed_peers(network) assert (await network.validator.run_once()).outcome == "submitted" assert network.chain.submissions == [(541, ((1, 65535),), 1)] @@ -479,7 +471,7 @@ async def test_class_a_submits_independent_vector_and_signed_dissent_after_peer_ async def test_verify_only_reports_vector_dissent_as_a_degraded_outcome(network): body = replace(network.bundle.body, final_vector=((2, 65535),)) - network.bundle = Bundle(body, sign_raw(bytes([7]) * 32, BUNDLE_DOMAIN, body.encode())) + network.bundle = Bundle(body) signed_peers(network) network.validator.verify_only = True @@ -490,11 +482,11 @@ async def test_verify_only_reports_vector_dissent_as_a_degraded_outcome(network) @pytest.mark.parametrize("challenge,can_submit", [(b"bounty", True), (b"proof", False)]) -async def test_quarantine_uses_surviving_signed_mass_threshold(network, challenge, can_submit): +async def test_quarantine_uses_surviving_mass_threshold(network, challenge, can_submit): from cortex.protocol.merkle import merkle_root leaves = tuple( - replace(leaf, challenge_sig=bytes(64)) if leaf.challenge_id == challenge else leaf + replace(leaf, epoch=13) if leaf.challenge_id == challenge else leaf for leaf in network.bundle.body.leaves ) body = replace( @@ -502,7 +494,7 @@ async def test_quarantine_uses_surviving_signed_mass_threshold(network, challeng leaves=leaves, merkle_root=merkle_root(leaf.encode() for leaf in leaves), ) - network.bundle = Bundle(body, sign_raw(bytes([7]) * 32, BUNDLE_DOMAIN, body.encode())) + network.bundle = Bundle(body) signed_peers(network) if can_submit: assert (await network.validator.run_once()).outcome == "submitted" From 6606ebb00b2c4cac9f19e8ee09748053ebb988f1 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:08:09 +0000 Subject: [PATCH 09/56] Remove cortex keygen command Challenge leaf keys no longer exist. Co-Authored-By: Claude Sonnet 5.5 --- src/cortex/cli.py | 14 -------------- src/cortex/keygen.py | 32 -------------------------------- tests/test_keygen.py | 28 ---------------------------- 3 files changed, 74 deletions(-) delete mode 100644 src/cortex/keygen.py delete mode 100644 tests/test_keygen.py diff --git a/src/cortex/cli.py b/src/cortex/cli.py index 30673e084..b15441477 100644 --- a/src/cortex/cli.py +++ b/src/cortex/cli.py @@ -40,11 +40,6 @@ def parser() -> argparse.ArgumentParser: reconcile.add_argument("--attempt-id", required=True) reconcile.add_argument("--result", choices=("submitted", "not_broadcast"), required=True) reconcile.add_argument("--evidence-digest", required=True) - keygen = commands.add_parser( - "keygen", help="generate an sr25519 seed and public key for a challenge leaf key" - ) - keygen.add_argument("--seed-out", type=Path, required=True) - keygen.add_argument("--public-out", type=Path, required=True) vm = commands.add_parser("vm-host", help="serve the dedicated Firecracker host") vm.add_argument("--config", type=Path, required=True) vm.add_argument( @@ -217,13 +212,6 @@ async def run_master(args: argparse.Namespace) -> None: await asyncio.to_thread(close_subtensor, subtensor) -def run_keygen(args: argparse.Namespace) -> None: - from cortex.keygen import generate_key - - public = generate_key(args.seed_out, args.public_out) - print(json.dumps({"public_key": public, "seed_file": str(args.seed_out)})) - - def run_validator_reconcile(args: argparse.Namespace) -> None: from cortex.validator import SubmissionJournal @@ -269,8 +257,6 @@ def main(argv: list[str] | None = None) -> None: from cortex.vm.server import serve serve(args.config) - elif args.command == "keygen": - run_keygen(args) elif args.command == "validator-reconcile": run_validator_reconcile(args) else: diff --git a/src/cortex/keygen.py b/src/cortex/keygen.py deleted file mode 100644 index bb12d1240..000000000 --- a/src/cortex/keygen.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Generate a raw sr25519 seed and its public key for a challenge leaf or topic key.""" - -import os -import secrets -from pathlib import Path - -from cortex.protocol.crypto import public_key - - -def _exclusive_write(path: Path, value: bytes, mode: int) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode) - try: - with os.fdopen(descriptor, "wb", closefd=False) as stream: - stream.write(value) - stream.flush() - os.fsync(stream.fileno()) - finally: - os.close(descriptor) - - -def generate_key(seed_path: Path, public_path: Path) -> str: - """Create a new raw seed (0600) and its hexadecimal public key (0644), never overwriting.""" - seed = secrets.token_bytes(32) - encoded_public = public_key(seed).hex() - _exclusive_write(seed_path, seed, 0o600) - try: - _exclusive_write(public_path, (encoded_public + "\n").encode(), 0o644) - except BaseException: - seed_path.unlink(missing_ok=True) - raise - return encoded_public diff --git a/tests/test_keygen.py b/tests/test_keygen.py deleted file mode 100644 index 05a1520ef..000000000 --- a/tests/test_keygen.py +++ /dev/null @@ -1,28 +0,0 @@ -import stat - -import pytest - -from cortex.config import read_seed -from cortex.keygen import generate_key -from cortex.protocol.crypto import public_key - - -def test_generates_private_seed_matching_public_key(tmp_path): - seed, public = tmp_path / "leaf.seed", tmp_path / "leaf.pubkey" - encoded = generate_key(seed, public) - assert stat.S_IMODE(seed.stat().st_mode) == 0o600 - assert public.read_text().strip() == encoded == public_key(read_seed(seed)).hex() - - -def test_never_overwrites_and_leaves_no_orphan_seed(tmp_path): - seed, public = tmp_path / "leaf.seed", tmp_path / "leaf.pubkey" - generate_key(seed, public) - original = seed.read_bytes() - with pytest.raises(FileExistsError): - generate_key(seed, tmp_path / "other.pubkey") - assert seed.read_bytes() == original - public.write_text("x") - other = tmp_path / "second.seed" - with pytest.raises(FileExistsError): - generate_key(other, public) - assert not other.exists() From 33d2cf8fae3059e5fd46d2e3ffa9fb8ac55c7cb3 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:28:13 +0000 Subject: [PATCH 10/56] Refuse a v1 gateway store that only holds trust profiles The wire-format guard looked at raw leaves and bundles only, so a v1 store with just gateway_trust_profiles was stamped wire_format=2 and later failed with 'corrupt stored trust profile'. Treat trust profiles as v1 data too. Co-Authored-By: Claude Sonnet 5.5 --- src/cortex/gateway/store.py | 3 ++- tests/gateway/test_gateway.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/cortex/gateway/store.py b/src/cortex/gateway/store.py index 6c22bdc7b..3a6759bd0 100644 --- a/src/cortex/gateway/store.py +++ b/src/cortex/gateway/store.py @@ -117,7 +117,8 @@ def _require_wire_format(self) -> None: if row is None: used = connection.execute( "SELECT 1 FROM gateway_raw_leaves UNION ALL " - "SELECT 1 FROM gateway_bundles LIMIT 1" + "SELECT 1 FROM gateway_bundles UNION ALL " + "SELECT 1 FROM gateway_trust_profiles LIMIT 1" ).fetchone() if used: raise ServiceError( diff --git a/tests/gateway/test_gateway.py b/tests/gateway/test_gateway.py index 0c562ee77..2ca453c2c 100644 --- a/tests/gateway/test_gateway.py +++ b/tests/gateway/test_gateway.py @@ -747,3 +747,18 @@ async def test_store_with_another_wire_format_marker_is_refused(network): connection.execute("UPDATE gateway_metadata SET value='1' WHERE name='wire_format'") with pytest.raises(ServiceError, match="wire format"): GatewayStore(network.db_path) + + +def test_store_holding_only_v1_trust_profiles_is_refused_and_not_stamped(tmp_path): + path = tmp_path / "v1.db" + GatewayStore(path).close() + with sqlite3.connect(path) as connection: + connection.execute("DELETE FROM gateway_metadata WHERE name='wire_format'") + connection.execute("INSERT INTO gateway_trust_profiles VALUES (?,?)", (b"d", "{}")) + with pytest.raises(ServiceError, match="new state dir"): + GatewayStore(path) + with sqlite3.connect(path) as connection: + assert ( + connection.execute("SELECT 1 FROM gateway_metadata WHERE name='wire_format'").fetchone() + is None + ) From c699f8793adf6493315d8b459d917b1b218d144f Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:28:36 +0000 Subject: [PATCH 11/56] Dissent unsupported version for a v1 bundle before decoding it A v1 body failed Bundle.decode with 'SCALE vector too large', which mapped to AGGREGATION_OVERFLOW (13). Check the leading version first so the dissent is PROTOCOL_VERSION_UNSUPPORTED (8). Co-Authored-By: Claude Sonnet 5.5 --- src/cortex/validator/service.py | 5 ++++- tests/validator/test_validator.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/cortex/validator/service.py b/src/cortex/validator/service.py index 16a7b1299..47713b2fc 100644 --- a/src/cortex/validator/service.py +++ b/src/cortex/validator/service.py @@ -17,7 +17,7 @@ from cortex.errors import ServiceError from cortex.http import decode_json from cortex.protocol import Bundle, MetagraphRow, ProtocolError, TrustRoot -from cortex.protocol.bundle import bundle_digest, recompute, verify_inputs +from cortex.protocol.bundle import BUNDLE_PROTOCOL_VERSION, bundle_digest, recompute, verify_inputs from cortex.protocol.consensus import Dissent, DissentReason, RootStatement from cortex.protocol.crypto import public_key from cortex.protocol.models import encode_final_vector @@ -529,6 +529,9 @@ async def _run_once(self) -> TickResult: if latest.get("netuid", self.netuid) != self.netuid: raise ProtocolError("gateway subnet mismatch") data = await self._read(f"/v1/bundle/{epoch}", 32 * 1024 * 1024) + # The version leads the encoding: a v1 body must not die in decode with a misleading error. + if int.from_bytes(data[:2], "little") != BUNDLE_PROTOCOL_VERSION: + raise ProtocolError("unsupported bundle version") bundle = Bundle.decode(data) self._observed = bundle tip = await self.chain.current_block() diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py index e7cdc75d1..901b9dd8f 100644 --- a/tests/validator/test_validator.py +++ b/tests/validator/test_validator.py @@ -577,3 +577,17 @@ async def test_peer_api_serves_signed_evidence_and_never_claims_dcap_verified(ne refused = await http.post(f"/v1/attest/{path}", json={}) assert refused.status_code == 503 assert refused.json()["verified"] is False + + +async def test_v1_bundle_dissents_as_unsupported_version_before_decoding(network): + from cortex.protocol.consensus import DissentReason + + real = network.bundle + # v1 layout differs after the version, so decoding it as v2 would fail with another error. + network.bundle = SimpleNamespace(body=real.body, encode=lambda: b"\x01\x00" + b"\xff" * 64) + signed_peers(network) + with pytest.raises(ProtocolError, match="unsupported bundle version"): + await network.validator.run_once() + assert network.chain.submissions == [] + reason = network.journal.evidence.dissents()[-1].reason + assert reason == DissentReason.PROTOCOL_VERSION_UNSUPPORTED From 395166030d57fdad945b232ac00fe56acaac7f96 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:28:46 +0000 Subject: [PATCH 12/56] docs: remove stale gateway/leaf-key mentions, document operator-bearer forge risk Lens review fixes: THREAT_MODEL, OPERATOR_SECURITY, validator-quickstart, AGENTS key-role table, configuration, CHALLENGES and miner docs now match wire v2. Accepted risk (founder 2026-10-19): the operator bearer feeds /v1/weights/raw and sealing; no independent bundle cross-check exists. Co-Authored-By: Claude Sonnet 5.5 --- AGENTS.md | 14 ++++++-------- docs/CHALLENGES.md | 4 ++-- docs/OPERATOR_SECURITY.md | 9 +++++++-- docs/THREAT_MODEL.md | 16 ++++++++++++++-- docs/external-miner/bounty.md | 2 +- docs/external-miner/proof.md | 2 +- docs/external-miner/troubleshoot.md | 2 +- docs/reference/configuration.md | 4 ++-- docs/validator-quickstart.md | 5 ++--- 9 files changed, 36 insertions(+), 22 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0ee45615b..0850808fa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ canonical documentation instead of duplicating runbooks. Cortex is an autonomous research subnet on Bittensor. Proof runs inside the master. Every other challenge is a Docker container that implements [the challenge contract](docs/CHALLENGES.md); the master loads it from an -operator registry and signs its leaves. +operator registry and records its leaves. | Challenge | Source | Purpose | | --- | --- | --- | @@ -69,23 +69,21 @@ or permit UID is not a submit path. | Key or token | Role | | --- | --- | -| gateway seed | signs immutable epoch bundles | -| challenge seed | `.key`; signs that container challenge's leaves and must match the challenge profile | -| Proof seed | signs Proof topics and leaves and must match the challenge profile | -| operator bearer | protects master administrative routes | +| Proof seed | signs Proof topics (`PROOF_SK_FILE`); no bundle, leaf or gateway key exists | +| operator bearer | protects master administrative routes, including `/v1/weights/raw` and sealing | | VM orchestrator bearer | authenticates master to the dedicated KVM host; not a wallet | | validator hotkey | weight-setting role only: signs root/dissent evidence and Bittensor submissions | | challenge internal token | master bearer for a container's `get_weights`; never a signing key | | miner hotkey | signs Bounty pairing or Proof submission payloads | -Do not conflate gateway sealing, master ownership and validator chain signing. +Do not conflate gateway sealing (unsigned), master ownership and validator chain signing. Validators build their view from the gateway profile at `GET /v1/trust`; the gateway is the single source of truth. ## Challenge container contracts - The challenge profile decides emission; the unsigned registry (`deploy/challenges/registry.toml`) only decides what runs. A container never - receives a leaf-signing seed. + receives a signing key. - `challenge-supervisor` is the only process with the Docker socket. It verifies image labels and GitHub build provenance, canaries without secrets, rolls back on failure and never reads a secret. @@ -179,7 +177,7 @@ uv build --no-build-isolation ``` Challenge verification must exercise a real intake request, failure probes, -signed leaves, gateway seal, a `sealed: true` fetch and validator recomputation. +leaves, gateway seal, a `sealed: true` fetch and validator recomputation. Health endpoints alone do not verify a challenge. CI must not contact OpenRouter, rent provider compute, boot Firecracker or submit an on-chain transaction. diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index 77ebd3ff7..99931f203 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -92,7 +92,7 @@ returns: that epoch. - The call must answer within 180 seconds. -### Leaves the master signs +### Leaves the master records Let `E` be the owner-policy participant set of the sealed metagraph, `w_i` the returned weight (0 when absent), `W = sum(w_i for i in E)` and @@ -108,7 +108,7 @@ changes `D`. Every other hotkey in `E` receives `NoScore(NotAttempted)`. A failed or invalid call gives `NoScore(ChallengeInternal)` to every hotkey in `E`: one invalid weight invalidates the whole answer, so an answer is never partially paid. -Algorithm 1 signed Bounty's legacy champion score, which no container computes, +Algorithm 1 scored Bounty's legacy champion score, which no container computes, so containers are not scored under it; trust watermarks never return to it. Unpaid mass always burns to UID0 and never moves to another challenge. Under algorithm 3, Bounty with `full_share_mass = 10` pays exactly what algorithm 2 pays: diff --git a/docs/OPERATOR_SECURITY.md b/docs/OPERATOR_SECURITY.md index 3f781cfc6..0abcfc791 100644 --- a/docs/OPERATOR_SECURITY.md +++ b/docs/OPERATOR_SECURITY.md @@ -11,6 +11,11 @@ recovery. It complements the [threat model](THREAT_MODEL.md). build argument, Terraform state, cloud-init payload, log or shell history. - [ ] Master operator, per-challenge internal/admin, Proof and VM-host tokens are distinct and rotated independently. +- [ ] The master operator bearer is held only by the master and the operator. It + feeds `POST /v1/weights/raw` and seals epochs (`/v1/admin/seal`), so a holder can + forge any score and seal it (accepted risk, founder decision 2026-10-19; see the + [threat model](THREAT_MODEL.md)). Rotate it on any suspicion. Wire v2 has no + independent bundle cross-check: its vectors come from the Python implementation. - [ ] Miner BYOK vault storage is durable only as long as queued work requires, and terminal jobs have no remaining secret files. - [ ] The OpenRouter owner key stays on the VM host and is not exposed to miner @@ -58,10 +63,10 @@ recovery. It complements the [threat model](THREAT_MODEL.md). - [ ] Every registered challenge answers `GET /challenge//version`, and its own readiness probe passes (for Bounty, `/challenge/bounty/v1/status` reports `can_score: true` after a real stable feed probe). -- [ ] Stopping a challenge container makes the next epoch contain signed +- [ ] Stopping a challenge container makes the next epoch contain `ChallengeInternal` leaves for it and burn its share without blocking others. - [ ] While Proof is unwired, `PROOF_VM_ORCHESTRATOR_URL` is empty, no Proof topic - is open, and its share burns to UID 0 through signed `ChallengeInternal` leaves. + is open, and its share burns to UID 0 through `ChallengeInternal` leaves. - [ ] When Proof is enabled, `/v1/status` reports a valid topic, sealed baseline, registered runner, pinned image, open inference offer and compatible executor offer; its failure matrix returns 503 without a scored row. diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index 67e8dc166..75e78c3ee 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -82,7 +82,7 @@ an owner signature approves exact content and visibility. ## Reward controls -Challenge leaves are signed under keys in the owner root. The expected set comes +Challenge leaves carry no signature (wire v2); the master records them after the gateway authenticates the operator bearer. The expected set comes from the historical metagraph; a challenge cannot shrink it by omission. A missing score becomes an explicit `NoScore`, and an unavailable challenge burns its share rather than blocking every other challenge. @@ -93,7 +93,7 @@ the complete participant set for one challenge and epoch. A challenge-container outage or invalid answer therefore replaces every participant of that challenge with `NoScore(ChallengeInternal)` and burns its full share without retaining stale positives. A container's first answer per epoch is final, and the master -signs it once, so a later change cannot rewrite a sealed epoch. `GET /v1/weights/latest` +records it once, so a later change cannot rewrite a sealed epoch. `GET /v1/weights/latest` returns an unsealed UID0 fallback when no valid seal exists; validators refuse that fallback. A sealed UID0 burn is valid and must still be submitted after independent verification. @@ -103,6 +103,18 @@ is not automatically retried, because a blind retry can submit twice. No test or health endpoint proves an on-chain payment; that requires observing the actual extrinsic and resulting chain state. +### Accepted risk: one operator bearer forges and seals (founder decision 2026-10-19) + +The operator bearer guards both `POST /v1/weights/raw` and `POST /v1/admin/seal`. +Whoever holds it can write any score for any miner and seal the epoch. Before +wire v2, forging a leaf also needed a challenge key. Nothing else separates the two +powers now, and no independent cross-check of a bundle exists: `wire_v2.json` is +generated by the Python implementation, so no second implementation pins the bytes. + +Mitigation: the operator token is held only by the master and the operator, never +by miners, validators or challenge containers; rotate it on any suspicion. Future +option: separate raw-intake and seal tokens. + ## Availability and deployment The master and SQLite databases are availability dependencies. Durable volumes, diff --git a/docs/external-miner/bounty.md b/docs/external-miner/bounty.md index 7fa90a62e..4b271d825 100644 --- a/docs/external-miner/bounty.md +++ b/docs/external-miner/bounty.md @@ -28,6 +28,6 @@ Substrate signing context, not the Cortex one. `bounty-report` posts to `/challenge/bounty/v1/reports`. Only a report that CortexLM/backend publishes as `valid` earns weight: one point per valid report. Bounty pays `share * min(N / 10, 1)`, split in proportion to each author's count, and the -rest burns to UID0. The master turns the container's weights into signed leaves +rest burns to UID0. The master turns the container's weights into leaves under the [challenge contract](../CHALLENGES.md). Validators never contact the container. diff --git a/docs/external-miner/proof.md b/docs/external-miner/proof.md index e45ccf03a..7182e9d63 100644 --- a/docs/external-miner/proof.md +++ b/docs/external-miner/proof.md @@ -245,6 +245,6 @@ split in the challenge profile. Legacy 2000/8000 deployments retain algorithm 1 until the profile version is bumped ([Changing emission shares](../../deploy/README.md#changing-emission-shares)). -Actual payment additionally needs signed leaves, a valid gateway seal and +Actual payment additionally needs recorded leaves, a valid gateway seal and validator submission on Bittensor. See [validators](validators.md) and [troubleshooting](troubleshoot.md). diff --git a/docs/external-miner/troubleshoot.md b/docs/external-miner/troubleshoot.md index 0d4040fc4..b36b0bd76 100644 --- a/docs/external-miner/troubleshoot.md +++ b/docs/external-miner/troubleshoot.md @@ -81,7 +81,7 @@ digest even when its extracted files look identical. | `503` credential vault or stored artifact error | Required private inputs are unavailable; the operator must restore or reconcile the accepted job | | `503` teardown or evaluation infrastructure failure | No score may be fabricated; failed work may still have a durable job and consumed nonce | | `201` with `queued` | The topic explicitly sets `params.defer_scoring: "true"`; the response is not a completed score | -| `201` with `accepted` but no payment | Topic payout, signed leaves, sealing and chain submission are additional stages | +| `201` with `accepted` but no payment | Topic payout, leaves, sealing and chain submission are additional stages | | POST times out or client disconnects | Accepted work continues; check its returned id or reconcile with the operator before resubmitting | Normal Proof POST responses are **201-after-score**. The CLI waits 7200 seconds diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index c75eb53cf..74e589649 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -46,8 +46,8 @@ never reassigned. Changing the challenge profile is described in The gateway serves the public `/v1` routes `GET /v1/weights/latest`, `GET /v1/weights/{epoch}`, `GET /v1/bundle/{epoch}` and `GET /v1/trust`. -`GET /v1/trust` returns the current challenge profile (challenges with keys, -shares and policy, measurements digest, gateway hotkey, versions and +`GET /v1/trust` returns the current challenge profile (challenges with +shares and policy, measurements digest, versions and `introduced_epoch`) as JSON. It is public, unsigned and read-only; validators build their trust view from it over HTTPS. diff --git a/docs/validator-quickstart.md b/docs/validator-quickstart.md index 65e16db3c..f1e6486d3 100644 --- a/docs/validator-quickstart.md +++ b/docs/validator-quickstart.md @@ -62,7 +62,7 @@ Flags: ## Verify-only mode -`--verify-only` runs one verification and exits without submitting. Use it first, to confirm your trust files, pinned key and network work. `--dry-run` goes one step less: it prints the `cortex validator` command, with the wallet name and hotkey masked, and starts nothing. +`--verify-only` runs one verification and exits without submitting. Use it first, to confirm your gateway, chain access and network work. `--dry-run` goes one step less: it prints the `cortex validator` command, with the wallet name and hotkey masked, and starts nothing. ## What the validator checks @@ -70,10 +70,9 @@ Before starting, the script: - fetches `GET $GATEWAY/v1/weights/latest` and requires HTTP 200; - requires the answer to be JSON with the expected `netuid`; -- reads the minimum trust-file versions from `config/challenges.toml` and `config/measurements.toml`; - reads the subnet `weights_version` from the chain, unless you set `VERSION_KEY`. -Then `cortex validator` takes over. It uses the owner public key and the signed trust files in `config/`, verifies the sealed bundle against them and the chain, and recomputes the weights from the signed leaves. It doesn't trust the display fields in the gateway answer. See the [validator guide](external-miner/validators.md) for the full rules. +Then `cortex validator` takes over. It fetches the challenge profile from the gateway, verifies the sealed bundle's hashes and Merkle root against it and the chain, and recomputes the weights from the leaves. It doesn't trust the display fields in the gateway answer. See the [validator guide](external-miner/validators.md) for the full rules. ## Safety notes From b6de50249800f2fece799c05179924ee99ec9d89 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:28:55 +0000 Subject: [PATCH 13/56] Test the strict leaf epoch check of verify_inputs With leaf signatures gone, leaf.epoch == body.epoch in _verify_leaves is the only guard on a leaf's epoch; removing it left the suite green. Co-Authored-By: Claude Sonnet 5.5 --- tests/protocol/test_bundle.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/protocol/test_bundle.py b/tests/protocol/test_bundle.py index 88f38b4ca..c4b32b2c9 100644 --- a/tests/protocol/test_bundle.py +++ b/tests/protocol/test_bundle.py @@ -15,7 +15,7 @@ build_bundle, verify_bundle, ) -from cortex.protocol.bundle import recompute +from cortex.protocol.bundle import recompute, verify_inputs from cortex.protocol.merkle import merkle_root from cortex.protocol.scale import Reader @@ -189,6 +189,20 @@ def test_tampered_merkle_root_is_detected(network): verify_bundle(Bundle(replace(bundle.body, merkle_root=bytes(32))), **arguments) +def test_leaf_from_another_epoch_is_refused_strictly_and_quarantined_when_allowed(network): + bundle, arguments = network + leaves = list(bundle.body.leaves) + # Merkle root is recomputed so that only the epoch is wrong. + leaves[0] = replace(leaves[0], epoch=13) + body = replace( + bundle.body, leaves=tuple(leaves), merkle_root=merkle_root(leaf.encode() for leaf in leaves) + ) + with pytest.raises(ProtocolError, match="leaf epoch mismatch"): + verify_bundle(Bundle(body), **arguments) + _, quarantined = verify_inputs(Bundle(body), **arguments, allow_quarantine=True) + assert quarantined == {leaves[0].challenge_id} + + def test_decode_rejects_trailing_bytes_and_compact_malleability(network): with pytest.raises(ProtocolError, match="trailing"): Bundle.decode(network[0].encode() + b"\x00") From 0d534b9bb1d9395c17af15d724e3980e6caba593 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:29:01 +0000 Subject: [PATCH 14/56] Refresh wire-format docstrings for v2 Co-Authored-By: Claude Sonnet 5.5 --- src/cortex/gateway/models.py | 2 +- src/cortex/protocol/models.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/cortex/gateway/models.py b/src/cortex/gateway/models.py index bfab094d1..ae73bd2db 100644 --- a/src/cortex/gateway/models.py +++ b/src/cortex/gateway/models.py @@ -1,4 +1,4 @@ -"""Public gateway JSON contracts, mapped to the signed SCALE leaf payload.""" +"""Public gateway JSON contracts, mapped to the unsigned SCALE leaf payload.""" from typing import Annotated diff --git a/src/cortex/protocol/models.py b/src/cortex/protocol/models.py index 0d835f902..11fdab5c6 100644 --- a/src/cortex/protocol/models.py +++ b/src/cortex/protocol/models.py @@ -1,4 +1,8 @@ -"""Immutable SCALE v1 types. Field order mirrors the frozen Rust contract.""" +"""Immutable SCALE types of wire v2 (no signatures). + +Field order is frozen by tests/protocol/vectors/wire_v2.json, generated by Python with +scripts/gen_wire_vector.py and checked byte for byte (plus round trip) in test_bundle.py. +""" import re from dataclasses import dataclass From 6af357f5e921a901bb4344e1edd599d9d007efaf Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:31:48 +0000 Subject: [PATCH 15/56] Persist trust-version watermarks only after verification The challenges/measurements watermarks were written from the unauthenticated /v1/trust profile before verify_inputs, so one hostile or mistyped profile (version 2**32-1) made every later honest profile fail with 'consensus rollback refused'. Check the rollback first, persist after the profile and bundle verified. Co-Authored-By: Claude Sonnet 5.5 --- src/cortex/validator/evidence.py | 5 ++++- src/cortex/validator/service.py | 9 +++++++-- tests/validator/test_validator.py | 25 +++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/cortex/validator/evidence.py b/src/cortex/validator/evidence.py index 6075daa4f..9b9a7e774 100644 --- a/src/cortex/validator/evidence.py +++ b/src/cortex/validator/evidence.py @@ -31,12 +31,15 @@ def __init__(self, connection: sqlite3.Connection): ); """) - def watermark(self, name: str, value: int) -> None: + def check_watermark(self, name: str, value: int) -> None: row = self.connection.execute( "SELECT value FROM consensus_watermarks WHERE name=?", (name,) ).fetchone() if row and int(row[0]) > value: raise ProtocolError("consensus rollback refused") + + def watermark(self, name: str, value: int) -> None: + self.check_watermark(name, value) self.connection.execute( "INSERT INTO consensus_watermarks VALUES (?,?) " "ON CONFLICT(name) DO UPDATE SET value=excluded.value", diff --git a/src/cortex/validator/service.py b/src/cortex/validator/service.py index 47713b2fc..c86bb9df7 100644 --- a/src/cortex/validator/service.py +++ b/src/cortex/validator/service.py @@ -548,8 +548,11 @@ async def _run_once(self) -> TickResult: # The gateway is ours and is the source of truth for the emission profile and the # bundle; TLS is the anchor. Integrity of the bundle is recomputed, not signed. self.trust = trust_from_json(decode_json(await self._read("/v1/trust", 1024 * 1024))) - self.journal.evidence.watermark("challenges_version", self.trust.challenges_version) - self.journal.evidence.watermark("measurements_version", self.trust.measurements_version) + # Refuse a rollback now, but persist the versions only once the profile has verified + # against the chain and the bundle: an unauthenticated profile must not move them. + evidence = self.journal.evidence + evidence.check_watermark("challenges_version", self.trust.challenges_version) + evidence.check_watermark("measurements_version", self.trust.measurements_version) body, quarantined = verify_inputs( bundle, rows=snapshot.rows, @@ -559,6 +562,8 @@ async def _run_once(self) -> TickResult: epoch=epoch, allow_quarantine=True, ) + evidence.watermark("challenges_version", self.trust.challenges_version) + evidence.watermark("measurements_version", self.trust.measurements_version) self.journal.evidence.watermark("epoch", epoch) self.journal.evidence.bundle(bundle) if latest.get("merkle_root") != body.merkle_root.hex(): diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py index 901b9dd8f..4797a3a1a 100644 --- a/tests/validator/test_validator.py +++ b/tests/validator/test_validator.py @@ -591,3 +591,28 @@ async def test_v1_bundle_dissents_as_unsupported_version_before_decoding(network assert network.chain.submissions == [] reason = network.journal.evidence.dissents()[-1].reason assert reason == DissentReason.PROTOCOL_VERSION_UNSUPPORTED + + +async def test_hostile_trust_profile_does_not_move_the_version_watermark(network): + from cortex.protocol.trust import trust_to_json + + hostile = {**trust_to_json(network.validator.trust), "challenges_version": 2**32 - 1} + served = {"profile": hostile} + + def transport(request): + if request.url.path == "/v1/trust": + return httpx.Response(200, json=served["profile"]) + return network.handler(request) + + validator = Validator( + gateway_url="https://master.invalid", + netuid=541, + chain=network.chain, + journal=network.journal, + http=httpx.AsyncClient(transport=httpx.MockTransport(transport)), + ) + with pytest.raises(ProtocolError): # profile does not match the bundle's shares/version + await validator.run_once() + assert network.chain.submissions == [] + served["profile"] = trust_to_json(network.validator.trust) + assert (await validator.run_once()).outcome == "submitted" From 97fd3a9529d916813b77c328ac8abb77a6c00dec Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:32:46 +0000 Subject: [PATCH 16/56] Test the validator-reconcile CLI end to end Co-Authored-By: Claude Sonnet 5.5 --- tests/validator/test_validator.py | 42 +++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py index 4797a3a1a..314b86aa1 100644 --- a/tests/validator/test_validator.py +++ b/tests/validator/test_validator.py @@ -616,3 +616,45 @@ def transport(request): assert network.chain.submissions == [] served["profile"] = trust_to_json(network.validator.trust) assert (await validator.run_once()).outcome == "submitted" + + +async def test_cli_validator_reconcile_resolves_an_uncertain_dispatch(network, capsys): + from cortex.cli import main + + network.chain.error = DispatchUncertain("RPC disconnected", extrinsic_hash="0x" + "ab" * 32) + result = await network.validator.run_once() + assert result.outcome == "dispatch_pending" + main( + [ + "validator-reconcile", + "--state-db", str(network.db_path), + "--netuid", "541", + "--epoch", "12", + "--digest", bundle_digest(network.bundle), + "--attempt-id", result.attempt_id, + "--result", "not_broadcast", + "--evidence-digest", "cd" * 32, + ] + ) # fmt: skip + assert '"state": "reconciled_not_broadcast"' in capsys.readouterr().out + assert network.journal.pending(541, 12) is None + network.chain.error = None + assert (await network.validator.run_once()).outcome == "submitted" + + +def test_cli_validator_reconcile_refuses_an_unknown_dispatch(network): + from cortex.cli import main + + with pytest.raises(SystemExit, match="Cortex command failed"): + main( + [ + "validator-reconcile", + "--state-db", str(network.db_path), + "--netuid", "541", + "--epoch", "12", + "--digest", "ab" * 32, + "--attempt-id", "nope", + "--result", "submitted", + "--evidence-digest", "cd" * 32, + ] + ) # fmt: skip From fe4ecc67cdb949c358d07f1a833dcba2bc6febb3 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:33:05 +0000 Subject: [PATCH 17/56] Reject a non-integer profile epoch/version with a clear error The example profiles ship introduced_epoch = "CHOOSE_ACTIVATION_EPOCH"; load_trust died with an uncaught TypeError in max(). Validate version and introduced_epoch as non-negative integers (ProtocolError, a ValueError the master already maps to 503). Adds the first load_trust tests: TOML -> TrustRoot, JSON roundtrip, placeholder refusal, the example and default profiles. Co-Authored-By: Claude Sonnet 5.5 --- src/cortex/protocol/trust.py | 16 +++++++-- tests/protocol/test_trust_load.py | 58 +++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 tests/protocol/test_trust_load.py diff --git a/src/cortex/protocol/trust.py b/src/cortex/protocol/trust.py index a4b5c68f7..be352cdc6 100644 --- a/src/cortex/protocol/trust.py +++ b/src/cortex/protocol/trust.py @@ -57,6 +57,13 @@ def measurements_body(document: dict) -> bytes: return vector(entries, lambda value: value) +def _whole(document: dict, name: str, default: int | None = None) -> int: + value = document.get(name, default) + if type(value) is not int or value < 0: + raise ProtocolError(f"{name} must be a non-negative integer, got {value!r}") + return value + + def load_trust(*, challenges_path: Path, measurements_path: Path) -> TrustRoot: """Build the profile from the master's own TOML files. No signature: the files are local.""" challenges_doc = tomllib.loads(challenges_path.read_text()) @@ -64,9 +71,12 @@ def load_trust(*, challenges_path: Path, measurements_path: Path) -> TrustRoot: trust = TrustRoot( challenge_entries(challenges_doc), sha256(measurements_body(measurements_doc)).digest(), - challenges_doc["version"], - measurements_doc["version"], - max(challenges_doc.get("introduced_epoch", 0), measurements_doc.get("introduced_epoch", 0)), + _whole(challenges_doc, "version"), + _whole(measurements_doc, "version"), + max( + _whole(challenges_doc, "introduced_epoch", 0), + _whole(measurements_doc, "introduced_epoch", 0), + ), ) trust.validate() return trust diff --git a/tests/protocol/test_trust_load.py b/tests/protocol/test_trust_load.py new file mode 100644 index 000000000..8f861b439 --- /dev/null +++ b/tests/protocol/test_trust_load.py @@ -0,0 +1,58 @@ +from hashlib import sha256 +from pathlib import Path + +import pytest + +from cortex.protocol import ProtocolError +from cortex.protocol.trust import load_trust, trust_from_json, trust_to_json + +CHALLENGES = """ +version = 3 +introduced_epoch = {epoch} + +[[challenges]] +id = "bounty" +emission_share_bps = 3000 +policy = "all_metagraph_hotkeys" + +[[challenges]] +id = "proof" +emission_share_bps = 7000 +policy = "all_metagraph_hotkeys" +""" + + +def files(tmp_path: Path, epoch: str) -> dict[str, Path]: + (tmp_path / "c.toml").write_text(CHALLENGES.format(epoch=epoch)) + (tmp_path / "m.toml").write_text("version = 2\n") + return {"challenges_path": tmp_path / "c.toml", "measurements_path": tmp_path / "m.toml"} + + +def test_load_trust_builds_the_profile_from_the_toml_files_and_roundtrips(tmp_path): + trust = load_trust(**files(tmp_path, "14")) + assert trust.shares == ((b"bounty", 3000), (b"proof", 7000)) + assert (trust.challenges_version, trust.measurements_version) == (3, 2) + assert trust.introduced_epoch == 14 + assert trust.measurements_digest == sha256(b"\x00").digest() # empty allowlist + assert trust_from_json(trust_to_json(trust)) == trust + + +@pytest.mark.parametrize("epoch", ['"CHOOSE_ACTIVATION_EPOCH"', "-1", "1.5", "true"]) +def test_load_trust_refuses_a_placeholder_or_non_integer_activation_epoch(tmp_path, epoch): + with pytest.raises(ProtocolError, match="introduced_epoch"): + load_trust(**files(tmp_path, epoch)) + + +@pytest.mark.parametrize("name", ["challenges-v2.example.toml", "challenges-v3.example.toml"]) +def test_example_profiles_fail_with_a_clear_error_until_the_epoch_is_chosen(name): + root = Path(__file__).parents[2] / "config" + with pytest.raises(ProtocolError, match="introduced_epoch"): + load_trust(challenges_path=root / name, measurements_path=root / "measurements.toml") + + +def test_repository_default_profile_loads(): + root = Path(__file__).parents[2] / "config" + trust = load_trust( + challenges_path=root / "challenges.toml", measurements_path=root / "measurements.toml" + ) + assert sum(value for _, value in trust.shares) == 10000 From f03b843ae3fc952e101054a18f7de3fb4ccf0acd Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:35:02 +0000 Subject: [PATCH 18/56] docs: never set introduced_epoch in the future; third challenge needs challenges_version >= 3 Lens I3: the profile is global, so a future introduced_epoch stops sealing and decoding of older epochs and burns the emission. Also drop the stale 'owner signs the roots' sentence. Co-Authored-By: Claude Sonnet 5.5 --- config/challenges.toml | 4 +++- deploy/README.md | 13 ++++++++++--- docs/THREAT_MODEL.md | 7 ++++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/config/challenges.toml b/config/challenges.toml index 10dd0d700..16fa6e01d 100644 --- a/config/challenges.toml +++ b/config/challenges.toml @@ -1,6 +1,8 @@ # Development challenge profile (emission shares). The master reads # this unsigned file; the gateway serves it at GET /v1/trust and validators build their -# view from that. To change shares: edit, bump `version`, restart the master. +# view from that. To change shares: edit, bump `version`, restart the master between two +# epochs. Never set `introduced_epoch` above the current epoch (sealing and decoding stop +# for older epochs). A third challenge or free shares need `challenges_version >= 3`. # # Two live challenges. Emission: bounty = 2000, proof = 8000 bps (sum 10000). # Readiness is independent of these shares: a missing feed, topic, baseline, diff --git a/deploy/README.md b/deploy/README.md index 5f538b27c..0b491a04b 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -208,9 +208,16 @@ The challenge profile is the unsigned `config/challenges.toml` (plus of truth and serves it at `GET /v1/trust`. 1. Edit the `[[challenges]]` shares (basis points, sum 10000) in `config/challenges.toml`. -2. Bump `version` (and set `introduced_epoch` to a future epoch if the change must - not apply to the current one). The gateway store refuses a version rollback. -3. Run `python scripts/check_repo.py`, then restart the master. +2. Bump `version`. Adding a third challenge or arbitrary shares needs + `challenges_version >= 3` (1..64 unique ids summing to 10000); versions 1-2 only + accept the 2000/8000 and 3000/7000 splits. Do NOT set `introduced_epoch` in the + future: the profile is global, so the master then cannot seal epochs before it + and the gateway cannot decode stored seals older than it (`/v1/weights/latest` + answers "unsealed" and the emission burns). Keep the existing value, or the epoch + of the first bundle the new profile must govern, never above the current epoch. + The gateway store refuses a version rollback. +3. Run `python scripts/check_repo.py`, then restart the master between two epochs + (just after a seal), because the new shares apply to the next bundle it builds. 4. Validators pick up the new profile from `/v1/trust` at the next epoch; no validator restart or file copy is needed. Check `GET /v1/trust` shows the new `challenges_version`. diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index 75e78c3ee..6372efa51 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -14,9 +14,10 @@ signed dissent can be retained as local evidence. The Merkle root is not part of the Bittensor weight extrinsic. Peer consensus is therefore not a public chain transparency log. A colluding validator set can -agree on one bad root, and an operator can delete local evidence. The owner signs -the roots and runs the master, so a malicious owner can authorize dishonest -challenge shares or generated topic rules that validators will faithfully accept. +agree on one bad root, and an operator can delete local evidence. The operator +runs the master and the gateway, and the challenge profile is unsigned, so a malicious +operator can authorize dishonest challenge shares or generated topic rules that validators +will faithfully accept. ## Trust boundaries From d846671a2cbcfd9c94a83d3c9ee1c3fd371cccc9 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:07:42 +0000 Subject: [PATCH 19/56] feat(gateway): require a separate bearer for raw leaf intake One operator bearer guarded both POST /v1/weights/raw and POST /v1/admin/seal, so holding it granted both forging and sealing. Founder decision 2026-10-19: split the two powers. Raw intake now requires BASE_GATEWAY_INTAKE_TOKEN_FILE; sealing keeps the operator (admin) bearer, as do the Proof routes. Each route refuses the other token with the existing generic 401. build_master refuses to start when both files resolve to the same path or hold the same value, without echoing either value. Rejected: a new auth class or scopes inside OperatorAuth; a second instance is enough. Deploy wiring (compose env, check_deploy audited paths and inline-value ban, examples, secrets docs) adds intake.token as a path only. Threat model and operator checklist now state what remains: an intake holder can still inject raw leaves the normal seal includes. Co-Authored-By: Claude Opus 5.5 --- .env.example | 2 ++ CHANGELOG.md | 5 ++++ deploy/README.md | 3 ++- deploy/compose/role-master.yml | 1 + deploy/secrets/README.md | 3 ++- docs/OPERATOR_SECURITY.md | 18 ++++++++------ docs/THREAT_MODEL.md | 29 +++++++++++++--------- docs/reference/configuration.md | 3 ++- scripts/check_deploy.py | 2 ++ src/cortex/config.py | 2 ++ src/cortex/gateway/api.py | 8 +++--- src/cortex/master.py | 9 +++++-- tests/gateway/test_gateway.py | 44 ++++++++++++++++++++++++--------- tests/test_deploy_contract.py | 19 ++++++++++++++ tests/test_master.py | 40 ++++++++++++++++++++++++++++++ tests/test_master_config.py | 1 + 16 files changed, 151 insertions(+), 38 deletions(-) diff --git a/.env.example b/.env.example index 03f7aa355..52c36876e 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,8 @@ BASE_STATE_DIR=.local/state BASE_CHALLENGES_FILE=config/challenges.toml BASE_MEASUREMENTS_FILE=config/measurements.toml BASE_GATEWAY_ADMIN_TOKEN_FILE=.local/secrets/operator.token +# Separate bearer for POST /v1/weights/raw; must differ from the admin bearer. +BASE_GATEWAY_INTAKE_TOKEN_FILE=.local/secrets/intake.token PROOF_SK_FILE=.local/secrets/proof.seed # Optional operator registry; see docs/CHALLENGES.md. Unset runs no container challenge. # BASE_CHALLENGE_REGISTRY_FILE=deploy/challenges/registry.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index e1c04746a..32dfb068d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Master: `POST /v1/weights/raw` now requires a separate intake bearer + (`BASE_GATEWAY_INTAKE_TOKEN_FILE`, `deploy/secrets/master/intake.token`); + `POST /v1/admin/seal` keeps the operator bearer. Each refuses the other with + 401, and startup refuses identical paths or values (BREAKING for deployments: + create `intake.token` before upgrading). - Wire format v2 (`protocol_version` 2, BREAKING): leaves and bundles carry no signatures and no gateway key. `Leaf` ends at its score, `Bundle` is the body, `ChallengeEntry` has no public key. The old v1 bundles and a gateway `state_dir` diff --git a/deploy/README.md b/deploy/README.md index 0b491a04b..ef1583425 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -85,7 +85,8 @@ Prepare a private directory, owned so container UID 65532 can read it: ```text deploy/secrets/master/ proof.key # signs Proof topics (miners pin its public key) - operator.token + operator.token # admin routes and sealing + intake.token # raw leaf intake only; a different value proof-vm.token proof-vm-ca.pem ``` diff --git a/deploy/compose/role-master.yml b/deploy/compose/role-master.yml index 896d52945..6ebe1e1d6 100644 --- a/deploy/compose/role-master.yml +++ b/deploy/compose/role-master.yml @@ -21,6 +21,7 @@ services: BASE_MEASUREMENTS_FILE: /etc/base/config/measurements.toml PROOF_SK_FILE: /run/secrets/proof.key BASE_GATEWAY_ADMIN_TOKEN_FILE: /run/secrets/operator.token + BASE_GATEWAY_INTAKE_TOKEN_FILE: /run/secrets/intake.token BASE_CHALLENGE_REGISTRY_FILE: /etc/base/challenges/registry.toml BASE_CHALLENGE_SECRETS_DIR: /run/challenge-secrets ports: diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index 80804ae89..684e9cc91 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -11,7 +11,8 @@ regular files, owned/readable by that identity, with mode 0400 or 0600. | File | Purpose | | --- | --- | | `proof.key` | 32-byte sr25519 seed that signs Proof topics; miners pin its public key (`--proof-public`) | -| `operator.token` | bearer for master administrative routes | +| `operator.token` | bearer for master administrative routes, including `/v1/admin/seal` | +| `intake.token` | separate bearer for raw leaf intake (`/v1/weights/raw`); must differ from `operator.token` | | `proof-vm.token` | bearer shared with the dedicated VM host | | `proof-vm-ca.pem` | CA used to verify the VM-host TLS certificate | diff --git a/docs/OPERATOR_SECURITY.md b/docs/OPERATOR_SECURITY.md index 0abcfc791..5493d5d1b 100644 --- a/docs/OPERATOR_SECURITY.md +++ b/docs/OPERATOR_SECURITY.md @@ -9,13 +9,17 @@ recovery. It complements the [threat model](THREAT_MODEL.md). untracked regular file with mode 0400 or 0600 inside a 0700 directory. - [ ] No credential is present in an environment example, image layer, compose build argument, Terraform state, cloud-init payload, log or shell history. -- [ ] Master operator, per-challenge internal/admin, Proof and VM-host tokens are - distinct and rotated independently. -- [ ] The master operator bearer is held only by the master and the operator. It - feeds `POST /v1/weights/raw` and seals epochs (`/v1/admin/seal`), so a holder can - forge any score and seal it (accepted risk, founder decision 2026-10-19; see the - [threat model](THREAT_MODEL.md)). Rotate it on any suspicion. Wire v2 has no - independent bundle cross-check: its vectors come from the Python implementation. +- [ ] Master operator, master intake, per-challenge internal/admin, Proof and + VM-host tokens are distinct and rotated independently. +- [ ] The master operator bearer (`operator.token`) seals epochs + (`/v1/admin/seal`) and guards the other administrative routes; the separate + intake bearer (`intake.token`) only feeds `POST /v1/weights/raw`. The master + refuses to start when both files are the same path or hold the same value. + Neither bearer grants the other's route, but an intake holder can still write + forged raw leaves that the master's normal epoch seal will include (see the + [threat model](THREAT_MODEL.md)). Hold each only where needed and rotate each + on any suspicion. Wire v2 has no independent bundle cross-check: its vectors + come from the Python implementation. - [ ] Miner BYOK vault storage is durable only as long as queued work requires, and terminal jobs have no remaining secret files. - [ ] The OpenRouter owner key stays on the VM host and is not exposed to miner diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index 6372efa51..78cdf7b98 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -104,17 +104,24 @@ is not automatically retried, because a blind retry can submit twice. No test or health endpoint proves an on-chain payment; that requires observing the actual extrinsic and resulting chain state. -### Accepted risk: one operator bearer forges and seals (founder decision 2026-10-19) - -The operator bearer guards both `POST /v1/weights/raw` and `POST /v1/admin/seal`. -Whoever holds it can write any score for any miner and seal the epoch. Before -wire v2, forging a leaf also needed a challenge key. Nothing else separates the two -powers now, and no independent cross-check of a bundle exists: `wire_v2.json` is -generated by the Python implementation, so no second implementation pins the bytes. - -Mitigation: the operator token is held only by the master and the operator, never -by miners, validators or challenge containers; rotate it on any suspicion. Future -option: separate raw-intake and seal tokens. +### Separate intake and seal bearers (founder decision 2026-10-19) + +`POST /v1/weights/raw` requires the intake bearer (`BASE_GATEWAY_INTAKE_TOKEN_FILE`) +and `POST /v1/admin/seal` requires the operator bearer +(`BASE_GATEWAY_ADMIN_TOKEN_FILE`); each route refuses the other token with 401. +The master refuses to start when both files resolve to the same path or hold the +same value. + +What remains: before wire v2, forging a leaf also needed a challenge key; now an +intake-token holder can still write any raw score for any miner, and the master's +own epoch seal will include it. The separation only stops that holder from sealing +on demand. A seal-token holder can seal early but cannot inject scores through raw +intake. No independent cross-check of a bundle exists: `wire_v2.json` is generated +by the Python implementation, so no second implementation pins the bytes. + +Mitigation: neither token is given to miners, validators or challenge containers; +hold the intake token only where raw intake is actually needed and rotate each +token independently on any suspicion. ## Availability and deployment diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 74e589649..230de919c 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -15,7 +15,8 @@ conflicting values are an error. Existing names and crypto domains are preserved | `BASE_CHALLENGES_FILE` | Unsigned challenge profile (emission shares); served by the gateway at `GET /v1/trust` | | `BASE_MEASUREMENTS_FILE` | Unsigned measurement profile; served with the challenge profile | | `PROOF_SK_FILE` | Proof topic seed file | -| `BASE_GATEWAY_ADMIN_TOKEN_FILE` | Required operator bearer file | +| `BASE_GATEWAY_ADMIN_TOKEN_FILE` | Required operator bearer file (admin routes and `/v1/admin/seal`) | +| `BASE_GATEWAY_INTAKE_TOKEN_FILE` | Required raw-intake bearer file (`/v1/weights/raw` only); path and value must differ from the admin bearer | | `BASE_CHALLENGE_REGISTRY_FILE` | Optional [challenge registry](../CHALLENGES.md); unset runs no container challenge | | `BASE_CHALLENGE_SECRETS_DIR` | Per-challenge `/internal.token` bearers; default `/run/challenge-secrets` | | `PROOF_VM_ORCHESTRATOR_URL` | Dedicated VM host HTTPS origin | diff --git a/scripts/check_deploy.py b/scripts/check_deploy.py index 30e23b117..cacc017f1 100644 --- a/scripts/check_deploy.py +++ b/scripts/check_deploy.py @@ -52,6 +52,7 @@ "BASE_MEASUREMENTS_FILE": "/etc/base/config/measurements.toml", "PROOF_SK_FILE": "/run/secrets/proof.key", "BASE_GATEWAY_ADMIN_TOKEN_FILE": "/run/secrets/operator.token", + "BASE_GATEWAY_INTAKE_TOKEN_FILE": "/run/secrets/intake.token", "BASE_CHALLENGE_REGISTRY_FILE": "/etc/base/challenges/registry.toml", "BASE_CHALLENGE_SECRETS_DIR": "/run/challenge-secrets", } @@ -368,6 +369,7 @@ def validate_compose(config: dict, role: str) -> None: "OPENROUTER_API_KEY", "PROOF_SK", "BASE_GATEWAY_ADMIN_TOKEN", + "BASE_GATEWAY_INTAKE_TOKEN", "PROOF_VM_ORCHESTRATOR_TOKEN", } if forbidden_values.intersection(environment): diff --git a/src/cortex/config.py b/src/cortex/config.py index c6c2085a8..ddeb8b9e1 100644 --- a/src/cortex/config.py +++ b/src/cortex/config.py @@ -103,6 +103,7 @@ class MasterConfig: measurements_file: Path proof_seed_file: Path operator_token_file: Path + intake_token_file: Path challenge_registry_file: Path | None = None challenge_secrets_dir: Path = Path("/run/challenge-secrets") chain_endpoint: str = "finney" @@ -180,6 +181,7 @@ def optional_path(name: str) -> Path | None: measurements_file=Path(value("BASE_MEASUREMENTS_FILE", "config/measurements.toml")), proof_seed_file=required_path("PROOF_SK_FILE"), operator_token_file=required_path("BASE_GATEWAY_ADMIN_TOKEN_FILE"), + intake_token_file=required_path("BASE_GATEWAY_INTAKE_TOKEN_FILE"), challenge_registry_file=optional_path("BASE_CHALLENGE_REGISTRY_FILE"), challenge_secrets_dir=Path( value("BASE_CHALLENGE_SECRETS_DIR", "/run/challenge-secrets") diff --git a/src/cortex/gateway/api.py b/src/cortex/gateway/api.py index 539898eef..7a37b4dc6 100644 --- a/src/cortex/gateway/api.py +++ b/src/cortex/gateway/api.py @@ -1,4 +1,4 @@ -"""Master gateway routes; writes (raw weights, sealing) require the rotating operator bearer.""" +"""Master gateway routes; raw intake and sealing each require their own rotating bearer.""" from fastapi import APIRouter, Request from fastapi.responses import JSONResponse, Response @@ -21,13 +21,15 @@ def _error(error: ServiceError) -> JSONResponse: return JSONResponse(body, status_code=error.status) -def create_router(service: GatewayService, operator_auth: OperatorAuth) -> APIRouter: +def create_router( + service: GatewayService, operator_auth: OperatorAuth, intake_auth: OperatorAuth +) -> APIRouter: router = APIRouter(tags=["gateway"]) @router.post("/v1/weights/raw", status_code=202) async def raw_weight(request: Request): try: - operator_auth.require(request) + intake_auth.require(request) body = RawWeightRequest.model_validate(decode_json(await bounded_body(request, 4096))) return service.accept_leaf(body.to_leaf()) except ValidationError: diff --git a/src/cortex/master.py b/src/cortex/master.py index 4724fbfaf..c4ea72c82 100644 --- a/src/cortex/master.py +++ b/src/cortex/master.py @@ -520,7 +520,8 @@ async def service_error(request, error: ServiceError): return JSONResponse({"error": error.reason}, status_code=error.status) operator = OperatorAuth(self.config.operator_token_file) - app.include_router(gateway_router(self.gateway, operator)) + intake = OperatorAuth(self.config.intake_token_file) + app.include_router(gateway_router(self.gateway, operator, intake)) proof = proof_router(self.proof, operator, self.topic_setup) app.include_router(proof) app.include_router(proof, prefix="/challenge/proof", include_in_schema=False) @@ -557,7 +558,11 @@ async def build_master( clock = EpochClock(epochs, config.netuid, stale_seconds=config.epoch_stale_seconds) state = await clock.refresh() local_trust = trust or config.challenge_profile(state.epoch) - read_private_file(config.operator_token_file) + # Holding the intake bearer must never grant sealing, and vice versa. + if config.operator_token_file.resolve() == config.intake_token_file.resolve() or ( + read_private_file(config.operator_token_file) == read_private_file(config.intake_token_file) + ): + raise ServiceError(503, "intake and admin bearers must differ") load_registry(config.challenge_registry_file) # fail fast on an invalid operator registry with ExitStack() as cleanup: gateway_store = GatewayStore(config.state_dir / "gateway.sqlite3") diff --git a/tests/gateway/test_gateway.py b/tests/gateway/test_gateway.py index 2ca453c2c..1bf0665d6 100644 --- a/tests/gateway/test_gateway.py +++ b/tests/gateway/test_gateway.py @@ -68,6 +68,9 @@ async def network(tmp_path): token_file = tmp_path / "operator-token" token_file.write_text("first-test-token") token_file.chmod(0o600) + intake_file = tmp_path / "intake-token" + intake_file.write_text("intake-test-token") + intake_file.chmod(0o600) store = GatewayStore(tmp_path / "gateway.db") rows = tuple(MetagraphRow(bytes([tag]) * 32, tag) for tag in range(3)) trust = TrustRoot( @@ -86,7 +89,7 @@ async def network(tmp_path): ) service = GatewayService(store=store, **settings) app = FastAPI() - app.include_router(create_router(service, OperatorAuth(token_file))) + app.include_router(create_router(service, OperatorAuth(token_file), OperatorAuth(intake_file))) journal = SubmissionJournal(tmp_path / "validator.db") async with httpx.AsyncClient( transport=httpx.ASGITransport(app=app), base_url="https://master.invalid" @@ -103,6 +106,7 @@ async def network(tmp_path): db_path=tmp_path / "gateway.db", journal=journal, auth={"authorization": "Bearer first-test-token"}, + intake_auth={"authorization": "Bearer intake-test-token"}, ) store.close() journal.close() @@ -124,7 +128,7 @@ def leaves(network, epoch=12, winner=1, reason=NoScoreReason.NOT_ATTEMPTED): async def intake(network, epoch=12, winner=1, reason=NoScoreReason.NOT_ATTEMPTED): for leaf in leaves(network, epoch, winner, reason): response = await network.client.post( - "/v1/weights/raw", json=leaf_request(leaf), headers=network.auth + "/v1/weights/raw", json=leaf_request(leaf), headers=network.intake_auth ) assert response.status_code == 202, response.text @@ -312,7 +316,7 @@ async def test_seal_requires_exact_participant_coverage(network): for leaf in leaves(network)[:-1]: assert ( await network.client.post( - "/v1/weights/raw", json=leaf_request(leaf), headers=network.auth + "/v1/weights/raw", json=leaf_request(leaf), headers=network.intake_auth ) ).status_code == 202 response = await seal(network) @@ -326,7 +330,9 @@ async def test_extra_participant_cannot_enter_the_seal(network): await intake(network) extra = Leaf(b"bounty", bytes([99]) * 32, 12, Score(1)) assert ( - await network.client.post("/v1/weights/raw", json=leaf_request(extra), headers=network.auth) + await network.client.post( + "/v1/weights/raw", json=leaf_request(extra), headers=network.intake_auth + ) ).status_code == 202 assert (await seal(network)).status_code == 409 @@ -356,7 +362,7 @@ async def test_invalid_intake_leaves_no_row(network, mutation, status): body["score_or_absence"] = {"no_score": {"reason": 8}} else: del body["miner_hotkey"] - response = await network.client.post("/v1/weights/raw", json=body, headers=network.auth) + response = await network.client.post("/v1/weights/raw", json=body, headers=network.intake_auth) assert response.status_code == status assert network.store.leaves(12) == () @@ -364,7 +370,7 @@ async def test_invalid_intake_leaves_no_row(network, mutation, status): async def test_duplicate_and_nonpositive_leaves_cannot_revoke_positive_score(network): positive = leaves(network)[1] first = await network.client.post( - "/v1/weights/raw", json=leaf_request(positive), headers=network.auth + "/v1/weights/raw", json=leaf_request(positive), headers=network.intake_auth ) repeated = Leaf(b"bounty", positive.miner_hotkey, 12, Score(50)) burns = [ @@ -373,7 +379,7 @@ async def test_duplicate_and_nonpositive_leaves_cannot_revoke_positive_score(net zero = Leaf(b"bounty", positive.miner_hotkey, 12, Score(0)) for incoming in (repeated, *burns, zero): response = await network.client.post( - "/v1/weights/raw", json=leaf_request(incoming), headers=network.auth + "/v1/weights/raw", json=leaf_request(incoming), headers=network.intake_auth ) assert response.status_code == 409 assert response.json()["original"]["id"] == first.json()["id"] @@ -504,7 +510,7 @@ async def test_changed_payload_supersedes_and_sealed_bytes_stay_immutable(networ original = (await network.client.get("/v1/bundle/12")).content changed = Leaf(b"bounty", network.rows[1].hotkey, 12, Score(100)) response = await network.client.post( - "/v1/weights/raw", json=leaf_request(changed), headers=network.auth + "/v1/weights/raw", json=leaf_request(changed), headers=network.intake_auth ) assert response.status_code == 202 and response.json()["superseded"] is True assert (await seal(network)).status_code == 200 @@ -650,10 +656,10 @@ async def test_historical_root_refuses_corrupt_or_missing_accepted_profile(netwo async def test_duplicate_json_and_oversized_request_are_rejected_before_store(network): duplicate = await network.client.post( - "/v1/weights/raw", content='{"epoch": 1, "epoch": 2}', headers=network.auth + "/v1/weights/raw", content='{"epoch": 1, "epoch": 2}', headers=network.intake_auth ) oversized = await network.client.post( - "/v1/weights/raw", content=b"x" * 4097, headers=network.auth + "/v1/weights/raw", content=b"x" * 4097, headers=network.intake_auth ) assert duplicate.status_code == 400 assert oversized.status_code == 413 @@ -710,7 +716,21 @@ async def test_content_addressed_bundle_lookup_returns_identical_signed_bytes(ne assert (await network.client.get("/v1/bundle/root/invalid")).status_code == 400 -async def test_raw_weight_intake_requires_the_operator_bearer(network): +async def test_raw_intake_and_seal_each_refuse_the_other_bearer(network): + body = leaf_request(leaves(network)[0]) + assert ( + await network.client.post("/v1/weights/raw", json=body, headers=network.auth) + ).status_code == 401 + assert network.store.leaves(12) == () + await intake(network) + assert ( + await network.client.post("/v1/admin/seal", json={"epoch": 12}, headers=network.intake_auth) + ).status_code == 401 + assert network.service.latest()["sealed"] is False + assert (await seal(network)).status_code == 200 + + +async def test_raw_weight_intake_requires_the_intake_bearer(network): body = leaf_request(leaves(network)[0]) assert (await network.client.post("/v1/weights/raw", json=body)).status_code == 401 wrong = {"authorization": "Bearer wrong"} @@ -719,7 +739,7 @@ async def test_raw_weight_intake_requires_the_operator_bearer(network): ).status_code == 401 assert network.store.leaves(12) == () assert ( - await network.client.post("/v1/weights/raw", json=body, headers=network.auth) + await network.client.post("/v1/weights/raw", json=body, headers=network.intake_auth) ).status_code == 202 diff --git a/tests/test_deploy_contract.py b/tests/test_deploy_contract.py index 4226e8506..6ea4b6d4a 100644 --- a/tests/test_deploy_contract.py +++ b/tests/test_deploy_contract.py @@ -162,6 +162,7 @@ def master_config(): "BASE_MEASUREMENTS_FILE": "/etc/base/config/measurements.toml", "PROOF_SK_FILE": "/run/secrets/proof.key", "BASE_GATEWAY_ADMIN_TOKEN_FILE": "/run/secrets/operator.token", + "BASE_GATEWAY_INTAKE_TOKEN_FILE": "/run/secrets/intake.token", "BASE_CHALLENGE_REGISTRY_FILE": "/etc/base/challenges/registry.toml", "BASE_CHALLENGE_SECRETS_DIR": "/run/challenge-secrets", }, @@ -237,6 +238,24 @@ def test_compose_healthcheck_cannot_be_satisfied_by_an_unrelated_command(role): CHECK["validate_compose"](config, role) +@pytest.mark.parametrize( + "intake", + [None, "/run/secrets/operator.token", "/tmp/intake.token", "inline"], +) +def test_master_requires_its_own_audited_intake_bearer_file(intake): + config = master_config() + environment = config["services"]["gateway"]["environment"] + if intake is None: + del environment["BASE_GATEWAY_INTAKE_TOKEN_FILE"] + elif intake == "inline": + environment["BASE_GATEWAY_INTAKE_TOKEN"] = "placeholder" + else: + environment["BASE_GATEWAY_INTAKE_TOKEN_FILE"] = intake + + with pytest.raises(ValueError, match="credentials"): + CHECK["validate_compose"](config, "master") + + @pytest.mark.parametrize("role", ["master", "validator"]) def test_compose_accepts_isolated_python_roles_with_durable_state(role): config = master_config() if role == "master" else validator_config() diff --git a/tests/test_master.py b/tests/test_master.py index b82efe1d1..d4787dc51 100644 --- a/tests/test_master.py +++ b/tests/test_master.py @@ -96,6 +96,7 @@ def secret(name, value): measurements_file=tmp_path / "measurements.toml", proof_seed_file=secret("proof.key", bytes([2]) * 32), operator_token_file=secret("operator.token", b"fixture-operator-token"), + intake_token_file=secret("intake.token", b"fixture-intake-token"), challenge_secrets_dir=tmp_path / "challenge-secrets", ) @@ -401,6 +402,42 @@ async def test_private_routes_require_rotating_file_and_master_routes_are_compos assert ( await client.post(drain, headers={"authorization": "Bearer rotated-token"}) ).status_code != 401 + raw = {"authorization": "Bearer rotated-token"} + assert (await client.post("/v1/weights/raw", json={}, headers=raw)).status_code == 401 + intake = {"authorization": "Bearer fixture-intake-token"} + assert (await client.post("/v1/weights/raw", json={}, headers=intake)).status_code == 400 + seal = await client.post("/v1/admin/seal", json={"epoch": 12}, headers=intake) + assert seal.status_code == 401 + + +def fixture_trust(): + return TrustRoot( + (ChallengeEntry(b"bounty", 2000), ChallengeEntry(b"proof", 8000)), + sha256(b"\x00").digest(), + ) + + +@pytest.mark.parametrize("shared", ["same-file", "same-value"]) +async def test_master_refuses_one_bearer_for_intake_and_sealing(tmp_path, shared): + config = master_config(tmp_path) + if shared == "same-file": + config = replace(config, intake_token_file=config.operator_token_file) + else: + config.intake_token_file.write_bytes(b"fixture-operator-token\n") + chain = FakeEpochChain() + with pytest.raises(ServiceError, match="must differ"): + await build_master(config, chain=chain, epochs=chain, trust=fixture_trust()) + + +async def test_master_refuses_a_missing_or_shared_mode_intake_bearer(tmp_path): + config = master_config(tmp_path) + chain = FakeEpochChain() + config.intake_token_file.chmod(0o644) + with pytest.raises(ServiceError, match="private"): + await build_master(config, chain=chain, epochs=chain, trust=fixture_trust()) + config.intake_token_file.unlink() + with pytest.raises(ServiceError, match="unavailable"): + await build_master(config, chain=chain, epochs=chain, trust=fixture_trust()) def test_private_seed_rejects_symlinks_permissions_and_accepts_raw_or_hex(tmp_path): @@ -422,11 +459,14 @@ def test_config_base_aliases_fail_closed_and_do_not_accept_inline_secret_values( "BASE_NETUID": "541", "PROOF_SK_FILE": "/private/proof", "BASE_GATEWAY_ADMIN_TOKEN_FILE": "/private/token", + "BASE_GATEWAY_INTAKE_TOKEN_FILE": "/private/intake", } alias = {key.replace("BASE_", "CORTEX_"): value for key, value in env.items()} assert MasterConfig.from_env(env) == MasterConfig.from_env(alias) with pytest.raises(ValueError, match="conflicting"): MasterConfig.from_env({**env, "CORTEX_NETUID": "1"}) + with pytest.raises(ValueError, match="BASE_GATEWAY_INTAKE_TOKEN_FILE"): + MasterConfig.from_env({k: v for k, v in env.items() if "INTAKE" not in k}) del env["PROOF_SK_FILE"] env["PROOF_SK"] = "secret-inline-material" with pytest.raises(ValueError, match="PROOF_SK_FILE"): diff --git a/tests/test_master_config.py b/tests/test_master_config.py index f4286eee8..23260e4cd 100644 --- a/tests/test_master_config.py +++ b/tests/test_master_config.py @@ -13,6 +13,7 @@ def configured_environment(): "BASE_NETUID": "541", "PROOF_SK_FILE": "/run/secrets/proof", "BASE_GATEWAY_ADMIN_TOKEN_FILE": "/run/secrets/operator", + "BASE_GATEWAY_INTAKE_TOKEN_FILE": "/run/secrets/intake", "PROOF_VM_ORCHESTRATOR_URL": "https://vm.example", "PROOF_VM_ORCHESTRATOR_TOKEN_FILE": "/run/secrets/vm-token", "PROOF_VM_ORCHESTRATOR_CA_FILE": "/run/secrets/vm-ca.pem", From 6263a2a0fa028372deae5ae39fae98fb213fa2bf Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:09:54 +0000 Subject: [PATCH 20/56] feat(validator): add an operator-only watermark reset command A consensus watermark advanced wrongly (for example by a profile version published by mistake) makes every later tick fail with "consensus rollback refused", and the only remedy was deleting the state database, which also discards the dispatch journal and evidence. cortex validator-watermark-reset lowers challenges_version, measurements_version or epoch in the local state DB only: no network, no wallet. It requires --confirm, refuses unknown names, negative values, an unset watermark, a missing DB file and any raise, prints old and new values, and appends a consensus_watermark_resets audit row in the same BEGIN IMMEDIATE transaction as the update. Rejected: allowing raises (normal ticks already advance watermarks) and deleting the row (an audit trail and an explicit floor are safer). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +++ docs/external-miner/troubleshoot.md | 1 + docs/external-miner/validators.md | 25 +++++++++++++++ src/cortex/cli.py | 30 ++++++++++++++++++ src/cortex/validator/evidence.py | 35 +++++++++++++++++++++ tests/validator/test_validator.py | 48 +++++++++++++++++++++++++++++ 6 files changed, 143 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32dfb068d..f8a2e6020 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 each paying `share * min(sum(leaves), 10^12) / 10^12`, remainder burned. Validators need no change beyond the release; they recompute signed leaves. - `GET /v1/metagraph/latest` serves the sealed hotkey map to challenges. +- `cortex validator-watermark-reset`: operator-only, local repair that lowers a + wrongly advanced `challenges_version`, `measurements_version` or `epoch` + watermark (requires `--confirm`, never raises one) and audits it in + `consensus_watermark_resets`. ### Changed diff --git a/docs/external-miner/troubleshoot.md b/docs/external-miner/troubleshoot.md index b36b0bd76..b4d8e8d4b 100644 --- a/docs/external-miner/troubleshoot.md +++ b/docs/external-miner/troubleshoot.md @@ -105,6 +105,7 @@ catalog; follow the documentation and checklist in the current signed topic. | Missing/invalid peer sample | Configure independently selected HTTPS peers with metagraph `validator_permit`; do not disable verification | | Conflicting signed roots or equivocation | Preserve the journal and signed evidence; reconcile the network state | | Stale block, future epoch, reorg or changed latest | Wait for a current valid seal; do not force-submit the old one | +| `consensus rollback refused` | The gateway serves a lower `challenges_version`, `measurements_version` or epoch than this validator already accepted. Ask the master operator first; only if the higher value was advanced wrongly, lower it with `cortex validator-watermark-reset` (see [the validator guide](validators.md#repairing-a-consensus-watermark)). Do not delete the database | | Ambiguous chain dispatch remains pending | Reconcile inclusion/finalization before retrying; retain the SQLite journal | | CRv4 or drand failure | Do not fall back to public weights while chain commit-reveal is enabled | | `/v1/attest/*` returns `503` and `verified: false` | DCAP verification is not implemented in the Python validator | diff --git a/docs/external-miner/validators.md b/docs/external-miner/validators.md index b7eb72abf..fed445c0d 100644 --- a/docs/external-miner/validators.md +++ b/docs/external-miner/validators.md @@ -234,6 +234,31 @@ Only `not_broadcast` releases the epoch for a retry. Keep the underlying chain receipt or nonce query private; the journal stores its digest. Do not delete the database to force a retry. +### Repairing a consensus watermark + +The journal keeps rollback watermarks for `challenges_version`, +`measurements_version` and `epoch`; a lower served value fails with +`consensus rollback refused`. If a watermark was advanced wrongly (for example a +profile version published by mistake and then withdrawn), stop the validator and +lower it locally. The command touches only the state database: no network, no +wallet. + +```bash +cortex validator-watermark-reset \ + --state-db /var/lib/base/validator.sqlite3 \ + --name challenges_version --to 2 --confirm +``` + +It refuses without `--confirm`, refuses a name outside those three, a negative +value, a watermark that was never set and any value above the current one (it +only lowers or repairs). It prints the old and new values and appends an audit +row to `consensus_watermark_resets(name, old_value, new_value, at)` in the same +transaction. + +**Warning:** resetting below a version or epoch this validator really accepted +re-opens rollback to that value. Confirm the correct value with the master +operator first. + The peer listener exposes: | Route | Response | diff --git a/src/cortex/cli.py b/src/cortex/cli.py index b15441477..ca0f392bf 100644 --- a/src/cortex/cli.py +++ b/src/cortex/cli.py @@ -16,7 +16,9 @@ from cortex.errors import ServiceError from cortex.http import read_private_file from cortex.miner import MinerClient, load_seed +from cortex.protocol import ProtocolError from cortex.protocol.crypto import decode_hotkey +from cortex.validator.evidence import RESETTABLE_WATERMARKS from cortex.wallet import HotkeySigner, load_wallet_hotkey @@ -40,6 +42,14 @@ def parser() -> argparse.ArgumentParser: reconcile.add_argument("--attempt-id", required=True) reconcile.add_argument("--result", choices=("submitted", "not_broadcast"), required=True) reconcile.add_argument("--evidence-digest", required=True) + reset = commands.add_parser( + "validator-watermark-reset", + help="lower one wrongly advanced consensus watermark (local state only)", + ) + reset.add_argument("--state-db", type=Path, required=True) + reset.add_argument("--name", choices=RESETTABLE_WATERMARKS, required=True) + reset.add_argument("--to", type=int, required=True) + reset.add_argument("--confirm", action="store_true") vm = commands.add_parser("vm-host", help="serve the dedicated Firecracker host") vm.add_argument("--config", type=Path, required=True) vm.add_argument( @@ -230,6 +240,24 @@ def run_validator_reconcile(args: argparse.Namespace) -> None: print(json.dumps(result, indent=2, sort_keys=True)) +def run_validator_watermark_reset(args: argparse.Namespace) -> None: + from cortex.validator import SubmissionJournal + + # Refusal reasons are fixed strings with no credential, so show them to the operator. + if not args.confirm: + raise SystemExit("Cortex command failed: --confirm required; lowering re-opens rollback") + if not args.state_db.is_file(): + raise SystemExit("Cortex command failed: validator state database not found") + journal = SubmissionJournal(args.state_db) + try: + old = journal.evidence.reset_watermark(args.name, args.to) + except ProtocolError as error: + raise SystemExit(f"Cortex command failed: {error}") from None + finally: + journal.close() + print(json.dumps({"name": args.name, "old_value": old, "new_value": args.to}, indent=2)) + + def main(argv: list[str] | None = None) -> None: values = sys.argv[1:] if argv is None else argv if values and values[0] == "validator": @@ -259,6 +287,8 @@ def main(argv: list[str] | None = None) -> None: serve(args.config) elif args.command == "validator-reconcile": run_validator_reconcile(args) + elif args.command == "validator-watermark-reset": + run_validator_watermark_reset(args) else: action = {"master": run_master, "miner": run_miner, "topic-create": run_topic_create} asyncio.run(action[args.command](args)) diff --git a/src/cortex/validator/evidence.py b/src/cortex/validator/evidence.py index 9b9a7e774..a7805b1cc 100644 --- a/src/cortex/validator/evidence.py +++ b/src/cortex/validator/evidence.py @@ -9,6 +9,8 @@ from cortex.protocol import Bundle, ProtocolError from cortex.protocol.consensus import Dissent, RootStatement +RESETTABLE_WATERMARKS = ("challenges_version", "measurements_version", "epoch") + class EvidenceStore: def __init__(self, connection: sqlite3.Connection): @@ -29,6 +31,10 @@ def __init__(self, connection: sqlite3.Connection): CREATE TABLE IF NOT EXISTS consensus_watermarks ( name TEXT PRIMARY KEY, value TEXT NOT NULL ); + CREATE TABLE IF NOT EXISTS consensus_watermark_resets ( + name TEXT NOT NULL, old_value TEXT NOT NULL, new_value TEXT NOT NULL, + at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) + ); """) def check_watermark(self, name: str, value: int) -> None: @@ -46,6 +52,35 @@ def watermark(self, name: str, value: int) -> None: (name, str(value)), ) + def reset_watermark(self, name: str, value: int) -> int: + """Operator repair: lower one rollback watermark and audit it; never raises one.""" + if name not in RESETTABLE_WATERMARKS: + raise ProtocolError("unknown consensus watermark") + if type(value) is not int or value < 0: + raise ProtocolError("watermark must be a non-negative integer") + self.connection.execute("BEGIN IMMEDIATE") + try: + row = self.connection.execute( + "SELECT value FROM consensus_watermarks WHERE name=?", (name,) + ).fetchone() + if row is None: + raise ProtocolError("consensus watermark not set") + old = int(row[0]) + if value > old: + raise ProtocolError("watermark reset may only lower a value") + self.connection.execute( + "UPDATE consensus_watermarks SET value=? WHERE name=?", (str(value), name) + ) + self.connection.execute( + "INSERT INTO consensus_watermark_resets (name,old_value,new_value) VALUES (?,?,?)", + (name, str(old), str(value)), + ) + except BaseException: + self.connection.execute("ROLLBACK") + raise + self.connection.execute("COMMIT") + return old + def root(self, statement: RootStatement, *, local: bool = False) -> bool: statement.verify() epoch = f"{statement.epoch:020d}" diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py index 314b86aa1..a62229049 100644 --- a/tests/validator/test_validator.py +++ b/tests/validator/test_validator.py @@ -1,3 +1,4 @@ +import json from dataclasses import replace from hashlib import sha256 from types import SimpleNamespace @@ -557,6 +558,53 @@ async def test_trust_version_rollback_is_refused_across_restart(network): assert len(network.chain.submissions) == 1 +def reset(db_path, *extra): + from cortex.cli import main + + main(["validator-watermark-reset", "--state-db", str(db_path), *extra]) + + +async def test_cli_watermark_reset_lowers_a_wrong_version_and_audits_it(network, capsys): + import sqlite3 + + original = network.validator.trust + network.validator.trust_loader = lambda _: replace(original, challenges_version=2) + await network.validator.run_once() + network.journal.close() + args = ("--name", "challenges_version", "--to", "1") + for refused in ( + args, # no --confirm + ("--name", "challenges_version", "--to", "3", "--confirm"), # raising + ("--name", "netuid", "--to", "1", "--confirm"), + ("--name", "challenges_version", "--to", "-1", "--confirm"), + ): + with pytest.raises(SystemExit): + reset(network.db_path, *refused) + capsys.readouterr() + reset(network.db_path, *args, "--confirm") + assert json.loads(capsys.readouterr().out) == { + "name": "challenges_version", + "old_value": 2, + "new_value": 1, + } + with sqlite3.connect(network.db_path) as connection: + assert connection.execute( + "SELECT name, old_value, new_value FROM consensus_watermark_resets" + ).fetchall() == [("challenges_version", "2", "1")] + network.validator.journal = SubmissionJournal(network.db_path) + network.validator.trust_loader = lambda _: original + assert (await network.validator.run_once()).outcome == "already_submitted_or_pending" + + +def test_cli_watermark_reset_refuses_an_unset_watermark_and_allows_equal(network, capsys): + network.journal.evidence.watermark("epoch", 12) + network.journal.close() + with pytest.raises(SystemExit): + reset(network.db_path, "--name", "measurements_version", "--to", "0", "--confirm") + reset(network.db_path, "--name", "epoch", "--to", "12", "--confirm") + assert json.loads(capsys.readouterr().out)["new_value"] == 12 + + async def test_peer_api_serves_signed_evidence_and_never_claims_dcap_verified(network): from cortex.protocol.consensus import RootStatement from cortex.validator.evidence import peer_app From 0f594d32719b01e6bae4be8877221564b0be5c4e Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:13:54 +0000 Subject: [PATCH 21/56] docs: describe raw intake under the separate intake bearer Review of the bearer split found three places still saying the operator bearer gates raw intake: the AGENTS.md key-roles table, the threat model's reward controls and an older Unreleased CHANGELOG line that contradicted the new entry. AGENTS.md was outside the brief's file list; the founder approved this edit. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 3 ++- CHANGELOG.md | 5 ++--- docs/THREAT_MODEL.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0850808fa..4486302ca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -70,7 +70,8 @@ or permit UID is not a submit path. | Key or token | Role | | --- | --- | | Proof seed | signs Proof topics (`PROOF_SK_FILE`); no bundle, leaf or gateway key exists | -| operator bearer | protects master administrative routes, including `/v1/weights/raw` and sealing | +| operator bearer | protects master administrative routes, including sealing (`/v1/admin/seal`) | +| intake bearer | protects raw leaf intake (`/v1/weights/raw`) only; must differ from the operator bearer | | VM orchestrator bearer | authenticates master to the dedicated KVM host; not a wallet | | validator hotkey | weight-setting role only: signs root/dissent evidence and Bittensor submissions | | challenge internal token | master bearer for a container's `get_weights`; never a signing key | diff --git a/CHANGELOG.md b/CHANGELOG.md index f8a2e6020..8c5d1224f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Master: `POST /v1/weights/raw` now requires a separate intake bearer +- Master: `POST /v1/weights/raw`, protected before wire v2 only by the challenge + leaf signature, now requires a separate intake bearer (`BASE_GATEWAY_INTAKE_TOKEN_FILE`, `deploy/secrets/master/intake.token`); `POST /v1/admin/seal` keeps the operator bearer. Each refuses the other with 401, and startup refuses identical paths or values (BREAKING for deployments: @@ -41,8 +42,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 created under v1 are refused: start the gateway on a new `state_dir`. Hash integrity (Merkle root, metagraph root, block hash, recomputed final vector) is unchanged, and so is the validator hotkey signature on the chain submission. -- `POST /v1/weights/raw` requires the operator bearer (it was protected only by the - challenge leaf signature). - The validator hotkey (`--wallet-name`/`--wallet-hotkey`) is required only to submit weights or run `--peer-consensus`; `--verify-only` runs without it. `scripts/run-validator.sh` needs `WALLET_NAME`/`WALLET_HOTKEY` only to submit. diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index 78cdf7b98..14f6bd23a 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -83,7 +83,7 @@ an owner signature approves exact content and visibility. ## Reward controls -Challenge leaves carry no signature (wire v2); the master records them after the gateway authenticates the operator bearer. The expected set comes +Challenge leaves carry no signature (wire v2); the master records them after the gateway authenticates the intake bearer (sealing needs the separate operator bearer). The expected set comes from the historical metagraph; a challenge cannot shrink it by omission. A missing score becomes an explicit `NoScore`, and an unavailable challenge burns its share rather than blocking every other challenge. From 927eb556c5c2a249d1534b6cb6ccb0d61cb3fdab Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:13:54 +0000 Subject: [PATCH 22/56] fix(cli): report a sqlite failure in watermark reset without traceback A locked or non-SQLite --state-db raised sqlite3 errors that main() does not catch, printing a raw traceback instead of the fixed "Cortex command failed" message. Nothing was written in either case; only the error path changes. Only the exception class name is shown. Co-Authored-By: Claude Opus 5.5 --- src/cortex/cli.py | 8 +++++++- tests/validator/test_validator.py | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/cortex/cli.py b/src/cortex/cli.py index ca0f392bf..1bc1b9a8d 100644 --- a/src/cortex/cli.py +++ b/src/cortex/cli.py @@ -7,6 +7,7 @@ import json import logging import os +import sqlite3 import sys from pathlib import Path @@ -248,11 +249,16 @@ def run_validator_watermark_reset(args: argparse.Namespace) -> None: raise SystemExit("Cortex command failed: --confirm required; lowering re-opens rollback") if not args.state_db.is_file(): raise SystemExit("Cortex command failed: validator state database not found") - journal = SubmissionJournal(args.state_db) + try: + journal = SubmissionJournal(args.state_db) + except sqlite3.Error as error: + raise SystemExit(f"Cortex command failed: {type(error).__name__}") from None try: old = journal.evidence.reset_watermark(args.name, args.to) except ProtocolError as error: raise SystemExit(f"Cortex command failed: {error}") from None + except sqlite3.Error as error: + raise SystemExit(f"Cortex command failed: {type(error).__name__}") from None finally: journal.close() print(json.dumps({"name": args.name, "old_value": old, "new_value": args.to}, indent=2)) diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py index a62229049..62b822c8b 100644 --- a/tests/validator/test_validator.py +++ b/tests/validator/test_validator.py @@ -605,6 +605,13 @@ def test_cli_watermark_reset_refuses_an_unset_watermark_and_allows_equal(network assert json.loads(capsys.readouterr().out)["new_value"] == 12 +def test_cli_watermark_reset_reports_a_non_sqlite_state_file_without_traceback(tmp_path): + state = tmp_path / "state.db" + state.write_bytes(b"not a database" * 100) + with pytest.raises(SystemExit, match="Cortex command failed"): + reset(state, "--name", "epoch", "--to", "0", "--confirm") + + async def test_peer_api_serves_signed_evidence_and_never_claims_dcap_verified(network): from cortex.protocol.consensus import RootStatement from cortex.validator.evidence import peer_app From 6dc89eb613d3d0c801b98ac54b117ff12b6422e5 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:27:18 +0000 Subject: [PATCH 23/56] fix(validator): fail fast on an unreadable hotkey or half a wallet The bittensor wallet reads its hotkey lazily, so a missing or mistyped hotkey file let the weight-setter start and log dispatch_failed every epoch. Read it once at startup and exit with the error type only (no path, no wallet name, no key material). --wallet-name without --wallet-hotkey (or the reverse) was silently dropped, which also dropped the chain preflight the operator asked for. It is now a startup error. --verify-only with a full wallet is kept on purpose: it is the only way to run the registration, permit and rate-limit preflight without submitting (founder decision; a blanket refusal of wallet arguments was rejected). run-validator.sh --verify-only no longer reads weights_version from the chain when WALLET_NAME is left in the environment: verify-only never submits, and an RPC failure there aborted a pure verification. Co-Authored-By: Claude Sonnet 5.5 --- scripts/run-validator.sh | 3 ++- src/cortex/validator/__main__.py | 16 +++++++++++--- tests/test_run_validator_script.py | 11 ++++++++++ tests/validator/test_main.py | 34 ++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/scripts/run-validator.sh b/scripts/run-validator.sh index b2a2d435d..a964da5e8 100755 --- a/scripts/run-validator.sh +++ b/scripts/run-validator.sh @@ -98,7 +98,8 @@ if [[ -n "${WALLET_NAME:-}" && -n "${WALLET_HOTKEY:-}" ]]; then fi fi -if [[ -n "${WALLET_NAME:-}" && -z "${VERSION_KEY:-}" ]]; then +# --verify-only never submits, so it needs no live version key and no RPC for it. +if ((!verify_only)) && [[ -n "${WALLET_NAME:-}" && -z "${VERSION_KEY:-}" ]]; then VERSION_KEY="$("${UV[@]}" python -c ' import sys from bittensor import Subtensor diff --git a/src/cortex/validator/__main__.py b/src/cortex/validator/__main__.py index d175ab211..1ca237e6a 100644 --- a/src/cortex/validator/__main__.py +++ b/src/cortex/validator/__main__.py @@ -216,6 +216,8 @@ def main(argv: list[str] | None = None) -> None: arguments = parser().parse_args(argv) if not isfinite(arguments.poll_seconds) or arguments.poll_seconds <= 0: raise SystemExit("--poll-seconds must be positive") + if bool(arguments.wallet_name) != bool(arguments.wallet_hotkey): + raise SystemExit("--wallet-name and --wallet-hotkey must be given together") submits = not arguments.verify_only if (submits or arguments.peer_consensus) and not ( arguments.wallet_name and arguments.wallet_hotkey @@ -238,9 +240,6 @@ def main(argv: list[str] | None = None) -> None: except ImportError as error: raise SystemExit("Install cortex-subnet[chain] to run the validator") from error logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") - subtensor = Subtensor( - network=arguments.network, fallback_endpoints=arguments.fallback_endpoints - ) # A hotkey exported from a Polkadot-style keystore has no mnemonic, so the # wallet library cannot load it (see .hotkey). That file is an alternative # source, and exactly one source must be given: a wallet whose hotkey is not @@ -265,6 +264,17 @@ def main(argv: list[str] | None = None) -> None: hotkey=arguments.wallet_hotkey, path=str(wallet_path), ) + # The library reads the hotkey lazily: without this a missing or + # unreadable file only surfaces as dispatch_failed every epoch. + try: + _ = wallet.hotkey + except Exception as error: + raise SystemExit( + f"validator hotkey unreadable under --wallet-path: {type(error).__name__}" + ) from None + subtensor = Subtensor( + network=arguments.network, fallback_endpoints=arguments.fallback_endpoints + ) try: result = asyncio.run(run(arguments, subtensor, wallet)) if ( diff --git a/tests/test_run_validator_script.py b/tests/test_run_validator_script.py index 851cf73ce..126de937a 100644 --- a/tests/test_run_validator_script.py +++ b/tests/test_run_validator_script.py @@ -106,3 +106,14 @@ def test_tilde_wallet_path_is_expanded_before_the_hotkey_check(env, tmp_path): result = run(env | {"WALLET_PATH": "~/w"}, "--dry-run") assert result.returncode == 0 assert "hotkey file not found" not in result.stderr + + +def test_verify_only_never_reads_the_version_key_from_the_chain(env, tmp_path): + # A uv that fails every call stands in for an unreachable RPC. + uv = tmp_path / "bin" / "uv" + uv.write_text('#!/usr/bin/env bash\n[[ " $* " == *" - "* ]] && exec python3 "${@:6}"\nexit 1\n') + uv.chmod(0o755) + del env["VERSION_KEY"] + result = run(env, "--dry-run", "--verify-only") + assert result.returncode == 0, result.stderr + assert "--version-key 0" in result.stdout diff --git a/tests/validator/test_main.py b/tests/validator/test_main.py index 92b082cd7..ef98d8d9e 100644 --- a/tests/validator/test_main.py +++ b/tests/validator/test_main.py @@ -80,6 +80,8 @@ def close(self): pass class Wallet: + hotkey = object() + def __init__(self, **kwargs): self.kwargs = kwargs @@ -121,6 +123,8 @@ def close(self): pass class Wallet: + hotkey = object() + def __init__(self, **kwargs): pass @@ -213,3 +217,33 @@ def test_removed_trust_signing_options_are_gone(tmp_path): def test_main_rejects_unsafe_consensus_settings_before_connecting(flag, value, message, tmp_path): with pytest.raises(SystemExit, match=message): validator_main.main([*required_arguments(tmp_path), flag, value]) + + +@pytest.mark.parametrize("flag", ["--wallet-name", "--wallet-hotkey"]) +def test_half_a_wallet_is_refused_instead_of_silently_ignored(flag, monkeypatch, tmp_path): + fake_chain_modules(monkeypatch) + monkeypatch.setattr(validator_main, "run", lambda *args: pytest.fail("must not start")) + with pytest.raises(SystemExit, match="must be given together"): + validator_main.main([*without(required_arguments(tmp_path), flag), "--verify-only"]) + + +def fake_chain_modules(monkeypatch): + class Subtensor: + def __init__(self, **kwargs): + pass + + def close(self): + pass + + bittensor = ModuleType("bittensor") + bittensor.Subtensor = Subtensor + monkeypatch.setitem(sys.modules, "bittensor", bittensor) + + +def test_weight_setter_with_a_missing_hotkey_file_fails_at_startup(monkeypatch, tmp_path): + fake_chain_modules(monkeypatch) # the real bittensor_wallet reads the empty wallet dir + monkeypatch.setattr(validator_main, "run", lambda *args: pytest.fail("must not start")) + arguments = [*required_arguments(tmp_path), "--wallet-path", str(tmp_path / "wallets")] + with pytest.raises(SystemExit, match="hotkey unreadable") as refused: + validator_main.main(arguments) + assert "default" not in str(refused.value) and "wallets" not in str(refused.value) From f8f10a90b2dd1f5757f7316629eb29089bde0463 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:27:48 +0000 Subject: [PATCH 24/56] fix(validator): say the chain preflight is skipped without a hotkey A hotkey-less --verify-only run returned outcome=verified without asking the chain anything about the hotkey, and the deploy README and validators.md told operators to enable submission on that result. Registration, validator permit and rate limit were never checked. Log preflight=skipped at startup in that mode, and document the real order: hotkey-less verify, then verify-only with the hotkey (the full submission preflight, no extrinsic), then start the weight-setter. There is no submission-off flag on the weight-setter, so none is documented. Co-Authored-By: Claude Sonnet 5.5 --- deploy/README.md | 14 ++++++++++++++ docs/external-miner/validators.md | 11 +++++++++++ src/cortex/validator/__main__.py | 5 +++++ tests/validator/test_main.py | 14 ++++++++++++++ 4 files changed, 44 insertions(+) diff --git a/deploy/README.md b/deploy/README.md index ef1583425..a3e9e899b 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -196,6 +196,20 @@ reports `validator outcome=verified` for the same root/vector from historical chain state. The unsealed UID0 fallback is a readiness failure, not a weight to submit. +`outcome=verified` from the hotkey-less role says nothing about the hotkey: it +logs `preflight=skipped` and does not check hotkey registration, validator permit +or the weight rate limit. There is no submission-off switch on the weight-setter, +so check the hotkey before starting it, on the weight-setter host: + +1. Run `WALLET_NAME=... WALLET_HOTKEY=... scripts/run-validator.sh --verify-only` + (or `cortex validator --verify-only --once` with `--wallet-name`, + `--wallet-hotkey` and `--wallet-path`). With a hotkey, verify-only runs the same + chain preflight as a submission and never submits: the hotkey must be + registered on the netuid, hold a validator permit, have its weight rate limit + elapsed (`blocks_since_last_update > weights_rate_limit`) and the chain limits + must accept the sealed vector. A refusal prints `chain_preflight_failed`. +2. Only then start `role-weight-setter.yml`. + To run a validator without Compose, use `scripts/run-validator.sh` from a repository clone. It needs `WALLET_NAME` and `WALLET_HOTKEY` only to submit weights (`--verify-only` needs neither), reads the diff --git a/docs/external-miner/validators.md b/docs/external-miner/validators.md index fed445c0d..9a2d661aa 100644 --- a/docs/external-miner/validators.md +++ b/docs/external-miner/validators.md @@ -154,6 +154,17 @@ never claims its epoch in the journal and never creates an extrinsic. It exits successfully only for the exact `verified` outcome; unsealed, changed, degraded or peer-unavailable states return a nonzero process status. +Without a hotkey, verify-only logs `preflight=skipped`: it does **not** check +hotkey registration, validator permit or the weight rate limit, so `verified` +proves the seal, not that this hotkey can submit. Before the first submission, +run verify-only once more with `--wallet-name` and `--wallet-hotkey` (both or +neither; one alone is refused): it then runs the full chain preflight of a +submission (registered on the netuid, validator permit, rate limit elapsed, +chain weight limits) without submitting, and a refusal is +`chain_preflight_failed`. Start the submitting validator only after that run is +`verified`; it has no submission-off switch. A hotkey file that cannot be read +stops the process at startup. + For Compose use the Python [validator environment example](../../deploy/env/python-validator.env.example) with the verify-only [validator role](../../deploy/compose/role-validator.yml) (no diff --git a/src/cortex/validator/__main__.py b/src/cortex/validator/__main__.py index 1ca237e6a..738799052 100644 --- a/src/cortex/validator/__main__.py +++ b/src/cortex/validator/__main__.py @@ -272,6 +272,11 @@ def main(argv: list[str] | None = None) -> None: raise SystemExit( f"validator hotkey unreadable under --wallet-path: {type(error).__name__}" ) from None + elif arguments.verify_only: + logging.warning( + "validator preflight=skipped: no hotkey, so registration, validator permit " + "and weight rate limit are not checked" + ) subtensor = Subtensor( network=arguments.network, fallback_endpoints=arguments.fallback_endpoints ) diff --git a/tests/validator/test_main.py b/tests/validator/test_main.py index ef98d8d9e..67d63e25c 100644 --- a/tests/validator/test_main.py +++ b/tests/validator/test_main.py @@ -247,3 +247,17 @@ def test_weight_setter_with_a_missing_hotkey_file_fails_at_startup(monkeypatch, with pytest.raises(SystemExit, match="hotkey unreadable") as refused: validator_main.main(arguments) assert "default" not in str(refused.value) and "wallets" not in str(refused.value) + + +def test_verify_only_without_hotkey_logs_that_the_chain_preflight_is_skipped( + monkeypatch, tmp_path, caplog +): + fake_chain_modules(monkeypatch) + + async def verified(arguments, subtensor, wallet): + return TickResult("verified") + + monkeypatch.setattr(validator_main, "run", verified) + validator_main.main([*without_hotkey(tmp_path), "--verify-only", "--once"]) + assert "preflight=skipped" in caplog.text + assert "validator permit" in caplog.text From 6c6b843d4151f39d994b3e32c7b86b2fbb17271d Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:28:50 +0000 Subject: [PATCH 25/56] docs(deploy): add the validator to weight-setter upgrade note role-validator.yml became verify-only and weight setting moved to role-weight-setter.yml with a new state volume, but neither the CHANGELOG nor deploy/README said what an existing submitter must do. After the upgrade the old role stays healthy and silently stops setting weights. Document the switch, the env vars the verify-only role now ignores, what an empty journal and empty watermarks imply, and an optional volume copy. The copy command is not executed here (no docker on the host); the volume names are Compose defaults the operator must check. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 15 +++++++++++++++ deploy/README.md | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c5d1224f..1a5321bf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Upgrade (BREAKING for validators) + +- `deploy/compose/role-validator.yml` is now verify-only: it runs `--verify-only`, + mounts no wallet and never submits, while its healthcheck stays green. An + existing validator that set weights with it stops setting weights after the + upgrade. Deploy `deploy/compose/role-weight-setter.yml` instead, and never run + both with the same wallet. `role-validator.yml` now ignores `BASE_WALLET_NAME`, + `BASE_WALLET_HOTKEY`, `BASE_VALIDATOR_WALLETS_DIR` and `consensus.key` in + `BASE_VALIDATOR_IDENTITY_DIR`; `BASE_GATEWAY_HOTKEY` is removed everywhere. +- The weight-setter uses a new state volume, `base-weight-setter-state`: its + submission journal and consensus watermarks start empty, so rollback + protection restarts from the first profile it is served. To keep them, stop the + old role and copy its volume first; see the + [validator upgrade note](deploy/README.md#upgrading-from-the-single-validator-role). + ### Added - `GET /v1/trust`: the gateway serves the challenge profile (ids, shares, diff --git a/deploy/README.md b/deploy/README.md index a3e9e899b..75853d774 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -210,6 +210,38 @@ so check the hotkey before starting it, on the weight-setter host: must accept the sealed vector. A refusal prints `chain_preflight_failed`. 2. Only then start `role-weight-setter.yml`. +### Upgrading from the single validator role + +Before this release `role-validator.yml` held the wallet and submitted. It is now +verify-only: after the upgrade it keeps running, stays healthy and sets no weights. +It ignores `BASE_WALLET_NAME`, `BASE_WALLET_HOTKEY`, `BASE_VALIDATOR_WALLETS_DIR` +and the `consensus.key` in its identity directory (`BASE_GATEWAY_HOTKEY` is removed). +To keep setting weights, move to `role-weight-setter.yml`, whose state volume +`base-weight-setter-state` is new. Started empty, the weight-setter has no +submission journal and no consensus watermarks: rollback protection restarts +from the first profile `/v1/trust` serves it, and it does not know the epochs the +old role already submitted (the chain rate-limit preflight still refuses a second +set inside the limit). Reconcile any `dispatch_pending` attempt of the old role +(`cortex validator-reconcile`) before switching, or keep its state by copying the +volume while both roles are stopped: + +```bash +docker compose --project-directory . --env-file deploy/env/python-validator.env \ + -f deploy/compose/role-validator.yml down +docker run --rm --entrypoint cp \ + -v cortex-validator_base-validator-state:/from:ro \ + -v cortex-weight-setter_base-weight-setter-state:/var/lib/base \ + "$CORTEX_IMAGE" -a /from/. /var/lib/base/ +docker compose --project-directory . --env-file deploy/env/python-validator.env \ + -f deploy/compose/role-weight-setter.yml up -d +``` + +The volume names are Compose's `_` defaults for the `name:` in +each role file; check yours with `docker volume ls` first. Docker creates the +target volume on that `docker run` and Compose then warns it was not created by +Compose; that warning is expected. Copy only an old volume that you no longer +start: two processes must not share one journal. + To run a validator without Compose, use `scripts/run-validator.sh` from a repository clone. It needs `WALLET_NAME` and `WALLET_HOTKEY` only to submit weights (`--verify-only` needs neither), reads the From 2d8f3cb8254a05b3b5270cfcaa0b93ac2640389e Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:29:06 +0000 Subject: [PATCH 26/56] docs(deploy): keep consensus.key out of the verify-only identity dir role-validator.yml mounts BASE_VALIDATOR_IDENTITY_DIR but reads only tls.crt and tls.key, while the env example and secrets README defined that directory as also holding consensus.key. An operator reusing the weight-setter's directory exposed the seed to a role that never uses it. Documentation only: a dedicated BASE_VALIDATOR_TLS_DIR would also need check_deploy.py's env-example set, mount contract and rendering fixture changed, for the same result as one directory per role. Both roles share one env example, so the variable stays and the docs say to point it at a directory without consensus.key. Co-Authored-By: Claude Sonnet 5.5 --- deploy/README.md | 4 +++- deploy/env/python-validator.env.example | 4 +++- deploy/secrets/README.md | 8 +++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 75853d774..55a4714e6 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -168,7 +168,9 @@ validator wallet name and hotkey. The HTTPS gateway is the only anchor for the c credential-free `wss://` URLs. The weight-setter mounts the wallet directory read-only at `/run/wallets`; the -verify-only role mounts none. The validator identity directory contains: +verify-only role mounts none. The weight-setter's identity directory contains +the files below; the verify-only role reads only `tls.crt` and `tls.key`, so give +it a separate directory without `consensus.key`: ```text consensus.key diff --git a/deploy/env/python-validator.env.example b/deploy/env/python-validator.env.example index 2154ac302..b3f76beab 100644 --- a/deploy/env/python-validator.env.example +++ b/deploy/env/python-validator.env.example @@ -16,7 +16,9 @@ BASE_MAX_BLOCK_LAG=256 # Optional absolute overrides; blank values use config and deploy/secrets/wallets. BASE_VALIDATOR_WALLETS_DIR= BASE_CONFIG_DIR= -# Same hotkey as the wallet; directory holds consensus.key, tls.crt, tls.key. +# Weight-setter: directory holds consensus.key (same hotkey as the wallet), tls.crt, tls.key. +# Verify-only role: reads only tls.crt and tls.key; point it at a directory WITHOUT +# consensus.key (one directory per role; never reuse the weight-setter's). # Trust directory also contains peers.json: {"": "https://host:8091"}. BASE_VALIDATOR_IDENTITY_DIR= BASE_VALIDATOR_BIND_ADDRESS= diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index 684e9cc91..4c18c826b 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -35,9 +35,11 @@ mounts the whole directory read-only at `/run/challenge-secrets` and reads only ## Validator mounts -The wallet tree is mounted read-only at `/run/wallets`. A separate validator -identity directory is mounted at `/run/validator` and contains -`consensus.key`, `tls.crt` and `tls.key`. The validator never receives gateway, +The weight-setter mounts the wallet tree read-only at `/run/wallets` and a +separate validator identity directory at `/run/validator` containing +`consensus.key`, `tls.crt` and `tls.key`. The verify-only role mounts no wallet +and reads only `tls.crt` and `tls.key` from its identity directory: give it its +own directory without `consensus.key`, never the weight-setter's. The validator never receives gateway, challenge, Proof, operator or provider credentials. The master holds no gateway or challenge-leaf signing key: bundles and leaves are unsigned (wire v2). From d9b2885dba52e0ed8a6d6eb6a988f1a2d6e0bd23 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:30:32 +0000 Subject: [PATCH 27/56] docs: drop the last signed-profile and leaf-signing claims Most M-doc lines from the Lens review were already fixed by 76730e1 and 08153f3. Remaining residues: README said the script reads trust files from config/, docs/index.md still advertised "trust files", the miner README called challenge signing keys verification material, AGENTS.md called algorithm 1 "signed", and the unreleased CHANGELOG entries said the master signs leaves and algorithm 3 is owner-activated. Historical CHANGELOG entries are not rewritten, only marked as superseded where they describe the signed profile. Co-Authored-By: Claude Sonnet 5.5 --- AGENTS.md | 2 +- CHANGELOG.md | 13 ++++++++----- README.md | 2 +- docs/external-miner/README.md | 4 ++-- docs/external-miner/troubleshoot.md | 2 +- docs/index.md | 2 +- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4486302ca..effc5d06d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,7 @@ operator registry and records its leaves. | `bounty` | [CortexLM/bounty](https://github.com/CortexLM/bounty) container | useful vulnerability reports, scored from the CortexLM/backend public feed | | `opentype` | [OpentypeAI/challenge](https://github.com/OpentypeAI/challenge) container | exact-gold typed-decision duels on DiffusionGemma weights | -Shares always sum to 10,000 basis points. Algorithm 1 is the legacy signed +Shares always sum to 10,000 basis points. Algorithm 1 is the legacy bounty/proof 2,000/8,000 profile; algorithm 2 is bounty/proof 3,000/7,000 (document version >=2); algorithm 3 (document version >=3) admits any 1..64 unique ids and pays each challenge `share * min(sum(leaves), 10^12) / 10^12`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a5321bf3..da6fa9e97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,13 +30,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 hotkey and consensus seed; `role-validator.yml` is verify-only. - Challenge containers (`docs/CHALLENGES.md`): the master loads any Docker challenge from an operator registry, polls its authenticated `get_weights` - once per epoch, signs its leaves and proxies its public routes under + once per epoch, records its leaves and proxies its public routes under `/challenge//`. `cortex challenge-supervisor` pulls `stable`/`edge`/pinned GHCR images, checks labels and GitHub build provenance, canaries without secrets, updates and rolls back automatically. It alone holds the Docker socket. -- Owner-activated algorithm 3 (trust document version >=3): 1..64 challenges, +- Algorithm 3 (challenge profile version >=3): 1..64 challenges, each paying `share * min(sum(leaves), 10^12) / 10^12`, remainder burned. - Validators need no change beyond the release; they recompute signed leaves. + Validators need no change beyond the release; they recompute the unsigned leaves. - `GET /v1/metagraph/latest` serves the sealed hotkey map to challenges. - `cortex validator-watermark-reset`: operator-only, local repair that lowers a wrongly advanced `challenges_version`, `measurements_version` or `epoch` @@ -83,7 +83,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 emission. Proof retains 70%; unused Bounty mass burns to UID0. The legacy signed 20/80 profile, algorithm 1 and historical sealed bytes remain intact. Activation requires coordinated gateway/validator upgrades and an offline - owner signature over the new profile and epoch. + owner signature over the new profile and epoch (superseded: the profile is now + unsigned and activation is a version bump in `config/challenges.toml`). - Bounty-only launch mode with a revision-pinned, fully paginated CortexLM backend feed, bounded public writes and readiness checks that fail closed. - Validator production preflight, resilient bounded master/validator Bittensor RPC failover and @@ -108,7 +109,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Initial production operation keeps the signed 2000/8000 Bounty/Proof split; - unwired Proof burns its share instead of scaling Bounty to 100%. + unwired Proof burns its share instead of scaling Bounty to 100%. (Superseded: + the profile is no longer signed.) - **BREAKING:** Proof topic and VM JSON documents use schema version 2. Operators must generate compatible signed topics and guest images; old JSON signatures are not reinterpreted. Frozen bundle and signature domains remain unchanged. @@ -126,6 +128,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Historical bundle-root lookup retains accepted trust profiles across rotation and restart; current weights still require the current profile. The unsigned activation template requires an explicit epoch before it can be signed. + (Superseded: the profile is no longer signed.) - Bounty reconstructs a backend-capped leaderboard from the complete report snapshot, avoiding a permanent burn after the public log exceeds 1,000 hotkeys. - Bounty adjudication rejects missing or misplaced severity, and external diff --git a/README.md b/README.md index 034f53464..6f488f34a 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ scripts/run-validator.sh --verify-only # one verification, never submits scripts/run-validator.sh # verify and submit every epoch ``` -The script checks the gateway answer, reads the trust files from `config/` and starts `cortex validator`. It refuses master-only commands and master environments. Flags are `--verify-only`, `--once`, `--dry-run` and `--help`. Full walkthrough: [docs/validator-quickstart.md](docs/validator-quickstart.md). +The script checks the gateway answer and starts `cortex validator`, which reads the unsigned challenge profile from the gateway (`GET /v1/trust`, HTTPS); no local trust file is read. It refuses master-only commands and master environments. Flags are `--verify-only`, `--once`, `--dry-run` and `--help`. Full walkthrough: [docs/validator-quickstart.md](docs/validator-quickstart.md). ## Quickstart for developers diff --git a/docs/external-miner/README.md b/docs/external-miner/README.md index 2055f07a0..83d0ad225 100644 --- a/docs/external-miner/README.md +++ b/docs/external-miner/README.md @@ -65,8 +65,8 @@ Place these options before the miner action. Password, development seed, BYOK and Bounty session files must be private (for example mode `0600`) and must not be symlinks. Wallet loading never creates a wallet or rewrites its key files. Never put a mnemonic, private key, session token or provider credential in Git -or a support ticket. Public challenge signing keys are verification material, -not miner secrets. +or a support ticket. Challenge leaves are unsigned (wire v2); the only public key +a miner pins is the Proof topic key (`--proof-public`). ## Support diff --git a/docs/external-miner/troubleshoot.md b/docs/external-miner/troubleshoot.md index b4d8e8d4b..24798a7a1 100644 --- a/docs/external-miner/troubleshoot.md +++ b/docs/external-miner/troubleshoot.md @@ -15,7 +15,7 @@ files, provider keys, Bounty sessions or private artifacts in diagnostics. | `Bittensor hotkey unavailable` | Check wallet name, hotkey name, wallet root and the password file; loading does not create keys | | Encrypted hotkey password file required | Add miner `--wallet-password-file` before the action; the miner does not prompt interactively | | Credential file unavailable or not private | Use a regular, non-symlink file with no group/other permissions, such as `0600` | -| Proof owner public key required | Add miner `--proof-public` with the independently pinned Proof key, not the gateway key | +| Proof owner public key required | Add miner `--proof-public` with the independently pinned Proof topic key (the gateway has no key) | | Invalid Proof signature from `wallet.sign()` | Proof needs Cortex context `base-sr25519-v1`, domain `base-proof-submit-v1`; use the Python wallet signer | | HTTP connection failure | Verify the selected gateway URL and TLS; there is no implicit deployment URL | diff --git a/docs/index.md b/docs/index.md index 6d417df05..e048747f1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,7 +19,7 @@ Start here: [Cortex ecosystem](ecosystem.md) and [Validator quickstart](validato ## Run a validator - [Validator quickstart](validator-quickstart.md): one script, verify-only mode and FAQ. -- [Validator guide](external-miner/validators.md): the underlying CLI and trust files. +- [Validator guide](external-miner/validators.md): the underlying CLI, the gateway-served challenge profile and preflight. ## Mine From 3151f6e105fbf319d0dc418881dabea2ffe17bb7 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:32:09 +0000 Subject: [PATCH 28/56] test(config): pin the 503 for a placeholder challenge profile a4fff3a made load_trust reject a non-integer introduced_epoch or version with ProtocolError, but no test went through the master's Config.challenge_profile, the boundary the review reported as an uncaught TypeError. Without a4fff3a both cases below raise TypeError past the (OSError, ValueError, KeyError) handler. Co-Authored-By: Claude Sonnet 5.5 --- tests/protocol/test_trust_load.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/protocol/test_trust_load.py b/tests/protocol/test_trust_load.py index 8f861b439..f358fce26 100644 --- a/tests/protocol/test_trust_load.py +++ b/tests/protocol/test_trust_load.py @@ -56,3 +56,31 @@ def test_repository_default_profile_loads(): challenges_path=root / "challenges.toml", measurements_path=root / "measurements.toml" ) assert sum(value for _, value in trust.shares) == 10000 + + +@pytest.mark.parametrize( + ("challenges", "measurements"), + [ + (CHALLENGES.format(epoch='"CHOOSE_ACTIVATION_EPOCH"'), "version = 2\n"), + (CHALLENGES.format(epoch="0").replace("version = 3", 'version = "3"'), "version = 2\n"), + ], +) +def test_master_turns_a_placeholder_profile_into_a_clean_503(tmp_path, challenges, measurements): + from cortex.config import MasterConfig + from cortex.errors import ServiceError + + (tmp_path / "c.toml").write_text(challenges) + (tmp_path / "m.toml").write_text(measurements) + config = MasterConfig( + netuid=541, + state_dir=tmp_path, + challenges_file=tmp_path / "c.toml", + measurements_file=tmp_path / "m.toml", + proof_seed_file=tmp_path / "unused", + operator_token_file=tmp_path / "unused", + intake_token_file=tmp_path / "unused", + challenge_secrets_dir=tmp_path, + ) + with pytest.raises(ServiceError, match="challenge profile unavailable") as refused: + config.challenge_profile(0) + assert refused.value.status == 503 From 05bd7c2344d83b3b6ba1c43946320f82bd8e1877 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:26:26 +0000 Subject: [PATCH 29/56] fix(protocol): cap measurements_version before the watermark moves A served profile with measurements_version = 2**32 - 1 passed verify_inputs, the tick submitted and the watermark persisted; every honest profile afterwards was refused as a rollback until an operator ran validator-watermark-reset. TrustRoot.validate now refuses a measurements_version above MAX_MEASUREMENTS_VERSION (1000, chosen by the founder). The validator decodes /v1/trust through validate, so a profile above the cap fails as "invalid trust profile" before check_watermark or watermark run. Non-integer values were already refused by uint(); the new test pins bool/float/str/None/list/2**32 too. The master refuses such a profile at load for the same reason. Rejected: a bounded step above the current watermark, which needs the watermark inside protocol validation. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 3 +++ docs/external-miner/validators.md | 5 +++- src/cortex/protocol/models.py | 5 ++++ tests/validator/test_validator.py | 44 +++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da6fa9e97..f2a1b3661 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `scripts/run-validator.sh` needs `WALLET_NAME`/`WALLET_HOTKEY` only to submit. - The challenge profile is the unsigned `config/challenges.toml` and `measurements.toml`; changing shares is an edit, a version bump and a master restart. +- A challenge profile whose `measurements_version` exceeds + `MAX_MEASUREMENTS_VERSION` (1000) is refused by the master and by validators + before the rollback watermark moves. ### Removed diff --git a/docs/external-miner/validators.md b/docs/external-miner/validators.md index 9a2d661aa..637237ab6 100644 --- a/docs/external-miner/validators.md +++ b/docs/external-miner/validators.md @@ -249,7 +249,10 @@ database to force a retry. The journal keeps rollback watermarks for `challenges_version`, `measurements_version` and `epoch`; a lower served value fails with -`consensus rollback refused`. If a watermark was advanced wrongly (for example a +`consensus rollback refused`. A profile whose `measurements_version` is not an +integer or exceeds `MAX_MEASUREMENTS_VERSION` (1000, `src/cortex/protocol/models.py`) +is refused as `invalid trust profile` before any watermark moves; the master +refuses to load such a profile too. If a watermark was advanced wrongly (for example a profile version published by mistake and then withdrawn), stop the validator and lower it locally. The command touches only the state database: no network, no wallet. diff --git a/src/cortex/protocol/models.py b/src/cortex/protocol/models.py index 11fdab5c6..0bfb41b77 100644 --- a/src/cortex/protocol/models.py +++ b/src/cortex/protocol/models.py @@ -16,6 +16,9 @@ # Algorithm 3: a challenge's leaves sum to at most this; the shortfall burns to UID0. FULL_SHARE_SCORE = 10**12 CHALLENGE_ID = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +# Highest measurement allowlist version this release accepts. A served profile above it is +# refused before the rollback watermark moves; raising it needs a validator release. +MAX_MEASUREMENTS_VERSION = 1000 class NoScoreReason(IntEnum): @@ -187,6 +190,8 @@ def validate(self) -> None: fixed(self.measurements_digest, 32) uint(self.challenges_version, 4) uint(self.measurements_version, 4) + if self.measurements_version > MAX_MEASUREMENTS_VERSION: + raise ProtocolError("measurements version above the supported maximum") uint(self.introduced_epoch, 8) for challenge in self.challenges: challenge.encode() diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py index 62b822c8b..641fe1bd9 100644 --- a/tests/validator/test_validator.py +++ b/tests/validator/test_validator.py @@ -713,3 +713,47 @@ def test_cli_validator_reconcile_refuses_an_unknown_dispatch(network): "--evidence-digest", "cd" * 32, ] ) # fmt: skip + + +@pytest.mark.parametrize("value", [True, 1.0, "1", None, [1], 2**32, 2**32 - 1, "max+1"], ids=repr) +async def test_unsupported_measurements_version_never_moves_the_watermark(network, value): + import sqlite3 + + from cortex.protocol.models import MAX_MEASUREMENTS_VERSION + from cortex.protocol.trust import trust_to_json + + if value == "max+1": + value = MAX_MEASUREMENTS_VERSION + 1 + honest = trust_to_json(network.validator.trust) + served = {"profile": {**honest, "measurements_version": value}} + + def transport(request): + if request.url.path == "/v1/trust": + return httpx.Response(200, json=served["profile"]) + return network.handler(request) + + validator = Validator( + gateway_url="https://master.invalid", + netuid=541, + chain=network.chain, + journal=network.journal, + http=httpx.AsyncClient(transport=httpx.MockTransport(transport)), + ) + with pytest.raises(ProtocolError, match="invalid trust profile"): + await validator.run_once() + assert network.chain.submissions == [] + with sqlite3.connect(network.db_path) as connection: + assert ( + connection.execute( + "SELECT value FROM consensus_watermarks WHERE name='measurements_version'" + ).fetchall() + == [] + ) + served["profile"] = honest + assert (await validator.run_once()).outcome == "submitted" + + +def test_measurements_version_at_the_supported_maximum_is_accepted(network): + from cortex.protocol.models import MAX_MEASUREMENTS_VERSION + + replace(network.validator.trust, measurements_version=MAX_MEASUREMENTS_VERSION).validate() From 7a8ce5638bed4b5cc4139df3903b69929cd51de1 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:29:27 +0000 Subject: [PATCH 30/56] fix(gateway): refuse raw intake while an epoch is being sealed The emitter replaces a completed epoch's challenge leaves and then awaits the chain snapshot inside seal(). The event loop serves HTTP during that await, so the intake bearer alone could POST /v1/weights/raw for the epoch and have the forged leaf sealed (Lens repro: 202, Score(1e9) in leaves(12), epoch sealed). GatewayService.sealing is set by EpochEmitter.tick around _emit and seal and reset in finally, so a failed seal reopens intake; accept_leaf answers 409 "epoch sealing in progress" while it is set. The emitter's own lock already serialises ticks, so one flag is enough. Routes and auth are unchanged; sealing still needs the operator bearer. The Lens race repro becomes test_master.py tests for the in-window 409, the out-of-window 202 and the reset after a failed seal. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 + docs/THREAT_MODEL.md | 4 ++ src/cortex/gateway/service.py | 5 +++ src/cortex/master.py | 9 ++++- tests/test_master.py | 71 +++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2a1b3661..fb3474e4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `scripts/run-validator.sh` needs `WALLET_NAME`/`WALLET_HOTKEY` only to submit. - The challenge profile is the unsigned `config/challenges.toml` and `measurements.toml`; changing shares is an edit, a version bump and a master restart. +- `POST /v1/weights/raw` answers 409 `epoch sealing in progress` while the + master replaces a completed epoch's challenge leaves and seals it. - A challenge profile whose `measurements_version` exceeds `MAX_MEASUREMENTS_VERSION` (1000) is refused by the master and by validators before the rollback watermark moves. diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index 14f6bd23a..c8fdfdf54 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -111,6 +111,10 @@ and `POST /v1/admin/seal` requires the operator bearer (`BASE_GATEWAY_ADMIN_TOKEN_FILE`); each route refuses the other token with 401. The master refuses to start when both files resolve to the same path or hold the same value. +While the master replaces a completed epoch's challenge leaves and seals it, raw +intake answers 409 `epoch sealing in progress`, so a raw leaf cannot land after +the authoritative replacement and before the seal; intake reopens when the seal +ends or fails. What remains: before wire v2, forging a leaf also needed a challenge key; now an intake-token holder can still write any raw score for any miner, and the master's diff --git a/src/cortex/gateway/service.py b/src/cortex/gateway/service.py index cf6405570..b530a3cc3 100644 --- a/src/cortex/gateway/service.py +++ b/src/cortex/gateway/service.py @@ -52,6 +52,8 @@ def __init__( self.clock = clock or (lambda: datetime.now(UTC)) self.chain_endpoint = chain_endpoint self.trust_loader = trust_loader + # Set by the master between replacing an epoch's challenge leaves and sealing it. + self.sealing = False self.store.accept_trust(trust) def _chain_endpoint(self) -> str: @@ -77,6 +79,9 @@ def refresh_trust(self, epoch: int) -> None: self.trust = trust def accept_leaf(self, leaf: Leaf) -> dict: + if self.sealing: + # Raw intake must not land between the authoritative replacement and the seal. + raise ServiceError(409, "epoch sealing in progress") self.refresh_trust(leaf.epoch) entry = next( (entry for entry in self.trust.challenges if entry.id == leaf.challenge_id), None diff --git a/src/cortex/master.py b/src/cortex/master.py index c4ea72c82..c73ce3607 100644 --- a/src/cortex/master.py +++ b/src/cortex/master.py @@ -431,8 +431,13 @@ async def tick(self) -> list[int]: pinned = await self.gateway.chain.snapshot(block, self.gateway.netuid) if pinned.block_hash != block_hash or pinned.block != block: raise ServiceError(503, "completed epoch pin changed") - await self._emit(epoch, pinned) - await self.gateway.seal(epoch, block_b=block) + # The event loop serves intake during every await below; close it until sealed. + self.gateway.sealing = True + try: + await self._emit(epoch, pinned) + await self.gateway.seal(epoch, block_b=block) + finally: + self.gateway.sealing = False self.journal.sealed(epoch) completed.append(epoch) # Keep each newly completed seal observable for one polling interval. diff --git a/tests/test_master.py b/tests/test_master.py index d4787dc51..64a80d381 100644 --- a/tests/test_master.py +++ b/tests/test_master.py @@ -14,6 +14,7 @@ from cortex.proof.models import Baseline, Endpoint, EvaluationReport, Metric, Rule, Topic, digest from cortex.proof.service import Readiness, sign_submission, sign_topic from cortex.protocol import ( + Bundle, ChallengeEntry, Leaf, MetagraphRow, @@ -625,3 +626,73 @@ async def test_bounty_only_algorithm_three_still_seals_without_epoch_time(decay_ assert len(master.seen) == 1 assert dict(master.seen[0].url.params) == {"epoch": "12"} assert runtime.gateway.latest()["sealed"] is True + + +async def test_intake_bearer_cannot_inject_a_leaf_while_the_epoch_is_being_sealed(master): + from cortex.gateway import leaf_request + + runtime, chain = master.runtime, master.chain + intake = {"authorization": "Bearer fixture-intake-token"} + operator = {"authorization": "Bearer fixture-operator-token"} + hotkey = chain.rows[0].hotkey + snapshot, posted = chain.snapshot, [] + async with httpx.AsyncClient( + transport=httpx.ASGITransport(runtime.app()), base_url="https://master" + ) as client: + + async def racing_snapshot(block, netuid): + # Each chain read of the completed-epoch cycle is an await an intake can land in. + forged = leaf_request(Leaf(b"bounty", hotkey, 12, Score(10**9 + len(posted)))) + posted.append(await client.post("/v1/weights/raw", json=forged, headers=intake)) + return await snapshot(block, netuid) + + await runtime.emitter.tick() + chain.state = EpochState(13, 100, 105) + chain.snapshot = racing_snapshot + assert await runtime.emitter.tick() == [12] + chain.snapshot = snapshot + # Pin and pin re-read precede the challenge replacement, which overwrites them; + # the seal's own snapshot read is inside the window. + assert [response.status_code for response in posted] == [202, 202, 409] + assert posted[-1].json() == {"error": "epoch sealing in progress"} + sealed = Bundle.decode(runtime.gateway.store.bundle(12).encoded) + leaves = sealed.body.leaves + assert all(leaf.score == NoScore(NoScoreReason.CHALLENGE_INTERNAL) for leaf in leaves) + # Outside the window the same bearer is accepted for the open epoch. + open_leaf = leaf_request(Leaf(b"bounty", hotkey, 13, Score(5))) + response = await client.post("/v1/weights/raw", json=open_leaf, headers=intake) + assert response.status_code == 202 + # Sealing remains the operator's alone. + body = {"epoch": 12, "block_b": sealed.body.block_b} + assert (await client.post("/v1/admin/seal", json=body, headers=intake)).status_code == 401 + assert (await client.post("/v1/admin/seal", json=body, headers=operator)).status_code == 200 + + +async def test_a_failed_seal_reopens_intake_for_its_epoch(master): + from cortex.gateway import leaf_request + + runtime, chain = master.runtime, master.chain + intake = {"authorization": "Bearer fixture-intake-token"} + leaf = leaf_request(Leaf(b"bounty", chain.rows[0].hotkey, 12, Score(7))) + snapshot, calls, posted = chain.snapshot, [], [] + async with httpx.AsyncClient( + transport=httpx.ASGITransport(runtime.app()), base_url="https://master" + ) as client: + + async def failing_seal_snapshot(block, netuid): + calls.append(block) + if len(calls) == 3: # the seal's own read + posted.append(await client.post("/v1/weights/raw", json=leaf, headers=intake)) + raise OSError("chain unavailable") + return await snapshot(block, netuid) + + await runtime.emitter.tick() + chain.state = EpochState(13, 100, 105) + chain.snapshot = failing_seal_snapshot + with pytest.raises(ServiceError, match="seal inputs unavailable"): + await runtime.emitter.tick() + chain.snapshot = snapshot + assert [response.status_code for response in posted] == [409] + assert runtime.gateway.store.bundle(12) is None + response = await client.post("/v1/weights/raw", json=leaf, headers=intake) + assert response.status_code == 202 From 5e1a0e3c19f1a2724c2f44fbb8e6f780b30ba63c Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:33:28 +0000 Subject: [PATCH 31/56] fix(http): re-check bearer distinctness and length on every request build_master compared the operator and intake bearers once, while OperatorAuth re-reads its file per request: a rotation that wrote the same value into both files was accepted, so one value opened raw intake and sealing again. Bearers also had no minimum length, and a read failure said only "credential file unavailable". OperatorAuth now reads through read_bearer, which requires at least 32 bytes after strip and prefixes errors with the setting name (BASE_GATEWAY_ADMIN_TOKEN_FILE / BASE_GATEWAY_INTAKE_TOKEN_FILE), never the value. The master wires each gateway bearer with the other file, so every private route answers 503 "intake and admin bearers must differ" while they match. Startup uses the same reader. Test fixture bearers are lengthened to meet the minimum. The CHANGELOG moves the token split to "Upgrade (BREAKING for masters)" with the operator steps. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 24 +++++++--- deploy/README.md | 2 +- deploy/secrets/README.md | 2 +- docs/OPERATOR_SECURITY.md | 4 +- docs/reference/configuration.md | 4 +- src/cortex/http.py | 30 ++++++++++-- src/cortex/master.py | 13 +++-- tests/gateway/test_gateway.py | 14 +++--- tests/proof/conftest.py | 2 +- tests/proof/test_executor.py | 4 +- tests/proof/test_setup.py | 2 +- tests/test_master.py | 84 +++++++++++++++++++++++++++++---- tests/test_network_e2e.py | 2 +- 13 files changed, 148 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb3474e4c..291704d6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 old role and copy its volume first; see the [validator upgrade note](deploy/README.md#upgrading-from-the-single-validator-role). +### Upgrade (BREAKING for masters) + +- `POST /v1/weights/raw` now requires a separate intake bearer + (`BASE_GATEWAY_INTAKE_TOKEN_FILE`); `POST /v1/admin/seal` keeps the operator + bearer, and each refuses the other with 401. Both bearers need at least 32 + bytes, and the master refuses to start, and answers 503 on every private + route, while they share a path or a value. Before upgrading: + 1. Create `intake.token` in the master secrets directory + (`BASE_MASTER_SECRETS_DIR`, default `deploy/secrets/master/`) with a new + value of at least 32 bytes, for example `openssl rand -hex 32`, mode 0600 + (or 0400) and readable by container UID 65532. + 2. Check that `operator.token` is also at least 32 bytes; rotate it if not. + 3. Keep the `role-master.yml` secrets bind to `/run/secrets`: it already + mounts `intake.token` and sets + `BASE_GATEWAY_INTAKE_TOKEN_FILE=/run/secrets/intake.token`. A custom + deployment must mount the file and set that variable itself. + 4. Give `intake.token` to the raw-intake client only, then restart the master. + ### Added - `GET /v1/trust`: the gateway serves the challenge profile (ids, shares, @@ -45,12 +63,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Master: `POST /v1/weights/raw`, protected before wire v2 only by the challenge - leaf signature, now requires a separate intake bearer - (`BASE_GATEWAY_INTAKE_TOKEN_FILE`, `deploy/secrets/master/intake.token`); - `POST /v1/admin/seal` keeps the operator bearer. Each refuses the other with - 401, and startup refuses identical paths or values (BREAKING for deployments: - create `intake.token` before upgrading). - Wire format v2 (`protocol_version` 2, BREAKING): leaves and bundles carry no signatures and no gateway key. `Leaf` ends at its score, `Bundle` is the body, `ChallengeEntry` has no public key. The old v1 bundles and a gateway `state_dir` diff --git a/deploy/README.md b/deploy/README.md index 55a4714e6..85a3a32b5 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -91,7 +91,7 @@ deploy/secrets/master/ proof-vm-ca.pem ``` -The seed is raw 32 bytes or 64 hexadecimal characters. Nothing else on the master signs: the gateway seals unsigned wire-v2 bundles, so there is no gateway key and no per-challenge leaf key. Bearers are nonempty opaque values. Files are regular, +The seed is raw 32 bytes or 64 hexadecimal characters. Nothing else on the master signs: the gateway seals unsigned wire-v2 bundles, so there is no gateway key and no per-challenge leaf key. Bearers are opaque values of at least 32 bytes (for example `openssl rand -hex 32`); `operator.token` and `intake.token` must differ, and the master re-checks both on every request, so a rotation that makes them equal answers 503. Files are regular, not hardlinks or symlinks, and mode 0400 or 0600. The Proof VM token and CA are needed only when `PROOF_VM_ORCHESTRATOR_URL` is set. diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index 4c18c826b..80b3c865c 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -12,7 +12,7 @@ regular files, owned/readable by that identity, with mode 0400 or 0600. | --- | --- | | `proof.key` | 32-byte sr25519 seed that signs Proof topics; miners pin its public key (`--proof-public`) | | `operator.token` | bearer for master administrative routes, including `/v1/admin/seal` | -| `intake.token` | separate bearer for raw leaf intake (`/v1/weights/raw`); must differ from `operator.token` | +| `intake.token` | separate bearer for raw leaf intake (`/v1/weights/raw`); must differ from `operator.token`; each bearer at least 32 bytes | | `proof-vm.token` | bearer shared with the dedicated VM host | | `proof-vm-ca.pem` | CA used to verify the VM-host TLS certificate | diff --git a/docs/OPERATOR_SECURITY.md b/docs/OPERATOR_SECURITY.md index 5493d5d1b..9a817cdbd 100644 --- a/docs/OPERATOR_SECURITY.md +++ b/docs/OPERATOR_SECURITY.md @@ -14,7 +14,9 @@ recovery. It complements the [threat model](THREAT_MODEL.md). - [ ] The master operator bearer (`operator.token`) seals epochs (`/v1/admin/seal`) and guards the other administrative routes; the separate intake bearer (`intake.token`) only feeds `POST /v1/weights/raw`. The master - refuses to start when both files are the same path or hold the same value. + refuses to start when both files are the same path or hold the same value, and + answers 503 on every private route if a rotation later makes them equal. Each + bearer needs at least 32 bytes. Neither bearer grants the other's route, but an intake holder can still write forged raw leaves that the master's normal epoch seal will include (see the [threat model](THREAT_MODEL.md)). Hold each only where needed and rotate each diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 230de919c..78eb94c7e 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -15,8 +15,8 @@ conflicting values are an error. Existing names and crypto domains are preserved | `BASE_CHALLENGES_FILE` | Unsigned challenge profile (emission shares); served by the gateway at `GET /v1/trust` | | `BASE_MEASUREMENTS_FILE` | Unsigned measurement profile; served with the challenge profile | | `PROOF_SK_FILE` | Proof topic seed file | -| `BASE_GATEWAY_ADMIN_TOKEN_FILE` | Required operator bearer file (admin routes and `/v1/admin/seal`) | -| `BASE_GATEWAY_INTAKE_TOKEN_FILE` | Required raw-intake bearer file (`/v1/weights/raw` only); path and value must differ from the admin bearer | +| `BASE_GATEWAY_ADMIN_TOKEN_FILE` | Required operator bearer file (admin routes and `/v1/admin/seal`); at least 32 bytes | +| `BASE_GATEWAY_INTAKE_TOKEN_FILE` | Required raw-intake bearer file (`/v1/weights/raw` only); path and value must differ from the admin bearer (re-checked per request); at least 32 bytes | | `BASE_CHALLENGE_REGISTRY_FILE` | Optional [challenge registry](../CHALLENGES.md); unset runs no container challenge | | `BASE_CHALLENGE_SECRETS_DIR` | Per-challenge `/internal.token` bearers; default `/run/challenge-secrets` | | `PROOF_VM_ORCHESTRATOR_URL` | Dedicated VM host HTTPS origin | diff --git a/src/cortex/http.py b/src/cortex/http.py index 5bfb6400b..8ddf9816b 100644 --- a/src/cortex/http.py +++ b/src/cortex/http.py @@ -28,12 +28,36 @@ def read_private_file(path: Path, limit: int = 8192) -> str: raise ServiceError(503, "credential file unavailable") from None +MIN_BEARER_BYTES = 32 + + +def read_bearer(path: Path, label: str) -> str: + """Read a rotating bearer file; errors name the setting, never the value.""" + try: + value = read_private_file(path) + except ServiceError as error: + raise ServiceError(error.status, f"{label}: {error.reason}") from None + if len(value.encode()) < MIN_BEARER_BYTES: + raise ServiceError(503, f"{label}: bearer must be at least {MIN_BEARER_BYTES} bytes") + return value + + class OperatorAuth: - def __init__(self, token_file: Path): - self.token_file = token_file + def __init__( + self, + token_file: Path, + label: str = "BASE_GATEWAY_ADMIN_TOKEN_FILE", + distinct_from: tuple[Path, str] | None = None, + ): + self.token_file, self.label, self.distinct_from = token_file, label, distinct_from def require(self, request: Request) -> None: - expected = read_private_file(self.token_file) + expected = read_bearer(self.token_file, self.label) + # Both files rotate independently, so the startup distinctness check is not enough. + if self.distinct_from is not None and hmac.compare_digest( + expected.encode(), read_bearer(*self.distinct_from).encode() + ): + raise ServiceError(503, "intake and admin bearers must differ") supplied = request.headers.get("authorization", "") if not hmac.compare_digest(supplied.encode(), ("Bearer " + expected).encode()): raise ServiceError(401, "operator bearer required") diff --git a/src/cortex/master.py b/src/cortex/master.py index c73ce3607..c503c0333 100644 --- a/src/cortex/master.py +++ b/src/cortex/master.py @@ -24,7 +24,7 @@ from cortex.errors import ServiceError from cortex.gateway import GatewayService, GatewayStore from cortex.gateway import create_router as gateway_router -from cortex.http import OperatorAuth, read_private_file +from cortex.http import OperatorAuth, read_bearer from cortex.proof.api import create_router as proof_router from cortex.proof.artifacts import FileVault from cortex.proof.models import Submission @@ -76,6 +76,8 @@ def validate(self) -> None: # reported rather than guessed at. This bound is generous relative to a tempo of # a few hundred blocks and keeps a stuck loop from walking the whole chain. _END_BLOCK_SEARCH = 8192 +ADMIN_TOKEN_SETTING = "BASE_GATEWAY_ADMIN_TOKEN_FILE" +INTAKE_TOKEN_SETTING = "BASE_GATEWAY_INTAKE_TOKEN_FILE" class EpochProvider(Protocol): @@ -524,8 +526,10 @@ async def lifespan(app): async def service_error(request, error: ServiceError): return JSONResponse({"error": error.reason}, status_code=error.status) - operator = OperatorAuth(self.config.operator_token_file) - intake = OperatorAuth(self.config.intake_token_file) + admin_file = (self.config.operator_token_file, ADMIN_TOKEN_SETTING) + intake_file = (self.config.intake_token_file, INTAKE_TOKEN_SETTING) + operator = OperatorAuth(*admin_file, distinct_from=intake_file) + intake = OperatorAuth(*intake_file, distinct_from=admin_file) app.include_router(gateway_router(self.gateway, operator, intake)) proof = proof_router(self.proof, operator, self.topic_setup) app.include_router(proof) @@ -565,7 +569,8 @@ async def build_master( local_trust = trust or config.challenge_profile(state.epoch) # Holding the intake bearer must never grant sealing, and vice versa. if config.operator_token_file.resolve() == config.intake_token_file.resolve() or ( - read_private_file(config.operator_token_file) == read_private_file(config.intake_token_file) + read_bearer(config.operator_token_file, ADMIN_TOKEN_SETTING) + == read_bearer(config.intake_token_file, INTAKE_TOKEN_SETTING) ): raise ServiceError(503, "intake and admin bearers must differ") load_registry(config.challenge_registry_file) # fail fast on an invalid operator registry diff --git a/tests/gateway/test_gateway.py b/tests/gateway/test_gateway.py index 1bf0665d6..d38f00a12 100644 --- a/tests/gateway/test_gateway.py +++ b/tests/gateway/test_gateway.py @@ -66,10 +66,10 @@ async def submit(self, netuid, vector, version_key): @pytest.fixture async def network(tmp_path): token_file = tmp_path / "operator-token" - token_file.write_text("first-test-token") + token_file.write_text("first-test-token-0123456789abcdef") token_file.chmod(0o600) intake_file = tmp_path / "intake-token" - intake_file.write_text("intake-test-token") + intake_file.write_text("intake-test-token-0123456789abcdef") intake_file.chmod(0o600) store = GatewayStore(tmp_path / "gateway.db") rows = tuple(MetagraphRow(bytes([tag]) * 32, tag) for tag in range(3)) @@ -105,8 +105,8 @@ async def network(tmp_path): token_file=token_file, db_path=tmp_path / "gateway.db", journal=journal, - auth={"authorization": "Bearer first-test-token"}, - intake_auth={"authorization": "Bearer intake-test-token"}, + auth={"authorization": "Bearer first-test-token-0123456789abcdef"}, + intake_auth={"authorization": "Bearer intake-test-token-0123456789abcdef"}, ) store.close() journal.close() @@ -598,10 +598,12 @@ async def test_operator_token_is_required_and_reread_on_rotation(network): await intake(network) path = "/v1/admin/seal" assert (await network.client.post(path, json={"epoch": 12})).status_code == 401 - network.token_file.write_text("rotated-token") + network.token_file.write_text("rotated-token-0123456789abcdef-xyz") assert (await seal(network)).status_code == 401 response = await network.client.post( - path, json={"epoch": 12}, headers={"authorization": "Bearer rotated-token"} + path, + json={"epoch": 12}, + headers={"authorization": "Bearer rotated-token-0123456789abcdef-xyz"}, ) assert response.status_code == 200 network.token_file.chmod(0o644) diff --git a/tests/proof/conftest.py b/tests/proof/conftest.py index 6cb71f633..d3698f7bf 100644 --- a/tests/proof/conftest.py +++ b/tests/proof/conftest.py @@ -83,7 +83,7 @@ def setup(tmp_path): backend = FakeVmBoundary() vault = FileVault(tmp_path / "vault") token = tmp_path / "operator-token" - token.write_text("fixture-operator-token") + token.write_text("fixture-operator-token-0123456789abcdef") token.chmod(0o600) service = ProofService( store=store, diff --git a/tests/proof/test_executor.py b/tests/proof/test_executor.py index 97a2f42b3..5c5f28549 100644 --- a/tests/proof/test_executor.py +++ b/tests/proof/test_executor.py @@ -591,12 +591,12 @@ async def test_executor_api_rotates_or_closes_only_with_operator_auth(setup): denied = await client.post("/v1/admin/proof/executor", json=closed.model_dump()) rotated = await client.post( "/v1/admin/proof/executor", - headers={"Authorization": "Bearer fixture-operator-token"}, + headers={"Authorization": "Bearer fixture-operator-token-0123456789abcdef"}, json=closed.model_dump(), ) forged = await client.post( "/v1/admin/proof/executor", - headers={"Authorization": "Bearer fixture-operator-token"}, + headers={"Authorization": "Bearer fixture-operator-token-0123456789abcdef"}, json=closed.model_copy(update={"machine_shape": "2x"}).model_dump(), ) after = await client.get("/v1/proof/executor") diff --git a/tests/proof/test_setup.py b/tests/proof/test_setup.py index 4b76a43dd..d29c70b0e 100644 --- a/tests/proof/test_setup.py +++ b/tests/proof/test_setup.py @@ -513,7 +513,7 @@ async def public_error(request, error): accepted = await client.post( "/v1/admin/proof/setup", json={"policy": policy().model_dump(mode="json")}, - headers={"Authorization": "Bearer fixture-operator-token"}, + headers={"Authorization": "Bearer fixture-operator-token-0123456789abcdef"}, ) assert accepted.status_code == 201 and accepted.json()["status"] == "open" diff --git a/tests/test_master.py b/tests/test_master.py index 64a80d381..f51c0f6bd 100644 --- a/tests/test_master.py +++ b/tests/test_master.py @@ -1,5 +1,6 @@ import asyncio import io +import secrets import tarfile from dataclasses import replace from hashlib import sha256 @@ -96,8 +97,8 @@ def secret(name, value): challenges_file=tmp_path / "challenges.toml", measurements_file=tmp_path / "measurements.toml", proof_seed_file=secret("proof.key", bytes([2]) * 32), - operator_token_file=secret("operator.token", b"fixture-operator-token"), - intake_token_file=secret("intake.token", b"fixture-intake-token"), + operator_token_file=secret("operator.token", b"fixture-operator-token-0123456789abcdef"), + intake_token_file=secret("intake.token", b"fixture-intake-token-0123456789abcdef"), challenge_secrets_dir=tmp_path / "challenge-secrets", ) @@ -399,13 +400,15 @@ async def test_private_routes_require_rotating_file_and_master_routes_are_compos assert (await client.get("/challenge/bounty/v1/status")).status_code == 404 drain = "/v1/admin/proof/drain" assert (await client.post(drain)).status_code == 401 - master.config.operator_token_file.write_text("rotated-token") + master.config.operator_token_file.write_text("rotated-token-0123456789abcdef-xyz") assert ( - await client.post(drain, headers={"authorization": "Bearer rotated-token"}) + await client.post( + drain, headers={"authorization": "Bearer rotated-token-0123456789abcdef-xyz"} + ) ).status_code != 401 - raw = {"authorization": "Bearer rotated-token"} + raw = {"authorization": "Bearer rotated-token-0123456789abcdef-xyz"} assert (await client.post("/v1/weights/raw", json={}, headers=raw)).status_code == 401 - intake = {"authorization": "Bearer fixture-intake-token"} + intake = {"authorization": "Bearer fixture-intake-token-0123456789abcdef"} assert (await client.post("/v1/weights/raw", json={}, headers=intake)).status_code == 400 seal = await client.post("/v1/admin/seal", json={"epoch": 12}, headers=intake) assert seal.status_code == 401 @@ -424,7 +427,7 @@ async def test_master_refuses_one_bearer_for_intake_and_sealing(tmp_path, shared if shared == "same-file": config = replace(config, intake_token_file=config.operator_token_file) else: - config.intake_token_file.write_bytes(b"fixture-operator-token\n") + config.intake_token_file.write_bytes(b"fixture-operator-token-0123456789abcdef\n") chain = FakeEpochChain() with pytest.raises(ServiceError, match="must differ"): await build_master(config, chain=chain, epochs=chain, trust=fixture_trust()) @@ -632,8 +635,8 @@ async def test_intake_bearer_cannot_inject_a_leaf_while_the_epoch_is_being_seale from cortex.gateway import leaf_request runtime, chain = master.runtime, master.chain - intake = {"authorization": "Bearer fixture-intake-token"} - operator = {"authorization": "Bearer fixture-operator-token"} + intake = {"authorization": "Bearer fixture-intake-token-0123456789abcdef"} + operator = {"authorization": "Bearer fixture-operator-token-0123456789abcdef"} hotkey = chain.rows[0].hotkey snapshot, posted = chain.snapshot, [] async with httpx.AsyncClient( @@ -672,7 +675,7 @@ async def test_a_failed_seal_reopens_intake_for_its_epoch(master): from cortex.gateway import leaf_request runtime, chain = master.runtime, master.chain - intake = {"authorization": "Bearer fixture-intake-token"} + intake = {"authorization": "Bearer fixture-intake-token-0123456789abcdef"} leaf = leaf_request(Leaf(b"bounty", chain.rows[0].hotkey, 12, Score(7))) snapshot, calls, posted = chain.snapshot, [], [] async with httpx.AsyncClient( @@ -696,3 +699,64 @@ async def failing_seal_snapshot(block, netuid): assert runtime.gateway.store.bundle(12) is None response = await client.post("/v1/weights/raw", json=leaf, headers=intake) assert response.status_code == 202 + + +def bearer(path, value): + path.write_text(value) + path.chmod(0o600) + + +async def test_a_rotation_to_one_bearer_is_refused_on_every_private_route(master): + shared = secrets.token_hex(32) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(master.runtime.app()), base_url="https://master" + ) as client: + bearer(master.config.operator_token_file, shared) + bearer(master.config.intake_token_file, shared) + headers = {"authorization": f"Bearer {shared}"} + for path in ("/v1/weights/raw", "/v1/admin/seal", "/v1/admin/proof/drain"): + response = await client.post(path, json={}, headers=headers) + assert response.status_code == 503, path + assert response.json() == {"error": "intake and admin bearers must differ"} + + +@pytest.mark.parametrize( + ("which", "label"), + [ + ("operator_token_file", "BASE_GATEWAY_ADMIN_TOKEN_FILE"), + ("intake_token_file", "BASE_GATEWAY_INTAKE_TOKEN_FILE"), + ], +) +async def test_a_short_bearer_is_refused_at_startup_and_after_rotation(tmp_path, which, label): + config = master_config(tmp_path) + short = secrets.token_hex(15) + " \n" # 30 bytes once stripped + bearer(getattr(config, which), short) + chain = FakeEpochChain() + with pytest.raises(ServiceError, match=f"^{label}: bearer must be at least 32 bytes$"): + await build_master(config, chain=chain, epochs=chain, trust=fixture_trust()) + + (tmp_path / "rotated").mkdir() + config = master_config(tmp_path / "rotated") + runtime = await build_master(config, chain=chain, epochs=chain, trust=fixture_trust()) + try: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(runtime.app()), base_url="https://master" + ) as client: + bearer(getattr(config, which), short) + headers = {"authorization": f"Bearer {short.strip()}"} + path = "/v1/admin/seal" if which == "operator_token_file" else "/v1/weights/raw" + response = await client.post(path, json={}, headers=headers) + assert response.status_code == 503 + assert response.json() == {"error": f"{label}: bearer must be at least 32 bytes"} + assert short.strip() not in response.text + finally: + await runtime.close() + + +async def test_a_missing_bearer_names_its_setting(tmp_path): + config = master_config(tmp_path) + config.intake_token_file.unlink() + chain = FakeEpochChain() + with pytest.raises(ServiceError) as raised: + await build_master(config, chain=chain, epochs=chain, trust=fixture_trust()) + assert raised.value.reason == "BASE_GATEWAY_INTAKE_TOKEN_FILE: credential file unavailable" diff --git a/tests/test_network_e2e.py b/tests/test_network_e2e.py index 30fc4be2f..76bd5621a 100644 --- a/tests/test_network_e2e.py +++ b/tests/test_network_e2e.py @@ -493,7 +493,7 @@ def socket_chmod(path, mode, **kwargs): ) as http: setup = await http.post( "/v1/admin/proof/setup", - headers={"authorization": "Bearer fixture-operator-token"}, + headers={"authorization": "Bearer fixture-operator-token-0123456789abcdef"}, json={ "policy": { "topic_id": "topic-a", From 9e504c417ba704d10514003dfd22123b70358724 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:35:40 +0000 Subject: [PATCH 32/56] fix(cli): open the watermark-reset target read-only before writing validator-watermark-reset opened --state-db through SubmissionJournal, which switches to WAL and creates every validator table before the row lookup. A mistyped path to another SQLite file failed with "consensus watermark not set" but kept the new tables and WAL mode. A running validator holding the lock surfaced as a bare OperationalError after the default 5 s busy wait, and nothing checked the subnet. The command now opens the file with mode=ro, requires the consensus_watermarks table and the named row, and only then opens it for write through EvidenceStore alone (no journal schema, no WAL pragma, 1 s busy timeout). A lock prints "state db locked; stop the validator first". The new --netuid flag is compared with the netuid row bind_netuid records; a journal without that row skips the check and says so on stderr. The runbook gains the docker compose stop / run --rm / start recipe that mounts the role's state volume, with placeholders only. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 4 ++- docs/external-miner/validators.md | 24 ++++++++++++- src/cortex/cli.py | 48 +++++++++++++++++++++---- tests/validator/test_validator.py | 60 +++++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 291704d6c..d71e2d1f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,7 +59,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `cortex validator-watermark-reset`: operator-only, local repair that lowers a wrongly advanced `challenges_version`, `measurements_version` or `epoch` watermark (requires `--confirm`, never raises one) and audits it in - `consensus_watermark_resets`. + `consensus_watermark_resets`. It checks the file read-only first and never + writes to a foreign or empty database; `--netuid` refuses another subnet's + journal. ### Changed diff --git a/docs/external-miner/validators.md b/docs/external-miner/validators.md index 637237ab6..68fa0d6e7 100644 --- a/docs/external-miner/validators.md +++ b/docs/external-miner/validators.md @@ -260,9 +260,31 @@ wallet. ```bash cortex validator-watermark-reset \ --state-db /var/lib/base/validator.sqlite3 \ - --name challenges_version --to 2 --confirm + --name challenges_version --to 2 --netuid --confirm ``` +`/var/lib/base/validator.sqlite3` is the path inside the validator container. With +the Compose roles, stop the role and run the command in a one-off container of +the same service, which mounts the same state volume (`role-weight-setter.yml` +shown; use `role-validator.yml` for the verify-only role): + +```bash +docker compose --project-directory . --env-file deploy/env/python-validator.env \ + -f deploy/compose/role-weight-setter.yml stop validator +docker compose --project-directory . --env-file deploy/env/python-validator.env \ + -f deploy/compose/role-weight-setter.yml run --rm --no-deps validator \ + validator-watermark-reset --state-db /var/lib/base/validator.sqlite3 \ + --name --to --netuid --confirm +docker compose --project-directory . --env-file deploy/env/python-validator.env \ + -f deploy/compose/role-weight-setter.yml start validator +``` + +The command first opens the file read-only and refuses, without writing, a +file that is not a validator state database or lacks the named watermark. A +running validator holding the database makes it fail with `state db locked; +stop the validator first`. `--netuid` refuses a journal recorded for another +subnet; a journal that records no netuid skips that check and says so. + It refuses without `--confirm`, refuses a name outside those three, a negative value, a watermark that was never set and any value above the current one (it only lowers or repairs). It prints the old and new values and appends an audit diff --git a/src/cortex/cli.py b/src/cortex/cli.py index 1bc1b9a8d..932d2597a 100644 --- a/src/cortex/cli.py +++ b/src/cortex/cli.py @@ -51,6 +51,7 @@ def parser() -> argparse.ArgumentParser: reset.add_argument("--name", choices=RESETTABLE_WATERMARKS, required=True) reset.add_argument("--to", type=int, required=True) reset.add_argument("--confirm", action="store_true") + reset.add_argument("--netuid", type=int, help="refuse a journal recorded for another subnet") vm = commands.add_parser("vm-host", help="serve the dedicated Firecracker host") vm.add_argument("--config", type=Path, required=True) vm.add_argument( @@ -241,26 +242,59 @@ def run_validator_reconcile(args: argparse.Namespace) -> None: print(json.dumps(result, indent=2, sort_keys=True)) +def _watermark_reset_preflight(args: argparse.Namespace) -> None: + """Prove the file is a validator journal holding the row, without writing to it.""" + reader = sqlite3.connect(f"{args.state_db.resolve().as_uri()}?mode=ro", uri=True) + try: + table = reader.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='consensus_watermarks'" + ).fetchone() + if table is None: + raise SystemExit("Cortex command failed: not a validator state database") + rows = dict( + reader.execute( + "SELECT name, value FROM consensus_watermarks WHERE name IN (?, 'netuid')", + (args.name,), + ).fetchall() + ) + finally: + reader.close() + if args.name not in rows: + raise SystemExit("Cortex command failed: consensus watermark not set") + if args.netuid is not None: + if "netuid" not in rows: + print("state db records no netuid; --netuid check skipped", file=sys.stderr) + elif rows["netuid"] != str(args.netuid): + raise SystemExit("Cortex command failed: validator journal belongs to another subnet") + + def run_validator_watermark_reset(args: argparse.Namespace) -> None: - from cortex.validator import SubmissionJournal + from cortex.validator.evidence import EvidenceStore # Refusal reasons are fixed strings with no credential, so show them to the operator. if not args.confirm: raise SystemExit("Cortex command failed: --confirm required; lowering re-opens rollback") if not args.state_db.is_file(): raise SystemExit("Cortex command failed: validator state database not found") + connection = None try: - journal = SubmissionJournal(args.state_db) - except sqlite3.Error as error: - raise SystemExit(f"Cortex command failed: {type(error).__name__}") from None - try: - old = journal.evidence.reset_watermark(args.name, args.to) + _watermark_reset_preflight(args) + # The journal schema and WAL mode belong to the validator; open only the existing file. + connection = sqlite3.connect(args.state_db, isolation_level=None, timeout=1) + old = EvidenceStore(connection).reset_watermark(args.name, args.to) except ProtocolError as error: raise SystemExit(f"Cortex command failed: {error}") from None + except sqlite3.OperationalError as error: + if "locked" in str(error) or "busy" in str(error): + raise SystemExit( + "Cortex command failed: state db locked; stop the validator first" + ) from None + raise SystemExit(f"Cortex command failed: {type(error).__name__}") from None except sqlite3.Error as error: raise SystemExit(f"Cortex command failed: {type(error).__name__}") from None finally: - journal.close() + if connection is not None: + connection.close() print(json.dumps({"name": args.name, "old_value": old, "new_value": args.to}, indent=2)) diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py index 641fe1bd9..342c71329 100644 --- a/tests/validator/test_validator.py +++ b/tests/validator/test_validator.py @@ -757,3 +757,63 @@ def test_measurements_version_at_the_supported_maximum_is_accepted(network): from cortex.protocol.models import MAX_MEASUREMENTS_VERSION replace(network.validator.trust, measurements_version=MAX_MEASUREMENTS_VERSION).validate() + + +def snapshot_file(path): + import sqlite3 + + stat = path.stat() + connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True) + try: + mode = connection.execute("PRAGMA journal_mode").fetchone()[0] + tables = connection.execute("SELECT name FROM sqlite_master ORDER BY name").fetchall() + finally: + connection.close() + siblings = sorted(item.name for item in path.parent.iterdir()) + return path.read_bytes(), stat.st_mtime_ns, mode, tables, siblings + + +@pytest.mark.parametrize("kind", ["empty", "foreign"]) +def test_cli_watermark_reset_never_mutates_a_foreign_state_file(tmp_path, kind): + import sqlite3 + + state = tmp_path / "other" / "state.db" + state.parent.mkdir() + if kind == "empty": + state.touch() + else: + with sqlite3.connect(state) as connection: + connection.execute("CREATE TABLE notes (body TEXT)") + connection.execute("INSERT INTO notes VALUES ('kept')") + connection.close() + before = snapshot_file(state) + with pytest.raises(SystemExit, match="not a validator state database"): + reset(state, "--name", "epoch", "--to", "0", "--confirm") + assert snapshot_file(state) == before + + +def test_cli_watermark_reset_reports_a_locked_state_db(network): + import sqlite3 + + network.journal.evidence.watermark("epoch", 12) + network.journal.close() + holder = sqlite3.connect(network.db_path, isolation_level=None) + holder.execute("BEGIN IMMEDIATE") + try: + with pytest.raises(SystemExit, match="state db locked; stop the validator first"): + reset(network.db_path, "--name", "epoch", "--to", "11", "--confirm") + finally: + holder.execute("ROLLBACK") + holder.close() + with sqlite3.connect(network.db_path) as connection: + value = connection.execute("SELECT value FROM consensus_watermarks WHERE name='epoch'") + assert value.fetchone() == ("12",) + + +def test_cli_watermark_reset_checks_the_recorded_netuid(network, capsys): + network.journal.evidence.watermark("epoch", 12) + network.journal.close() + with pytest.raises(SystemExit, match="belongs to another subnet"): + reset(network.db_path, "--name", "epoch", "--to", "11", "--netuid", "7", "--confirm") + reset(network.db_path, "--name", "epoch", "--to", "11", "--netuid", "541", "--confirm") + assert json.loads(capsys.readouterr().out)["new_value"] == 11 From a08d5f622cc81a1d738b9a55729adb8821e22e4f Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:37:26 +0000 Subject: [PATCH 33/56] fix(config): validate challenge profile field types before decoding challenge_profile caught only OSError, ValueError and KeyError, so a profile with `id = 5` escaped as AttributeError and `emission_share_bps = "10000"` as TypeError instead of the clean 503 "challenge profile unavailable". Policy hotkeys and measurement fields of the wrong type crashed the same way. load_trust now checks each row explicitly: challenges is an array of tables, id a string, emission_share_bps an int (not bool) in 0..10000, policy hotkeys an array of strings, measurement fields hex strings, all as ProtocolError. challenge_profile also maps TypeError to the same 503 as a backstop. The tests cover id=5, bps="3000", bps=true, out-of-range bps, missing keys, a non-table policy, malformed hotkeys and measurements, and TOML syntax errors. Co-Authored-By: Claude Sonnet 5.5 --- src/cortex/config.py | 4 ++- src/cortex/protocol/trust.py | 37 ++++++++++++++++++-------- tests/protocol/test_trust_load.py | 44 ++++++++++++++++++++++++++----- 3 files changed, 66 insertions(+), 19 deletions(-) diff --git a/src/cortex/config.py b/src/cortex/config.py index ddeb8b9e1..914fe78b7 100644 --- a/src/cortex/config.py +++ b/src/cortex/config.py @@ -208,5 +208,7 @@ def challenge_profile(self, epoch: int) -> TrustRoot: return load_trust( challenges_path=self.challenges_file, measurements_path=self.measurements_file ) - except (OSError, ValueError, KeyError): + except (OSError, ValueError, KeyError, TypeError): + # ProtocolError and TOMLDecodeError are ValueErrors; TypeError covers a field + # type that slips past the explicit checks in load_trust. raise ServiceError(503, "challenge profile unavailable") from None diff --git a/src/cortex/protocol/trust.py b/src/cortex/protocol/trust.py index be352cdc6..69503f44d 100644 --- a/src/cortex/protocol/trust.py +++ b/src/cortex/protocol/trust.py @@ -11,7 +11,7 @@ from .scale import ProtocolError, fixed, vector -def _policy(value: str | dict) -> ParticipantPolicy: +def _policy(value: object) -> ParticipantPolicy: if value == "all_metagraph_hotkeys": return ParticipantPolicy() if not isinstance(value, dict): @@ -22,31 +22,46 @@ def _policy(value: str | dict) -> ParticipantPolicy: if kind == "stake_at_least": return ParticipantPolicy(1, min_stake=value["min_stake"]) if kind in ("explicit_allowlist", "all_except_deny_list"): + hotkeys = value.get("hotkeys") + if not isinstance(hotkeys, list) or not all(isinstance(k, str) for k in hotkeys): + raise ProtocolError("participant policy hotkeys must be an array of strings") return ParticipantPolicy( 2 if kind == "explicit_allowlist" else 3, - hotkeys=tuple(decode_hotkey(k) for k in value["hotkeys"]), + hotkeys=tuple(decode_hotkey(k) for k in hotkeys), ) raise ProtocolError("unknown participant policy") +def _row(row: object) -> ChallengeEntry: + if not isinstance(row, dict): + raise ProtocolError("challenge row must be a table") + key, share = row.get("id"), row.get("emission_share_bps") + if not isinstance(key, str): + raise ProtocolError(f"challenge id must be a string, got {key!r}") + if type(share) is not int or not 0 <= share <= 10000: + raise ProtocolError(f"emission_share_bps must be an integer in 0..10000, got {share!r}") + return ChallengeEntry(key.encode(), share, _policy(row.get("policy"))) + + def challenge_entries(document: dict) -> tuple[ChallengeEntry, ...]: """Decode the challenge rows (id, share, participant policy).""" - return tuple( - ChallengeEntry( - row["id"].encode(), - row["emission_share_bps"], - _policy(row["policy"]), - ) - for row in document["challenges"] - ) + rows = document.get("challenges") + if not isinstance(rows, list): + raise ProtocolError("challenges must be an array of tables") + return tuple(_row(row) for row in rows) def measurements_body(document: dict) -> bytes: """Encode the measurement allowlist; its sha256 is the profile's measurements digest.""" entries = [] - for row in document.get("measurements", []): + rows = document.get("measurements", []) + if not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows): + raise ProtocolError("measurements must be an array of tables") + for row in rows: fields = [] for name in ("mr_td", "rtmr0", "rtmr1", "rtmr2", "rtmr3", "compose_hash"): + if not isinstance(row.get(name), str): + raise ProtocolError(f"measurement {name} must be a hex string") fields.append( fixed( bytes.fromhex(row[name].removeprefix("0x")), diff --git a/tests/protocol/test_trust_load.py b/tests/protocol/test_trust_load.py index f358fce26..6904a31b6 100644 --- a/tests/protocol/test_trust_load.py +++ b/tests/protocol/test_trust_load.py @@ -58,13 +58,43 @@ def test_repository_default_profile_loads(): assert sum(value for _, value in trust.shares) == 10000 -@pytest.mark.parametrize( - ("challenges", "measurements"), - [ - (CHALLENGES.format(epoch='"CHOOSE_ACTIVATION_EPOCH"'), "version = 2\n"), - (CHALLENGES.format(epoch="0").replace("version = 3", 'version = "3"'), "version = 2\n"), - ], -) +GOOD = CHALLENGES.format(epoch="0") + + +def policy(value: str) -> str: + return GOOD.replace('policy = "all_metagraph_hotkeys"', f"policy = {value}", 1) + + +MALFORMED = { + "epoch-placeholder": (CHALLENGES.format(epoch='"CHOOSE_ACTIVATION_EPOCH"'), "version = 2"), + "version-str": (GOOD.replace("version = 3", 'version = "3"'), "version = 2"), + "id-int": (GOOD.replace('id = "bounty"', "id = 5"), "version = 2"), + "bps-str": (GOOD.replace("= 3000", '= "3000"'), "version = 2"), + "bps-bool": (GOOD.replace("= 3000", "= true"), "version = 2"), + "bps-range": (GOOD.replace("= 3000", "= 70000"), "version = 2"), + "id-missing": (GOOD.replace('id = "bounty"\n', ""), "version = 2"), + "bps-missing": (GOOD.replace("emission_share_bps = 3000\n", ""), "version = 2"), + "policy-int": (GOOD.replace('policy = "all_metagraph_hotkeys"', "policy = 3"), "version = 2"), + "challenges-int": (GOOD.replace("[[challenges]]", "challenges = 1\n[[x]]", 1), "version = 2"), + "measurement-int": (GOOD, "version = 2\nmeasurements = [1]"), + "measurement-field-int": (GOOD, "version = 2\nmeasurements = [{ mr_td = 1 }]"), + "toml-syntax": (GOOD, "version = [2"), + "hotkeys-int": ( + policy('{ type = "explicit_allowlist", hotkeys = 5 }'), + "version = 2", + ), + "hotkey-int": ( + policy('{ type = "explicit_allowlist", hotkeys = [5] }'), + "version = 2", + ), + "min-stake-str": ( + policy('{ type = "stake_at_least", min_stake = "x" }'), + "version = 2", + ), +} + + +@pytest.mark.parametrize(("challenges", "measurements"), MALFORMED.values(), ids=MALFORMED) def test_master_turns_a_placeholder_profile_into_a_clean_503(tmp_path, challenges, measurements): from cortex.config import MasterConfig from cortex.errors import ServiceError From 748eaaed17f6c4875c952084726b03a1a86e8308 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:39:39 +0000 Subject: [PATCH 34/56] fix(deploy): give the weight-setter its own identity directory variable role-validator.yml and role-weight-setter.yml both read BASE_VALIDATOR_IDENTITY_DIR, and the README passes one env file to both, so the verify-only role mounted the directory that holds consensus.key unless the operator split it by hand. check_deploy.py checked only that the mount existed. role-weight-setter.yml now reads the required BASE_WEIGHT_SETTER_IDENTITY_DIR; the env example carries both variables. check_deploy.py gains validate_role_separation, run on the rendered roles by --check-examples, which refuses one normalised identity path or one named state volume for both roles. The CHANGELOG lists the new variable as a validator upgrade step. Not run here: --check-examples needs docker compose to render the roles; the new check is covered on fixture configs in tests/test_deploy_contract.py. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 7 +++++ deploy/README.md | 13 +++++--- deploy/compose/role-weight-setter.yml | 2 +- deploy/env/python-validator.env.example | 11 ++++--- docs/reference/configuration.md | 2 ++ scripts/check_deploy.py | 23 +++++++++++++- tests/test_deploy_contract.py | 42 +++++++++++++++++++++++++ 7 files changed, 90 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d71e2d1f3..6a4cc62dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 protection restarts from the first profile it is served. To keep them, stop the old role and copy its volume first; see the [validator upgrade note](deploy/README.md#upgrading-from-the-single-validator-role). +- `role-weight-setter.yml` reads its identity directory from the new, required + `BASE_WEIGHT_SETTER_IDENTITY_DIR` instead of `BASE_VALIDATOR_IDENTITY_DIR`, + which is now the verify-only role's alone. Set it in + `deploy/env/python-validator.env` to the directory holding `consensus.key`, + `tls.crt` and `tls.key`, and point `BASE_VALIDATOR_IDENTITY_DIR` at a + separate directory without `consensus.key`; `scripts/check_deploy.py` refuses + one path, or one state volume, for both roles. ### Upgrade (BREAKING for masters) diff --git a/deploy/README.md b/deploy/README.md index 85a3a32b5..0a482537c 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -169,8 +169,11 @@ credential-free `wss://` URLs. The weight-setter mounts the wallet directory read-only at `/run/wallets`; the verify-only role mounts none. The weight-setter's identity directory contains -the files below; the verify-only role reads only `tls.crt` and `tls.key`, so give -it a separate directory without `consensus.key`: +the files below and is set with `BASE_WEIGHT_SETTER_IDENTITY_DIR`; the verify-only +role reads only `tls.crt` and `tls.key` from its own `BASE_VALIDATOR_IDENTITY_DIR`, +a separate directory without `consensus.key`. `scripts/check_deploy.py` refuses +the same path for both roles. Both variables live in the one +`deploy/env/python-validator.env` passed with `--env-file`: ```text consensus.key @@ -218,8 +221,10 @@ Before this release `role-validator.yml` held the wallet and submitted. It is no verify-only: after the upgrade it keeps running, stays healthy and sets no weights. It ignores `BASE_WALLET_NAME`, `BASE_WALLET_HOTKEY`, `BASE_VALIDATOR_WALLETS_DIR` and the `consensus.key` in its identity directory (`BASE_GATEWAY_HOTKEY` is removed). -To keep setting weights, move to `role-weight-setter.yml`, whose state volume -`base-weight-setter-state` is new. Started empty, the weight-setter has no +To keep setting weights, move to `role-weight-setter.yml`: set +`BASE_WEIGHT_SETTER_IDENTITY_DIR` to the old identity directory (with +`consensus.key`) and give `BASE_VALIDATOR_IDENTITY_DIR` a new directory with only +`tls.crt` and `tls.key`. Its state volume `base-weight-setter-state` is new. Started empty, the weight-setter has no submission journal and no consensus watermarks: rollback protection restarts from the first profile `/v1/trust` serves it, and it does not know the epochs the old role already submitted (the chain rate-limit preflight still refuses a second diff --git a/deploy/compose/role-weight-setter.yml b/deploy/compose/role-weight-setter.yml index 72ca9d1ea..2341ba166 100644 --- a/deploy/compose/role-weight-setter.yml +++ b/deploy/compose/role-weight-setter.yml @@ -66,7 +66,7 @@ services: bind: create_host_path: false - type: bind - source: ${BASE_VALIDATOR_IDENTITY_DIR:?Set private validator identity directory} + source: ${BASE_WEIGHT_SETTER_IDENTITY_DIR:?Set the weight-setter identity directory, not the verify-only one} target: /run/validator read_only: true bind: diff --git a/deploy/env/python-validator.env.example b/deploy/env/python-validator.env.example index b3f76beab..1946f69b3 100644 --- a/deploy/env/python-validator.env.example +++ b/deploy/env/python-validator.env.example @@ -16,9 +16,12 @@ BASE_MAX_BLOCK_LAG=256 # Optional absolute overrides; blank values use config and deploy/secrets/wallets. BASE_VALIDATOR_WALLETS_DIR= BASE_CONFIG_DIR= -# Weight-setter: directory holds consensus.key (same hotkey as the wallet), tls.crt, tls.key. -# Verify-only role: reads only tls.crt and tls.key; point it at a directory WITHOUT -# consensus.key (one directory per role; never reuse the weight-setter's). -# Trust directory also contains peers.json: {"": "https://host:8091"}. +# Verify-only role (role-validator.yml): directory with tls.crt and tls.key only, +# WITHOUT consensus.key. BASE_VALIDATOR_IDENTITY_DIR= +# Weight-setter (role-weight-setter.yml): its own directory with consensus.key (same +# hotkey as the wallet), tls.crt and tls.key. check_deploy.py refuses the same path for +# both roles. +BASE_WEIGHT_SETTER_IDENTITY_DIR= +# The config directory also contains peers.json: {"": "https://host:8091"}. BASE_VALIDATOR_BIND_ADDRESS= diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 78eb94c7e..ac340207d 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -94,6 +94,8 @@ with a validator hotkey and consensus seed: | `BASE_VERSION_KEY` | Exact live subnet validator version key | | `BASE_MIN_PEER_SAMPLE` | Required independent permitted-validator peer sample, 0-64 | | `BASE_MAX_BLOCK_LAG` | Maximum age of a sealed metagraph block, at least 1 | +| `BASE_VALIDATOR_IDENTITY_DIR` | Verify-only role identity directory: `tls.crt`, `tls.key`, no `consensus.key` | +| `BASE_WEIGHT_SETTER_IDENTITY_DIR` | Weight-setter identity directory: `consensus.key`, `tls.crt`, `tls.key`; must differ from the verify-only one | The version key is intentionally blank in the example. Populate it from the live subnet; do not infer it from tests. The CLI flags `--wallet-name` and diff --git a/scripts/check_deploy.py b/scripts/check_deploy.py index cacc017f1..914f64682 100644 --- a/scripts/check_deploy.py +++ b/scripts/check_deploy.py @@ -916,6 +916,7 @@ def validate_env_examples(master_source: str, validator_source: str) -> None: "BASE_VALIDATOR_WALLETS_DIR", "BASE_CONFIG_DIR", "BASE_VALIDATOR_IDENTITY_DIR", + "BASE_WEIGHT_SETTER_IDENTITY_DIR", "BASE_VALIDATOR_BIND_ADDRESS", } if set(validator) != validator_required: @@ -949,6 +950,22 @@ def validate_env_examples(master_source: str, validator_source: str) -> None: _wss_endpoints(validator["BASE_CHAIN_FALLBACK_ENDPOINTS"], "validator fallback endpoints") +def validate_role_separation(validator: dict, weight_setter: dict) -> None: + """The verify-only role must not read the weight-setter's identity or journal.""" + + def sources(config: dict) -> dict[str, str]: + mounts = config["services"][ROLES["validator"]]["volumes"] + return {item["target"]: str(item.get("source", "")) for item in mounts} + + verify_only, setter = sources(validator), sources(weight_setter) + identity = "/run/validator" + if os.path.normpath(verify_only[identity]) == os.path.normpath(setter[identity]): + raise ValueError("validator and weight-setter must use separate identity directories") + state = "/var/lib/base" + if verify_only[state] and verify_only[state] == setter[state]: + raise ValueError("validator and weight-setter must use separate state volumes") + + def _bind_sources(role: str, compose_dir: Path = ROOT / "deploy/compose") -> None: """Every bind mount in the committed role file opts out of host-path creation.""" text = (compose_dir / f"role-{role}.yml").read_text() @@ -1018,7 +1035,8 @@ def check_examples() -> None: "BASE_WALLET_HOTKEY": "validator", "BASE_VERSION_KEY": "1", "BASE_VALIDATOR_BIND_ADDRESS": "127.0.0.2", - "BASE_VALIDATOR_IDENTITY_DIR": "/fixture/identity", + "BASE_VALIDATOR_IDENTITY_DIR": "/fixture/verify-only-identity", + "BASE_WEIGHT_SETTER_IDENTITY_DIR": "/fixture/weight-setter-identity", } ) # The runner's Compose still stats the default `env_file` even under @@ -1038,6 +1056,7 @@ def check_examples() -> None: def _render_examples(env: dict[str, str]) -> None: + rendered = {} for role in ("master", "validator", "weight-setter"): result = subprocess.run( [ @@ -1063,6 +1082,8 @@ def _render_examples(env: dict[str, str]) -> None: validate_compose(config, role) _bind_sources(role) _default_sources(config, role) + rendered[role] = config + validate_role_separation(rendered["validator"], rendered["weight-setter"]) def main() -> None: diff --git a/tests/test_deploy_contract.py b/tests/test_deploy_contract.py index 6ea4b6d4a..00b8294c4 100644 --- a/tests/test_deploy_contract.py +++ b/tests/test_deploy_contract.py @@ -636,3 +636,45 @@ def test_committed_bind_mounts_opt_out_of_host_path_creation(role, tmp_path): with pytest.raises(ValueError, match="bind paths must already exist"): CHECK["_bind_sources"](role, tmp_path) + + +def identity_source(config, source): + for mount in config["services"]["validator"]["volumes"]: + if mount["target"] == "/run/validator": + mount["source"] = source + return config + + +@pytest.mark.parametrize( + ("validator_dir", "setter_dir"), + [("/fixture/identity", "/fixture/identity"), ("/fixture/identity", "/fixture/x/../identity")], +) +def test_deploy_check_refuses_one_identity_dir_for_both_validator_roles(validator_dir, setter_dir): + with pytest.raises(ValueError, match="identity"): + CHECK["validate_role_separation"]( + identity_source(validator_config(), validator_dir), + identity_source(weight_setter_config(), setter_dir), + ) + + +def test_deploy_check_refuses_one_state_volume_for_both_validator_roles(): + validator = identity_source(validator_config(), "/fixture/verify-only") + setter = identity_source(weight_setter_config(), "/fixture/weight-setter") + for config in (validator, setter): + for mount in config["services"]["validator"]["volumes"]: + if mount["target"] == "/var/lib/base": + mount["source"] = "shared-state" + with pytest.raises(ValueError, match="state"): + CHECK["validate_role_separation"](validator, setter) + + +def test_deploy_check_accepts_separate_identity_dirs_and_the_roles_use_distinct_variables(): + CHECK["validate_role_separation"]( + identity_source(validator_config(), "/fixture/verify-only"), + identity_source(weight_setter_config(), "/fixture/weight-setter"), + ) + validator = (ROOT / "deploy/compose/role-validator.yml").read_text() + setter = (ROOT / "deploy/compose/role-weight-setter.yml").read_text() + assert "${BASE_VALIDATOR_IDENTITY_DIR:?" in validator + assert "${BASE_WEIGHT_SETTER_IDENTITY_DIR:?" in setter + assert "BASE_VALIDATOR_IDENTITY_DIR" not in setter From d90925589558b91cab561f5eb3f7c07301bad1f4 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:39:49 +0000 Subject: [PATCH 35/56] docs(changelog): describe algorithm 2 activation in the past tense The algorithm 2 entry still said activation "requires" an offline owner signature and described a "legacy signed 20/80 profile" in the present tense. Wire v2 removed profile signing, so the entry now records what shipped then and states that activation is a version bump in the unsigned config/challenges.toml. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a4cc62dc..6545eae8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,13 +104,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `BOUNTY_*` master settings). Bounty now runs as the CortexLM/bounty container; its operator guide migrates `bounty.sqlite3` and the session key unchanged. -- Owner-activated algorithm 2: each valid Bounty report earns one point, all - authors share proportionally, and ten valid reports unlock its full 30% of - emission. Proof retains 70%; unused Bounty mass burns to UID0. The legacy - signed 20/80 profile, algorithm 1 and historical sealed bytes remain intact. - Activation requires coordinated gateway/validator upgrades and an offline - owner signature over the new profile and epoch (superseded: the profile is now - unsigned and activation is a version bump in `config/challenges.toml`). +- Algorithm 2: each valid Bounty report earned one point, all authors shared + proportionally, and ten valid reports unlocked Bounty's full 30% of emission; + Proof kept 70% and unused Bounty mass burned to UID0. Algorithm 1 and its + 20/80 profile stayed supported for historical sealed bytes. Activation then + needed an offline owner signature over the profile; that signature was later + removed with wire v2, and activation is now a version bump in the unsigned + `config/challenges.toml`. - Bounty-only launch mode with a revision-pinned, fully paginated CortexLM backend feed, bounded public writes and readiness checks that fail closed. - Validator production preflight, resilient bounded master/validator Bittensor RPC failover and From 922112af45d90d5eac27ebb6f70ba351869a9eec Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:48:21 +0000 Subject: [PATCH 36/56] fix(gateway): scope the sealing refusal to the epoch being sealed The flag added in 5bcca27 was a boolean, so raw intake for every epoch, including the open tip epoch, got 409 for the whole of _emit and seal. That span covers every challenge get_weights call (up to 3 attempts of 180 s each). Only leaves for the epoch being sealed can reach that bundle. GatewayService.sealing now holds the epoch being sealed, or None, and accept_leaf refuses only a leaf for that epoch. The race test also posts an open-epoch leaf at every chain read and expects 202. Co-Authored-By: Claude Sonnet 5.5 --- src/cortex/gateway/service.py | 6 +++--- src/cortex/master.py | 4 ++-- tests/test_master.py | 8 ++++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/cortex/gateway/service.py b/src/cortex/gateway/service.py index b530a3cc3..2be1dcad3 100644 --- a/src/cortex/gateway/service.py +++ b/src/cortex/gateway/service.py @@ -52,8 +52,8 @@ def __init__( self.clock = clock or (lambda: datetime.now(UTC)) self.chain_endpoint = chain_endpoint self.trust_loader = trust_loader - # Set by the master between replacing an epoch's challenge leaves and sealing it. - self.sealing = False + # Epoch the master is between replacing challenge leaves for and sealing, if any. + self.sealing: int | None = None self.store.accept_trust(trust) def _chain_endpoint(self) -> str: @@ -79,7 +79,7 @@ def refresh_trust(self, epoch: int) -> None: self.trust = trust def accept_leaf(self, leaf: Leaf) -> dict: - if self.sealing: + if self.sealing == leaf.epoch: # Raw intake must not land between the authoritative replacement and the seal. raise ServiceError(409, "epoch sealing in progress") self.refresh_trust(leaf.epoch) diff --git a/src/cortex/master.py b/src/cortex/master.py index c503c0333..dc53924e7 100644 --- a/src/cortex/master.py +++ b/src/cortex/master.py @@ -434,12 +434,12 @@ async def tick(self) -> list[int]: if pinned.block_hash != block_hash or pinned.block != block: raise ServiceError(503, "completed epoch pin changed") # The event loop serves intake during every await below; close it until sealed. - self.gateway.sealing = True + self.gateway.sealing = epoch try: await self._emit(epoch, pinned) await self.gateway.seal(epoch, block_b=block) finally: - self.gateway.sealing = False + self.gateway.sealing = None self.journal.sealed(epoch) completed.append(epoch) # Keep each newly completed seal observable for one polling interval. diff --git a/tests/test_master.py b/tests/test_master.py index f51c0f6bd..05bb7bf8d 100644 --- a/tests/test_master.py +++ b/tests/test_master.py @@ -638,7 +638,7 @@ async def test_intake_bearer_cannot_inject_a_leaf_while_the_epoch_is_being_seale intake = {"authorization": "Bearer fixture-intake-token-0123456789abcdef"} operator = {"authorization": "Bearer fixture-operator-token-0123456789abcdef"} hotkey = chain.rows[0].hotkey - snapshot, posted = chain.snapshot, [] + snapshot, posted, open_epoch = chain.snapshot, [], [] async with httpx.AsyncClient( transport=httpx.ASGITransport(runtime.app()), base_url="https://master" ) as client: @@ -647,6 +647,9 @@ async def racing_snapshot(block, netuid): # Each chain read of the completed-epoch cycle is an await an intake can land in. forged = leaf_request(Leaf(b"bounty", hotkey, 12, Score(10**9 + len(posted)))) posted.append(await client.post("/v1/weights/raw", json=forged, headers=intake)) + # The open epoch is not part of this seal and stays writable. + tip = leaf_request(Leaf(b"bounty", hotkey, 13, Score(len(posted)))) + open_epoch.append(await client.post("/v1/weights/raw", json=tip, headers=intake)) return await snapshot(block, netuid) await runtime.emitter.tick() @@ -658,11 +661,12 @@ async def racing_snapshot(block, netuid): # the seal's own snapshot read is inside the window. assert [response.status_code for response in posted] == [202, 202, 409] assert posted[-1].json() == {"error": "epoch sealing in progress"} + assert [response.status_code for response in open_epoch] == [202, 202, 202] sealed = Bundle.decode(runtime.gateway.store.bundle(12).encoded) leaves = sealed.body.leaves assert all(leaf.score == NoScore(NoScoreReason.CHALLENGE_INTERNAL) for leaf in leaves) # Outside the window the same bearer is accepted for the open epoch. - open_leaf = leaf_request(Leaf(b"bounty", hotkey, 13, Score(5))) + open_leaf = leaf_request(Leaf(b"bounty", hotkey, 13, Score(50))) response = await client.post("/v1/weights/raw", json=open_leaf, headers=intake) assert response.status_code == 202 # Sealing remains the operator's alone. From 1a7754fe24478216f9b43f572568b33fcd4409f9 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:49:52 +0000 Subject: [PATCH 37/56] fix(cli): keep watermark-reset preflight free of sidecars and schema Review of 6533e1a found two writes left on a refused target. A mode=ro open of a foreign WAL database still creates -shm/-wal files beside it. A file holding consensus_watermarks and the row, but none of the journal's other tables, passed the preflight and then gained four tables from EvidenceStore's CREATE TABLE IF NOT EXISTS. The preflight reads with immutable=1 when no -wal file exists (every committed page is then in the main file) and falls back to mode=ro otherwise. It requires both consensus_watermarks and consensus_watermark_resets, and the write path builds EvidenceStore with create_schema=False, so the reset never adds schema. Co-Authored-By: Claude Sonnet 5.5 --- src/cortex/cli.py | 15 ++++++++++----- src/cortex/validator/evidence.py | 4 +++- tests/validator/test_validator.py | 14 +++++++++++--- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/cortex/cli.py b/src/cortex/cli.py index 932d2597a..d4065747b 100644 --- a/src/cortex/cli.py +++ b/src/cortex/cli.py @@ -244,12 +244,17 @@ def run_validator_reconcile(args: argparse.Namespace) -> None: def _watermark_reset_preflight(args: argparse.Namespace) -> None: """Prove the file is a validator journal holding the row, without writing to it.""" - reader = sqlite3.connect(f"{args.state_db.resolve().as_uri()}?mode=ro", uri=True) + path = args.state_db.resolve() + # mode=ro still creates -shm/-wal beside a WAL database. Without a -wal file every + # committed page is in the main file, so an immutable read is exact and creates none. + wal = path.with_name(path.name + "-wal").exists() + reader = sqlite3.connect(f"{path.as_uri()}?{'mode=ro' if wal else 'immutable=1'}", uri=True) try: - table = reader.execute( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name='consensus_watermarks'" + tables = reader.execute( + "SELECT count(*) FROM sqlite_master WHERE type='table' " + "AND name IN ('consensus_watermarks', 'consensus_watermark_resets')" ).fetchone() - if table is None: + if tables[0] != 2: raise SystemExit("Cortex command failed: not a validator state database") rows = dict( reader.execute( @@ -281,7 +286,7 @@ def run_validator_watermark_reset(args: argparse.Namespace) -> None: _watermark_reset_preflight(args) # The journal schema and WAL mode belong to the validator; open only the existing file. connection = sqlite3.connect(args.state_db, isolation_level=None, timeout=1) - old = EvidenceStore(connection).reset_watermark(args.name, args.to) + old = EvidenceStore(connection, create_schema=False).reset_watermark(args.name, args.to) except ProtocolError as error: raise SystemExit(f"Cortex command failed: {error}") from None except sqlite3.OperationalError as error: diff --git a/src/cortex/validator/evidence.py b/src/cortex/validator/evidence.py index a7805b1cc..9bb034e6d 100644 --- a/src/cortex/validator/evidence.py +++ b/src/cortex/validator/evidence.py @@ -13,8 +13,10 @@ class EvidenceStore: - def __init__(self, connection: sqlite3.Connection): + def __init__(self, connection: sqlite3.Connection, *, create_schema: bool = True): self.connection = connection + if not create_schema: + return connection.executescript(""" CREATE TABLE IF NOT EXISTS consensus_roots ( epoch TEXT NOT NULL, hotkey BLOB NOT NULL, root BLOB NOT NULL, diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py index 342c71329..be7277662 100644 --- a/tests/validator/test_validator.py +++ b/tests/validator/test_validator.py @@ -763,7 +763,8 @@ def snapshot_file(path): import sqlite3 stat = path.stat() - connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True) + # immutable=1 reads without creating -shm/-wal sidecars, unlike mode=ro. + connection = sqlite3.connect(f"{path.resolve().as_uri()}?immutable=1", uri=True) try: mode = connection.execute("PRAGMA journal_mode").fetchone()[0] tables = connection.execute("SELECT name FROM sqlite_master ORDER BY name").fetchall() @@ -773,7 +774,7 @@ def snapshot_file(path): return path.read_bytes(), stat.st_mtime_ns, mode, tables, siblings -@pytest.mark.parametrize("kind", ["empty", "foreign"]) +@pytest.mark.parametrize("kind", ["empty", "foreign", "foreign-wal", "partial"]) def test_cli_watermark_reset_never_mutates_a_foreign_state_file(tmp_path, kind): import sqlite3 @@ -782,9 +783,16 @@ def test_cli_watermark_reset_never_mutates_a_foreign_state_file(tmp_path, kind): if kind == "empty": state.touch() else: - with sqlite3.connect(state) as connection: + connection = sqlite3.connect(state) + if kind == "foreign-wal": + connection.execute("PRAGMA journal_mode=WAL") + if kind == "partial": # the watermark row alone, without the journal's other tables + connection.execute("CREATE TABLE consensus_watermarks (name TEXT, value TEXT)") + connection.execute("INSERT INTO consensus_watermarks VALUES ('epoch', '12')") + else: connection.execute("CREATE TABLE notes (body TEXT)") connection.execute("INSERT INTO notes VALUES ('kept')") + connection.commit() connection.close() before = snapshot_file(state) with pytest.raises(SystemExit, match="not a validator state database"): From 5fb36cdaa70bbe97382d0e5d676fe4ebcb90ca6c Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:50:45 +0000 Subject: [PATCH 38/56] fix(deploy): check the operator env file and real volume names Review of 7c71399 found that validate_role_separation only ran inside --check-examples on hard-coded fixture paths, so an operator's python-validator.env with one identity directory for both roles was never checked. The state comparison used the rendered mount source, which is the volume key, not the project-prefixed Docker name. New --validator-env mode refuses an env file whose two identity variables resolve to one path. The state check now compares the rendered top-level volumes[key].name when present. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 5 +++-- deploy/README.md | 5 +++-- scripts/check_deploy.py | 25 +++++++++++++++++++++++-- tests/test_deploy_contract.py | 25 +++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6545eae8e..00f78695c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,8 +26,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 which is now the verify-only role's alone. Set it in `deploy/env/python-validator.env` to the directory holding `consensus.key`, `tls.crt` and `tls.key`, and point `BASE_VALIDATOR_IDENTITY_DIR` at a - separate directory without `consensus.key`; `scripts/check_deploy.py` refuses - one path, or one state volume, for both roles. + separate directory without `consensus.key`. `scripts/check_deploy.py + --validator-env deploy/env/python-validator.env` refuses one path for both + roles; `--check-examples` also refuses one rendered state volume. ### Upgrade (BREAKING for masters) diff --git a/deploy/README.md b/deploy/README.md index 0a482537c..305cd9aa3 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -171,8 +171,9 @@ The weight-setter mounts the wallet directory read-only at `/run/wallets`; the verify-only role mounts none. The weight-setter's identity directory contains the files below and is set with `BASE_WEIGHT_SETTER_IDENTITY_DIR`; the verify-only role reads only `tls.crt` and `tls.key` from its own `BASE_VALIDATOR_IDENTITY_DIR`, -a separate directory without `consensus.key`. `scripts/check_deploy.py` refuses -the same path for both roles. Both variables live in the one +a separate directory without `consensus.key`. Check your env file with +`uv run python scripts/check_deploy.py --validator-env deploy/env/python-validator.env`; +it refuses the same path for both roles. Both variables live in the one `deploy/env/python-validator.env` passed with `--env-file`: ```text diff --git a/scripts/check_deploy.py b/scripts/check_deploy.py index 914f64682..6e5961733 100644 --- a/scripts/check_deploy.py +++ b/scripts/check_deploy.py @@ -955,7 +955,14 @@ def validate_role_separation(validator: dict, weight_setter: dict) -> None: def sources(config: dict) -> dict[str, str]: mounts = config["services"][ROLES["validator"]]["volumes"] - return {item["target"]: str(item.get("source", "")) for item in mounts} + named = config.get("volumes") or {} + # A rendered named volume's `source` is its key; Docker uses the project-prefixed name. + return { + item["target"]: str( + (named.get(item.get("source")) or {}).get("name") or item.get("source", "") + ) + for item in mounts + } verify_only, setter = sources(validator), sources(weight_setter) identity = "/run/validator" @@ -966,6 +973,15 @@ def sources(config: dict) -> dict[str, str]: raise ValueError("validator and weight-setter must use separate state volumes") +def validate_validator_env(source: str) -> None: + """An operator's validator env file must not give both roles one identity directory.""" + values = _env_file(source) + verify_only = values.get("BASE_VALIDATOR_IDENTITY_DIR", "") + setter = values.get("BASE_WEIGHT_SETTER_IDENTITY_DIR", "") + if verify_only and setter and Path(verify_only).resolve() == Path(setter).resolve(): + raise ValueError("validator and weight-setter must use separate identity directories") + + def _bind_sources(role: str, compose_dir: Path = ROOT / "deploy/compose") -> None: """Every bind mount in the committed role file opts out of host-path creation.""" text = (compose_dir / f"role-{role}.yml").read_text() @@ -1093,6 +1109,9 @@ def main() -> None: modes.add_argument("--check-examples", action="store_true") modes.add_argument("--config", type=Path, help="rendered Compose JSON; use - for stdin") modes.add_argument("--vm-host-config", type=Path, help="materialized Proof host TOML") + modes.add_argument( + "--validator-env", type=Path, help="operator validator env file (python-validator.env)" + ) parser.add_argument("--role", choices=list(ROLES)) args = parser.parse_args() try: @@ -1103,11 +1122,13 @@ def main() -> None: elif args.config and args.role: source = sys.stdin.read() if str(args.config) == "-" else args.config.read_text() validate_compose(json.loads(source), args.role) + elif args.validator_env and not args.role: + validate_validator_env(args.validator_env.read_text()) elif args.vm_host_config and not args.role: with args.vm_host_config.open("rb") as stream: validate_vm_host(tomllib.load(stream)) else: - parser.error("--config requires --role; --vm-host-config forbids it") + parser.error("--config requires --role; --vm-host-config and --validator-env forbid it") except ValueError as error: raise SystemExit(f"Deployment check failed: {error}") from None except subprocess.CalledProcessError as error: diff --git a/tests/test_deploy_contract.py b/tests/test_deploy_contract.py index 00b8294c4..16709e8f8 100644 --- a/tests/test_deploy_contract.py +++ b/tests/test_deploy_contract.py @@ -678,3 +678,28 @@ def test_deploy_check_accepts_separate_identity_dirs_and_the_roles_use_distinct_ assert "${BASE_VALIDATOR_IDENTITY_DIR:?" in validator assert "${BASE_WEIGHT_SETTER_IDENTITY_DIR:?" in setter assert "BASE_VALIDATOR_IDENTITY_DIR" not in setter + + +def test_rendered_named_volumes_compare_by_their_docker_name(): + validator = identity_source(validator_config(), "/fixture/verify-only") + setter = identity_source(weight_setter_config(), "/fixture/weight-setter") + for config, key in ((validator, "base-validator-state"), (setter, "base-weight-setter-state")): + for mount in config["services"]["validator"]["volumes"]: + if mount["target"] == "/var/lib/base": + mount["source"] = key + config["volumes"] = {key: {"name": f"{key}-project"}} + CHECK["validate_role_separation"](validator, setter) + setter["volumes"]["base-weight-setter-state"]["name"] = "base-validator-state-project" + with pytest.raises(ValueError, match="state"): + CHECK["validate_role_separation"](validator, setter) + + +def test_operator_validator_env_refuses_one_identity_dir_for_both_roles(tmp_path): + example = (ROOT / "deploy/env/python-validator.env.example").read_text() + shared = example.replace( + "BASE_VALIDATOR_IDENTITY_DIR=", f"BASE_VALIDATOR_IDENTITY_DIR={tmp_path}/id" + ).replace("BASE_WEIGHT_SETTER_IDENTITY_DIR=", f"BASE_WEIGHT_SETTER_IDENTITY_DIR={tmp_path}/id/") + with pytest.raises(ValueError, match="identity"): + CHECK["validate_validator_env"](shared) + separate = shared.replace(f"={tmp_path}/id/", f"={tmp_path}/setter") + CHECK["validate_validator_env"](separate) From b3ec7dbe9549cf123ab73916d3ea0d62f1088ec7 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:44:34 +0000 Subject: [PATCH 39/56] fix(protocol): cap challenges_version like measurements_version A v3 profile with challenges_version=2**32-1 passed validation, moved the validator watermark and then made every honest profile fail with "consensus rollback refused". TrustRoot.validate() now refuses a value above MAX_CHALLENGES_VERSION (1000) next to the measurements check, so master load_trust, gateway accept_trust and validator trust_from_json all reject it before any watermark write. Co-Authored-By: Claude Sonnet 5.5 --- src/cortex/protocol/models.py | 8 +++- tests/validator/test_validator.py | 64 +++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/cortex/protocol/models.py b/src/cortex/protocol/models.py index 0bfb41b77..93fe8ad63 100644 --- a/src/cortex/protocol/models.py +++ b/src/cortex/protocol/models.py @@ -16,9 +16,11 @@ # Algorithm 3: a challenge's leaves sum to at most this; the shortfall burns to UID0. FULL_SHARE_SCORE = 10**12 CHALLENGE_ID = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") -# Highest measurement allowlist version this release accepts. A served profile above it is -# refused before the rollback watermark moves; raising it needs a validator release. +# Highest measurement allowlist and challenge profile versions this release accepts. A served +# profile above either is refused before the rollback watermark moves; raising one needs a +# validator release. MAX_MEASUREMENTS_VERSION = 1000 +MAX_CHALLENGES_VERSION = 1000 class NoScoreReason(IntEnum): @@ -190,6 +192,8 @@ def validate(self) -> None: fixed(self.measurements_digest, 32) uint(self.challenges_version, 4) uint(self.measurements_version, 4) + if self.challenges_version > MAX_CHALLENGES_VERSION: + raise ProtocolError("challenges version above the supported maximum") if self.measurements_version > MAX_MEASUREMENTS_VERSION: raise ProtocolError("measurements version above the supported maximum") uint(self.introduced_epoch, 8) diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py index be7277662..96f394349 100644 --- a/tests/validator/test_validator.py +++ b/tests/validator/test_validator.py @@ -759,6 +759,70 @@ def test_measurements_version_at_the_supported_maximum_is_accepted(network): replace(network.validator.trust, measurements_version=MAX_MEASUREMENTS_VERSION).validate() +def serve_v3_profile(network, challenges_version): + """A v3-sealed epoch whose /v1/trust answer carries the given challenges_version.""" + from cortex.protocol.trust import trust_to_json + + honest = replace(network.validator.trust, challenges_version=3) + leaves = tuple( + Leaf(name, row.hotkey, 12, Score(50) if row.uid == 1 else NoScore()) + for name in (b"bounty", b"proof") + for row in network.chain.view.rows + ) + network.bundle = build_bundle( + epoch=12, + netuid=541, + block_b=99, + block_hash=bytes([9]) * 32, + rows=network.chain.view.rows, + leaves=leaves, + trust=honest, + ) + floats = aggregate_leaves( + network.bundle.body.leaves, honest.shares, network.bundle.body.uid_map, algorithm_version=3 + ) + network.latest_overrides.update(uids=list(floats.uids), weights=list(floats.weights)) + served = {"profile": {**trust_to_json(honest), "challenges_version": challenges_version}} + + def transport(request): + if request.url.path == "/v1/trust": + return httpx.Response(200, json=served["profile"]) + return network.handler(request) + + validator = Validator( + gateway_url="https://master.invalid", + netuid=541, + chain=network.chain, + journal=network.journal, + http=httpx.AsyncClient(transport=httpx.MockTransport(transport)), + ) + return validator, served, trust_to_json(honest) + + +@pytest.mark.parametrize("value", [1001, 2**32 - 1, True, None, "5", 1.0, -1], ids=repr) +async def test_challenges_version_above_cap_is_refused_before_watermark(network, value): + import sqlite3 + + validator, served, honest = serve_v3_profile(network, value) + with pytest.raises(ProtocolError, match="invalid trust profile"): + await validator.run_once() + assert network.chain.submissions == [] + with sqlite3.connect(network.db_path) as connection: + assert connection.execute("SELECT name FROM consensus_watermarks").fetchall() == [ + ("netuid",) + ] + served["profile"] = honest + assert (await validator.run_once()).outcome == "submitted" + + +async def test_challenges_version_boundary_1000_accepted(network): + from cortex.protocol.models import MAX_CHALLENGES_VERSION + + assert MAX_CHALLENGES_VERSION == 1000 + validator, _, _ = serve_v3_profile(network, MAX_CHALLENGES_VERSION) + assert (await validator.run_once()).outcome == "submitted" + + def snapshot_file(path): import sqlite3 From 3eda46604f1e0b223dd4b2d420ed52d0779038d5 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:45:14 +0000 Subject: [PATCH 40/56] docs(changelog): explain the version cap upgrade consequence A state written before the cap may hold a version above 1000; after the upgrade the gateway refuses it as a corrupt stored profile and the validator refuses every honest profile as a rollback. Operators need the remedy in the upgrade block, not only the new refusal in Changed. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00f78695c..8233a715d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 separate directory without `consensus.key`. `scripts/check_deploy.py --validator-env deploy/env/python-validator.env` refuses one path for both roles; `--check-examples` also refuses one rendered state volume. +- A `measurements_version` or `challenges_version` above 1000 is refused. A + gateway `state_dir` that stored such a profile fails with `corrupt stored + trust profile`, and a validator journal whose watermark holds such a value + refuses every honest profile with `consensus rollback refused`. Start the + gateway on a fresh `state_dir` (wire v2 already requires one); start the + validator on a fresh state database or lower the watermark with `cortex + validator-watermark-reset`. ### Upgrade (BREAKING for masters) @@ -87,7 +94,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `POST /v1/weights/raw` answers 409 `epoch sealing in progress` while the master replaces a completed epoch's challenge leaves and seals it. - A challenge profile whose `measurements_version` exceeds - `MAX_MEASUREMENTS_VERSION` (1000) is refused by the master and by validators + `MAX_MEASUREMENTS_VERSION` (1000) or whose `challenges_version` exceeds + `MAX_CHALLENGES_VERSION` (1000) is refused by the master and by validators before the rollback watermark moves. ### Removed From dfad4d305d8758dbbd6a2297f4351dcfce645788 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:46:11 +0000 Subject: [PATCH 41/56] test(master): pin the intake refusal across the whole emit window The existing race test posts during the chain reads and the seal's own snapshot read, so moving `gateway.sealing = epoch` after `_emit` still passed it. This test races an intake POST inside Proof readiness, after bounty's leaves are replaced, and fails with 202 under that mutation. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_master.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_master.py b/tests/test_master.py index 05bb7bf8d..f7890d6ca 100644 --- a/tests/test_master.py +++ b/tests/test_master.py @@ -675,6 +675,33 @@ async def racing_snapshot(block, netuid): assert (await client.post("/v1/admin/seal", json=body, headers=operator)).status_code == 200 +async def test_emit_window_refuses_intake(master): + from cortex.gateway import leaf_request + + runtime, chain = master.runtime, master.chain + intake = {"authorization": "Bearer fixture-intake-token-0123456789abcdef"} + forged = leaf_request(Leaf(b"bounty", chain.rows[0].hotkey, 12, Score(10**9))) + backend, posted = runtime.emitter.proof.backend, [] + async with httpx.AsyncClient( + transport=httpx.ASGITransport(runtime.app()), base_url="https://master" + ) as client: + + class RacingBackend: + async def readiness(self): + # Proof readiness is awaited inside _emit, after bounty's leaves are replaced. + posted.append(await client.post("/v1/weights/raw", json=forged, headers=intake)) + return await backend.readiness() + + await runtime.emitter.tick() + chain.state = EpochState(13, 100, 105) + runtime.emitter.proof.backend = RacingBackend() + assert await runtime.emitter.tick() == [12] + runtime.emitter.proof.backend = backend + assert [response.status_code for response in posted] == [409] + leaves = Bundle.decode(runtime.gateway.store.bundle(12).encoded).body.leaves + assert all(leaf.score == NoScore(NoScoreReason.CHALLENGE_INTERNAL) for leaf in leaves) + + async def test_a_failed_seal_reopens_intake_for_its_epoch(master): from cortex.gateway import leaf_request From 18268913df49a1a14734b7b94182c52ecfe1aa7b Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:46:39 +0000 Subject: [PATCH 42/56] fix(config): map a deeply nested challenge profile to the clean 503 tomllib recurses once per nested array or inline table, so a profile with 100000 nested arrays raised RecursionError past the ServiceError mapping. It now answers "challenge profile unavailable" like any other malformed profile. Catching Exception was rejected: it would also hide programming errors in load_trust. Co-Authored-By: Claude Sonnet 5.5 --- src/cortex/config.py | 5 +++-- tests/protocol/test_trust_load.py | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/cortex/config.py b/src/cortex/config.py index 914fe78b7..7c569cfc7 100644 --- a/src/cortex/config.py +++ b/src/cortex/config.py @@ -208,7 +208,8 @@ def challenge_profile(self, epoch: int) -> TrustRoot: return load_trust( challenges_path=self.challenges_file, measurements_path=self.measurements_file ) - except (OSError, ValueError, KeyError, TypeError): + except (OSError, ValueError, KeyError, TypeError, RecursionError): # ProtocolError and TOMLDecodeError are ValueErrors; TypeError covers a field - # type that slips past the explicit checks in load_trust. + # type that slips past the explicit checks in load_trust; tomllib recurses per + # nested array or inline table. raise ServiceError(503, "challenge profile unavailable") from None diff --git a/tests/protocol/test_trust_load.py b/tests/protocol/test_trust_load.py index 6904a31b6..9a3ddb66c 100644 --- a/tests/protocol/test_trust_load.py +++ b/tests/protocol/test_trust_load.py @@ -114,3 +114,26 @@ def test_master_turns_a_placeholder_profile_into_a_clean_503(tmp_path, challenge with pytest.raises(ServiceError, match="challenge profile unavailable") as refused: config.challenge_profile(0) assert refused.value.status == 503 + + +@pytest.mark.parametrize("nested", ["challenges", "measurements"]) +def test_nested_toml_recursion_is_clean_error(tmp_path, nested): + from cortex.config import MasterConfig + from cortex.errors import ServiceError + + deep = "x = " + "[" * 100000 + "]" * 100000 + "\n" + (tmp_path / "c.toml").write_text(deep if nested == "challenges" else "version = 2") + (tmp_path / "m.toml").write_text(deep if nested == "measurements" else "version = 2") + config = MasterConfig( + netuid=541, + state_dir=tmp_path, + challenges_file=tmp_path / "c.toml", + measurements_file=tmp_path / "m.toml", + proof_seed_file=tmp_path / "unused", + operator_token_file=tmp_path / "unused", + intake_token_file=tmp_path / "unused", + challenge_secrets_dir=tmp_path, + ) + with pytest.raises(ServiceError, match="challenge profile unavailable") as refused: + config.challenge_profile(0) + assert refused.value.status == 503 From 60f7ac78a7c17643272246bf0be128ff9a179856 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:49:04 +0000 Subject: [PATCH 43/56] fix(cli): give watermark-reset fixed reasons for unusable state files A garbage file, a read-only file and a read-only directory printed only "DatabaseError" or "OperationalError", and the read-only file still got -shm/-wal sidecars from the write open. The reset now refuses a file or directory it cannot write before opening it ("state db is read-only") and maps a non-SQLite file to "not a validator state database". No path or SQLite message is printed. Co-Authored-By: Claude Sonnet 5.5 --- src/cortex/cli.py | 6 ++++++ tests/validator/test_validator.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/cortex/cli.py b/src/cortex/cli.py index d4065747b..76f8bd483 100644 --- a/src/cortex/cli.py +++ b/src/cortex/cli.py @@ -284,6 +284,10 @@ def run_validator_watermark_reset(args: argparse.Namespace) -> None: connection = None try: _watermark_reset_preflight(args) + path = args.state_db.resolve() + # A refused write must leave no -wal/-shm beside the file; check before opening it. + if not (os.access(path, os.W_OK) and os.access(path.parent, os.W_OK | os.X_OK)): + raise SystemExit("Cortex command failed: state db is read-only") # The journal schema and WAL mode belong to the validator; open only the existing file. connection = sqlite3.connect(args.state_db, isolation_level=None, timeout=1) old = EvidenceStore(connection, create_schema=False).reset_watermark(args.name, args.to) @@ -295,6 +299,8 @@ def run_validator_watermark_reset(args: argparse.Namespace) -> None: "Cortex command failed: state db locked; stop the validator first" ) from None raise SystemExit(f"Cortex command failed: {type(error).__name__}") from None + except sqlite3.DatabaseError: + raise SystemExit("Cortex command failed: not a validator state database") from None except sqlite3.Error as error: raise SystemExit(f"Cortex command failed: {type(error).__name__}") from None finally: diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py index 96f394349..c10f29b13 100644 --- a/tests/validator/test_validator.py +++ b/tests/validator/test_validator.py @@ -612,6 +612,35 @@ def test_cli_watermark_reset_reports_a_non_sqlite_state_file_without_traceback(t reset(state, "--name", "epoch", "--to", "0", "--confirm") +@pytest.mark.parametrize( + ("kind", "reason"), + [ + ("garbage", "not a validator state database"), + ("read-only-file", "state db is read-only"), + ("read-only-dir", "state db is read-only"), + ], +) +def test_watermark_reset_garbage_db_message_has_no_path(network, kind, reason): + network.journal.evidence.watermark("epoch", 12) + network.journal.close() + state = network.db_path + if kind == "garbage": + state.write_bytes(b"not a database" * 100) + elif kind == "read-only-file": + state.chmod(0o444) + else: + state.parent.chmod(0o555) + before = sorted(item.name for item in state.parent.iterdir()) + try: + with pytest.raises(SystemExit) as refused: + reset(state, "--name", "epoch", "--to", "11", "--confirm") + assert sorted(item.name for item in state.parent.iterdir()) == before + finally: + state.parent.chmod(0o755) + state.chmod(0o644) + assert refused.value.code == f"Cortex command failed: {reason}" + + async def test_peer_api_serves_signed_evidence_and_never_claims_dcap_verified(network): from cortex.protocol.consensus import RootStatement from cortex.validator.evidence import peer_app From 11613191a7d9ae5e8edddde4d417a75730393e65 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:50:27 +0000 Subject: [PATCH 44/56] fix(cli): read the watermark-reset preflight without sidecars With an orphan -wal beside the file (a crash image or a copy taken under a live writer) the preflight fell back to mode=ro, and SQLite created -shm and replayed into it even for a foreign database. The preflight now always reads the main file with immutable=1. Rows that only live in the -wal are invisible to it, so a refusal then adds a checkpoint hint, and a missing netuid with a -wal present is refused rather than skipping --netuid. The write open still re-reads the real value under BEGIN IMMEDIATE. Rejected: copying the file and its -wal to a temporary directory; it reads every byte and still needs the same trust in the copy. Co-Authored-By: Claude Sonnet 5.5 --- src/cortex/cli.py | 20 +++++++++++++------ tests/validator/test_validator.py | 33 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/cortex/cli.py b/src/cortex/cli.py index 76f8bd483..35edadef3 100644 --- a/src/cortex/cli.py +++ b/src/cortex/cli.py @@ -245,17 +245,23 @@ def run_validator_reconcile(args: argparse.Namespace) -> None: def _watermark_reset_preflight(args: argparse.Namespace) -> None: """Prove the file is a validator journal holding the row, without writing to it.""" path = args.state_db.resolve() - # mode=ro still creates -shm/-wal beside a WAL database. Without a -wal file every - # committed page is in the main file, so an immutable read is exact and creates none. + # mode=ro still creates -shm (and opens -wal) beside a WAL database, even a foreign one, + # so read only the main file. Pages still in a -wal are invisible: a refusal then names + # the checkpoint, and the write open below re-checks the real values. wal = path.with_name(path.name + "-wal").exists() - reader = sqlite3.connect(f"{path.as_uri()}?{'mode=ro' if wal else 'immutable=1'}", uri=True) + hint = "; stop the validator so SQLite checkpoints its -wal" if wal else "" + + def refuse(reason: str) -> SystemExit: + return SystemExit(f"Cortex command failed: {reason}{hint}") + + reader = sqlite3.connect(f"{path.as_uri()}?immutable=1", uri=True) try: tables = reader.execute( "SELECT count(*) FROM sqlite_master WHERE type='table' " "AND name IN ('consensus_watermarks', 'consensus_watermark_resets')" ).fetchone() if tables[0] != 2: - raise SystemExit("Cortex command failed: not a validator state database") + raise refuse("not a validator state database") rows = dict( reader.execute( "SELECT name, value FROM consensus_watermarks WHERE name IN (?, 'netuid')", @@ -265,12 +271,14 @@ def _watermark_reset_preflight(args: argparse.Namespace) -> None: finally: reader.close() if args.name not in rows: - raise SystemExit("Cortex command failed: consensus watermark not set") + raise refuse("consensus watermark not set") if args.netuid is not None: + if "netuid" not in rows and wal: + raise refuse("validator journal netuid unreadable") if "netuid" not in rows: print("state db records no netuid; --netuid check skipped", file=sys.stderr) elif rows["netuid"] != str(args.netuid): - raise SystemExit("Cortex command failed: validator journal belongs to another subnet") + raise refuse("validator journal belongs to another subnet") def run_validator_watermark_reset(args: argparse.Namespace) -> None: diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py index c10f29b13..5948f0780 100644 --- a/tests/validator/test_validator.py +++ b/tests/validator/test_validator.py @@ -893,6 +893,39 @@ def test_cli_watermark_reset_never_mutates_a_foreign_state_file(tmp_path, kind): assert snapshot_file(state) == before +@pytest.mark.parametrize("schema", ["foreign", "validator"]) +def test_watermark_reset_preflight_leaves_no_shm(network, tmp_path, schema): + import shutil + import sqlite3 + + network.journal.evidence.watermark("epoch", 12) + network.journal.close() + live = tmp_path / "live" / "state.db" + live.parent.mkdir() + if schema == "validator": + shutil.copy(network.db_path, live) + writer = sqlite3.connect(live) + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("CREATE TABLE notes (body TEXT)") + writer.execute("INSERT INTO notes VALUES ('only in the wal')") + writer.commit() + # A copy taken while the writer is open is a crash image: an orphan -wal and no -shm. + state = tmp_path / "orphan" / "state.db" + state.parent.mkdir() + for suffix in ("", "-wal"): + shutil.copy(live.with_name(live.name + suffix), state.with_name(state.name + suffix)) + writer.close() + + def listing(): + return sorted((item.name, item.stat().st_mtime_ns) for item in state.parent.iterdir()) + + before = listing() + # The validator row is behind the orphan -wal's frames for "foreign", ahead of them otherwise. + with pytest.raises(SystemExit): + reset(state, "--name", "epoch", "--to", "11", "--netuid", "7", "--confirm") + assert listing() == before + + def test_cli_watermark_reset_reports_a_locked_state_db(network): import sqlite3 From 41452405482746ec2c3773bfeb54691dfe7da113 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:51:13 +0000 Subject: [PATCH 45/56] fix(deploy): read validator env values as compose does and refuse nesting The --validator-env guard compared raw values, so "/srv/id" against /srv/id, or a value with an inline " #" comment, passed although both roles would mount one directory. A weight-setter directory nested in the verify-only one (or the reverse) also passed and exposes consensus.key to the verify-only role. The guard now strips matching quotes and inline comments and refuses equal or nested resolved paths. deploy/README.md still says "refuses the same path"; it is outside this change's scope and left as is. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 6 ++++-- scripts/check_deploy.py | 20 +++++++++++++++++--- tests/test_deploy_contract.py | 25 +++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8233a715d..0754bf02a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,8 +27,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `deploy/env/python-validator.env` to the directory holding `consensus.key`, `tls.crt` and `tls.key`, and point `BASE_VALIDATOR_IDENTITY_DIR` at a separate directory without `consensus.key`. `scripts/check_deploy.py - --validator-env deploy/env/python-validator.env` refuses one path for both - roles; `--check-examples` also refuses one rendered state volume. + --validator-env deploy/env/python-validator.env` refuses one path, or one + directory inside the other, for both roles (quotes and inline ` #` comments + are read as Compose reads them); `--check-examples` also refuses one rendered + state volume. - A `measurements_version` or `challenges_version` above 1000 is refused. A gateway `state_dir` that stored such a profile fails with `corrupt stored trust profile`, and a validator journal whose watermark holds such a value diff --git a/scripts/check_deploy.py b/scripts/check_deploy.py index 6e5961733..2d482ebe6 100644 --- a/scripts/check_deploy.py +++ b/scripts/check_deploy.py @@ -973,12 +973,26 @@ def sources(config: dict) -> dict[str, str]: raise ValueError("validator and weight-setter must use separate state volumes") +def _env_value(value: str) -> str: + """The value Compose reads from an env file: quotes removed, inline ` #` comment dropped.""" + value = value.strip() + if value[:1] in ("'", '"'): + closing = value.find(value[0], 1) + if closing > 0: + return value[1:closing] + return value.split(" #", 1)[0].strip() + + def validate_validator_env(source: str) -> None: """An operator's validator env file must not give both roles one identity directory.""" values = _env_file(source) - verify_only = values.get("BASE_VALIDATOR_IDENTITY_DIR", "") - setter = values.get("BASE_WEIGHT_SETTER_IDENTITY_DIR", "") - if verify_only and setter and Path(verify_only).resolve() == Path(setter).resolve(): + verify_only = _env_value(values.get("BASE_VALIDATOR_IDENTITY_DIR", "")) + setter = _env_value(values.get("BASE_WEIGHT_SETTER_IDENTITY_DIR", "")) + if not (verify_only and setter): + return + first, second = Path(verify_only).resolve(), Path(setter).resolve() + # One role nested in the other's directory reads the other's consensus.key too. + if first == second or first.is_relative_to(second) or second.is_relative_to(first): raise ValueError("validator and weight-setter must use separate identity directories") diff --git a/tests/test_deploy_contract.py b/tests/test_deploy_contract.py index 16709e8f8..0ac375b50 100644 --- a/tests/test_deploy_contract.py +++ b/tests/test_deploy_contract.py @@ -703,3 +703,28 @@ def test_operator_validator_env_refuses_one_identity_dir_for_both_roles(tmp_path CHECK["validate_validator_env"](shared) separate = shared.replace(f"={tmp_path}/id/", f"={tmp_path}/setter") CHECK["validate_validator_env"](separate) + + +@pytest.mark.parametrize( + ("verify_only", "setter"), + [ + ('"{tmp}/id"', "'{tmp}/id'"), + ("{tmp}/id # verify-only", "{tmp}/id"), + ("{tmp}/id", "{tmp}/id/setter"), + ("{tmp}/id/verify", '"{tmp}/id" # parent of the other'), + ], + ids=["quoted-equal", "inline-comment", "setter-nested", "verify-only-nested"], +) +def test_validator_env_quoted_comment_and_nested_dirs(tmp_path, verify_only, setter): + example = (ROOT / "deploy/env/python-validator.env.example").read_text() + + def render(first, second): + return example.replace( + "BASE_VALIDATOR_IDENTITY_DIR=", f"BASE_VALIDATOR_IDENTITY_DIR={first}" + ).replace("BASE_WEIGHT_SETTER_IDENTITY_DIR=", f"BASE_WEIGHT_SETTER_IDENTITY_DIR={second}") + + hostile = render(verify_only.format(tmp=tmp_path), setter.format(tmp=tmp_path)) + with pytest.raises(ValueError, match="identity"): + CHECK["validate_validator_env"](hostile) + # Siblings sharing a name prefix are separate directories. + CHECK["validate_validator_env"](render(f'"{tmp_path}/id" # ok', f"{tmp_path}/id-setter")) From 3196cc3dd52cd53b5e0d2715a2e802ea6f40c0b7 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:51:50 +0000 Subject: [PATCH 46/56] docs(security): state the 32-byte bearer minimum and per-request check Lens noted the key-role table and the bearer section of the threat model never say how long a bearer must be or when it is checked. read_bearer (src/cortex/http.py) refuses fewer than 32 bytes, and OperatorAuth re-reads both bearer files on every private request. Co-Authored-By: Claude Sonnet 5.5 --- AGENTS.md | 2 ++ docs/THREAT_MODEL.md | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index effc5d06d..3a987ee42 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,6 +77,8 @@ or permit UID is not a submit path. | challenge internal token | master bearer for a container's `get_weights`; never a signing key | | miner hotkey | signs Bounty pairing or Proof submission payloads | +Operator and intake bearers are at least 32 bytes and re-checked on every request. + Do not conflate gateway sealing (unsigned), master ownership and validator chain signing. Validators build their view from the gateway profile at `GET /v1/trust`; the gateway is the single source of truth. diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index c8fdfdf54..d8696a5cd 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -110,7 +110,9 @@ extrinsic and resulting chain state. and `POST /v1/admin/seal` requires the operator bearer (`BASE_GATEWAY_ADMIN_TOKEN_FILE`); each route refuses the other token with 401. The master refuses to start when both files resolve to the same path or hold the -same value. +same value. Bearers are at least 32 bytes and re-checked on every request: each +private request re-reads both files and answers 503 while either is shorter or +they are equal, so a rotation cannot silently weaken or merge them. While the master replaces a completed epoch's challenge leaves and seals it, raw intake answers 409 `epoch sealing in progress`, so a raw leaf cannot land after the authoritative replacement and before the seal; intake reopens when the seal From 5a330f703ed892a5ea4ae080faa95cd26c75c91c Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:52:33 +0000 Subject: [PATCH 47/56] test(cli): cover watermark-reset --netuid on a journal without netuid The table records netuid since bind_netuid, so --netuid stays; journals written before that have no row and the check is skipped with a warning. That branch had no test: replacing it with a strict comparison passed the suite. Co-Authored-By: Claude Sonnet 5.5 --- tests/validator/test_validator.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py index 5948f0780..99072dca1 100644 --- a/tests/validator/test_validator.py +++ b/tests/validator/test_validator.py @@ -951,3 +951,18 @@ def test_cli_watermark_reset_checks_the_recorded_netuid(network, capsys): reset(network.db_path, "--name", "epoch", "--to", "11", "--netuid", "7", "--confirm") reset(network.db_path, "--name", "epoch", "--to", "11", "--netuid", "541", "--confirm") assert json.loads(capsys.readouterr().out)["new_value"] == 11 + + +def test_cli_watermark_reset_without_a_recorded_netuid_warns_and_proceeds(network, capsys): + import sqlite3 + + network.journal.evidence.watermark("epoch", 12) + network.journal.close() + connection = sqlite3.connect(network.db_path) + connection.execute("DELETE FROM consensus_watermarks WHERE name='netuid'") + connection.commit() + connection.close() # the last close checkpoints the -wal into the main file + reset(network.db_path, "--name", "epoch", "--to", "11", "--netuid", "7", "--confirm") + captured = capsys.readouterr() + assert "state db records no netuid; --netuid check skipped" in captured.err + assert json.loads(captured.out)["new_value"] == 11 From 94a680af9a26cf5d1c479c15cb080ddbf57b8762 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:53:09 +0000 Subject: [PATCH 48/56] test(master): pin the 32-byte bearer boundary on a rotated intake file No test fixed MIN_BEARER_BYTES at 32 or proved the bearer file is read again per request. Rotating the intake file to 31 then 32 bytes must give 503 then 202; setting the minimum to 31 or 33 fails the test. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_master.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_master.py b/tests/test_master.py index f7890d6ca..e637c8f0c 100644 --- a/tests/test_master.py +++ b/tests/test_master.py @@ -702,6 +702,23 @@ async def readiness(self): assert all(leaf.score == NoScore(NoScoreReason.CHALLENGE_INTERNAL) for leaf in leaves) +async def test_rotated_intake_bearer_is_rechecked_at_the_32_byte_boundary(master): + from cortex.gateway import leaf_request + + runtime, chain = master.runtime, master.chain + leaf = leaf_request(Leaf(b"bounty", chain.rows[0].hotkey, 12, Score(7))) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(runtime.app()), base_url="https://master" + ) as client: + statuses = [] + for value in ("i" * 31, "i" * 32): + master.config.intake_token_file.write_text(value) + headers = {"authorization": f"Bearer {value}"} + response = await client.post("/v1/weights/raw", json=leaf, headers=headers) + statuses.append(response.status_code) + assert statuses == [503, 202] + + async def test_a_failed_seal_reopens_intake_for_its_epoch(master): from cortex.gateway import leaf_request From 3d4e315ad85cb81ffcdd6e7f77af227298bd8706 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:27:00 +0000 Subject: [PATCH 49/56] docs(config): describe profile reload and cautious restart in comments The profile is unsigned and has no public_key fields, and the master reloads it before epoch emission (refresh_trust in _emit), so the restart is the documented cautious workflow, not a requirement. Name the TOML key `version`; `challenges_version` is only the /v1/trust JSON field. State that a change between two seals can leave latest unsealed until the next seal. Co-Authored-By: Claude Sonnet 5.5 --- config/challenges-v2.example.toml | 7 ++++--- config/challenges-v3.example.toml | 5 +++-- config/challenges.toml | 10 +++++++--- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/config/challenges-v2.example.toml b/config/challenges-v2.example.toml index 40270a5b9..044a9a7c6 100644 --- a/config/challenges-v2.example.toml +++ b/config/challenges-v2.example.toml @@ -1,6 +1,7 @@ -# Activation template. Replace development public keys, choose the next profile -# version and activation epoch, then install it as config/challenges.toml and -# restart the master. No production activation occurs from this file. +# Activation template. Choose the next profile `version` and activation epoch, +# then install it as config/challenges.toml. The master reloads the profile before +# epoch emission; the cautious workflow restarts it between two epochs, just after +# a seal. No production activation occurs from this file. version = 2 introduced_epoch = "CHOOSE_ACTIVATION_EPOCH" diff --git a/config/challenges-v3.example.toml b/config/challenges-v3.example.toml index eaa732e7a..d163078cb 100644 --- a/config/challenges-v3.example.toml +++ b/config/challenges-v3.example.toml @@ -1,6 +1,7 @@ # Algorithm 3 template (profile version >= 3). Choose the shares, the next profile -# version and the activation epoch, replace development public keys, then install -# it as config/challenges.toml and restart the master. +# `version` and the activation epoch, then install it as config/challenges.toml. +# The master reloads the profile before epoch emission; the cautious workflow +# restarts it between two epochs, just after a seal. # Each challenge pays share * min(sum(leaves), 10^12) / 10^12; the rest burns. # Container ids also need a row in deploy/challenges/registry.toml to run. version = 3 diff --git a/config/challenges.toml b/config/challenges.toml index 16fa6e01d..09a92c2ac 100644 --- a/config/challenges.toml +++ b/config/challenges.toml @@ -1,8 +1,12 @@ # Development challenge profile (emission shares). The master reads # this unsigned file; the gateway serves it at GET /v1/trust and validators build their -# view from that. To change shares: edit, bump `version`, restart the master between two -# epochs. Never set `introduced_epoch` above the current epoch (sealing and decoding stop -# for older epochs). A third challenge or free shares need `challenges_version >= 3`. +# view from that. To change shares: edit and bump `version`; the master reloads the +# profile before epoch emission, and the documented cautious workflow restarts it between +# two epochs, just after a seal. A profile change between two seals can make +# `GET /v1/weights/latest` answer "unsealed" until the next seal; previous bundles stay +# available by epoch or root. Never set `introduced_epoch` above the current epoch +# (sealing and decoding stop for older epochs). A third challenge or free shares need +# `version >= 3` (served as `challenges_version` by /v1/trust). # # Two live challenges. Emission: bounty = 2000, proof = 8000 bps (sum 10000). # Readiness is independent of these shares: a missing feed, topic, baseline, From b05daf06d8f25052915399ac3403e18d08c59dac Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:27:21 +0000 Subject: [PATCH 50/56] docs(deploy): describe profile activation as cautious, not required The master reloads the profile before epoch emission, so the restart between two epochs is caution. A change between seals can leave latest unsealed, so validators stay disabled or accept unsealed until a bundle under the new profile is sealed. Do not claim a share-only change makes sealing fail; it is not guaranteed to. Co-Authored-By: Claude Sonnet 5.5 --- deploy/README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 305cd9aa3..f42143247 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -264,18 +264,22 @@ of truth and serves it at `GET /v1/trust`. 1. Edit the `[[challenges]]` shares (basis points, sum 10000) in `config/challenges.toml`. 2. Bump `version`. Adding a third challenge or arbitrary shares needs - `challenges_version >= 3` (1..64 unique ids summing to 10000); versions 1-2 only + `version >= 3` (1..64 unique ids summing to 10000); versions 1-2 only accept the 2000/8000 and 3000/7000 splits. Do NOT set `introduced_epoch` in the future: the profile is global, so the master then cannot seal epochs before it and the gateway cannot decode stored seals older than it (`/v1/weights/latest` answers "unsealed" and the emission burns). Keep the existing value, or the epoch of the first bundle the new profile must govern, never above the current epoch. The gateway store refuses a version rollback. -3. Run `python scripts/check_repo.py`, then restart the master between two epochs - (just after a seal), because the new shares apply to the next bundle it builds. -4. Validators pick up the new profile from `/v1/trust` at the next epoch; no - validator restart or file copy is needed. Check `GET /v1/trust` shows the new - `challenges_version`. +3. Run `python scripts/check_repo.py`. The master reloads the profile before each + epoch emission; the cautious workflow still restarts it between two epochs (just + after a seal), because the new shares apply to the next bundle it builds. A change + between two seals can make `GET /v1/weights/latest` answer "unsealed" until the + next seal; previous bundles stay available by epoch or root. A share-only change + is not guaranteed to make sealing fail, so check the next bundle. +4. Validators read the new profile from `/v1/trust`; no file copy is needed. Keep + validators disabled (or accept an `unsealed` answer) until a bundle under the new + profile is sealed. Check `GET /v1/trust` shows the new `challenges_version`. A registered container id still needs a row in `deploy/challenges/registry.toml` to run. From cb8179d938f9deccc45ef537e72ba43477447e7c Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:27:21 +0000 Subject: [PATCH 51/56] docs(validator): state the reset lock guard limits and drop gateway key SubmissionJournal is autocommit and holds no transaction between ticks; reset_watermark's BEGIN IMMEDIATE with timeout=1 fails only while the validator is inside a journal write. Keep stopping the validator as the required step and say a skipped stop can be undone by the next tick. The quickstart gateway hotkey bullet has no consumer since --gateway-public was removed. Co-Authored-By: Claude Sonnet 5.5 --- docs/external-miner/validators.md | 8 +++++--- docs/validator-quickstart.md | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/external-miner/validators.md b/docs/external-miner/validators.md index 68fa0d6e7..a71ed32fb 100644 --- a/docs/external-miner/validators.md +++ b/docs/external-miner/validators.md @@ -280,9 +280,11 @@ docker compose --project-directory . --env-file deploy/env/python-validator.env ``` The command first opens the file read-only and refuses, without writing, a -file that is not a validator state database or lacks the named watermark. A -running validator holding the database makes it fail with `state db locked; -stop the validator first`. `--netuid` refuses a journal recorded for another +file that is not a validator state database or lacks the named watermark. +Stopping the validator first is required. The `state db locked; stop the +validator first` error is only a best-effort guard: it triggers while the +validator is inside a journal write, not between ticks, so a reset against a +running validator can succeed and then be undone by its next tick. `--netuid` refuses a journal recorded for another subnet; a journal that records no netuid skips that check and says so. It refuses without `--confirm`, refuses a name outside those three, a negative diff --git a/docs/validator-quickstart.md b/docs/validator-quickstart.md index f1e6486d3..ba02fcf66 100644 --- a/docs/validator-quickstart.md +++ b/docs/validator-quickstart.md @@ -9,7 +9,6 @@ For the underlying CLI and the challenge profile served by the gateway, see the - Linux, Python 3.12 or 3.13, [uv](https://docs.astral.sh/uv/), `curl` and libsodium 1.0.18 or newer. - A clone of this repository. - To submit weights only: a Bittensor wallet with a hotkey registered on netuid 100. Verifying needs no wallet. -- Optional: the gateway's public hotkey, pinned from an independent, trusted source, as an extra check. Don't copy it from the gateway response. ```bash sudo apt-get install libsodium23 From 9eec8ea2654b0153b183a5786c604553c2d6f30c Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:27:33 +0000 Subject: [PATCH 52/56] docs(changelog): describe the master restart on share change as caution The master reloads the profile before epoch emission; the restart is the cautious workflow, matching config and deploy wording. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0754bf02a..91a82d520 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,7 +92,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 submit weights or run `--peer-consensus`; `--verify-only` runs without it. `scripts/run-validator.sh` needs `WALLET_NAME`/`WALLET_HOTKEY` only to submit. - The challenge profile is the unsigned `config/challenges.toml` and - `measurements.toml`; changing shares is an edit, a version bump and a master restart. + `measurements.toml`; changing shares is an edit and a version bump (master restart is caution). - `POST /v1/weights/raw` answers 409 `epoch sealing in progress` while the master replaces a completed epoch's challenge leaves and seals it. - A challenge profile whose `measurements_version` exceeds From aed7efe3afbd2c35ab1568437309a2b1c1b3fcb7 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:28:01 +0000 Subject: [PATCH 53/56] docs(review): describe wire v2 rules in coderabbit and AGENTS The review rules still forbade the v2 wire the founder decided on 2026-10-19. Record the decision, the retired base-trustroot-v1, base-rawweight-v1 and base-bundle-v1 preimages, the re-pinned v2 BUNDLE_SPEC hash and the vector sources. Keep the rules strict: no edit of the pinned spec, no weaker check_repo.py, and any new wire change needs maintainer approval plus an independent vector when one exists. Replace the signed participant population wording with the derived set of BUNDLE_SPEC section 7. Co-Authored-By: Claude Sonnet 5.5 --- .coderabbit.yaml | 19 +++++++++++++++---- AGENTS.md | 18 +++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 09543dd47..57e914a0a 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -19,11 +19,22 @@ reviews: sum is always 10,000. Proof is built in; every other challenge is a Docker container per docs/CHALLENGES.md. Design, Prism and Relearn are retired and removed. - - Preserve frozen SCALE encodings, Merkle construction, aggregation and every - `base-*-v1` signature preimage. Cross-language vectors must remain green. + - Wire format v2 (no signatures on leaves, bundle or gateway) was decided by + the founder on 2026-10-19. docs/BUNDLE_SPEC.md was changed on purpose by the + PR that introduced v2 and is byte-pinned again at its v2 content + (`FROZEN_SPECS` in `scripts/check_repo.py` holds the hash). The + `base-trustroot-v1`, `base-rawweight-v1` and `base-bundle-v1` preimages are + retired; `tests/protocol/vectors/rust_wire.json` keeps only what still exists + and `tests/protocol/vectors/wire_v2.json` is generated by + `scripts/gen_wire_vector.py`. + - Preserve the v2 SCALE encodings, Merkle construction, aggregation and every + remaining `base-*-v1` signature preimage. Cross-language vectors must remain + green. Any new wire change needs explicit maintainer approval and a + second-implementation or independently pinned vector when one exists. Algorithm 2 counts each valid Bounty report once, distributes proportionally, scales its 30% share by min(total_valid/10, 1), and burns unused/quarantined - mass. The total includes only the signed expected participant population. + mass. The total includes only the expected participant set validators derive + from the unsigned challenge-profile policy (docs/BUNDLE_SPEC.md section 7). Algorithm 3 pays each challenge share * min(sum(leaves), 10^12) / 10^12 and burns the rest; unpaid mass never moves to another challenge. Never allow an algorithm 1 body under the new profile or activate before @@ -52,5 +63,5 @@ reviews: - CI remains offline: no OpenRouter call, Lium rent, Firecracker boot or chain submission. Fake-boundary tests cannot be described as live evidence. - API changes update the matching `docs/external-miner/` guide. - - Do not edit the byte-pinned frozen specification (docs/BUNDLE_SPEC.md). Do not weaken + - Do not edit the byte-pinned v2 specification (docs/BUNDLE_SPEC.md). Do not weaken `scripts/check_repo.py` to accept protocol drift. diff --git a/AGENTS.md b/AGENTS.md index 3a987ee42..b6f7ec8a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,7 +34,7 @@ reproduction or on-chain payment. | Path | Responsibility | | --- | --- | -| `src/cortex/protocol/` | frozen SCALE, signatures, Merkle and aggregation | +| `src/cortex/protocol/` | frozen v2 SCALE (unsigned wire), Merkle and aggregation | | `src/cortex/gateway/` | durable leaves, immutable seals and burn fallback | | `src/cortex/validator/` | independent recomputation, root consensus and chain dispatch | | `src/cortex/challenges/` | container registry, weights client, public proxy and auto-updating supervisor | @@ -48,10 +48,18 @@ The working branch is `main`. Production releases are annotated `v*.*.*` tags. ## Consensus and compatibility -- Preserve `BASE_*` names, deployed compatibility paths and every `base-*-v1` - signature domain. `CORTEX_*` is an accepted master alias only. -- Do not edit `docs/BUNDLE_SPEC.md`. Its exact SHA-256 value is checked by - `scripts/check_repo.py`. +- Wire format v2 (no signatures on leaves, bundle or gateway) was decided by the + founder on 2026-10-19. `docs/BUNDLE_SPEC.md` was changed on purpose by the PR + that introduced v2; the `base-trustroot-v1`, `base-rawweight-v1` and + `base-bundle-v1` preimages are retired. `tests/protocol/vectors/rust_wire.json` + keeps only what still exists; `tests/protocol/vectors/wire_v2.json` is generated + by `scripts/gen_wire_vector.py`. +- Preserve `BASE_*` names, deployed compatibility paths and every remaining + `base-*-v1` signature domain. `CORTEX_*` is an accepted master alias only. +- Do not edit the v2 `docs/BUNDLE_SPEC.md`. Its exact SHA-256 value is pinned in + `FROZEN_SPECS` in `scripts/check_repo.py`; do not weaken that check. Any new + wire change needs explicit maintainer approval and a second-implementation or + independently pinned vector when one exists. - Production image references are digest-only. Never invent a digest; an empty or unknown pin remains fail-closed. - Secrets are private files. Do not put seeds, mnemonics, bearer values, BYOK, From 1c4c1f48393b6de3de2312b8e133bf9ae23e1ca0 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:56:57 +0000 Subject: [PATCH 54/56] fix(deploy): refuse nested validator identity directories validate_role_separation compared the two identity bind sources only for equality, so a verify-only mount whose directory contained the weight-setter's identity passed the Compose check while exposing the weight-setter consensus key. Compare the normalized paths for containment in either direction, as validate_validator_env already does. Pure path comparison: the sources are rendered strings, not host paths to resolve. Co-Authored-By: Claude Sonnet 5.5 --- scripts/check_deploy.py | 7 +++++-- tests/test_deploy_contract.py | 11 ++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/scripts/check_deploy.py b/scripts/check_deploy.py index 2d482ebe6..52f006a60 100644 --- a/scripts/check_deploy.py +++ b/scripts/check_deploy.py @@ -11,7 +11,7 @@ import subprocess import sys import tomllib -from pathlib import Path +from pathlib import Path, PurePosixPath from urllib.parse import urlsplit ROOT = Path(__file__).resolve().parents[1] @@ -966,7 +966,10 @@ def sources(config: dict) -> dict[str, str]: verify_only, setter = sources(validator), sources(weight_setter) identity = "/run/validator" - if os.path.normpath(verify_only[identity]) == os.path.normpath(setter[identity]): + first = PurePosixPath(os.path.normpath(verify_only[identity])) + second = PurePosixPath(os.path.normpath(setter[identity])) + # One role nested in the other's directory reads the other's consensus.key too. + if first.is_relative_to(second) or second.is_relative_to(first): raise ValueError("validator and weight-setter must use separate identity directories") state = "/var/lib/base" if verify_only[state] and verify_only[state] == setter[state]: diff --git a/tests/test_deploy_contract.py b/tests/test_deploy_contract.py index 0ac375b50..cd91dff66 100644 --- a/tests/test_deploy_contract.py +++ b/tests/test_deploy_contract.py @@ -647,7 +647,12 @@ def identity_source(config, source): @pytest.mark.parametrize( ("validator_dir", "setter_dir"), - [("/fixture/identity", "/fixture/identity"), ("/fixture/identity", "/fixture/x/../identity")], + [ + ("/fixture/identity", "/fixture/identity"), + ("/fixture/identity", "/fixture/x/../identity"), + ("/fixture/identity", "/fixture/identity/setter"), + ("/fixture/identity/setter", "/fixture/identity"), + ], ) def test_deploy_check_refuses_one_identity_dir_for_both_validator_roles(validator_dir, setter_dir): with pytest.raises(ValueError, match="identity"): @@ -673,6 +678,10 @@ def test_deploy_check_accepts_separate_identity_dirs_and_the_roles_use_distinct_ identity_source(validator_config(), "/fixture/verify-only"), identity_source(weight_setter_config(), "/fixture/weight-setter"), ) + CHECK["validate_role_separation"]( + identity_source(validator_config(), "/fixture/verify"), + identity_source(weight_setter_config(), "/fixture/verify-only"), + ) validator = (ROOT / "deploy/compose/role-validator.yml").read_text() setter = (ROOT / "deploy/compose/role-weight-setter.yml").read_text() assert "${BASE_VALIDATOR_IDENTITY_DIR:?" in validator From 71b0cca4ec0d07a16b7f9480175cdf6611e4ffde Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:57:14 +0000 Subject: [PATCH 55/56] test(validator): skip read-only watermark reset cases as root The read-only-file and read-only-dir cases rely on chmod making the state database unwritable. For uid 0 os.access stays true, the reset succeeds and the SystemExit expectation fails. Skip only those two cases as root; the garbage case still runs. Co-Authored-By: Claude Sonnet 5.5 --- tests/validator/test_validator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/validator/test_validator.py b/tests/validator/test_validator.py index 99072dca1..30948b48e 100644 --- a/tests/validator/test_validator.py +++ b/tests/validator/test_validator.py @@ -1,4 +1,5 @@ import json +import os from dataclasses import replace from hashlib import sha256 from types import SimpleNamespace @@ -621,6 +622,8 @@ def test_cli_watermark_reset_reports_a_non_sqlite_state_file_without_traceback(t ], ) def test_watermark_reset_garbage_db_message_has_no_path(network, kind, reason): + if kind != "garbage" and os.geteuid() == 0: + pytest.skip("root ignores read-only mode bits") network.journal.evidence.watermark("epoch", 12) network.journal.close() state = network.db_path From b50cb7ac9cccd78eb523a03ea8c44edda742904f Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:57:30 +0000 Subject: [PATCH 56/56] docs(validator): cover both profile version caps and restart wording The validators guide named only the measurements_version cap, but TrustRoot.validate also refuses challenges_version above MAX_CHALLENGES_VERSION before any watermark moves; state both and that no watermark reset is needed for them. The changelog parenthetical about restarting the master was ungrammatical and did not say whether a restart is required: it is optional, since the master reloads the profile before epoch emission. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 4 +++- docs/external-miner/validators.md | 9 +++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91a82d520..5a2744ab3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,7 +92,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 submit weights or run `--peer-consensus`; `--verify-only` runs without it. `scripts/run-validator.sh` needs `WALLET_NAME`/`WALLET_HOTKEY` only to submit. - The challenge profile is the unsigned `config/challenges.toml` and - `measurements.toml`; changing shares is an edit and a version bump (master restart is caution). + `measurements.toml`; changing shares is an edit and a version bump. The master + reloads the profile before epoch emission, so restarting it after a share + change is optional, cautious operational practice, not a requirement. - `POST /v1/weights/raw` answers 409 `epoch sealing in progress` while the master replaces a completed epoch's challenge leaves and seals it. - A challenge profile whose `measurements_version` exceeds diff --git a/docs/external-miner/validators.md b/docs/external-miner/validators.md index a71ed32fb..46b7358d6 100644 --- a/docs/external-miner/validators.md +++ b/docs/external-miner/validators.md @@ -249,10 +249,11 @@ database to force a retry. The journal keeps rollback watermarks for `challenges_version`, `measurements_version` and `epoch`; a lower served value fails with -`consensus rollback refused`. A profile whose `measurements_version` is not an -integer or exceeds `MAX_MEASUREMENTS_VERSION` (1000, `src/cortex/protocol/models.py`) -is refused as `invalid trust profile` before any watermark moves; the master -refuses to load such a profile too. If a watermark was advanced wrongly (for example a +`consensus rollback refused`. A profile whose `measurements_version` or +`challenges_version` is not an integer or exceeds 1000 (`MAX_MEASUREMENTS_VERSION` +and `MAX_CHALLENGES_VERSION` in `src/cortex/protocol/models.py`) is refused as +`invalid trust profile` before any watermark moves, so it needs no +`validator-watermark-reset`; the master refuses to load such a profile too. If a watermark was advanced wrongly (for example a profile version published by mistake and then withdrawn), stop the validator and lower it locally. The command touches only the state database: no network, no wallet.