Record upstream error bodies and auto-disable terminally rejected accounts

app/admin_api.py

@@ -67,7 +67,7 @@ def _public_credential(item):
               "trial_supported", "trial",
               "daily_chat_supported", "auto_daily_chat", "daily_chat",
               "growth_supported", "auto_growth", "growth_auto",
-              "token_expired", "token_expires_at", "last_refresh_time", "sessions", "sticky_sessions", "last_error_code", "usage_stats"}
+              "token_expired", "token_expires_at", "last_refresh_time", "sessions", "sticky_sessions", "last_error_code", "last_error_body", "auto_disabled", "usage_stats"}
     result = {key: value for key, value in item.items() if key in fields}
     identity = item.get("account_key") or item.get("id")
     if identity and ("/" in str(identity) or "\\" in str(identity)):

app/audit_store.py

@@ -44,6 +44,32 @@ def safe_agent(value: Any) -> str | None:
         return None
     return None if _SECRET.search(cleaned) else cleaned
 
+
+def safe_diagnostic(value: Any, limit: int = 256) -> str | None:
+    """Short printable upstream diagnostic text; collapsed to one line and secret-checked."""
+    if not isinstance(value, str):
+        return None
+    cleaned = " ".join(value.split())[:limit]
+    if not cleaned or not cleaned.isprintable() or _SECRET.search(cleaned):
+        return None
+    return cleaned
+
+
+_UPSTREAM_BODY_LIMIT = 256 * 1024
+_JWT = re.compile(r"eyJ[A-Za-z0-9_-]{10,}")
+
+
+def safe_body(value: Any, limit: int = _UPSTREAM_BODY_LIMIT) -> str | None:
+    """Bounded upstream error JSON; redacted upstream, re-checked here for token leakage."""
+    if not isinstance(value, str) or not value:
+        return None
+    if len(value) > limit:
+        value = value[:limit] + "...[truncated]"
+    if _JWT.search(value):
+        return None
+    return value
+
+
 def number(value: Any) -> int | float | None:
     if isinstance(value, bool) or not isinstance(value, (int, float)):
         return None
@@ -59,10 +85,18 @@ def safe_attempt(value: Any) -> dict:
     result = {}
     for key in ("stage", "code", "error_code", "model", "upstream_model", "profile",
                 "credential", "usage_source", "outcome", "consent_source", "agreement_revision", "conversation_id", "request_id",
-                "attempt_id", "upstream_request_id", "session_source"):
+                "attempt_id", "upstream_request_id", "session_source", "upstream_code"):
         clean = safe_label(value.get(key))
         if clean is not None:
             result[key] = clean
+    for key in ("upstream_msg",):
+        clean = safe_diagnostic(value.get(key))
+        if clean is not None:
+            result[key] = clean
+    for key in ("upstream_body",):
+        clean = safe_body(value.get(key))
+        if clean is not None:
+            result[key] = clean
     for key in ("status_code", "duration_ms", "attempt", "retry_after", "max_attempts", "total_tokens",
                 "attempt_index", "dropped", "merged_runs", "merged_messages"):
         clean = number(value.get(key))
@@ -142,7 +176,7 @@ class AuditStore:
     TRANSIENT_WINDOW_SECONDS = 3600
 
     def __init__(self, path, max_bytes=256 * 1024 * 1024, retention_days=30,
-                 preview_limit=8192):
+                 preview_limit=1536 * 1024):
         self._validate(max_bytes, retention_days, preview_limit)
         self.max_bytes, self.retention_days, self.preview_limit = max_bytes, retention_days, preview_limit
         self._lock = threading.Lock()
@@ -248,7 +282,7 @@ def _validate(max_bytes, retention_days, preview_limit):
             raise ValueError("invalid max_bytes")
         if isinstance(retention_days, bool) or not isinstance(retention_days, int) or not 1 <= retention_days <= 36500:
             raise ValueError("invalid retention_days")
-        if isinstance(preview_limit, bool) or not isinstance(preview_limit, int) or not 0 <= preview_limit <= 65536:
+        if isinstance(preview_limit, bool) or not isinstance(preview_limit, int) or not 0 <= preview_limit <= 4 * 1024 * 1024:
             raise ValueError("invalid preview_limit")
 
     def _fault(self, exc, dropped=False):

app/control_store.py

@@ -186,10 +186,16 @@ def _load(self):
         for identity, metadata in data["credentials"].items():
             _identifier(identity, "账号指纹")
             growth = metadata.get("growth_auto")
+            growth = metadata.get("growth_auto")
             if (not isinstance(metadata, dict)
                     or set(metadata) - {"enabled", "label", "auto_checkin", "auto_travel",
-                                        "auto_daily_chat", "auto_growth", "growth_auto"}
+                                        "auto_daily_chat", "auto_growth", "growth_auto",
+                                        "auto_disabled", "auto_reason", "auto_body", "auto_at"}
                     or type(metadata.get("enabled")) is not bool
+                    or metadata.get("auto_disabled") is not None and type(metadata["auto_disabled"]) is not bool
+                    or metadata.get("auto_reason") is not None and not isinstance(metadata["auto_reason"], str)
+                    or metadata.get("auto_body") is not None and not isinstance(metadata["auto_body"], str)
+                    or metadata.get("auto_at") is not None and not isinstance(metadata["auto_at"], (int, float))
                     or metadata.get("auto_growth") is not None and type(metadata["auto_growth"]) is not bool
                     or any(key in metadata and type(metadata[key]) is not bool for key in
                            ("auto_checkin", "auto_travel", "auto_daily_chat"))
@@ -253,10 +259,32 @@ def change(state):
 
 
     def set_credential(self, account_key, enabled):
+        """Manual toggle: any operator action clears the automatic-disable provenance."""
         _identifier(account_key, "账号指纹")
         if type(enabled) is not bool:
             raise ValueError("enabled 必须为布尔值")
-        return self._update(None, lambda state: state["credentials"].setdefault(account_key, {}).update(enabled=enabled))
+
+        def change(state):
+            record = state["credentials"].setdefault(account_key, {})
+            record["enabled"] = enabled
+            record.pop("auto_disabled", None)  # Any manual toggle clears the provenance.
+        return self._update(None, change)
+
+    def disable_credential_auto(self, account_key, reason="", body=None, at=None):
+        """Record an automatic disable with its upstream evidence so it survives restarts."""
+        _identifier(account_key, "账号指纹")
+
+        def change(state):
+            record = state["credentials"].setdefault(account_key, {})
+            record.update(enabled=False, auto_disabled=True)
+            for key, value in (("auto_reason", str(reason or "")[:256] or None),
+                               ("auto_body", str(body)[:8192] if body else None),
+                               ("auto_at", at)):
+                if value is None:
+                    record.pop(key, None)
+                else:
+                    record[key] = value
+        return self._update(None, change)
 
     def unbind_credential(self, account_key):
         """Drop an account key from every rule, returning {source: old_ids} for rollback."""

app/gateway_management.py

@@ -37,21 +37,27 @@ def admin_credential_inventory(self):
         with pool._lock:
             entries = {entry["id"]: entry for entry in pool.entries()}
             result = []
+            policy = model_policy.view(self.CONFIG)
             for row in pool.snapshot():
                 path = row.pop("auth_file", "")
                 entry = entries[path]
                 identity = entry.get("account_key") or hashlib.sha256(path.encode()).hexdigest()
                 ledger = self.CONFIG.get("ledger")
                 balance = ledger.entry(path) if ledger else {}
-                enabled = model_policy.credential_enabled(self.CONFIG, entry)
+                record = policy.credential(entry)
+                enabled = record.get("enabled", True)
                 until = entry.get("fail_until", 0)
                 cooldowns = [{"model": model, "until": deadline,
                               "remaining_seconds": max(0, round(deadline - now))}
                              for (cid, model), deadline in pool._model_fail.items()
                              if cid == path and deadline > now]
-                state = ("disabled" if not enabled else "error" if row.get("error") else
+                # Provenance lives in the store so it survives restarts and cooldown resets.
+                auto_disabled = record.get("auto_disabled") is True and not enabled
+                state = ("auto_disabled" if auto_disabled else
+                         "disabled" if not enabled else "error" if row.get("error") else
                          "circuit_open" if until > now else "expired" if row.get("token_expired") else "ready")
                 row.update(id=identity, name=Path(path).name, enabled=enabled, health=state,
+                           auto_disabled=auto_disabled,
                            auto_checkin=model_policy.credential_auto_checkin(self.CONFIG, entry),
                            checkin=checkin.view(balance.get("checkin") or {}),
                            auto_travel=model_policy.credential_auto_travel(self.CONFIG, entry),
@@ -74,6 +80,7 @@ def admin_credential_inventory(self):
                                             "http_403" if entry.get("last_error") == "backend HTTP 403" else
                                             "credential_error" if entry.get("last_error") else None),
                            last_failure_at=entry.get("last_failure_at"),
+                           last_error_body=entry.get("last_error_body"),
                            sync_error=(balance.get("error") or entry.get("last_error") or None),
                            cooldowns=cooldowns, credits=balance.get("credits") or None,
                            sync_pending=path in pool._sync_pending or path in pool._syncing or path in pool._sync_retry,
@@ -94,9 +101,16 @@ def admin_set_credential_enabled(self, identity, enabled):
             self.CONFIG["control_store"].set_credential(identity, enabled)
             pool._sticky.clear()
             if enabled:
+                # Re-enabling returns the credential to a fresh state: reset_cooldowns_for
+                # lifts every cooldown and drops the recorded failure evidence.
+                pool.reset_cooldowns_for(identity)
                 pool._queue_sync(entry["id"])
             else:
                 pool._sync_pending.discard(entry["id"])
+                # entries() hands out copies; clear the flag on the live entry.
+                live = pool.entry_by_id(entry["id"])
+                if live is not None:
+                    live["auto_disabled"] = None
                 pool._sync_retry.pop(entry["id"], None)
                 if not pool._sync_pending:
                     pool._sync_event.clear()

app/model_policy.py

@@ -75,6 +75,7 @@ def credential_enabled(config, entry, policy=None):
     return _policy(config, policy)["credentials"].get(entry.get("account_key"), {}).get("enabled", True)
 
 
+
 def credential_auto_checkin(config, entry, policy=None):
     """International accounts opt in explicitly; the preference is independent of routing."""
     default = entry.get("profile") in {"cn-cli", "cn-work"}

app/settings.py

@@ -147,7 +147,7 @@ def _item(default, type_, label, *, mode="hot", env=None, minimum=None, maximum=
                          env="CODEBUDDY2API_STREAM_MODE", choices=["compatible", "realtime"]),
     "audit_max_bytes": _item(256 * 1024 * 1024, "integer", "审计明细预算", minimum=1024**2, maximum=1024**4),
     "audit_retention_days": _item(30, "integer", "审计明细保留天数", minimum=1, maximum=36500),
-    "audit_diagnostic_bytes": _item(8192, "integer", "失败诊断最大字节", minimum=0, maximum=8192),
+    "audit_diagnostic_bytes": _item(1536 * 1024, "integer", "失败诊断最大字节", minimum=0, maximum=4 * 1024 * 1024),
     "trusted_proxies": _item("", "string", "可信代理地址（IP/CIDR，逗号分隔）",
                              env="CODEBUDDY2API_TRUSTED_PROXIES", allow_empty=True,
                              max_length=2000, validator=normalize_trusted_proxies),