From fdb47cc4bcb3dcfdafed80ca836d38c4b1eb6d07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?=
 <277378677+maiphucgiang@users.noreply.github.com>
Date: Fri, 23 Oct 2026 18:55:42 +0800
Subject: [PATCH 1/8] Record upstream error bodies and disable restricted
 accounts

- Persist the full upstream error JSON (256k cap, redacted) on every
  non-200 as upstream_code/upstream_msg/upstream_body audit fields;
  raise the audit diagnostic budget to fit it.
- Keep the last error body on the credential entry and expose it at
  /admin/credentials; the WebUI shows it on hover (desktop) or tap
  (mobile), click pins the popover open.
- Upstream restriction 11140 (account limited for unusual activity)
  now disables the credential through the WebUI policy instead of
  relying on the 300s auth cooldown; plain 401/403 unchanged.
---
 app/admin_api.py                   |   2 +-
 app/audit_store.py                 |  40 ++++++++-
 app/gateway_management.py          |   1 +
 app/settings.py                    |   2 +-
 converter.py                       |  64 ++++++++++++++
 tests/test_credential_cooldowns.py |  47 ++++++++++
 tests/test_upstream_diagnostics.py | 137 +++++++++++++++++++++++++++++
 web/src/pages/Credentials.tsx      |  97 +++++++++++++++++++-
 web/src/ui.module.scss             |  30 +++++++
 9 files changed, 411 insertions(+), 9 deletions(-)
 create mode 100644 tests/test_upstream_diagnostics.py

diff --git a/app/admin_api.py b/app/admin_api.py
index 21fa1f4..b76f67a 100644
--- a/app/admin_api.py
+++ b/app/admin_api.py
@@ -67,7 +67,7 @@ def _public_credential(item):
               "trial_supported", "trial",
               "daily_chat_supported", "auto_daily_chat", "daily_chat",
               "growth_supported", "auto_growth", "growth_auto",
-              "token_expired", "token_expires_at", "last_refresh_time", "sessions", "sticky_sessions", "last_error_code", "usage_stats"}
+              "token_expired", "token_expires_at", "last_refresh_time", "sessions", "sticky_sessions", "last_error_code", "last_error_body", "usage_stats"}
     result = {key: value for key, value in item.items() if key in fields}
     identity = item.get("account_key") or item.get("id")
     if identity and ("/" in str(identity) or "\\" in str(identity)):
diff --git a/app/audit_store.py b/app/audit_store.py
index 03a76a3..973b84b 100644
--- a/app/audit_store.py
+++ b/app/audit_store.py
@@ -44,6 +44,32 @@ def safe_agent(value: Any) -> str | None:
         return None
     return None if _SECRET.search(cleaned) else cleaned
 
+
+def safe_diagnostic(value: Any, limit: int = 256) -> str | None:
+    """Short printable upstream diagnostic text; collapsed to one line and secret-checked."""
+    if not isinstance(value, str):
+        return None
+    cleaned = " ".join(value.split())[:limit]
+    if not cleaned or not cleaned.isprintable() or _SECRET.search(cleaned):
+        return None
+    return cleaned
+
+
+_UPSTREAM_BODY_LIMIT = 256 * 1024
+_JWT = re.compile(r"eyJ[A-Za-z0-9_-]{10,}")
+
+
+def safe_body(value: Any, limit: int = _UPSTREAM_BODY_LIMIT) -> str | None:
+    """Bounded upstream error JSON; redacted upstream, re-checked here for token leakage."""
+    if not isinstance(value, str) or not value:
+        return None
+    if len(value) > limit:
+        value = value[:limit] + "...[truncated]"
+    if _JWT.search(value):
+        return None
+    return value
+
+
 def number(value: Any) -> int | float | None:
     if isinstance(value, bool) or not isinstance(value, (int, float)):
         return None
@@ -59,10 +85,18 @@ def safe_attempt(value: Any) -> dict:
     result = {}
     for key in ("stage", "code", "error_code", "model", "upstream_model", "profile",
                 "credential", "usage_source", "outcome", "consent_source", "agreement_revision", "conversation_id", "request_id",
-                "attempt_id", "upstream_request_id", "session_source"):
+                "attempt_id", "upstream_request_id", "session_source", "upstream_code"):
         clean = safe_label(value.get(key))
         if clean is not None:
             result[key] = clean
+    for key in ("upstream_msg",):
+        clean = safe_diagnostic(value.get(key))
+        if clean is not None:
+            result[key] = clean
+    for key in ("upstream_body",):
+        clean = safe_body(value.get(key))
+        if clean is not None:
+            result[key] = clean
     for key in ("status_code", "duration_ms", "attempt", "retry_after", "max_attempts", "total_tokens",
                 "attempt_index", "dropped", "merged_runs", "merged_messages"):
         clean = number(value.get(key))
@@ -142,7 +176,7 @@ class AuditStore:
     TRANSIENT_WINDOW_SECONDS = 3600
 
     def __init__(self, path, max_bytes=256 * 1024 * 1024, retention_days=30,
-                 preview_limit=8192):
+                 preview_limit=1536 * 1024):
         self._validate(max_bytes, retention_days, preview_limit)
         self.max_bytes, self.retention_days, self.preview_limit = max_bytes, retention_days, preview_limit
         self._lock = threading.Lock()
@@ -248,7 +282,7 @@ def _validate(max_bytes, retention_days, preview_limit):
             raise ValueError("invalid max_bytes")
         if isinstance(retention_days, bool) or not isinstance(retention_days, int) or not 1 <= retention_days <= 36500:
             raise ValueError("invalid retention_days")
-        if isinstance(preview_limit, bool) or not isinstance(preview_limit, int) or not 0 <= preview_limit <= 65536:
+        if isinstance(preview_limit, bool) or not isinstance(preview_limit, int) or not 0 <= preview_limit <= 4 * 1024 * 1024:
             raise ValueError("invalid preview_limit")
 
     def _fault(self, exc, dropped=False):
diff --git a/app/gateway_management.py b/app/gateway_management.py
index 0d1afe6..1840b8e 100644
--- a/app/gateway_management.py
+++ b/app/gateway_management.py
@@ -74,6 +74,7 @@ def admin_credential_inventory(self):
                                             "http_403" if entry.get("last_error") == "backend HTTP 403" else
                                             "credential_error" if entry.get("last_error") else None),
                            last_failure_at=entry.get("last_failure_at"),
+                           last_error_body=entry.get("last_error_body"),
                            sync_error=(balance.get("error") or entry.get("last_error") or None),
                            cooldowns=cooldowns, credits=balance.get("credits") or None,
                            sync_pending=path in pool._sync_pending or path in pool._syncing or path in pool._sync_retry,
diff --git a/app/settings.py b/app/settings.py
index 29a3d5d..7b0c4dd 100644
--- a/app/settings.py
+++ b/app/settings.py
@@ -147,7 +147,7 @@ def _item(default, type_, label, *, mode="hot", env=None, minimum=None, maximum=
                          env="CODEBUDDY2API_STREAM_MODE", choices=["compatible", "realtime"]),
     "audit_max_bytes": _item(256 * 1024 * 1024, "integer", "审计明细预算", minimum=1024**2, maximum=1024**4),
     "audit_retention_days": _item(30, "integer", "审计明细保留天数", minimum=1, maximum=36500),
-    "audit_diagnostic_bytes": _item(8192, "integer", "失败诊断最大字节", minimum=0, maximum=8192),
+    "audit_diagnostic_bytes": _item(1536 * 1024, "integer", "失败诊断最大字节", minimum=0, maximum=4 * 1024 * 1024),
     "trusted_proxies": _item("", "string", "可信代理地址（IP/CIDR，逗号分隔）",
                              env="CODEBUDDY2API_TRUSTED_PROXIES", allow_empty=True,
                              max_length=2000, validator=normalize_trusted_proxies),
diff --git a/converter.py b/converter.py
index e2d8824..f3fde1a 100644
--- a/converter.py
+++ b/converter.py
@@ -446,6 +446,8 @@ def summary(self) -> dict:
 CRED_COOLDOWN = 300         # Credential cooldown in seconds
 MODEL_COOLDOWN = 600        # Model cooldown when a 429 omits reset time
 MODEL_COOLDOWN_MAX = 86400  # Maximum model cooldown in seconds
+UPSTREAM_BODY_AUDIT_LIMIT = 256 * 1024  # Bounded upstream error JSON kept per failed attempt.
+RESTRICTED_CODES = frozenset({"11140"})  # Upstream account-restriction codes; disable, never requeue.
 MODEL_SITE_BLOCK_S = 6 * 3600      # Initial unsupported-model backoff
 MODEL_SITE_BLOCK_MAX_S = 24 * 3600  # Maximum unsupported-model backoff
 # An unsupported model must be routed to a different backend.
@@ -539,6 +541,33 @@ def _parse_not_servable(raw: bytes, status: int):
     return None
 
 
+def _upstream_diagnostic(raw: bytes):
+    """Extract the business code, message and bounded JSON body from an upstream error."""
+    if not raw:
+        return None, None, None
+    try:
+        payload = json.loads(raw.decode("utf-8", "replace"))
+    except (ValueError, UnicodeError, RecursionError):
+        return None, None, None
+    if not isinstance(payload, dict):
+        return None, None, None
+    nodes = [payload]
+    inner = payload.get("error")
+    if isinstance(inner, dict):
+        nodes.append(inner)
+    code = msg = ""
+    for node in nodes:
+        for key in ("code", "errCode", "error_code"):
+            value = node.get(key)
+            if value is not None and str(value).strip():
+                code = code or str(value).strip()
+        for key in ("msg", "message"):
+            value = node.get(key)
+            if isinstance(value, str) and value.strip():
+                msg = msg or value.strip()
+    return (code[:64] or None), (msg[:256] or None), (format_log_body(payload, UPSTREAM_BODY_AUDIT_LIMIT) or None)
+
+
 def _block_model(model: str | None) -> str | None:
     """Track blocked models by public ID, normalizing the international auto alias."""
     return "auto" if model == "default-model" else model
@@ -1346,6 +1375,34 @@ def _lease_matches(cm, generation):
             return generation == cm._generation
         return generation == cm._generation
 
+    def _note_error_body(self, cm, raw, *, generation=None):
+        """Keep the last upstream error JSON for the credentials tooltip; bounded, redacted."""
+        _, _, body = _upstream_diagnostic(raw or b"")
+        if not body:
+            return
+        with self._lock, (cm._lock if generation is not None else nullcontext()):
+            if not self._lease_matches(cm, generation):
+                return
+            entry = self._index_cm.get(cm)
+            if entry is not None:
+                entry["last_error_body"] = body[:8192]
+
+    def _disable_restricted(self, cm, code, *, generation=None):
+        """An upstream restriction is terminal: disable via the WebUI policy, not a cooldown."""
+        entry = self._index_cm.get(cm)
+        store = CONFIG.get("control_store")
+        management = CONFIG.get("management")
+        identity = entry.get("account_key") if entry else None
+        if not identity or store is None or management is None:
+            return
+        with self._lock, (cm._lock if generation is not None else nullcontext()):
+            if not self._lease_matches(cm, generation):
+                return
+            store.set_credential(identity, False)
+            self._sticky.clear()
+        management.gateway.invalidate_model_table()
+        _log(f"[cred] 上游限制 {code}，账号已禁用: {Path(cm.path).name}")
+
     def cooldown(self, cm: CredentialManager, reason: str = "", *, generation=None):
         with self._lock, (cm._lock if generation is not None else nullcontext()):
             if not self._lease_matches(cm, generation):
@@ -1366,6 +1423,10 @@ def note_status(self, cm: CredentialManager | None, status: int,
             return
         if status in (401, 403):
             self.cooldown(cm, reason=f"backend HTTP {status}", generation=generation)
+            self._note_error_body(cm, raw, generation=generation)
+            code, _, _ = _upstream_diagnostic(raw or b"")
+            if status == 403 and code in RESTRICTED_CODES:
+                self._disable_restricted(cm, code, generation=generation)
             return
         if status == 400 and is_channel_check_error(raw):
             # Only a rejection that survived the console retry is account-wide; endpoints without a
@@ -4105,6 +4166,9 @@ def _safe_err_raw(raw: bytes, status: int) -> dict:
 def _check_upstream_status(status, raw, cred, model, *, headers=None, from_channel_fallback=False):
     if status != 200:
         retry_after = parse_retry_after((headers or {}).get("Retry-After"))
+        code, message, error_body = _upstream_diagnostic(raw)
+        observe_attempt("upstream_error", status_code=status, upstream_code=code,
+                        upstream_msg=message, upstream_body=error_body)
         if not is_filter_error(raw):
             _note_cred_status(cred, status, model=model, raw=raw, retry_after=retry_after,
                               from_channel_fallback=from_channel_fallback)
diff --git a/tests/test_credential_cooldowns.py b/tests/test_credential_cooldowns.py
index 6c5e4f7..18f26fb 100644
--- a/tests/test_credential_cooldowns.py
+++ b/tests/test_credential_cooldowns.py
@@ -6,13 +6,16 @@
 import sys
 import tempfile
 import time
+import time
 import unittest
 from pathlib import Path
+from types import SimpleNamespace
 from unittest.mock import patch
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))  # Allow direct execution.
 
 import converter
+from app.control_store import ControlStore as converter_local_ControlStore
 from app.credential_cooldowns import (AUTH_CEILING_S, MAX_ACCOUNTS, MAX_BYTES, MAX_MODELS,
                                       MODEL_CEILING_S, VERSION, CredentialCooldowns)
 
@@ -304,6 +307,50 @@ def pool(self, *paths):
         return converter.CredentialPool(list(paths), blocks_path=self.root / "blocks.json",
                                         cooldowns_path=self.path)
 
+
+    def test_auth_rejection_keeps_the_upstream_error_body_for_the_tooltip(self):
+        pool = self.pool(self.credential())
+        body = json.dumps({"code": 11140, "msg": "request illegal",
+                           "displayMsg": {"zh": "账号存在异常行为，已被限制使用"}}).encode()
+        pool.note_status(pool._entries[0]["cm"], 403, raw=body)
+        entry = pool._entries[0]
+        self.assertIn("11140", entry["last_error_body"])
+        self.assertIn("已被限制使用", entry["last_error_body"])
+        # Non-JSON rejections (WAF pages) must not leave a body behind.
+        pool.note_status(pool._entries[0]["cm"], 403, raw=b"<html>blocked</html>")
+        self.assertIn("11140", entry["last_error_body"])
+
+    def test_upstream_restriction_disables_the_account_through_the_webui_policy(self):
+        """11140 is a terminal account restriction: disable via the policy, not just a cooldown."""
+        path = self.credential()
+        pool = self.pool(path)
+        store = converter_local_ControlStore(self.root / "control.sqlite3")
+        self.addCleanup(store.close)
+        identity = pool._entries[0]["account_key"]
+        store.set_credential(identity, True)
+        invalidated = []
+        management = SimpleNamespace(gateway=SimpleNamespace(
+            invalidate_model_table=lambda: invalidated.append(1)))
+        self.enterContext(patch.dict(converter.CONFIG, {"control_store": store, "management": management}))
+        body = json.dumps({"code": 11140, "msg": "request illegal"}).encode()
+        pool.note_status(pool._entries[0]["cm"], 403, raw=body)
+        self.assertFalse(converter.model_policy.credential_enabled(converter.CONFIG, pool._entries[0]))
+        self.assertEqual(invalidated, [1])
+        # The policy survives a pool rebuild, matching a WebUI-initiated disable.
+        revived = self.pool(self.credential())
+        self.assertFalse(converter.model_policy.credential_enabled(converter.CONFIG, revived._entries[0]))
+
+    def test_plain_403_cooldowns_without_disabling(self):
+        pool = self.pool(self.credential())
+        store = converter_local_ControlStore(self.root / "control.sqlite3")
+        self.addCleanup(store.close)
+        identity = pool._entries[0]["account_key"]
+        store.set_credential(identity, True)
+        self.enterContext(patch.dict(converter.CONFIG, {"control_store": store, "management": None}))
+        pool.note_status(pool._entries[0]["cm"], 403, raw=b"forbidden")
+        self.assertTrue(converter.model_policy.credential_enabled(converter.CONFIG, pool._entries[0]))
+        self.assertFalse(pool._healthy(pool._entries[0]))
+
     def test_auth_circuit_breaker_survives_a_restart(self):
         first = self.pool(self.credential())
         first.cooldown(first._entries[0]["cm"], reason="backend HTTP 401")
diff --git a/tests/test_upstream_diagnostics.py b/tests/test_upstream_diagnostics.py
new file mode 100644
index 0000000..80b10b1
--- /dev/null
+++ b/tests/test_upstream_diagnostics.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""Test that upstream error codes and messages reach the audit trail, sanitized."""
+
+import contextlib
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))  # Allow direct execution.
+import httpx
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+import converter
+from app import upstream_io
+from app.audit_store import AuditStore
+
+ROUTE = "/v1/chat/completions"
+
+
+class UpstreamDiagnosticTests(unittest.TestCase):
+    """Upstream error bodies are the only way to tell a WAF block from a business rejection."""
+
+    def setUp(self):
+        self.directory = self.enterContext(tempfile.TemporaryDirectory())
+        self.audit = self.enterContext(contextlib.closing(
+            AuditStore(Path(self.directory) / "logs.sqlite3")))
+        self.enterContext(patch.dict(converter.CONFIG, {
+            "api_key": "", "cred": None, "cred_pool": None, "model_guard": False,
+            "max_images": 16, "image_policy": "truncate", "max_request_bytes": 32 * 1024 * 1024,
+            "log_body_limit": 65536, "log_path": None, "desensitize": False, "no_compact": False,
+            "audit_store": self.audit}))
+        self.enterContext(patch.object(converter, "_cred_for", return_value=(None, {})))
+        self.enterContext(patch.object(converter, "_log"))
+        self.requests = []
+        self.respond = lambda request: httpx.Response(200, content=b'data: [DONE]\n\n')
+        real_client = httpx.AsyncClient
+        transport = httpx.MockTransport(self.handle)
+        self.enterContext(patch.object(upstream_io.httpx, "AsyncClient",
+                                       side_effect=lambda **kw: real_client(transport=transport, **kw)))
+        self.client = self.enterContext(TestClient(self._build_app()))
+
+    def handle(self, request):
+        self.requests.append(request)
+        return self.respond(request)
+
+    def _build_app(self):
+        """Rebuild the inference app with the audit middleware the raw app does not carry."""
+        from app.observability import AuditMiddleware
+        from app.request_context import RequestContextMiddleware
+        app = FastAPI()
+        for route in converter.app.routes:
+            app.routes.append(route)
+        app.add_middleware(AuditMiddleware, config=converter.CONFIG)
+        app.add_middleware(RequestContextMiddleware, config=converter.CONFIG)
+        return app
+
+    def _attempts(self):
+        for record in self.audit.list_records("request", limit=20)["items"]:
+            for attempt in record.get("attempts") or []:
+                yield attempt
+
+    def test_business_error_code_and_message_reach_the_audit_trail(self):
+        self.respond = lambda request: httpx.Response(
+            403, json={"code": "11-128", "msg": "conversation id missing"})
+        response = self.client.post(ROUTE, json={
+            "model": "auto", "messages": [{"role": "user", "content": "hi"}]})
+        self.assertEqual(response.status_code, 403)
+        entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error")
+        self.assertEqual(entry["status_code"], 403)
+        self.assertEqual(entry["upstream_code"], "11-128")
+        self.assertEqual(entry["upstream_msg"], "conversation id missing")
+
+    def test_nested_envelope_is_unwrapped(self):
+        self.respond = lambda request: httpx.Response(
+            403, json={"error": {"code": "11140", "message": "request illegal"}})
+        self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]})
+        entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error")
+        self.assertEqual(entry["upstream_code"], "11140")
+        self.assertEqual(entry["upstream_msg"], "request illegal")
+
+    def test_non_json_body_records_no_diagnostic_but_keeps_the_status(self):
+        self.respond = lambda request: httpx.Response(403, text="<html>blocked</html>")
+        self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]})
+        entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error")
+        self.assertEqual(entry["status_code"], 403)
+        self.assertNotIn("upstream_code", entry)
+        self.assertNotIn("upstream_msg", entry)
+        self.assertNotIn("upstream_body", entry)
+
+    def test_restriction_body_is_recorded_in_full(self):
+        """The 11140 restriction envelope is the evidence that decides account handling."""
+        self.respond = lambda request: httpx.Response(403, json={
+            "code": 11140, "msg": "request illegal", "requestId": "01a162757c61710a82d9c17d3e9f060e",
+            "displayMsg": {"en": "Your account has been restricted due to unusual activity",
+                           "zh": "账号存在异常行为，已被限制使用"},
+            "displayTips": {"zh": "系统检测到该账号存在异常使用行为，如需申诉请提交反馈"},
+            "actions": ["EDIT_INPUT", "SUBMIT_FEEDBACK"]})
+        self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]})
+        entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error")
+        self.assertEqual(entry["upstream_code"], "11140")
+        for fragment in ("已被限制使用", "SUBMIT_FEEDBACK", "01a162757c61710a82d9c17d3e9f060e"):
+            self.assertIn(fragment, entry["upstream_body"])
+
+    def test_body_is_capped_at_256k(self):
+        payload = {f"key{index}": "x" * 4096 for index in range(80)}
+        self.respond = lambda request: httpx.Response(403, json={"code": "1", "msg": "wide", **payload})
+        self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]})
+        entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error")
+        self.assertLessEqual(len(entry["upstream_body"]), 256 * 1024 + 64)
+        self.assertIn("truncated", entry["upstream_body"])
+
+    def test_body_tokens_are_redacted_before_storage(self):
+        self.respond = lambda request: httpx.Response(403, json={
+            "code": "11-128", "msg": "denied", "accessToken": "synthetic-private-token"})
+        self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]})
+        entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error")
+        self.assertNotIn("synthetic-private-token", entry["upstream_body"])
+        raw = b"".join(f.read_bytes() for f in Path(self.directory).glob("logs.sqlite3*"))
+
+    def test_secrets_in_upstream_messages_are_dropped(self):
+        self.respond = lambda request: httpx.Response(
+            403, json={"code": "11-128", "msg": "denied for cb-" + "a" * 32})
+        self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]})
+        entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error")
+        self.assertEqual(entry["upstream_code"], "11-128")
+        self.assertNotIn("upstream_msg", entry)
+        raw = b"".join(f.read_bytes() for f in Path(self.directory).glob("logs.sqlite3*"))
+        self.assertNotIn(b"cb-" + b"a" * 32, raw)
+
+
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)
diff --git a/web/src/pages/Credentials.tsx b/web/src/pages/Credentials.tsx
index 9fcc67e..4254360 100644
--- a/web/src/pages/Credentials.tsx
+++ b/web/src/pages/Credentials.tsx
@@ -66,6 +66,98 @@ function lastErrorLabel(value: unknown) {
   return lastErrorLabels[key] ?? key;
 }
 
+function prettyErrorJson(value: unknown) {
+  const raw = text(value);
+  if (raw === "") return "";
+  try {
+    return JSON.stringify(JSON.parse(raw), null, 2);
+  } catch {
+    return raw;
+  }
+}
+
+/** Last upstream error; hover (desktop) or tap (mobile) opens the recorded JSON. Click pins it open. */
+function LastErrorJson({ code, at, body }: { code: unknown; at: unknown; body: unknown }) {
+  const json = prettyErrorJson(body);
+  const [pinned, setPinned] = useState(false);
+  const [hovered, setHovered] = useState(false);
+  const leaveTimer = useRef<number | null>(null);
+  const trigger = useRef<HTMLButtonElement | null>(null);
+  const tip = useRef<HTMLDivElement | null>(null);
+  const open = pinned || hovered;
+  const close = useCallback(() => {
+    setPinned(false);
+    setHovered(false);
+  }, []);
+  const placement = usePopover(open, close, trigger, tip, "start");
+  useEffect(
+    () => () => {
+      if (leaveTimer.current !== null) window.clearTimeout(leaveTimer.current);
+    },
+    [],
+  );
+  // A short grace period bridges the gap between the trigger and the portaled tip.
+  const scheduleClose = () => {
+    if (leaveTimer.current !== null) window.clearTimeout(leaveTimer.current);
+    leaveTimer.current = window.setTimeout(() => setHovered(false), 160);
+  };
+  const holdOpen = () => {
+    if (leaveTimer.current !== null) {
+      window.clearTimeout(leaveTimer.current);
+      leaveTimer.current = null;
+    }
+    setHovered(true);
+  };
+  const hoverProps = {
+    onPointerEnter: (event: { pointerType: string }) => {
+      if (event.pointerType === "mouse") holdOpen();
+    },
+    onPointerLeave: (event: { pointerType: string }) => {
+      if (event.pointerType === "mouse") scheduleClose();
+    },
+  };
+  const label = (
+    <>
+      最近错误：{text(lastErrorLabel(code))}
+      {number(at) !== null ? ` · ${expiry(at)}` : ""}
+    </>
+  );
+  if (json === "") return <small>{label}</small>;
+  return (
+    <>
+      <small>
+        <button
+          ref={trigger}
+          type="button"
+          className={s.errorJsonTrigger}
+          aria-expanded={open}
+          onClick={() => {
+            holdOpen();
+            setPinned((value) => !value);
+          }}
+          {...hoverProps}
+        >
+          {label}
+        </button>
+      </small>
+      {open &&
+        createPortal(
+          <div
+            ref={tip}
+            className={s.errorJsonTip}
+            style={
+              placement ? { top: placement.top, left: placement.left } : { visibility: "hidden" }
+            }
+            {...hoverProps}
+          >
+            {json}
+          </div>,
+          popoverHost(trigger.current),
+        )}
+    </>
+  );
+}
+
 export { safeOAuthUrl } from "../OAuth";
 
 function boundModels(credential: Credential): string[] {
@@ -778,10 +870,7 @@ export function Credentials() {
           ) : null}
         </div>
         {c.last_error_code != null && (
-          <small>
-            最近错误：{text(lastErrorLabel(c.last_error_code))}
-            {number(c.last_failure_at) !== null ? ` · ${expiry(c.last_failure_at)}` : ""}
-          </small>
+          <LastErrorJson code={c.last_error_code} at={c.last_failure_at} body={c.last_error_body} />
         )}
         {typeof c.sync_error === "string" && c.sync_error !== "" && (
           <small>失败原因：{text(c.sync_error)}</small>
diff --git a/web/src/ui.module.scss b/web/src/ui.module.scss
index 2814b76..e7d4668 100644
--- a/web/src/ui.module.scss
+++ b/web/src/ui.module.scss
@@ -602,6 +602,36 @@ $white: var(--page);
   border-radius: 14px;
   white-space: normal;
 }
+
+.errorJsonTip {
+  /* Fixed beside the trigger so a scrolling table never clips it. */
+  position: fixed;
+  z-index: 40;
+  max-width: min(560px, calc(100vw - 16px));
+  max-height: min(420px, calc(100vh - 16px));
+  overflow: auto;
+  margin: 0;
+  padding: 10px 12px;
+  @include glass(0.92, 16px);
+  border-radius: 12px;
+  font-size: 12px;
+  line-height: 1.5;
+  white-space: pre-wrap;
+  word-break: break-all;
+  font-family: var(--mono, ui-monospace, monospace);
+  cursor: text;
+}
+
+.errorJsonTrigger {
+  border: none;
+  background: none;
+  padding: 0;
+  color: inherit;
+  font: inherit;
+  cursor: help;
+  text-decoration: underline dotted;
+  text-underline-offset: 3px;
+}
 .channelTrack button > span,
 .channelTrack small {
   font-size: 11px;

From a33aadfd5c0db148e33a4356df23acf53332adb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?=
 <277378677+maiphucgiang@users.noreply.github.com>
Date: Fri, 23 Oct 2026 19:01:15 +0800
Subject: [PATCH 2/8] Re-enabling or resetting a credential returns it to a
 fresh state

reset_cooldowns_for now also drops the recorded failure evidence
(last_failure_at, last_error_body), so the WebUI re-enable toggle and
the reset-cooldown endpoint both hand back a credential with no
cooldown, no error label and no stale tooltip JSON.
---
 app/gateway_management.py          |  3 +++
 converter.py                       |  3 +++
 tests/test_credential_cooldowns.py | 16 ++++++++++++++++
 3 files changed, 22 insertions(+)

diff --git a/app/gateway_management.py b/app/gateway_management.py
index 1840b8e..af46b00 100644
--- a/app/gateway_management.py
+++ b/app/gateway_management.py
@@ -95,6 +95,9 @@ def admin_set_credential_enabled(self, identity, enabled):
             self.CONFIG["control_store"].set_credential(identity, enabled)
             pool._sticky.clear()
             if enabled:
+                # Re-enabling returns the credential to a fresh state: reset_cooldowns_for
+                # lifts every cooldown and drops the recorded failure evidence.
+                pool.reset_cooldowns_for(identity)
                 pool._queue_sync(entry["id"])
             else:
                 pool._sync_pending.discard(entry["id"])
diff --git a/converter.py b/converter.py
index f3fde1a..b6cf193 100644
--- a/converter.py
+++ b/converter.py
@@ -1047,7 +1047,10 @@ def reset_cooldowns_for(self, identity: str) -> dict:
             now = time.time()
             changed = entry["fail_until"] > now
             entry["fail_until"] = 0.0
+            entry["fail_until"] = 0.0
             entry["last_error"] = None
+            entry["last_failure_at"] = None
+            entry["last_error_body"] = None
             for key in [k for k in self._model_fail if k[0] == entry["id"]]:
                 if self._model_fail[key] > now:
                     changed = True
diff --git a/tests/test_credential_cooldowns.py b/tests/test_credential_cooldowns.py
index 18f26fb..129326d 100644
--- a/tests/test_credential_cooldowns.py
+++ b/tests/test_credential_cooldowns.py
@@ -351,6 +351,22 @@ def test_plain_403_cooldowns_without_disabling(self):
         self.assertTrue(converter.model_policy.credential_enabled(converter.CONFIG, pool._entries[0]))
         self.assertFalse(pool._healthy(pool._entries[0]))
 
+
+    def test_reset_returns_the_account_to_a_fresh_state(self):
+        """An operator reset (or re-enable) lifts cooldowns and drops every failure record."""
+        pool = self.pool(self.credential())
+        body = json.dumps({"code": 11140, "msg": "request illegal"}).encode()
+        entry = pool._entries[0]
+        pool.note_status(entry["cm"], 403, raw=body)
+        self.assertIsNotNone(entry["last_error_body"])
+        self.assertIsNotNone(entry["last_failure_at"])
+        pool.reset_cooldowns_for(entry["account_key"])
+        self.assertEqual(entry["fail_until"], 0.0)
+        self.assertIsNone(entry["last_error"])
+        self.assertIsNone(entry["last_failure_at"])
+        self.assertIsNone(entry["last_error_body"])
+        self.assertTrue(pool._healthy(entry))
+
     def test_auth_circuit_breaker_survives_a_restart(self):
         first = self.pool(self.credential())
         first.cooldown(first._entries[0]["cm"], reason="backend HTTP 401")

From 77054a1a665d4b3ceafccdb89d98f5569c693b2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?=
 <277378677+maiphucgiang@users.noreply.github.com>
Date: Fri, 23 Oct 2026 19:19:54 +0800
Subject: [PATCH 3/8] Distinguish auto-disabled credentials from manual ones
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

An upstream 11140 restriction now marks the entry auto_disabled; the
inventory exposes it, health reports auto_disabled, and the WebUI
shows 自动禁用 on the badge instead of the manual disabled label.
Re-enable and reset clear the flag along with the failure evidence.
---
 app/admin_api.py                   |  2 +-
 app/gateway_management.py          |  6 +++++-
 converter.py                       |  4 +++-
 tests/test_credential_cooldowns.py |  3 +++
 web/src/pages/Credentials.tsx      | 11 ++++++++++-
 web/src/values.tsx                 |  1 +
 6 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/app/admin_api.py b/app/admin_api.py
index b76f67a..ee187fd 100644
--- a/app/admin_api.py
+++ b/app/admin_api.py
@@ -67,7 +67,7 @@ def _public_credential(item):
               "trial_supported", "trial",
               "daily_chat_supported", "auto_daily_chat", "daily_chat",
               "growth_supported", "auto_growth", "growth_auto",
-              "token_expired", "token_expires_at", "last_refresh_time", "sessions", "sticky_sessions", "last_error_code", "last_error_body", "usage_stats"}
+              "token_expired", "token_expires_at", "last_refresh_time", "sessions", "sticky_sessions", "last_error_code", "last_error_body", "auto_disabled", "usage_stats"}
     result = {key: value for key, value in item.items() if key in fields}
     identity = item.get("account_key") or item.get("id")
     if identity and ("/" in str(identity) or "\\" in str(identity)):
diff --git a/app/gateway_management.py b/app/gateway_management.py
index af46b00..c83945d 100644
--- a/app/gateway_management.py
+++ b/app/gateway_management.py
@@ -49,9 +49,12 @@ def admin_credential_inventory(self):
                               "remaining_seconds": max(0, round(deadline - now))}
                              for (cid, model), deadline in pool._model_fail.items()
                              if cid == path and deadline > now]
-                state = ("disabled" if not enabled else "error" if row.get("error") else
+                auto_disabled = bool(entry.get("auto_disabled")) and not enabled
+                state = ("auto_disabled" if auto_disabled else
+                         "disabled" if not enabled else "error" if row.get("error") else
                          "circuit_open" if until > now else "expired" if row.get("token_expired") else "ready")
                 row.update(id=identity, name=Path(path).name, enabled=enabled, health=state,
+                           auto_disabled=auto_disabled,
                            auto_checkin=model_policy.credential_auto_checkin(self.CONFIG, entry),
                            checkin=checkin.view(balance.get("checkin") or {}),
                            auto_travel=model_policy.credential_auto_travel(self.CONFIG, entry),
@@ -101,6 +104,7 @@ def admin_set_credential_enabled(self, identity, enabled):
                 pool._queue_sync(entry["id"])
             else:
                 pool._sync_pending.discard(entry["id"])
+                entry["auto_disabled"] = None
                 pool._sync_retry.pop(entry["id"], None)
                 if not pool._sync_pending:
                     pool._sync_event.clear()
diff --git a/converter.py b/converter.py
index b6cf193..7d8b76e 100644
--- a/converter.py
+++ b/converter.py
@@ -1051,6 +1051,7 @@ def reset_cooldowns_for(self, identity: str) -> dict:
             entry["last_error"] = None
             entry["last_failure_at"] = None
             entry["last_error_body"] = None
+            entry["auto_disabled"] = None
             for key in [k for k in self._model_fail if k[0] == entry["id"]]:
                 if self._model_fail[key] > now:
                     changed = True
@@ -1404,7 +1405,8 @@ def _disable_restricted(self, cm, code, *, generation=None):
             store.set_credential(identity, False)
             self._sticky.clear()
         management.gateway.invalidate_model_table()
-        _log(f"[cred] 上游限制 {code}，账号已禁用: {Path(cm.path).name}")
+        entry["auto_disabled"] = True
+        _log(f"[cred] 上游限制 {code}，账号已自动禁用: {Path(cm.path).name}")
 
     def cooldown(self, cm: CredentialManager, reason: str = "", *, generation=None):
         with self._lock, (cm._lock if generation is not None else nullcontext()):
diff --git a/tests/test_credential_cooldowns.py b/tests/test_credential_cooldowns.py
index 129326d..f37e06f 100644
--- a/tests/test_credential_cooldowns.py
+++ b/tests/test_credential_cooldowns.py
@@ -334,6 +334,7 @@ def test_upstream_restriction_disables_the_account_through_the_webui_policy(self
         self.enterContext(patch.dict(converter.CONFIG, {"control_store": store, "management": management}))
         body = json.dumps({"code": 11140, "msg": "request illegal"}).encode()
         pool.note_status(pool._entries[0]["cm"], 403, raw=body)
+        self.assertTrue(pool._entries[0]["auto_disabled"])
         self.assertFalse(converter.model_policy.credential_enabled(converter.CONFIG, pool._entries[0]))
         self.assertEqual(invalidated, [1])
         # The policy survives a pool rebuild, matching a WebUI-initiated disable.
@@ -361,10 +362,12 @@ def test_reset_returns_the_account_to_a_fresh_state(self):
         self.assertIsNotNone(entry["last_error_body"])
         self.assertIsNotNone(entry["last_failure_at"])
         pool.reset_cooldowns_for(entry["account_key"])
+        pool.reset_cooldowns_for(entry["account_key"])
         self.assertEqual(entry["fail_until"], 0.0)
         self.assertIsNone(entry["last_error"])
         self.assertIsNone(entry["last_failure_at"])
         self.assertIsNone(entry["last_error_body"])
+        self.assertIsNone(entry["auto_disabled"])
         self.assertTrue(pool._healthy(entry))
 
     def test_auth_circuit_breaker_survives_a_restart(self):
diff --git a/web/src/pages/Credentials.tsx b/web/src/pages/Credentials.tsx
index 4254360..fdeb634 100644
--- a/web/src/pages/Credentials.tsx
+++ b/web/src/pages/Credentials.tsx
@@ -61,6 +61,15 @@ const lastErrorLabels: Record<string, string> = {
   credential_error: "凭证同步失败（详见操作结果）",
 };
 
+const healthLabels: Record<string, string> = {
+  ready: "就绪",
+  disabled: "已停用",
+  auto_disabled: "自动禁用",
+  circuit_open: "认证熔断",
+  expired: "已过期",
+  error: "错误",
+};
+
 function lastErrorLabel(value: unknown) {
   const key = text(value);
   return lastErrorLabels[key] ?? key;
@@ -824,7 +833,7 @@ export function Credentials() {
           ? `认证熔断 ${remaining}s`
           : c.health === "ready"
             ? "认证正常"
-            : text(c.health)}
+            : (healthLabels[typeof c.health === "string" ? c.health : ""] ?? text(c.health))}
       </Badge>
     );
   };
diff --git a/web/src/values.tsx b/web/src/values.tsx
index a883eab..83071ab 100644
--- a/web/src/values.tsx
+++ b/web/src/values.tsx
@@ -169,6 +169,7 @@ const profiles: Record<string, string> = {
 const statuses: Record<string, string> = {
   ready: "就绪",
   disabled: "已停用",
+  auto_disabled: "自动禁用",
   error: "失败",
   success: "成功",
   cancelled: "已取消",

From 608a989c6beeb3b942cbb3e3383223fe9e6aad9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?=
 <277378677+maiphucgiang@users.noreply.github.com>
Date: Fri, 23 Oct 2026 19:46:37 +0800
Subject: [PATCH 4/8] Auto-disable accounts the upstream has terminally
 rejected

Two new terminal signals feed the same auto-disable chain as the
11140 restriction: three consecutive refresh-token business
rejections (network faults excluded, counted per credential, reset
on success), and three 401s against a token refreshed within the
last two minutes inside an hour. Refresh rejections now raise
TokenRefreshRejected with the structured code/msg/body so the
tooltip shows the upstream verdict instead of only a log line.
Reset or re-enable clears both counters with the rest of the state.
---
 converter.py                       | 97 +++++++++++++++++++++++++-----
 tests/test_credential_cooldowns.py | 51 ++++++++++++++++
 2 files changed, 134 insertions(+), 14 deletions(-)

diff --git a/converter.py b/converter.py
index 7d8b76e..a588882 100644
--- a/converter.py
+++ b/converter.py
@@ -371,7 +371,9 @@ def _refresh_locked(self):
         except Exception as e:
             raise RuntimeError(f"刷新 token 网络失败：{e}")
         if data.get("code") != 0 or not data.get("data"):
-            raise RuntimeError(f"刷新 token 失败：{data.get('msg', data)}")
+            code, msg, body = _diagnostic_from_payload(data)
+            raise TokenRefreshRejected(f"刷新 token 失败：{data.get('msg', data)}",
+                                       code=code, msg=msg, body=body)
         new_auth = dict(data["data"])
         if not isinstance(new_auth.get("accessToken"), str) or not new_auth["accessToken"]:
             raise RuntimeError("刷新接口未返回有效的 accessToken")
@@ -414,6 +416,13 @@ def invalidate(self):
             self._generation += 1
 
 
+    def token_age_s(self) -> float | None:
+        """Seconds since the stored token was last refreshed; None when unknown."""
+        with self._lock:
+            auth = self._session().get("auth") or {}
+            last = auth.get("lastRefreshTime")
+            return None if not last else max(0.0, time.time() - last / 1000)
+
     def summary(self) -> dict:
         perf_metrics.inc("credential_summary_calls")
         with self._lock:
@@ -448,6 +457,9 @@ def summary(self) -> dict:
 MODEL_COOLDOWN_MAX = 86400  # Maximum model cooldown in seconds
 UPSTREAM_BODY_AUDIT_LIMIT = 256 * 1024  # Bounded upstream error JSON kept per failed attempt.
 RESTRICTED_CODES = frozenset({"11140"})  # Upstream account-restriction codes; disable, never requeue.
+FRESH_401_LIMIT = 3         # Fresh-token 401s inside the window before auto-disable
+FRESH_401_WINDOW = 3600     # Sliding window for fresh-token 401 counting
+REFRESH_REJECT_LIMIT = 3    # Consecutive business refresh rejections before auto-disable
 MODEL_SITE_BLOCK_S = 6 * 3600      # Initial unsupported-model backoff
 MODEL_SITE_BLOCK_MAX_S = 24 * 3600  # Maximum unsupported-model backoff
 # An unsupported model must be routed to a different backend.
@@ -541,14 +553,8 @@ def _parse_not_servable(raw: bytes, status: int):
     return None
 
 
-def _upstream_diagnostic(raw: bytes):
-    """Extract the business code, message and bounded JSON body from an upstream error."""
-    if not raw:
-        return None, None, None
-    try:
-        payload = json.loads(raw.decode("utf-8", "replace"))
-    except (ValueError, UnicodeError, RecursionError):
-        return None, None, None
+def _diagnostic_from_payload(payload):
+    """Extract (code, msg, bounded body) from an upstream JSON payload, for audit only."""
     if not isinstance(payload, dict):
         return None, None, None
     nodes = [payload]
@@ -568,6 +574,25 @@ def _upstream_diagnostic(raw: bytes):
     return (code[:64] or None), (msg[:256] or None), (format_log_body(payload, UPSTREAM_BODY_AUDIT_LIMIT) or None)
 
 
+def _upstream_diagnostic(raw: bytes):
+    """Extract the business code, message and bounded JSON body from an upstream error."""
+    if not raw:
+        return None, None, None
+    try:
+        payload = json.loads(raw.decode("utf-8", "replace"))
+    except (ValueError, UnicodeError, RecursionError):
+        return None, None, None
+    return _diagnostic_from_payload(payload)
+
+
+class TokenRefreshRejected(RuntimeError):
+    """The upstream refresh endpoint answered with a business rejection, not a network fault."""
+
+    def __init__(self, message, *, code=None, msg=None, body=None):
+        super().__init__(message)
+        self.code, self.msg, self.body = code, msg, body
+
+
 def _block_model(model: str | None) -> str | None:
     """Track blocked models by public ID, normalizing the international auto alias."""
     return "auto" if model == "default-model" else model
@@ -1052,6 +1077,8 @@ def reset_cooldowns_for(self, identity: str) -> dict:
             entry["last_failure_at"] = None
             entry["last_error_body"] = None
             entry["auto_disabled"] = None
+            entry["fresh_401s"] = []
+            entry["refresh_rejects"] = 0
             for key in [k for k in self._model_fail if k[0] == entry["id"]]:
                 if self._model_fail[key] > now:
                     changed = True
@@ -1391,8 +1418,8 @@ def _note_error_body(self, cm, raw, *, generation=None):
             if entry is not None:
                 entry["last_error_body"] = body[:8192]
 
-    def _disable_restricted(self, cm, code, *, generation=None):
-        """An upstream restriction is terminal: disable via the WebUI policy, not a cooldown."""
+    def _auto_disable(self, cm, reason, *, generation=None):
+        """A terminal upstream verdict: disable via the WebUI policy, not a cooldown."""
         entry = self._index_cm.get(cm)
         store = CONFIG.get("control_store")
         management = CONFIG.get("management")
@@ -1404,9 +1431,45 @@ def _disable_restricted(self, cm, code, *, generation=None):
                 return
             store.set_credential(identity, False)
             self._sticky.clear()
+            entry["auto_disabled"] = True
         management.gateway.invalidate_model_table()
-        entry["auto_disabled"] = True
-        _log(f"[cred] 上游限制 {code}，账号已自动禁用: {Path(cm.path).name}")
+        _log(f"[cred] {reason}，账号已自动禁用: {Path(cm.path).name}")
+
+    def _note_fresh_401(self, cm, *, generation=None):
+        """A 401 against a just-refreshed token means the account was revoked server-side."""
+        try:
+            age = cm.token_age_s()
+        except Exception:
+            return
+        if age is None or age > 120:
+            return
+        with self._lock, (cm._lock if generation is not None else nullcontext()):
+            if not self._lease_matches(cm, generation):
+                return
+            entry = self._index_cm.get(cm)
+            if entry is None:
+                return
+            now = time.time()
+            strikes = [t for t in entry.get("fresh_401s", []) if now - t < FRESH_401_WINDOW]
+            strikes.append(now)
+            entry["fresh_401s"] = strikes
+            fresh = len(strikes) >= FRESH_401_LIMIT
+            if fresh:
+                entry["fresh_401s"] = []
+        if fresh:
+            self._auto_disable(cm, f"新令牌 {FRESH_401_WINDOW}s 内 {FRESH_401_LIMIT} 次被拒（401）",
+                               generation=generation)
+    def _note_refresh_reject(self, entry, error):
+        """Count consecutive upstream refresh rejections; the refresh token itself is dead."""
+        rejects = entry.get("refresh_rejects", 0) + 1
+        entry["refresh_rejects"] = rejects
+        if error.body:
+            entry["last_error_body"] = error.body[:8192]
+        if rejects < REFRESH_REJECT_LIMIT:
+            return
+        entry["refresh_rejects"] = 0
+        self._auto_disable(entry["cm"], "刷新令牌被上游连续拒绝")
+
 
     def cooldown(self, cm: CredentialManager, reason: str = "", *, generation=None):
         with self._lock, (cm._lock if generation is not None else nullcontext()):
@@ -1431,7 +1494,9 @@ def note_status(self, cm: CredentialManager | None, status: int,
             self._note_error_body(cm, raw, generation=generation)
             code, _, _ = _upstream_diagnostic(raw or b"")
             if status == 403 and code in RESTRICTED_CODES:
-                self._disable_restricted(cm, code, generation=generation)
+                self._auto_disable(cm, f"上游限制 {code}", generation=generation)
+            elif status == 401:
+                self._note_fresh_401(cm, generation=generation)
             return
         if status == 400 and is_channel_check_error(raw):
             # Only a rejection that survived the console retry is account-wide; endpoints without a
@@ -1583,6 +1648,9 @@ def refresh_due(self, margin_s: int = CRED_REFRESH_MARGIN, keepalive_s: int = CR
                         continue
                     refreshed = cm.refresh_if_due(margin_s, keepalive_s if keepalive_due else 0)
                     entry["keepalive_after"] = 0.0
+                except TokenRefreshRejected as error:
+                    failure = (str(error), cm._generation)
+                    self._note_refresh_reject(entry, error)
                 except Exception as error:
                     failure = (str(error), cm._generation)
                     if keepalive_due:
@@ -1590,6 +1658,7 @@ def refresh_due(self, margin_s: int = CRED_REFRESH_MARGIN, keepalive_s: int = CR
             if failure:
                 self.cooldown(cm, reason=failure[0], generation=failure[1])
             elif refreshed:
+                entry["refresh_rejects"] = 0
                 _log(f"[cred] {'每日保活刷新' if keepalive_due else '已主动刷新'}并回写: {Path(entry['id']).name}")
 
     def remove_file(self, name: str) -> bool:
diff --git a/tests/test_credential_cooldowns.py b/tests/test_credential_cooldowns.py
index f37e06f..347ac4f 100644
--- a/tests/test_credential_cooldowns.py
+++ b/tests/test_credential_cooldowns.py
@@ -370,6 +370,57 @@ def test_reset_returns_the_account_to_a_fresh_state(self):
         self.assertIsNone(entry["auto_disabled"])
         self.assertTrue(pool._healthy(entry))
 
+    def test_fresh_token_401s_auto_disable(self):
+        """401 against a just-refreshed token, three times, means the account was revoked."""
+        pool = self.pool(self.credential())
+        store = converter_local_ControlStore(self.root / "control.sqlite3")
+        self.addCleanup(store.close)
+        identity = pool._entries[0]["account_key"]
+        store.set_credential(identity, True)
+        invalidated = []
+        management = SimpleNamespace(gateway=SimpleNamespace(
+            invalidate_model_table=lambda: invalidated.append(1)))
+        self.enterContext(patch.dict(converter.CONFIG, {"control_store": store, "management": management}))
+        entry = pool._entries[0]
+        for _ in range(2):
+            pool.note_status(entry["cm"], 401, raw=b"unauthorized")
+        self.assertIsNone(entry.get("auto_disabled"))
+        pool.note_status(entry["cm"], 401, raw=b"unauthorized")
+        self.assertTrue(entry["auto_disabled"])
+        self.assertFalse(converter.model_policy.credential_enabled(converter.CONFIG, entry))
+        self.assertEqual(invalidated, [1])
+        # A stale token never counts toward the strikes.
+        fresh = self.pool(self.credential(name="stale.info", uid="stale-uid"))
+        with patch.object(fresh._entries[0]["cm"], "token_age_s", return_value=9999.0):
+            for _ in range(5):
+                fresh.note_status(fresh._entries[0]["cm"], 401, raw=b"unauthorized")
+        self.assertTrue(converter.model_policy.credential_enabled(converter.CONFIG, fresh._entries[0]))
+
+    def test_consecutive_refresh_rejections_auto_disable(self):
+        """Upstream keeps rejecting the refresh token: disable after the third rejection."""
+        pool = self.pool(self.credential())
+        store = converter_local_ControlStore(self.root / "control.sqlite3")
+        self.addCleanup(store.close)
+        identity = pool._entries[0]["account_key"]
+        store.set_credential(identity, True)
+        invalidated = []
+        management = SimpleNamespace(gateway=SimpleNamespace(
+            invalidate_model_table=lambda: invalidated.append(1)))
+        self.enterContext(patch.dict(converter.CONFIG, {"control_store": store, "management": management}))
+        entry = pool._entries[0]
+        body = json.dumps({"code": 10016, "msg": "refresh token invalid"}).encode()
+        error = converter.TokenRefreshRejected("刷新 token 失败：refresh token invalid",
+                                               code="10016", msg="refresh token invalid",
+                                               body=body.decode())
+        for _ in range(2):
+            pool._note_refresh_reject(entry, error)
+        self.assertEqual(entry["refresh_rejects"], 2)
+        pool._note_refresh_reject(entry, error)
+        self.assertTrue(entry["auto_disabled"])
+        self.assertEqual(entry["refresh_rejects"], 0)
+        self.assertIn("refresh token invalid", entry["last_error_body"])
+        self.assertFalse(converter.model_policy.credential_enabled(converter.CONFIG, entry))
+
     def test_auth_circuit_breaker_survives_a_restart(self):
         first = self.pool(self.credential())
         first.cooldown(first._entries[0]["cm"], reason="backend HTTP 401")

From e531479a465905a5f113d624fc0fe9a01d25eaef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?=
 <277378677+maiphucgiang@users.noreply.github.com>
Date: Fri, 23 Oct 2026 20:17:11 +0800
Subject: [PATCH 5/8] Hide transient cooldown labels on disabled credentials
 and fix new tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The health badge no longer counts down 认证熔断 for disabled
credentials: an auto-disabled or manually stopped account is in a
steady state, so the badge shows 自动禁用 / 已停用 directly. Test
expectations follow the widened audit diagnostic budget and the new
upstream_error attempt records.
---
 tests/test_control_store.py   |  3 ++-
 tests/test_stream_failover.py |  3 ++-
 web/src/pages/Credentials.tsx | 22 ++++++++++++++++------
 3 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/tests/test_control_store.py b/tests/test_control_store.py
index cb2764d..0bfd54d 100644
--- a/tests/test_control_store.py
+++ b/tests/test_control_store.py
@@ -155,7 +155,8 @@ def test_setting_precedence_and_source_redaction(self):
         self.assertEqual(items["image_policy"]["source"], "environment")
         self.assertIsNone(items["api_key"]["value"])
         self.assertIsNone(items["auth_dir"]["value"])
-        for invalid in ({"usd_rate": float("nan")}, {"port": 65536}, {"audit_diagnostic_bytes": 8193}):
+        for invalid in ({"usd_rate": float("nan")}, {"port": 65536},
+                        {"audit_diagnostic_bytes": 4 * 1024 * 1024 + 1}):
             with self.assertRaises(ValueError):
                 validate_settings(invalid)
 
diff --git a/tests/test_stream_failover.py b/tests/test_stream_failover.py
index 2313375..382ac1f 100644
--- a/tests/test_stream_failover.py
+++ b/tests/test_stream_failover.py
@@ -347,7 +347,8 @@ def test_replayed_request_is_audited_as_success(self):
         record = self.only_record(store)
         self.assertEqual(record["outcome"], "success", record)
         self.assertFalse(record.get("error_code"), record)
-        statuses = [attempt.get("status_code") for attempt in record["attempts"]]
+        statuses = [attempt.get("status_code") for attempt in record["attempts"]
+                    if attempt.get("stage") == "upstream_http"]
         self.assertEqual(statuses[:2], [429, 200], record["attempts"])
         self.assertIn("failover_recovered", json.dumps(record["attempts"], ensure_ascii=False))
         markers = [attempt for attempt in record["attempts"]
diff --git a/web/src/pages/Credentials.tsx b/web/src/pages/Credentials.tsx
index fdeb634..9f6bbbb 100644
--- a/web/src/pages/Credentials.tsx
+++ b/web/src/pages/Credentials.tsx
@@ -823,17 +823,27 @@ export function Credentials() {
     </label>
   );
   const renderHealthBadge = (c: Credential) => {
+    const disabled = c.enabled === false;
     const until = number(c.fail_until);
-    const remaining = until === null ? null : Math.max(0, Math.ceil(until - now / 1000));
+    const remaining = disabled
+      ? null
+      : until === null
+        ? null
+        : Math.max(0, Math.ceil(until - now / 1000));
+    const label = disabled
+      ? c.auto_disabled
+        ? healthLabels.auto_disabled
+        : healthLabels.disabled
+      : remaining !== null && remaining > 0
+        ? `认证熔断 ${remaining}s`
+        : c.health === "ready"
+          ? "认证正常"
+          : (healthLabels[typeof c.health === "string" ? c.health : ""] ?? text(c.health));
     return (
       <Badge
         tone={remaining !== null && remaining > 0 ? "bad" : c.health === "ready" ? "good" : "warn"}
       >
-        {remaining !== null && remaining > 0
-          ? `认证熔断 ${remaining}s`
-          : c.health === "ready"
-            ? "认证正常"
-            : (healthLabels[typeof c.health === "string" ? c.health : ""] ?? text(c.health))}
+        {label}
       </Badge>
     );
   };

From 2296a2644e885cc4ace05a39dd3c32410f168c8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?=
 <277378677+maiphucgiang@users.noreply.github.com>
Date: Fri, 23 Oct 2026 21:05:12 +0800
Subject: [PATCH 6/8] Address review: lock ordering, nested codes, store
 failures, touch toggle

- refresh_due defers _note_refresh_reject past the credential lock: the
  third rejection reaches _auto_disable, which takes the pool lock, so
  calling it under cm._lock could deadlock against pool paths that
  acquire pool -> credential in the opposite order.
- _diagnostic_from_payload prefers the nested error envelope over a
  generic outer code, so {"code":"error", "error":{"code":11140}}
  no longer masks the terminal restriction code.
- _auto_disable catches control-store failures and logs them; a store
  outage no longer replaces the upstream 401/403 being reported.
- Touch taps toggle the tooltip purely through the pin; holdOpen stays
  mouse-only so a tap can never strand the hover state open.
- Tests: refresh_due wiring (patched refresh_if_due raising
  TokenRefreshRejected), nested-code precedence, and the persisted-bytes
  secret assertion.
---
 converter.py                       | 34 ++++++++++++++++++++----------
 tests/test_credential_cooldowns.py | 24 +++++++++++++++++++++
 tests/test_upstream_diagnostics.py | 12 +++++++++++
 web/src/pages/Credentials.tsx      |  3 ++-
 4 files changed, 61 insertions(+), 12 deletions(-)

diff --git a/converter.py b/converter.py
index a588882..b7728fb 100644
--- a/converter.py
+++ b/converter.py
@@ -557,10 +557,11 @@ def _diagnostic_from_payload(payload):
     """Extract (code, msg, bounded body) from an upstream JSON payload, for audit only."""
     if not isinstance(payload, dict):
         return None, None, None
-    nodes = [payload]
+    # The nested error envelope carries the business verdict; a generic outer
+    # code (e.g. "error") must not mask a terminal nested code such as 11140.
     inner = payload.get("error")
-    if isinstance(inner, dict):
-        nodes.append(inner)
+    nodes = [inner] if isinstance(inner, dict) else []
+    nodes.append(payload)
     code = msg = ""
     for node in nodes:
         for key in ("code", "errCode", "error_code"):
@@ -1426,13 +1427,18 @@ def _auto_disable(self, cm, reason, *, generation=None):
         identity = entry.get("account_key") if entry else None
         if not identity or store is None or management is None:
             return
-        with self._lock, (cm._lock if generation is not None else nullcontext()):
-            if not self._lease_matches(cm, generation):
-                return
-            store.set_credential(identity, False)
-            self._sticky.clear()
-            entry["auto_disabled"] = True
-        management.gateway.invalidate_model_table()
+        try:
+            with self._lock, (cm._lock if generation is not None else nullcontext()):
+                if not self._lease_matches(cm, generation):
+                    return
+                store.set_credential(identity, False)
+                self._sticky.clear()
+                entry["auto_disabled"] = True
+            management.gateway.invalidate_model_table()
+        except Exception as error:
+            # A control-store outage must never replace the upstream rejection being reported.
+            _log(f"[cred] 自动禁用写入失败（{Path(cm.path).name}）: {sanitize_log_text(str(error), 200)}")
+            return
         _log(f"[cred] {reason}，账号已自动禁用: {Path(cm.path).name}")
 
     def _note_fresh_401(self, cm, *, generation=None):
@@ -1633,7 +1639,9 @@ def refresh_due(self, margin_s: int = CRED_REFRESH_MARGIN, keepalive_s: int = CR
             if now < entry.get("fail_until", 0.0) or not model_policy.credential_enabled(CONFIG, entry):
                 continue
             cm = entry["cm"]
+            cm = entry["cm"]
             failure = None
+            refresh_reject = None
             refreshed = keepalive_due = False
             with cm._lock:
                 try:
@@ -1650,11 +1658,15 @@ def refresh_due(self, margin_s: int = CRED_REFRESH_MARGIN, keepalive_s: int = CR
                     entry["keepalive_after"] = 0.0
                 except TokenRefreshRejected as error:
                     failure = (str(error), cm._generation)
-                    self._note_refresh_reject(entry, error)
+                    # Deferred past cm._lock: the third rejection reaches _auto_disable,
+                    # which takes the pool lock; pool paths acquire pool -> credential.
+                    refresh_reject = error
                 except Exception as error:
                     failure = (str(error), cm._generation)
                     if keepalive_due:
                         entry["keepalive_after"] = now + CRED_KEEPALIVE_RETRY_S
+            if refresh_reject is not None:
+                self._note_refresh_reject(entry, refresh_reject)
             if failure:
                 self.cooldown(cm, reason=failure[0], generation=failure[1])
             elif refreshed:
diff --git a/tests/test_credential_cooldowns.py b/tests/test_credential_cooldowns.py
index 347ac4f..7b74118 100644
--- a/tests/test_credential_cooldowns.py
+++ b/tests/test_credential_cooldowns.py
@@ -421,6 +421,30 @@ def test_consecutive_refresh_rejections_auto_disable(self):
         self.assertIn("refresh token invalid", entry["last_error_body"])
         self.assertFalse(converter.model_policy.credential_enabled(converter.CONFIG, entry))
 
+    def test_refresh_due_feeds_rejections_into_the_auto_disable_path(self):
+        """refresh_due itself must count TokenRefreshRejected; the helper is not enough."""
+        pool = self.pool(self.credential())
+        store = converter_local_ControlStore(self.root / "control.sqlite3")
+        self.addCleanup(store.close)
+        identity = pool._entries[0]["account_key"]
+        store.set_credential(identity, True)
+        invalidated = []
+        management = SimpleNamespace(gateway=SimpleNamespace(
+            invalidate_model_table=lambda: invalidated.append(1)))
+        self.enterContext(patch.dict(converter.CONFIG, {"control_store": store, "management": management}))
+        entry = pool._entries[0]
+        error = converter.TokenRefreshRejected("刷新 token 失败：refresh token invalid",
+                                               code="10016", msg="refresh token invalid")
+        with patch.object(entry["cm"], "refresh_if_due", side_effect=error):
+            for _ in range(2):
+                pool.refresh_due(margin_s=10 ** 9, keepalive_s=0)
+                entry["fail_until"] = 0.0  # The rejection cooldown would skip the next pass.
+            self.assertEqual(entry["refresh_rejects"], 2)
+            pool.refresh_due(margin_s=10 ** 9, keepalive_s=0)
+        self.assertTrue(entry["auto_disabled"])
+        self.assertFalse(converter.model_policy.credential_enabled(converter.CONFIG, entry))
+        self.assertEqual(invalidated, [1])
+
     def test_auth_circuit_breaker_survives_a_restart(self):
         first = self.pool(self.credential())
         first.cooldown(first._entries[0]["cm"], reason="backend HTTP 401")
diff --git a/tests/test_upstream_diagnostics.py b/tests/test_upstream_diagnostics.py
index 80b10b1..f6c757a 100644
--- a/tests/test_upstream_diagnostics.py
+++ b/tests/test_upstream_diagnostics.py
@@ -81,6 +81,16 @@ def test_nested_envelope_is_unwrapped(self):
         self.assertEqual(entry["upstream_code"], "11140")
         self.assertEqual(entry["upstream_msg"], "request illegal")
 
+    def test_nested_terminal_code_masks_generic_outer_code(self):
+        """A generic outer code must not hide a terminal nested restriction code."""
+        self.respond = lambda request: httpx.Response(
+            403, json={"code": "error", "msg": "forbidden",
+                       "error": {"code": "11140", "message": "request illegal"}})
+        self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]})
+        entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error")
+        self.assertEqual(entry["upstream_code"], "11140")
+        self.assertEqual(entry["upstream_msg"], "request illegal")
+
     def test_non_json_body_records_no_diagnostic_but_keeps_the_status(self):
         self.respond = lambda request: httpx.Response(403, text="<html>blocked</html>")
         self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]})
@@ -119,6 +129,8 @@ def test_body_tokens_are_redacted_before_storage(self):
         entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error")
         self.assertNotIn("synthetic-private-token", entry["upstream_body"])
         raw = b"".join(f.read_bytes() for f in Path(self.directory).glob("logs.sqlite3*"))
+        # Storage itself must never persist the secret, not only the record read path.
+        self.assertNotIn(b"synthetic-private-token", raw)
 
     def test_secrets_in_upstream_messages_are_dropped(self):
         self.respond = lambda request: httpx.Response(
diff --git a/web/src/pages/Credentials.tsx b/web/src/pages/Credentials.tsx
index 9f6bbbb..1d73844 100644
--- a/web/src/pages/Credentials.tsx
+++ b/web/src/pages/Credentials.tsx
@@ -140,8 +140,9 @@ function LastErrorJson({ code, at, body }: { code: unknown; at: unknown; body: u
           type="button"
           className={s.errorJsonTrigger}
           aria-expanded={open}
+          /* Touch toggles purely via the pin: hover only tracks mouse pointers,
+             so a tap never strands `hovered` open. */
           onClick={() => {
-            holdOpen();
             setPinned((value) => !value);
           }}
           {...hoverProps}

From 8431d090a2e5d2c69a73eeabde0343d15e586ab4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?=
 <277378677+maiphucgiang@users.noreply.github.com>
Date: Fri, 23 Oct 2026 21:25:22 +0800
Subject: [PATCH 7/8] Fix consecutive-refresh semantics and persist
 auto-disable provenance
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Refresh rejections are consecutive only now: a request-path refresh
  (get_headers -> _refresh) succeeding after the last rejection, or a
  gap beyond one hour, resets the counter — sporadic rejections over
  days no longer accumulate to the disable threshold.
- TokenRefreshRejected during a keepalive pass now applies the same
  CRED_KEEPALIVE_RETRY_S backoff as every other keepalive failure.
- The 401/403 body is parsed once in note_status and handed to the
  tooltip note, instead of re-parsing (and sanitizing) three times,
  once under the pool lock.
- auto-disabled provenance persists in the control store
  (disable_credential_auto) and is adopted on pool rebuild, so a
  restart or cooldown reset keeps the 自动禁用 label and its tooltip
  body; any manual toggle clears the provenance, and the switch label
  is a plain 停用 while the badge distinguishes 人工停用/自动禁用.
- Manual disable clears the flag on the live entry; entries() copies
  no longer swallow the write.
---
 app/control_store.py               | 32 ++++++++++++-
 app/gateway_management.py          | 12 +++--
 app/model_policy.py                |  4 ++
 converter.py                       | 52 ++++++++++++++++++--
 tests/test_credential_cooldowns.py | 77 ++++++++++++++++++++++++++++++
 web/src/pages/Credentials.tsx      |  4 +-
 web/src/values.tsx                 |  2 +-
 7 files changed, 170 insertions(+), 13 deletions(-)

diff --git a/app/control_store.py b/app/control_store.py
index ab3380a..2f95767 100644
--- a/app/control_store.py
+++ b/app/control_store.py
@@ -186,10 +186,16 @@ def _load(self):
         for identity, metadata in data["credentials"].items():
             _identifier(identity, "账号指纹")
             growth = metadata.get("growth_auto")
+            growth = metadata.get("growth_auto")
             if (not isinstance(metadata, dict)
                     or set(metadata) - {"enabled", "label", "auto_checkin", "auto_travel",
-                                        "auto_daily_chat", "auto_growth", "growth_auto"}
+                                        "auto_daily_chat", "auto_growth", "growth_auto",
+                                        "auto_disabled", "auto_reason", "auto_body", "auto_at"}
                     or type(metadata.get("enabled")) is not bool
+                    or metadata.get("auto_disabled") is not None and type(metadata["auto_disabled"]) is not bool
+                    or metadata.get("auto_reason") is not None and not isinstance(metadata["auto_reason"], str)
+                    or metadata.get("auto_body") is not None and not isinstance(metadata["auto_body"], str)
+                    or metadata.get("auto_at") is not None and not isinstance(metadata["auto_at"], (int, float))
                     or metadata.get("auto_growth") is not None and type(metadata["auto_growth"]) is not bool
                     or any(key in metadata and type(metadata[key]) is not bool for key in
                            ("auto_checkin", "auto_travel", "auto_daily_chat"))
@@ -253,10 +259,32 @@ def change(state):
 
 
     def set_credential(self, account_key, enabled):
+        """Manual toggle: any operator action clears the automatic-disable provenance."""
         _identifier(account_key, "账号指纹")
         if type(enabled) is not bool:
             raise ValueError("enabled 必须为布尔值")
-        return self._update(None, lambda state: state["credentials"].setdefault(account_key, {}).update(enabled=enabled))
+
+        def change(state):
+            record = state["credentials"].setdefault(account_key, {})
+            record["enabled"] = enabled
+            record.pop("auto_disabled", None)  # Any manual toggle clears the provenance.
+        return self._update(None, change)
+
+    def disable_credential_auto(self, account_key, reason="", body=None, at=None):
+        """Record an automatic disable with its upstream evidence so it survives restarts."""
+        _identifier(account_key, "账号指纹")
+
+        def change(state):
+            record = state["credentials"].setdefault(account_key, {})
+            record.update(enabled=False, auto_disabled=True)
+            for key, value in (("auto_reason", str(reason or "")[:256] or None),
+                               ("auto_body", str(body)[:8192] if body else None),
+                               ("auto_at", at)):
+                if value is None:
+                    record.pop(key, None)
+                else:
+                    record[key] = value
+        return self._update(None, change)
 
     def unbind_credential(self, account_key):
         """Drop an account key from every rule, returning {source: old_ids} for rollback."""
diff --git a/app/gateway_management.py b/app/gateway_management.py
index c83945d..e77488d 100644
--- a/app/gateway_management.py
+++ b/app/gateway_management.py
@@ -37,19 +37,22 @@ def admin_credential_inventory(self):
         with pool._lock:
             entries = {entry["id"]: entry for entry in pool.entries()}
             result = []
+            policy = model_policy.snapshot(self.CONFIG)
             for row in pool.snapshot():
                 path = row.pop("auth_file", "")
                 entry = entries[path]
                 identity = entry.get("account_key") or hashlib.sha256(path.encode()).hexdigest()
                 ledger = self.CONFIG.get("ledger")
                 balance = ledger.entry(path) if ledger else {}
-                enabled = model_policy.credential_enabled(self.CONFIG, entry)
+                record = model_policy.credential_record(self.CONFIG, entry, policy=policy)
+                enabled = record.get("enabled", True)
                 until = entry.get("fail_until", 0)
                 cooldowns = [{"model": model, "until": deadline,
                               "remaining_seconds": max(0, round(deadline - now))}
                              for (cid, model), deadline in pool._model_fail.items()
                              if cid == path and deadline > now]
-                auto_disabled = bool(entry.get("auto_disabled")) and not enabled
+                # Provenance lives in the store so it survives restarts and cooldown resets.
+                auto_disabled = record.get("auto_disabled") is True and not enabled
                 state = ("auto_disabled" if auto_disabled else
                          "disabled" if not enabled else "error" if row.get("error") else
                          "circuit_open" if until > now else "expired" if row.get("token_expired") else "ready")
@@ -104,7 +107,10 @@ def admin_set_credential_enabled(self, identity, enabled):
                 pool._queue_sync(entry["id"])
             else:
                 pool._sync_pending.discard(entry["id"])
-                entry["auto_disabled"] = None
+                # entries() hands out copies; clear the flag on the live entry.
+                live = pool.entry_by_id(entry["id"])
+                if live is not None:
+                    live["auto_disabled"] = None
                 pool._sync_retry.pop(entry["id"], None)
                 if not pool._sync_pending:
                     pool._sync_event.clear()
diff --git a/app/model_policy.py b/app/model_policy.py
index 9cb00e1..658034d 100644
--- a/app/model_policy.py
+++ b/app/model_policy.py
@@ -75,6 +75,10 @@ def credential_enabled(config, entry, policy=None):
     return _policy(config, policy)["credentials"].get(entry.get("account_key"), {}).get("enabled", True)
 
 
+def credential_record(config, entry, policy=None):
+    """The whole stored credential record; carries the automatic-disable provenance."""
+    return _policy(config, policy)["credentials"].get(entry.get("account_key"), {})
+
 def credential_auto_checkin(config, entry, policy=None):
     """International accounts opt in explicitly; the preference is independent of routing."""
     default = entry.get("profile") in {"cn-cli", "cn-work"}
diff --git a/converter.py b/converter.py
index b7728fb..b5b66bf 100644
--- a/converter.py
+++ b/converter.py
@@ -460,6 +460,7 @@ def summary(self) -> dict:
 FRESH_401_LIMIT = 3         # Fresh-token 401s inside the window before auto-disable
 FRESH_401_WINDOW = 3600     # Sliding window for fresh-token 401 counting
 REFRESH_REJECT_LIMIT = 3    # Consecutive business refresh rejections before auto-disable
+REFRESH_REJECT_WINDOW = 3600  # Rejections further apart than this are not consecutive
 MODEL_SITE_BLOCK_S = 6 * 3600      # Initial unsupported-model backoff
 MODEL_SITE_BLOCK_MAX_S = 24 * 3600  # Maximum unsupported-model backoff
 # An unsupported model must be routed to a different backend.
@@ -729,6 +730,7 @@ def reload(self, paths: list[Path], *, reset: bool = True):
                             # account's own breaker, which is hydrated just below.
                             self._forget_credential_cooldown(entry)
                         self._adopt_cooldowns(entry)
+                        self._adopt_auto_state(entry)
                         if entry.get("uid"):
                             have_uids[identity] = cid
                         self._queue_sync(cid)
@@ -764,6 +766,7 @@ def reload(self, paths: list[Path], *, reset: bool = True):
                          "account_key": identity_key, "catalog_dirty": True}
                 self._bind_entry(entry)
                 self._adopt_cooldowns(entry)
+                self._adopt_auto_state(entry)
                 self._entries.append(entry)
                 by_id[cid] = entry
                 self._ignored_duplicates.discard(cid)
@@ -969,6 +972,26 @@ def _adopt_cooldowns(self, entry):
             key = (entry["id"], model)
             self._model_fail[key] = max(self._model_fail.get(key, 0.0), until)
 
+
+    def _adopt_auto_state(self, entry):
+        """Restore the persisted auto-disable provenance so a restart keeps the label."""
+        store = CONFIG.get("control_store")
+        if store is None or not entry.get("account_key"):
+            return
+        try:
+            record = store.snapshot()["credentials"].get(entry["account_key"]) or {}
+        except Exception:
+            return
+        if record.get("auto_disabled") is not True or record.get("enabled", True):
+            return
+        entry["auto_disabled"] = True
+        if record.get("auto_reason"):
+            entry["last_error"] = record["auto_reason"]
+        if record.get("auto_body"):
+            entry["last_error_body"] = record["auto_body"][:8192]
+        if record.get("auto_at"):
+            entry["last_failure_at"] = record["auto_at"]
+
     def _durable_identity(self, entry) -> bool:
         """Whether this account's identity is complete enough to key durable state.
 
@@ -1407,9 +1430,9 @@ def _lease_matches(cm, generation):
             return generation == cm._generation
         return generation == cm._generation
 
-    def _note_error_body(self, cm, raw, *, generation=None):
+    def _note_error_body(self, cm, body, *, generation=None):
         """Keep the last upstream error JSON for the credentials tooltip; bounded, redacted."""
-        _, _, body = _upstream_diagnostic(raw or b"")
+        # The diagnostic arrives pre-parsed from note_status; never re-parse under the lock.
         if not body:
             return
         with self._lock, (cm._lock if generation is not None else nullcontext()):
@@ -1431,7 +1454,9 @@ def _auto_disable(self, cm, reason, *, generation=None):
             with self._lock, (cm._lock if generation is not None else nullcontext()):
                 if not self._lease_matches(cm, generation):
                     return
-                store.set_credential(identity, False)
+                store.disable_credential_auto(
+                    identity, reason, body=entry.get("last_error_body") if entry else None,
+                    at=time.time())
                 self._sticky.clear()
                 entry["auto_disabled"] = True
             management.gateway.invalidate_model_table()
@@ -1467,6 +1492,18 @@ def _note_fresh_401(self, cm, *, generation=None):
                                generation=generation)
     def _note_refresh_reject(self, entry, error):
         """Count consecutive upstream refresh rejections; the refresh token itself is dead."""
+        now = time.time()
+        # A request-path refresh (get_headers -> _refresh) succeeding between rejections
+        # proves the token alive; a long gap alone is not "consecutive" either.
+        try:
+            age = entry["cm"].token_age_s()
+        except Exception:
+            age = None
+        refreshed_after = now - age if age is not None else 0.0
+        if refreshed_after > entry.get("refresh_reject_at", 0.0) or \
+                now - entry.get("refresh_reject_at", 0.0) > REFRESH_REJECT_WINDOW:
+            entry["refresh_rejects"] = 0
+        entry["refresh_reject_at"] = now
         rejects = entry.get("refresh_rejects", 0) + 1
         entry["refresh_rejects"] = rejects
         if error.body:
@@ -1497,8 +1534,9 @@ def note_status(self, cm: CredentialManager | None, status: int,
             return
         if status in (401, 403):
             self.cooldown(cm, reason=f"backend HTTP {status}", generation=generation)
-            self._note_error_body(cm, raw, generation=generation)
-            code, _, _ = _upstream_diagnostic(raw or b"")
+            # One parse feeds the audit body, the restriction check and the tooltip.
+            code, _, body = _upstream_diagnostic(raw or b"")
+            self._note_error_body(cm, body, generation=generation)
             if status == 403 and code in RESTRICTED_CODES:
                 self._auto_disable(cm, f"上游限制 {code}", generation=generation)
             elif status == 401:
@@ -1658,6 +1696,10 @@ def refresh_due(self, margin_s: int = CRED_REFRESH_MARGIN, keepalive_s: int = CR
                     entry["keepalive_after"] = 0.0
                 except TokenRefreshRejected as error:
                     failure = (str(error), cm._generation)
+                    # Keepalive rejections back off like every other keepalive failure;
+                    # without this a 5-minute cadence stacks 3 strikes in ~10 minutes.
+                    if keepalive_due:
+                        entry["keepalive_after"] = now + CRED_KEEPALIVE_RETRY_S
                     # Deferred past cm._lock: the third rejection reaches _auto_disable,
                     # which takes the pool lock; pool paths acquire pool -> credential.
                     refresh_reject = error
diff --git a/tests/test_credential_cooldowns.py b/tests/test_credential_cooldowns.py
index 7b74118..8e7cd4b 100644
--- a/tests/test_credential_cooldowns.py
+++ b/tests/test_credential_cooldowns.py
@@ -445,6 +445,83 @@ def test_refresh_due_feeds_rejections_into_the_auto_disable_path(self):
         self.assertFalse(converter.model_policy.credential_enabled(converter.CONFIG, entry))
         self.assertEqual(invalidated, [1])
 
+    def _auto_disable_fixture(self):
+        """A pool + store + management wired for auto-disable, with the live entry returned."""
+        pool = self.pool(self.credential())
+        store = converter_local_ControlStore(self.root / "control.sqlite3")
+        self.addCleanup(store.close)
+        identity = pool._entries[0]["account_key"]
+        store.set_credential(identity, True)
+        invalidated = []
+        management = SimpleNamespace(gateway=SimpleNamespace(
+            invalidate_model_table=lambda: invalidated.append(1)))
+        self.enterContext(patch.dict(converter.CONFIG, {"control_store": store, "management": management}))
+        return pool, store, identity
+
+    def test_request_path_refresh_resets_the_rejection_count(self):
+        """A successful get_headers refresh between rejections proves the token alive."""
+        pool, store, identity = self._auto_disable_fixture()
+        entry = pool._entries[0]
+        error = converter.TokenRefreshRejected("刷新 token 失败：refresh token invalid")
+        pool._note_refresh_reject(entry, error)
+        pool._note_refresh_reject(entry, error)
+        self.assertEqual(entry["refresh_rejects"], 2)
+        # The last rejection predates a fresh token, as a request-path refresh would produce.
+        entry["refresh_reject_at"] = time.time() - 10
+        with patch.object(entry["cm"], "token_age_s", return_value=0.5):
+            pool._note_refresh_reject(entry, error)
+        self.assertEqual(entry["refresh_rejects"], 1)
+        self.assertIsNone(entry.get("auto_disabled"))
+
+    def test_stale_refresh_rejections_do_not_accumulate(self):
+        """Sporadic rejections days apart never reach the disable threshold."""
+        pool = self.pool(self.credential())
+        entry = pool._entries[0]
+        error = converter.TokenRefreshRejected("刷新 token 失败")
+        with patch.object(entry["cm"], "token_age_s", return_value=None):
+            for _ in range(5):
+                entry["refresh_reject_at"] = time.time() - converter.REFRESH_REJECT_WINDOW - 1
+                pool._note_refresh_reject(entry, error)
+        self.assertEqual(entry["refresh_rejects"], 1)
+        self.assertIsNone(entry.get("auto_disabled"))
+
+    def test_keepalive_rejection_backs_off_like_other_failures(self):
+        pool = self.pool(self.credential())
+        entry = pool._entries[0]
+        error = converter.TokenRefreshRejected("刷新 token 失败：refresh token invalid")
+        with patch.object(entry["cm"], "summary",
+                          return_value={"token_expires_at": (time.time() + 86400) * 1000,
+                                        "last_refresh_time": 0}), \
+                patch.object(entry["cm"], "refresh_if_due", side_effect=error):
+            pool.refresh_due(margin_s=600, keepalive_s=300)
+        self.assertGreater(entry["keepalive_after"], time.time())
+
+    def test_manual_toggle_clears_the_auto_provenance(self):
+        pool, store, identity = self._auto_disable_fixture()
+        body = json.dumps({"code": 11140, "msg": "request illegal"}).encode()
+        pool.note_status(pool._entries[0]["cm"], 403, raw=body)
+        self.assertTrue(store.snapshot()["credentials"][identity]["auto_disabled"])
+        store.set_credential(identity, False)  # The operator toggles the switch.
+        record = store.snapshot()["credentials"][identity]
+        self.assertFalse(record["enabled"])
+        self.assertNotIn("auto_disabled", record)
+
+    def test_auto_provenance_survives_a_restart_and_a_cooldown_reset(self):
+        pool, store, identity = self._auto_disable_fixture()
+        body = json.dumps({"code": 11140, "msg": "request illegal"}).encode()
+        pool.note_status(pool._entries[0]["cm"], 403, raw=body)
+        # A cooldown reset clears in-memory evidence but not the persisted label.
+        pool.reset_cooldowns_for(identity)
+        self.assertTrue(store.snapshot()["credentials"][identity]["auto_disabled"])
+        # A restart adopts the label and the tooltip body back from the store.
+        revived = self.pool(self.credential())
+        rentry = revived._entries[0]
+        self.assertTrue(rentry["auto_disabled"])
+        self.assertIn("11140", rentry["last_error_body"])
+        # Re-enabling clears the provenance for good.
+        store.set_credential(identity, True)
+        self.assertFalse(store.snapshot()["credentials"][identity].get("auto_disabled"))
+
     def test_auth_circuit_breaker_survives_a_restart(self):
         first = self.pool(self.credential())
         first.cooldown(first._entries[0]["cm"], reason="backend HTTP 401")
diff --git a/web/src/pages/Credentials.tsx b/web/src/pages/Credentials.tsx
index 1d73844..e215504 100644
--- a/web/src/pages/Credentials.tsx
+++ b/web/src/pages/Credentials.tsx
@@ -63,7 +63,7 @@ const lastErrorLabels: Record<string, string> = {
 
 const healthLabels: Record<string, string> = {
   ready: "就绪",
-  disabled: "已停用",
+  disabled: "人工停用",
   auto_disabled: "自动禁用",
   circuit_open: "认证熔断",
   expired: "已过期",
@@ -820,7 +820,7 @@ export function Credentials() {
           )
         }
       />
-      <span>{c.enabled === true ? "已启用" : c.enabled === false ? "人工停用" : "未知"}</span>
+      <span>{c.enabled === true ? "已启用" : c.enabled === false ? "停用" : "未知"}</span>
     </label>
   );
   const renderHealthBadge = (c: Credential) => {
diff --git a/web/src/values.tsx b/web/src/values.tsx
index 83071ab..d474176 100644
--- a/web/src/values.tsx
+++ b/web/src/values.tsx
@@ -168,7 +168,7 @@ const profiles: Record<string, string> = {
 };
 const statuses: Record<string, string> = {
   ready: "就绪",
-  disabled: "已停用",
+  disabled: "人工停用",
   auto_disabled: "自动禁用",
   error: "失败",
   success: "成功",

From 0aef23ac425b16b4fac952c7ad636f12182f5345 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?=
 <277378677+maiphucgiang@users.noreply.github.com>
Date: Fri, 23 Oct 2026 21:28:47 +0800
Subject: [PATCH 8/8] Fix inventory crash: pass a policy view, not a raw
 snapshot
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

admin_credential_inventory resolved model_policy.snapshot() (a dict)
into helpers expecting a PolicyView, so every credentials listing and
toggle answered 500 管理操作失败. Reuse PolicyView.credential on one
view per inventory call and drop the now-unused helper.
---
 app/gateway_management.py | 4 ++--
 app/model_policy.py       | 3 ---
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/app/gateway_management.py b/app/gateway_management.py
index e77488d..c185463 100644
--- a/app/gateway_management.py
+++ b/app/gateway_management.py
@@ -37,14 +37,14 @@ def admin_credential_inventory(self):
         with pool._lock:
             entries = {entry["id"]: entry for entry in pool.entries()}
             result = []
-            policy = model_policy.snapshot(self.CONFIG)
+            policy = model_policy.view(self.CONFIG)
             for row in pool.snapshot():
                 path = row.pop("auth_file", "")
                 entry = entries[path]
                 identity = entry.get("account_key") or hashlib.sha256(path.encode()).hexdigest()
                 ledger = self.CONFIG.get("ledger")
                 balance = ledger.entry(path) if ledger else {}
-                record = model_policy.credential_record(self.CONFIG, entry, policy=policy)
+                record = policy.credential(entry)
                 enabled = record.get("enabled", True)
                 until = entry.get("fail_until", 0)
                 cooldowns = [{"model": model, "until": deadline,
diff --git a/app/model_policy.py b/app/model_policy.py
index 658034d..3c149ab 100644
--- a/app/model_policy.py
+++ b/app/model_policy.py
@@ -75,9 +75,6 @@ def credential_enabled(config, entry, policy=None):
     return _policy(config, policy)["credentials"].get(entry.get("account_key"), {}).get("enabled", True)
 
 
-def credential_record(config, entry, policy=None):
-    """The whole stored credential record; carries the automatic-disable provenance."""
-    return _policy(config, policy)["credentials"].get(entry.get("account_key"), {})
 
 def credential_auto_checkin(config, entry, policy=None):
     """International accounts opt in explicitly; the preference is independent of routing."""