diff --git a/app/admin_api.py b/app/admin_api.py index 21fa1f4..ee187fd 100644 --- a/app/admin_api.py +++ b/app/admin_api.py @@ -67,7 +67,7 @@ def _public_credential(item): "trial_supported", "trial", "daily_chat_supported", "auto_daily_chat", "daily_chat", "growth_supported", "auto_growth", "growth_auto", - "token_expired", "token_expires_at", "last_refresh_time", "sessions", "sticky_sessions", "last_error_code", "usage_stats"} + "token_expired", "token_expires_at", "last_refresh_time", "sessions", "sticky_sessions", "last_error_code", "last_error_body", "auto_disabled", "usage_stats"} result = {key: value for key, value in item.items() if key in fields} identity = item.get("account_key") or item.get("id") if identity and ("/" in str(identity) or "\\" in str(identity)): diff --git a/app/audit_store.py b/app/audit_store.py index 03a76a3..973b84b 100644 --- a/app/audit_store.py +++ b/app/audit_store.py @@ -44,6 +44,32 @@ def safe_agent(value: Any) -> str | None: return None return None if _SECRET.search(cleaned) else cleaned + +def safe_diagnostic(value: Any, limit: int = 256) -> str | None: + """Short printable upstream diagnostic text; collapsed to one line and secret-checked.""" + if not isinstance(value, str): + return None + cleaned = " ".join(value.split())[:limit] + if not cleaned or not cleaned.isprintable() or _SECRET.search(cleaned): + return None + return cleaned + + +_UPSTREAM_BODY_LIMIT = 256 * 1024 +_JWT = re.compile(r"eyJ[A-Za-z0-9_-]{10,}") + + +def safe_body(value: Any, limit: int = _UPSTREAM_BODY_LIMIT) -> str | None: + """Bounded upstream error JSON; redacted upstream, re-checked here for token leakage.""" + if not isinstance(value, str) or not value: + return None + if len(value) > limit: + value = value[:limit] + "...[truncated]" + if _JWT.search(value): + return None + return value + + def number(value: Any) -> int | float | None: if isinstance(value, bool) or not isinstance(value, (int, float)): return None @@ -59,10 +85,18 @@ def safe_attempt(value: Any) -> dict: result = {} for key in ("stage", "code", "error_code", "model", "upstream_model", "profile", "credential", "usage_source", "outcome", "consent_source", "agreement_revision", "conversation_id", "request_id", - "attempt_id", "upstream_request_id", "session_source"): + "attempt_id", "upstream_request_id", "session_source", "upstream_code"): clean = safe_label(value.get(key)) if clean is not None: result[key] = clean + for key in ("upstream_msg",): + clean = safe_diagnostic(value.get(key)) + if clean is not None: + result[key] = clean + for key in ("upstream_body",): + clean = safe_body(value.get(key)) + if clean is not None: + result[key] = clean for key in ("status_code", "duration_ms", "attempt", "retry_after", "max_attempts", "total_tokens", "attempt_index", "dropped", "merged_runs", "merged_messages"): clean = number(value.get(key)) @@ -142,7 +176,7 @@ class AuditStore: TRANSIENT_WINDOW_SECONDS = 3600 def __init__(self, path, max_bytes=256 * 1024 * 1024, retention_days=30, - preview_limit=8192): + preview_limit=1536 * 1024): self._validate(max_bytes, retention_days, preview_limit) self.max_bytes, self.retention_days, self.preview_limit = max_bytes, retention_days, preview_limit self._lock = threading.Lock() @@ -248,7 +282,7 @@ def _validate(max_bytes, retention_days, preview_limit): raise ValueError("invalid max_bytes") if isinstance(retention_days, bool) or not isinstance(retention_days, int) or not 1 <= retention_days <= 36500: raise ValueError("invalid retention_days") - if isinstance(preview_limit, bool) or not isinstance(preview_limit, int) or not 0 <= preview_limit <= 65536: + if isinstance(preview_limit, bool) or not isinstance(preview_limit, int) or not 0 <= preview_limit <= 4 * 1024 * 1024: raise ValueError("invalid preview_limit") def _fault(self, exc, dropped=False): diff --git a/app/control_store.py b/app/control_store.py index ab3380a..2f95767 100644 --- a/app/control_store.py +++ b/app/control_store.py @@ -186,10 +186,16 @@ def _load(self): for identity, metadata in data["credentials"].items(): _identifier(identity, "账号指纹") growth = metadata.get("growth_auto") + growth = metadata.get("growth_auto") if (not isinstance(metadata, dict) or set(metadata) - {"enabled", "label", "auto_checkin", "auto_travel", - "auto_daily_chat", "auto_growth", "growth_auto"} + "auto_daily_chat", "auto_growth", "growth_auto", + "auto_disabled", "auto_reason", "auto_body", "auto_at"} or type(metadata.get("enabled")) is not bool + or metadata.get("auto_disabled") is not None and type(metadata["auto_disabled"]) is not bool + or metadata.get("auto_reason") is not None and not isinstance(metadata["auto_reason"], str) + or metadata.get("auto_body") is not None and not isinstance(metadata["auto_body"], str) + or metadata.get("auto_at") is not None and not isinstance(metadata["auto_at"], (int, float)) or metadata.get("auto_growth") is not None and type(metadata["auto_growth"]) is not bool or any(key in metadata and type(metadata[key]) is not bool for key in ("auto_checkin", "auto_travel", "auto_daily_chat")) @@ -253,10 +259,32 @@ def change(state): def set_credential(self, account_key, enabled): + """Manual toggle: any operator action clears the automatic-disable provenance.""" _identifier(account_key, "账号指纹") if type(enabled) is not bool: raise ValueError("enabled 必须为布尔值") - return self._update(None, lambda state: state["credentials"].setdefault(account_key, {}).update(enabled=enabled)) + + def change(state): + record = state["credentials"].setdefault(account_key, {}) + record["enabled"] = enabled + record.pop("auto_disabled", None) # Any manual toggle clears the provenance. + return self._update(None, change) + + def disable_credential_auto(self, account_key, reason="", body=None, at=None): + """Record an automatic disable with its upstream evidence so it survives restarts.""" + _identifier(account_key, "账号指纹") + + def change(state): + record = state["credentials"].setdefault(account_key, {}) + record.update(enabled=False, auto_disabled=True) + for key, value in (("auto_reason", str(reason or "")[:256] or None), + ("auto_body", str(body)[:8192] if body else None), + ("auto_at", at)): + if value is None: + record.pop(key, None) + else: + record[key] = value + return self._update(None, change) def unbind_credential(self, account_key): """Drop an account key from every rule, returning {source: old_ids} for rollback.""" diff --git a/app/gateway_management.py b/app/gateway_management.py index 0d1afe6..c185463 100644 --- a/app/gateway_management.py +++ b/app/gateway_management.py @@ -37,21 +37,27 @@ def admin_credential_inventory(self): with pool._lock: entries = {entry["id"]: entry for entry in pool.entries()} result = [] + policy = model_policy.view(self.CONFIG) for row in pool.snapshot(): path = row.pop("auth_file", "") entry = entries[path] identity = entry.get("account_key") or hashlib.sha256(path.encode()).hexdigest() ledger = self.CONFIG.get("ledger") balance = ledger.entry(path) if ledger else {} - enabled = model_policy.credential_enabled(self.CONFIG, entry) + record = policy.credential(entry) + enabled = record.get("enabled", True) until = entry.get("fail_until", 0) cooldowns = [{"model": model, "until": deadline, "remaining_seconds": max(0, round(deadline - now))} for (cid, model), deadline in pool._model_fail.items() if cid == path and deadline > now] - state = ("disabled" if not enabled else "error" if row.get("error") else + # Provenance lives in the store so it survives restarts and cooldown resets. + auto_disabled = record.get("auto_disabled") is True and not enabled + state = ("auto_disabled" if auto_disabled else + "disabled" if not enabled else "error" if row.get("error") else "circuit_open" if until > now else "expired" if row.get("token_expired") else "ready") row.update(id=identity, name=Path(path).name, enabled=enabled, health=state, + auto_disabled=auto_disabled, auto_checkin=model_policy.credential_auto_checkin(self.CONFIG, entry), checkin=checkin.view(balance.get("checkin") or {}), auto_travel=model_policy.credential_auto_travel(self.CONFIG, entry), @@ -74,6 +80,7 @@ def admin_credential_inventory(self): "http_403" if entry.get("last_error") == "backend HTTP 403" else "credential_error" if entry.get("last_error") else None), last_failure_at=entry.get("last_failure_at"), + last_error_body=entry.get("last_error_body"), sync_error=(balance.get("error") or entry.get("last_error") or None), cooldowns=cooldowns, credits=balance.get("credits") or None, sync_pending=path in pool._sync_pending or path in pool._syncing or path in pool._sync_retry, @@ -94,9 +101,16 @@ def admin_set_credential_enabled(self, identity, enabled): self.CONFIG["control_store"].set_credential(identity, enabled) pool._sticky.clear() if enabled: + # Re-enabling returns the credential to a fresh state: reset_cooldowns_for + # lifts every cooldown and drops the recorded failure evidence. + pool.reset_cooldowns_for(identity) pool._queue_sync(entry["id"]) else: pool._sync_pending.discard(entry["id"]) + # entries() hands out copies; clear the flag on the live entry. + live = pool.entry_by_id(entry["id"]) + if live is not None: + live["auto_disabled"] = None pool._sync_retry.pop(entry["id"], None) if not pool._sync_pending: pool._sync_event.clear() diff --git a/app/model_policy.py b/app/model_policy.py index 9cb00e1..3c149ab 100644 --- a/app/model_policy.py +++ b/app/model_policy.py @@ -75,6 +75,7 @@ def credential_enabled(config, entry, policy=None): return _policy(config, policy)["credentials"].get(entry.get("account_key"), {}).get("enabled", True) + def credential_auto_checkin(config, entry, policy=None): """International accounts opt in explicitly; the preference is independent of routing.""" default = entry.get("profile") in {"cn-cli", "cn-work"} diff --git a/app/settings.py b/app/settings.py index 29a3d5d..7b0c4dd 100644 --- a/app/settings.py +++ b/app/settings.py @@ -147,7 +147,7 @@ def _item(default, type_, label, *, mode="hot", env=None, minimum=None, maximum= env="CODEBUDDY2API_STREAM_MODE", choices=["compatible", "realtime"]), "audit_max_bytes": _item(256 * 1024 * 1024, "integer", "审计明细预算", minimum=1024**2, maximum=1024**4), "audit_retention_days": _item(30, "integer", "审计明细保留天数", minimum=1, maximum=36500), - "audit_diagnostic_bytes": _item(8192, "integer", "失败诊断最大字节", minimum=0, maximum=8192), + "audit_diagnostic_bytes": _item(1536 * 1024, "integer", "失败诊断最大字节", minimum=0, maximum=4 * 1024 * 1024), "trusted_proxies": _item("", "string", "可信代理地址（IP/CIDR，逗号分隔）", env="CODEBUDDY2API_TRUSTED_PROXIES", allow_empty=True, max_length=2000, validator=normalize_trusted_proxies), diff --git a/converter.py b/converter.py index e2d8824..b5b66bf 100644 --- a/converter.py +++ b/converter.py @@ -371,7 +371,9 @@ def _refresh_locked(self): except Exception as e: raise RuntimeError(f"刷新 token 网络失败：{e}") if data.get("code") != 0 or not data.get("data"): - raise RuntimeError(f"刷新 token 失败：{data.get('msg', data)}") + code, msg, body = _diagnostic_from_payload(data) + raise TokenRefreshRejected(f"刷新 token 失败：{data.get('msg', data)}", + code=code, msg=msg, body=body) new_auth = dict(data["data"]) if not isinstance(new_auth.get("accessToken"), str) or not new_auth["accessToken"]: raise RuntimeError("刷新接口未返回有效的 accessToken") @@ -414,6 +416,13 @@ def invalidate(self): self._generation += 1 + def token_age_s(self) -> float | None: + """Seconds since the stored token was last refreshed; None when unknown.""" + with self._lock: + auth = self._session().get("auth") or {} + last = auth.get("lastRefreshTime") + return None if not last else max(0.0, time.time() - last / 1000) + def summary(self) -> dict: perf_metrics.inc("credential_summary_calls") with self._lock: @@ -446,6 +455,12 @@ def summary(self) -> dict: CRED_COOLDOWN = 300 # Credential cooldown in seconds MODEL_COOLDOWN = 600 # Model cooldown when a 429 omits reset time MODEL_COOLDOWN_MAX = 86400 # Maximum model cooldown in seconds +UPSTREAM_BODY_AUDIT_LIMIT = 256 * 1024 # Bounded upstream error JSON kept per failed attempt. +RESTRICTED_CODES = frozenset({"11140"}) # Upstream account-restriction codes; disable, never requeue. +FRESH_401_LIMIT = 3 # Fresh-token 401s inside the window before auto-disable +FRESH_401_WINDOW = 3600 # Sliding window for fresh-token 401 counting +REFRESH_REJECT_LIMIT = 3 # Consecutive business refresh rejections before auto-disable +REFRESH_REJECT_WINDOW = 3600 # Rejections further apart than this are not consecutive MODEL_SITE_BLOCK_S = 6 * 3600 # Initial unsupported-model backoff MODEL_SITE_BLOCK_MAX_S = 24 * 3600 # Maximum unsupported-model backoff # An unsupported model must be routed to a different backend. @@ -539,6 +554,47 @@ def _parse_not_servable(raw: bytes, status: int): return None +def _diagnostic_from_payload(payload): + """Extract (code, msg, bounded body) from an upstream JSON payload, for audit only.""" + if not isinstance(payload, dict): + return None, None, None + # The nested error envelope carries the business verdict; a generic outer + # code (e.g. "error") must not mask a terminal nested code such as 11140. + inner = payload.get("error") + nodes = [inner] if isinstance(inner, dict) else [] + nodes.append(payload) + code = msg = "" + for node in nodes: + for key in ("code", "errCode", "error_code"): + value = node.get(key) + if value is not None and str(value).strip(): + code = code or str(value).strip() + for key in ("msg", "message"): + value = node.get(key) + if isinstance(value, str) and value.strip(): + msg = msg or value.strip() + return (code[:64] or None), (msg[:256] or None), (format_log_body(payload, UPSTREAM_BODY_AUDIT_LIMIT) or None) + + +def _upstream_diagnostic(raw: bytes): + """Extract the business code, message and bounded JSON body from an upstream error.""" + if not raw: + return None, None, None + try: + payload = json.loads(raw.decode("utf-8", "replace")) + except (ValueError, UnicodeError, RecursionError): + return None, None, None + return _diagnostic_from_payload(payload) + + +class TokenRefreshRejected(RuntimeError): + """The upstream refresh endpoint answered with a business rejection, not a network fault.""" + + def __init__(self, message, *, code=None, msg=None, body=None): + super().__init__(message) + self.code, self.msg, self.body = code, msg, body + + def _block_model(model: str | None) -> str | None: """Track blocked models by public ID, normalizing the international auto alias.""" return "auto" if model == "default-model" else model @@ -674,6 +730,7 @@ def reload(self, paths: list[Path], *, reset: bool = True): # account's own breaker, which is hydrated just below. self._forget_credential_cooldown(entry) self._adopt_cooldowns(entry) + self._adopt_auto_state(entry) if entry.get("uid"): have_uids[identity] = cid self._queue_sync(cid) @@ -709,6 +766,7 @@ def reload(self, paths: list[Path], *, reset: bool = True): "account_key": identity_key, "catalog_dirty": True} self._bind_entry(entry) self._adopt_cooldowns(entry) + self._adopt_auto_state(entry) self._entries.append(entry) by_id[cid] = entry self._ignored_duplicates.discard(cid) @@ -914,6 +972,26 @@ def _adopt_cooldowns(self, entry): key = (entry["id"], model) self._model_fail[key] = max(self._model_fail.get(key, 0.0), until) + + def _adopt_auto_state(self, entry): + """Restore the persisted auto-disable provenance so a restart keeps the label.""" + store = CONFIG.get("control_store") + if store is None or not entry.get("account_key"): + return + try: + record = store.snapshot()["credentials"].get(entry["account_key"]) or {} + except Exception: + return + if record.get("auto_disabled") is not True or record.get("enabled", True): + return + entry["auto_disabled"] = True + if record.get("auto_reason"): + entry["last_error"] = record["auto_reason"] + if record.get("auto_body"): + entry["last_error_body"] = record["auto_body"][:8192] + if record.get("auto_at"): + entry["last_failure_at"] = record["auto_at"] + def _durable_identity(self, entry) -> bool: """Whether this account's identity is complete enough to key durable state. @@ -1018,7 +1096,13 @@ def reset_cooldowns_for(self, identity: str) -> dict: now = time.time() changed = entry["fail_until"] > now entry["fail_until"] = 0.0 + entry["fail_until"] = 0.0 entry["last_error"] = None + entry["last_failure_at"] = None + entry["last_error_body"] = None + entry["auto_disabled"] = None + entry["fresh_401s"] = [] + entry["refresh_rejects"] = 0 for key in [k for k in self._model_fail if k[0] == entry["id"]]: if self._model_fail[key] > now: changed = True @@ -1346,6 +1430,90 @@ def _lease_matches(cm, generation): return generation == cm._generation return generation == cm._generation + def _note_error_body(self, cm, body, *, generation=None): + """Keep the last upstream error JSON for the credentials tooltip; bounded, redacted.""" + # The diagnostic arrives pre-parsed from note_status; never re-parse under the lock. + if not body: + return + with self._lock, (cm._lock if generation is not None else nullcontext()): + if not self._lease_matches(cm, generation): + return + entry = self._index_cm.get(cm) + if entry is not None: + entry["last_error_body"] = body[:8192] + + def _auto_disable(self, cm, reason, *, generation=None): + """A terminal upstream verdict: disable via the WebUI policy, not a cooldown.""" + entry = self._index_cm.get(cm) + store = CONFIG.get("control_store") + management = CONFIG.get("management") + identity = entry.get("account_key") if entry else None + if not identity or store is None or management is None: + return + try: + with self._lock, (cm._lock if generation is not None else nullcontext()): + if not self._lease_matches(cm, generation): + return + store.disable_credential_auto( + identity, reason, body=entry.get("last_error_body") if entry else None, + at=time.time()) + self._sticky.clear() + entry["auto_disabled"] = True + management.gateway.invalidate_model_table() + except Exception as error: + # A control-store outage must never replace the upstream rejection being reported. + _log(f"[cred] 自动禁用写入失败（{Path(cm.path).name}）: {sanitize_log_text(str(error), 200)}") + return + _log(f"[cred] {reason}，账号已自动禁用: {Path(cm.path).name}") + + def _note_fresh_401(self, cm, *, generation=None): + """A 401 against a just-refreshed token means the account was revoked server-side.""" + try: + age = cm.token_age_s() + except Exception: + return + if age is None or age > 120: + return + with self._lock, (cm._lock if generation is not None else nullcontext()): + if not self._lease_matches(cm, generation): + return + entry = self._index_cm.get(cm) + if entry is None: + return + now = time.time() + strikes = [t for t in entry.get("fresh_401s", []) if now - t < FRESH_401_WINDOW] + strikes.append(now) + entry["fresh_401s"] = strikes + fresh = len(strikes) >= FRESH_401_LIMIT + if fresh: + entry["fresh_401s"] = [] + if fresh: + self._auto_disable(cm, f"新令牌 {FRESH_401_WINDOW}s 内 {FRESH_401_LIMIT} 次被拒（401）", + generation=generation) + def _note_refresh_reject(self, entry, error): + """Count consecutive upstream refresh rejections; the refresh token itself is dead.""" + now = time.time() + # A request-path refresh (get_headers -> _refresh) succeeding between rejections + # proves the token alive; a long gap alone is not "consecutive" either. + try: + age = entry["cm"].token_age_s() + except Exception: + age = None + refreshed_after = now - age if age is not None else 0.0 + if refreshed_after > entry.get("refresh_reject_at", 0.0) or \ + now - entry.get("refresh_reject_at", 0.0) > REFRESH_REJECT_WINDOW: + entry["refresh_rejects"] = 0 + entry["refresh_reject_at"] = now + rejects = entry.get("refresh_rejects", 0) + 1 + entry["refresh_rejects"] = rejects + if error.body: + entry["last_error_body"] = error.body[:8192] + if rejects < REFRESH_REJECT_LIMIT: + return + entry["refresh_rejects"] = 0 + self._auto_disable(entry["cm"], "刷新令牌被上游连续拒绝") + + def cooldown(self, cm: CredentialManager, reason: str = "", *, generation=None): with self._lock, (cm._lock if generation is not None else nullcontext()): if not self._lease_matches(cm, generation): @@ -1366,6 +1534,13 @@ def note_status(self, cm: CredentialManager | None, status: int, return if status in (401, 403): self.cooldown(cm, reason=f"backend HTTP {status}", generation=generation) + # One parse feeds the audit body, the restriction check and the tooltip. + code, _, body = _upstream_diagnostic(raw or b"") + self._note_error_body(cm, body, generation=generation) + if status == 403 and code in RESTRICTED_CODES: + self._auto_disable(cm, f"上游限制 {code}", generation=generation) + elif status == 401: + self._note_fresh_401(cm, generation=generation) return if status == 400 and is_channel_check_error(raw): # Only a rejection that survived the console retry is account-wide; endpoints without a @@ -1502,7 +1677,9 @@ def refresh_due(self, margin_s: int = CRED_REFRESH_MARGIN, keepalive_s: int = CR if now < entry.get("fail_until", 0.0) or not model_policy.credential_enabled(CONFIG, entry): continue cm = entry["cm"] + cm = entry["cm"] failure = None + refresh_reject = None refreshed = keepalive_due = False with cm._lock: try: @@ -1517,13 +1694,25 @@ def refresh_due(self, margin_s: int = CRED_REFRESH_MARGIN, keepalive_s: int = CR continue refreshed = cm.refresh_if_due(margin_s, keepalive_s if keepalive_due else 0) entry["keepalive_after"] = 0.0 + except TokenRefreshRejected as error: + failure = (str(error), cm._generation) + # Keepalive rejections back off like every other keepalive failure; + # without this a 5-minute cadence stacks 3 strikes in ~10 minutes. + if keepalive_due: + entry["keepalive_after"] = now + CRED_KEEPALIVE_RETRY_S + # Deferred past cm._lock: the third rejection reaches _auto_disable, + # which takes the pool lock; pool paths acquire pool -> credential. + refresh_reject = error except Exception as error: failure = (str(error), cm._generation) if keepalive_due: entry["keepalive_after"] = now + CRED_KEEPALIVE_RETRY_S + if refresh_reject is not None: + self._note_refresh_reject(entry, refresh_reject) if failure: self.cooldown(cm, reason=failure[0], generation=failure[1]) elif refreshed: + entry["refresh_rejects"] = 0 _log(f"[cred] {'每日保活刷新' if keepalive_due else '已主动刷新'}并回写: {Path(entry['id']).name}") def remove_file(self, name: str) -> bool: @@ -4105,6 +4294,9 @@ def _safe_err_raw(raw: bytes, status: int) -> dict: def _check_upstream_status(status, raw, cred, model, *, headers=None, from_channel_fallback=False): if status != 200: retry_after = parse_retry_after((headers or {}).get("Retry-After")) + code, message, error_body = _upstream_diagnostic(raw) + observe_attempt("upstream_error", status_code=status, upstream_code=code, + upstream_msg=message, upstream_body=error_body) if not is_filter_error(raw): _note_cred_status(cred, status, model=model, raw=raw, retry_after=retry_after, from_channel_fallback=from_channel_fallback) diff --git a/tests/test_control_store.py b/tests/test_control_store.py index cb2764d..0bfd54d 100644 --- a/tests/test_control_store.py +++ b/tests/test_control_store.py @@ -155,7 +155,8 @@ def test_setting_precedence_and_source_redaction(self): self.assertEqual(items["image_policy"]["source"], "environment") self.assertIsNone(items["api_key"]["value"]) self.assertIsNone(items["auth_dir"]["value"]) - for invalid in ({"usd_rate": float("nan")}, {"port": 65536}, {"audit_diagnostic_bytes": 8193}): + for invalid in ({"usd_rate": float("nan")}, {"port": 65536}, + {"audit_diagnostic_bytes": 4 * 1024 * 1024 + 1}): with self.assertRaises(ValueError): validate_settings(invalid) diff --git a/tests/test_credential_cooldowns.py b/tests/test_credential_cooldowns.py index 6c5e4f7..8e7cd4b 100644 --- a/tests/test_credential_cooldowns.py +++ b/tests/test_credential_cooldowns.py @@ -6,13 +6,16 @@ import sys import tempfile import time +import time import unittest from pathlib import Path +from types import SimpleNamespace from unittest.mock import patch sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # Allow direct execution. import converter +from app.control_store import ControlStore as converter_local_ControlStore from app.credential_cooldowns import (AUTH_CEILING_S, MAX_ACCOUNTS, MAX_BYTES, MAX_MODELS, MODEL_CEILING_S, VERSION, CredentialCooldowns) @@ -304,6 +307,221 @@ def pool(self, *paths): return converter.CredentialPool(list(paths), blocks_path=self.root / "blocks.json", cooldowns_path=self.path) + + def test_auth_rejection_keeps_the_upstream_error_body_for_the_tooltip(self): + pool = self.pool(self.credential()) + body = json.dumps({"code": 11140, "msg": "request illegal", + "displayMsg": {"zh": "账号存在异常行为，已被限制使用"}}).encode() + pool.note_status(pool._entries[0]["cm"], 403, raw=body) + entry = pool._entries[0] + self.assertIn("11140", entry["last_error_body"]) + self.assertIn("已被限制使用", entry["last_error_body"]) + # Non-JSON rejections (WAF pages) must not leave a body behind. + pool.note_status(pool._entries[0]["cm"], 403, raw=b"blocked") + self.assertIn("11140", entry["last_error_body"]) + + def test_upstream_restriction_disables_the_account_through_the_webui_policy(self): + """11140 is a terminal account restriction: disable via the policy, not just a cooldown.""" + path = self.credential() + pool = self.pool(path) + store = converter_local_ControlStore(self.root / "control.sqlite3") + self.addCleanup(store.close) + identity = pool._entries[0]["account_key"] + store.set_credential(identity, True) + invalidated = [] + management = SimpleNamespace(gateway=SimpleNamespace( + invalidate_model_table=lambda: invalidated.append(1))) + self.enterContext(patch.dict(converter.CONFIG, {"control_store": store, "management": management})) + body = json.dumps({"code": 11140, "msg": "request illegal"}).encode() + pool.note_status(pool._entries[0]["cm"], 403, raw=body) + self.assertTrue(pool._entries[0]["auto_disabled"]) + self.assertFalse(converter.model_policy.credential_enabled(converter.CONFIG, pool._entries[0])) + self.assertEqual(invalidated, [1]) + # The policy survives a pool rebuild, matching a WebUI-initiated disable. + revived = self.pool(self.credential()) + self.assertFalse(converter.model_policy.credential_enabled(converter.CONFIG, revived._entries[0])) + + def test_plain_403_cooldowns_without_disabling(self): + pool = self.pool(self.credential()) + store = converter_local_ControlStore(self.root / "control.sqlite3") + self.addCleanup(store.close) + identity = pool._entries[0]["account_key"] + store.set_credential(identity, True) + self.enterContext(patch.dict(converter.CONFIG, {"control_store": store, "management": None})) + pool.note_status(pool._entries[0]["cm"], 403, raw=b"forbidden") + self.assertTrue(converter.model_policy.credential_enabled(converter.CONFIG, pool._entries[0])) + self.assertFalse(pool._healthy(pool._entries[0])) + + + def test_reset_returns_the_account_to_a_fresh_state(self): + """An operator reset (or re-enable) lifts cooldowns and drops every failure record.""" + pool = self.pool(self.credential()) + body = json.dumps({"code": 11140, "msg": "request illegal"}).encode() + entry = pool._entries[0] + pool.note_status(entry["cm"], 403, raw=body) + self.assertIsNotNone(entry["last_error_body"]) + self.assertIsNotNone(entry["last_failure_at"]) + pool.reset_cooldowns_for(entry["account_key"]) + pool.reset_cooldowns_for(entry["account_key"]) + self.assertEqual(entry["fail_until"], 0.0) + self.assertIsNone(entry["last_error"]) + self.assertIsNone(entry["last_failure_at"]) + self.assertIsNone(entry["last_error_body"]) + self.assertIsNone(entry["auto_disabled"]) + self.assertTrue(pool._healthy(entry)) + + def test_fresh_token_401s_auto_disable(self): + """401 against a just-refreshed token, three times, means the account was revoked.""" + pool = self.pool(self.credential()) + store = converter_local_ControlStore(self.root / "control.sqlite3") + self.addCleanup(store.close) + identity = pool._entries[0]["account_key"] + store.set_credential(identity, True) + invalidated = [] + management = SimpleNamespace(gateway=SimpleNamespace( + invalidate_model_table=lambda: invalidated.append(1))) + self.enterContext(patch.dict(converter.CONFIG, {"control_store": store, "management": management})) + entry = pool._entries[0] + for _ in range(2): + pool.note_status(entry["cm"], 401, raw=b"unauthorized") + self.assertIsNone(entry.get("auto_disabled")) + pool.note_status(entry["cm"], 401, raw=b"unauthorized") + self.assertTrue(entry["auto_disabled"]) + self.assertFalse(converter.model_policy.credential_enabled(converter.CONFIG, entry)) + self.assertEqual(invalidated, [1]) + # A stale token never counts toward the strikes. + fresh = self.pool(self.credential(name="stale.info", uid="stale-uid")) + with patch.object(fresh._entries[0]["cm"], "token_age_s", return_value=9999.0): + for _ in range(5): + fresh.note_status(fresh._entries[0]["cm"], 401, raw=b"unauthorized") + self.assertTrue(converter.model_policy.credential_enabled(converter.CONFIG, fresh._entries[0])) + + def test_consecutive_refresh_rejections_auto_disable(self): + """Upstream keeps rejecting the refresh token: disable after the third rejection.""" + pool = self.pool(self.credential()) + store = converter_local_ControlStore(self.root / "control.sqlite3") + self.addCleanup(store.close) + identity = pool._entries[0]["account_key"] + store.set_credential(identity, True) + invalidated = [] + management = SimpleNamespace(gateway=SimpleNamespace( + invalidate_model_table=lambda: invalidated.append(1))) + self.enterContext(patch.dict(converter.CONFIG, {"control_store": store, "management": management})) + entry = pool._entries[0] + body = json.dumps({"code": 10016, "msg": "refresh token invalid"}).encode() + error = converter.TokenRefreshRejected("刷新 token 失败：refresh token invalid", + code="10016", msg="refresh token invalid", + body=body.decode()) + for _ in range(2): + pool._note_refresh_reject(entry, error) + self.assertEqual(entry["refresh_rejects"], 2) + pool._note_refresh_reject(entry, error) + self.assertTrue(entry["auto_disabled"]) + self.assertEqual(entry["refresh_rejects"], 0) + self.assertIn("refresh token invalid", entry["last_error_body"]) + self.assertFalse(converter.model_policy.credential_enabled(converter.CONFIG, entry)) + + def test_refresh_due_feeds_rejections_into_the_auto_disable_path(self): + """refresh_due itself must count TokenRefreshRejected; the helper is not enough.""" + pool = self.pool(self.credential()) + store = converter_local_ControlStore(self.root / "control.sqlite3") + self.addCleanup(store.close) + identity = pool._entries[0]["account_key"] + store.set_credential(identity, True) + invalidated = [] + management = SimpleNamespace(gateway=SimpleNamespace( + invalidate_model_table=lambda: invalidated.append(1))) + self.enterContext(patch.dict(converter.CONFIG, {"control_store": store, "management": management})) + entry = pool._entries[0] + error = converter.TokenRefreshRejected("刷新 token 失败：refresh token invalid", + code="10016", msg="refresh token invalid") + with patch.object(entry["cm"], "refresh_if_due", side_effect=error): + for _ in range(2): + pool.refresh_due(margin_s=10 ** 9, keepalive_s=0) + entry["fail_until"] = 0.0 # The rejection cooldown would skip the next pass. + self.assertEqual(entry["refresh_rejects"], 2) + pool.refresh_due(margin_s=10 ** 9, keepalive_s=0) + self.assertTrue(entry["auto_disabled"]) + self.assertFalse(converter.model_policy.credential_enabled(converter.CONFIG, entry)) + self.assertEqual(invalidated, [1]) + + def _auto_disable_fixture(self): + """A pool + store + management wired for auto-disable, with the live entry returned.""" + pool = self.pool(self.credential()) + store = converter_local_ControlStore(self.root / "control.sqlite3") + self.addCleanup(store.close) + identity = pool._entries[0]["account_key"] + store.set_credential(identity, True) + invalidated = [] + management = SimpleNamespace(gateway=SimpleNamespace( + invalidate_model_table=lambda: invalidated.append(1))) + self.enterContext(patch.dict(converter.CONFIG, {"control_store": store, "management": management})) + return pool, store, identity + + def test_request_path_refresh_resets_the_rejection_count(self): + """A successful get_headers refresh between rejections proves the token alive.""" + pool, store, identity = self._auto_disable_fixture() + entry = pool._entries[0] + error = converter.TokenRefreshRejected("刷新 token 失败：refresh token invalid") + pool._note_refresh_reject(entry, error) + pool._note_refresh_reject(entry, error) + self.assertEqual(entry["refresh_rejects"], 2) + # The last rejection predates a fresh token, as a request-path refresh would produce. + entry["refresh_reject_at"] = time.time() - 10 + with patch.object(entry["cm"], "token_age_s", return_value=0.5): + pool._note_refresh_reject(entry, error) + self.assertEqual(entry["refresh_rejects"], 1) + self.assertIsNone(entry.get("auto_disabled")) + + def test_stale_refresh_rejections_do_not_accumulate(self): + """Sporadic rejections days apart never reach the disable threshold.""" + pool = self.pool(self.credential()) + entry = pool._entries[0] + error = converter.TokenRefreshRejected("刷新 token 失败") + with patch.object(entry["cm"], "token_age_s", return_value=None): + for _ in range(5): + entry["refresh_reject_at"] = time.time() - converter.REFRESH_REJECT_WINDOW - 1 + pool._note_refresh_reject(entry, error) + self.assertEqual(entry["refresh_rejects"], 1) + self.assertIsNone(entry.get("auto_disabled")) + + def test_keepalive_rejection_backs_off_like_other_failures(self): + pool = self.pool(self.credential()) + entry = pool._entries[0] + error = converter.TokenRefreshRejected("刷新 token 失败：refresh token invalid") + with patch.object(entry["cm"], "summary", + return_value={"token_expires_at": (time.time() + 86400) * 1000, + "last_refresh_time": 0}), \ + patch.object(entry["cm"], "refresh_if_due", side_effect=error): + pool.refresh_due(margin_s=600, keepalive_s=300) + self.assertGreater(entry["keepalive_after"], time.time()) + + def test_manual_toggle_clears_the_auto_provenance(self): + pool, store, identity = self._auto_disable_fixture() + body = json.dumps({"code": 11140, "msg": "request illegal"}).encode() + pool.note_status(pool._entries[0]["cm"], 403, raw=body) + self.assertTrue(store.snapshot()["credentials"][identity]["auto_disabled"]) + store.set_credential(identity, False) # The operator toggles the switch. + record = store.snapshot()["credentials"][identity] + self.assertFalse(record["enabled"]) + self.assertNotIn("auto_disabled", record) + + def test_auto_provenance_survives_a_restart_and_a_cooldown_reset(self): + pool, store, identity = self._auto_disable_fixture() + body = json.dumps({"code": 11140, "msg": "request illegal"}).encode() + pool.note_status(pool._entries[0]["cm"], 403, raw=body) + # A cooldown reset clears in-memory evidence but not the persisted label. + pool.reset_cooldowns_for(identity) + self.assertTrue(store.snapshot()["credentials"][identity]["auto_disabled"]) + # A restart adopts the label and the tooltip body back from the store. + revived = self.pool(self.credential()) + rentry = revived._entries[0] + self.assertTrue(rentry["auto_disabled"]) + self.assertIn("11140", rentry["last_error_body"]) + # Re-enabling clears the provenance for good. + store.set_credential(identity, True) + self.assertFalse(store.snapshot()["credentials"][identity].get("auto_disabled")) + def test_auth_circuit_breaker_survives_a_restart(self): first = self.pool(self.credential()) first.cooldown(first._entries[0]["cm"], reason="backend HTTP 401") diff --git a/tests/test_stream_failover.py b/tests/test_stream_failover.py index 2313375..382ac1f 100644 --- a/tests/test_stream_failover.py +++ b/tests/test_stream_failover.py @@ -347,7 +347,8 @@ def test_replayed_request_is_audited_as_success(self): record = self.only_record(store) self.assertEqual(record["outcome"], "success", record) self.assertFalse(record.get("error_code"), record) - statuses = [attempt.get("status_code") for attempt in record["attempts"]] + statuses = [attempt.get("status_code") for attempt in record["attempts"] + if attempt.get("stage") == "upstream_http"] self.assertEqual(statuses[:2], [429, 200], record["attempts"]) self.assertIn("failover_recovered", json.dumps(record["attempts"], ensure_ascii=False)) markers = [attempt for attempt in record["attempts"] diff --git a/tests/test_upstream_diagnostics.py b/tests/test_upstream_diagnostics.py new file mode 100644 index 0000000..f6c757a --- /dev/null +++ b/tests/test_upstream_diagnostics.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Test that upstream error codes and messages reach the audit trail, sanitized.""" + +import contextlib +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # Allow direct execution. +import httpx +from fastapi import FastAPI +from fastapi.testclient import TestClient + +import converter +from app import upstream_io +from app.audit_store import AuditStore + +ROUTE = "/v1/chat/completions" + + +class UpstreamDiagnosticTests(unittest.TestCase): + """Upstream error bodies are the only way to tell a WAF block from a business rejection.""" + + def setUp(self): + self.directory = self.enterContext(tempfile.TemporaryDirectory()) + self.audit = self.enterContext(contextlib.closing( + AuditStore(Path(self.directory) / "logs.sqlite3"))) + self.enterContext(patch.dict(converter.CONFIG, { + "api_key": "", "cred": None, "cred_pool": None, "model_guard": False, + "max_images": 16, "image_policy": "truncate", "max_request_bytes": 32 * 1024 * 1024, + "log_body_limit": 65536, "log_path": None, "desensitize": False, "no_compact": False, + "audit_store": self.audit})) + self.enterContext(patch.object(converter, "_cred_for", return_value=(None, {}))) + self.enterContext(patch.object(converter, "_log")) + self.requests = [] + self.respond = lambda request: httpx.Response(200, content=b'data: [DONE]\n\n') + real_client = httpx.AsyncClient + transport = httpx.MockTransport(self.handle) + self.enterContext(patch.object(upstream_io.httpx, "AsyncClient", + side_effect=lambda **kw: real_client(transport=transport, **kw))) + self.client = self.enterContext(TestClient(self._build_app())) + + def handle(self, request): + self.requests.append(request) + return self.respond(request) + + def _build_app(self): + """Rebuild the inference app with the audit middleware the raw app does not carry.""" + from app.observability import AuditMiddleware + from app.request_context import RequestContextMiddleware + app = FastAPI() + for route in converter.app.routes: + app.routes.append(route) + app.add_middleware(AuditMiddleware, config=converter.CONFIG) + app.add_middleware(RequestContextMiddleware, config=converter.CONFIG) + return app + + def _attempts(self): + for record in self.audit.list_records("request", limit=20)["items"]: + for attempt in record.get("attempts") or []: + yield attempt + + def test_business_error_code_and_message_reach_the_audit_trail(self): + self.respond = lambda request: httpx.Response( + 403, json={"code": "11-128", "msg": "conversation id missing"}) + response = self.client.post(ROUTE, json={ + "model": "auto", "messages": [{"role": "user", "content": "hi"}]}) + self.assertEqual(response.status_code, 403) + entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error") + self.assertEqual(entry["status_code"], 403) + self.assertEqual(entry["upstream_code"], "11-128") + self.assertEqual(entry["upstream_msg"], "conversation id missing") + + def test_nested_envelope_is_unwrapped(self): + self.respond = lambda request: httpx.Response( + 403, json={"error": {"code": "11140", "message": "request illegal"}}) + self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]}) + entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error") + self.assertEqual(entry["upstream_code"], "11140") + self.assertEqual(entry["upstream_msg"], "request illegal") + + def test_nested_terminal_code_masks_generic_outer_code(self): + """A generic outer code must not hide a terminal nested restriction code.""" + self.respond = lambda request: httpx.Response( + 403, json={"code": "error", "msg": "forbidden", + "error": {"code": "11140", "message": "request illegal"}}) + self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]}) + entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error") + self.assertEqual(entry["upstream_code"], "11140") + self.assertEqual(entry["upstream_msg"], "request illegal") + + def test_non_json_body_records_no_diagnostic_but_keeps_the_status(self): + self.respond = lambda request: httpx.Response(403, text="blocked") + self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]}) + entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error") + self.assertEqual(entry["status_code"], 403) + self.assertNotIn("upstream_code", entry) + self.assertNotIn("upstream_msg", entry) + self.assertNotIn("upstream_body", entry) + + def test_restriction_body_is_recorded_in_full(self): + """The 11140 restriction envelope is the evidence that decides account handling.""" + self.respond = lambda request: httpx.Response(403, json={ + "code": 11140, "msg": "request illegal", "requestId": "01a162757c61710a82d9c17d3e9f060e", + "displayMsg": {"en": "Your account has been restricted due to unusual activity", + "zh": "账号存在异常行为，已被限制使用"}, + "displayTips": {"zh": "系统检测到该账号存在异常使用行为，如需申诉请提交反馈"}, + "actions": ["EDIT_INPUT", "SUBMIT_FEEDBACK"]}) + self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]}) + entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error") + self.assertEqual(entry["upstream_code"], "11140") + for fragment in ("已被限制使用", "SUBMIT_FEEDBACK", "01a162757c61710a82d9c17d3e9f060e"): + self.assertIn(fragment, entry["upstream_body"]) + + def test_body_is_capped_at_256k(self): + payload = {f"key{index}": "x" * 4096 for index in range(80)} + self.respond = lambda request: httpx.Response(403, json={"code": "1", "msg": "wide", **payload}) + self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]}) + entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error") + self.assertLessEqual(len(entry["upstream_body"]), 256 * 1024 + 64) + self.assertIn("truncated", entry["upstream_body"]) + + def test_body_tokens_are_redacted_before_storage(self): + self.respond = lambda request: httpx.Response(403, json={ + "code": "11-128", "msg": "denied", "accessToken": "synthetic-private-token"}) + self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]}) + entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error") + self.assertNotIn("synthetic-private-token", entry["upstream_body"]) + raw = b"".join(f.read_bytes() for f in Path(self.directory).glob("logs.sqlite3*")) + # Storage itself must never persist the secret, not only the record read path. + self.assertNotIn(b"synthetic-private-token", raw) + + def test_secrets_in_upstream_messages_are_dropped(self): + self.respond = lambda request: httpx.Response( + 403, json={"code": "11-128", "msg": "denied for cb-" + "a" * 32}) + self.client.post(ROUTE, json={"model": "auto", "messages": [{"role": "user", "content": "hi"}]}) + entry = next(a for a in self._attempts() if a.get("stage") == "upstream_error") + self.assertEqual(entry["upstream_code"], "11-128") + self.assertNotIn("upstream_msg", entry) + raw = b"".join(f.read_bytes() for f in Path(self.directory).glob("logs.sqlite3*")) + self.assertNotIn(b"cb-" + b"a" * 32, raw) + + + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/web/src/pages/Credentials.tsx b/web/src/pages/Credentials.tsx index 9fcc67e..e215504 100644 --- a/web/src/pages/Credentials.tsx +++ b/web/src/pages/Credentials.tsx @@ -61,11 +61,113 @@ const lastErrorLabels: Record = { credential_error: "凭证同步失败（详见操作结果）", }; +const healthLabels: Record = { + ready: "就绪", + disabled: "人工停用", + auto_disabled: "自动禁用", + circuit_open: "认证熔断", + expired: "已过期", + error: "错误", +}; + function lastErrorLabel(value: unknown) { const key = text(value); return lastErrorLabels[key] ?? key; } +function prettyErrorJson(value: unknown) { + const raw = text(value); + if (raw === "") return ""; + try { + return JSON.stringify(JSON.parse(raw), null, 2); + } catch { + return raw; + } +} + +/** Last upstream error; hover (desktop) or tap (mobile) opens the recorded JSON. Click pins it open. */ +function LastErrorJson({ code, at, body }: { code: unknown; at: unknown; body: unknown }) { + const json = prettyErrorJson(body); + const [pinned, setPinned] = useState(false); + const [hovered, setHovered] = useState(false); + const leaveTimer = useRef(null); + const trigger = useRef(null); + const tip = useRef(null); + const open = pinned || hovered; + const close = useCallback(() => { + setPinned(false); + setHovered(false); + }, []); + const placement = usePopover(open, close, trigger, tip, "start"); + useEffect( + () => () => { + if (leaveTimer.current !== null) window.clearTimeout(leaveTimer.current); + }, + [], + ); + // A short grace period bridges the gap between the trigger and the portaled tip. + const scheduleClose = () => { + if (leaveTimer.current !== null) window.clearTimeout(leaveTimer.current); + leaveTimer.current = window.setTimeout(() => setHovered(false), 160); + }; + const holdOpen = () => { + if (leaveTimer.current !== null) { + window.clearTimeout(leaveTimer.current); + leaveTimer.current = null; + } + setHovered(true); + }; + const hoverProps = { + onPointerEnter: (event: { pointerType: string }) => { + if (event.pointerType === "mouse") holdOpen(); + }, + onPointerLeave: (event: { pointerType: string }) => { + if (event.pointerType === "mouse") scheduleClose(); + }, + }; + const label = ( + <> + 最近错误：{text(lastErrorLabel(code))} + {number(at) !== null ? ` · ${expiry(at)}` : ""} + + ); + if (json === "") return {label}; + return ( + <> + + + + {open && + createPortal( +
+ {json} +
, + popoverHost(trigger.current), + )} + + ); +} + export { safeOAuthUrl } from "../OAuth"; function boundModels(credential: Credential): string[] { @@ -718,21 +820,31 @@ export function Credentials() { ) } /> - {c.enabled === true ? "已启用" : c.enabled === false ? "人工停用" : "未知"} + {c.enabled === true ? "已启用" : c.enabled === false ? "停用" : "未知"} ); const renderHealthBadge = (c: Credential) => { + const disabled = c.enabled === false; const until = number(c.fail_until); - const remaining = until === null ? null : Math.max(0, Math.ceil(until - now / 1000)); + const remaining = disabled + ? null + : until === null + ? null + : Math.max(0, Math.ceil(until - now / 1000)); + const label = disabled + ? c.auto_disabled + ? healthLabels.auto_disabled + : healthLabels.disabled + : remaining !== null && remaining > 0 + ? `认证熔断 ${remaining}s` + : c.health === "ready" + ? "认证正常" + : (healthLabels[typeof c.health === "string" ? c.health : ""] ?? text(c.health)); return ( 0 ? "bad" : c.health === "ready" ? "good" : "warn"} > - {remaining !== null && remaining > 0 - ? `认证熔断 ${remaining}s` - : c.health === "ready" - ? "认证正常" - : text(c.health)} + {label} ); }; @@ -778,10 +890,7 @@ export function Credentials() { ) : null} {c.last_error_code != null && ( - - 最近错误：{text(lastErrorLabel(c.last_error_code))} - {number(c.last_failure_at) !== null ? ` · ${expiry(c.last_failure_at)}` : ""} - + )} {typeof c.sync_error === "string" && c.sync_error !== "" && ( 失败原因：{text(c.sync_error)} diff --git a/web/src/ui.module.scss b/web/src/ui.module.scss index 2814b76..e7d4668 100644 --- a/web/src/ui.module.scss +++ b/web/src/ui.module.scss @@ -602,6 +602,36 @@ $white: var(--page); border-radius: 14px; white-space: normal; } + +.errorJsonTip { + /* Fixed beside the trigger so a scrolling table never clips it. */ + position: fixed; + z-index: 40; + max-width: min(560px, calc(100vw - 16px)); + max-height: min(420px, calc(100vh - 16px)); + overflow: auto; + margin: 0; + padding: 10px 12px; + @include glass(0.92, 16px); + border-radius: 12px; + font-size: 12px; + line-height: 1.5; + white-space: pre-wrap; + word-break: break-all; + font-family: var(--mono, ui-monospace, monospace); + cursor: text; +} + +.errorJsonTrigger { + border: none; + background: none; + padding: 0; + color: inherit; + font: inherit; + cursor: help; + text-decoration: underline dotted; + text-underline-offset: 3px; +} .channelTrack button > span, .channelTrack small { font-size: 11px; diff --git a/web/src/values.tsx b/web/src/values.tsx index a883eab..d474176 100644 --- a/web/src/values.tsx +++ b/web/src/values.tsx @@ -168,7 +168,8 @@ const profiles: Record = { }; const statuses: Record = { ready: "就绪", - disabled: "已停用", + disabled: "人工停用", + auto_disabled: "自动禁用", error: "失败", success: "成功", cancelled: "已取消",