From 9e10bf11cfa1f46f55bd8a97724ebee09b787396 Mon Sep 17 00:00:00 2001 From: dark Date: Thu, 4 Jun 2026 11:55:38 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96llm=E5=88=86=E6=9E=90?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pam_deploy_graph/llm/openai_compatible.py | 57 ++++++++++++++++++---- pam_deploy_graph/llm/prompts.py | 2 + pam_deploy_graph/llm/rule_based.py | 3 +- prompts/action_review.txt | 2 + tests/test_llm_structured.py | 58 ++++++++++++++++++++++- 5 files changed, 109 insertions(+), 13 deletions(-) diff --git a/pam_deploy_graph/llm/openai_compatible.py b/pam_deploy_graph/llm/openai_compatible.py index 8098ee7..c490ce0 100644 --- a/pam_deploy_graph/llm/openai_compatible.py +++ b/pam_deploy_graph/llm/openai_compatible.py @@ -158,15 +158,7 @@ class OpenAICompatibleLlmClient: self.action_analysis_prompt, { "action": action, - "result": { - "backend": result.backend, - "ok": result.ok, - "exit_code": result.exit_code, - "tool_name": result.tool_name, - "values": _redact_sensitive(result.values), - "stderr": _truncate_text(result.stderr), - "error_summary": result.error_summary, - }, + "result": _action_review_result_payload(action, result), "state_summary": _redact_sensitive(state_summary), }, ) @@ -325,6 +317,45 @@ def _redact_sensitive(value: Any) -> Any: return value +def _action_review_result_payload(action: str, result: ActionResult) -> dict[str, Any]: + """构造 action 审核输入，避免把正常脚本日志当作错误喂给 LLM。""" + payload: dict[str, Any] = { + "backend": result.backend, + "ok": result.ok, + "exit_code": result.exit_code, + "tool_name": result.tool_name, + "values": _redact_sensitive(result.values), + "error_summary": result.error_summary, + } + if _needs_diagnostic_log(action, result): + diagnostic = _diagnostic_log_text(result) + if diagnostic: + payload["diagnostic_log"] = diagnostic + return payload + + +def _needs_diagnostic_log(action: str, result: ActionResult) -> bool: + """仅在失败或业务异常时把少量诊断日志交给 LLM。""" + if not result.ok or result.error_summary or result.values.get("PENDING_AGENT_CONFIRMATION"): + return True + if action == "verify-ip": + success = result.values.get("SUCCESS") + if success is not None and str(success).lower() not in ("true", "1", "yes"): + return True + return False + + +def _diagnostic_log_text(result: ActionResult) -> str: + """优先使用错误摘要；必要时取 stderr/stdout/raw_output 的尾部作为诊断上下文。""" + if result.error_summary: + return _truncate_text(result.error_summary) + for text in (result.stderr, result.stdout, result.raw_output): + stripped = text.strip() + if stripped: + return _tail_text(stripped) + return "" + + def _truncate_text(value: str, limit: int = 1000) -> str: """截断发送给 LLM 的长文本，避免传入完整日志。""" if len(value) <= limit: @@ -332,6 +363,14 @@ def _truncate_text(value: str, limit: int = 1000) -> str: return value[:limit] + "...[已截断]" +def _tail_text(value: str, limit: int = 1000) -> str: + """保留长诊断日志尾部，通常错误原因更靠近末尾。""" + if len(value) <= limit: + return value + marker = "[已截断]..." + return marker + value[-(limit - len(marker)) :] + + def _string(payload: dict[str, Any], key: str, default: str) -> str: """安全读取字符串字段。""" value = payload.get(key, default) diff --git a/pam_deploy_graph/llm/prompts.py b/pam_deploy_graph/llm/prompts.py index 2c59869..7d299aa 100644 --- a/pam_deploy_graph/llm/prompts.py +++ b/pam_deploy_graph/llm/prompts.py @@ -83,5 +83,7 @@ ACTION_ANALYSIS_PROMPT = """分析一次 PAM action 执行结果。 要求： - 必须明确给出 `should_continue`：没有问题时为 true；存在需要人工判断的问题时为 false。 - 如果 exit_code 非 0、ok=false、verify-ip SUCCESS=false、出现 pending_confirmation，应标记异常。 +- 主要依据结构化字段 `ok`、`exit_code`、`values`、`error_summary` 判断；只有输入里存在 `diagnostic_log` 时，才把它当作异常诊断上下文。 +- 脚本正常过程日志不会作为错误依据，不能因为日志来自 stderr 就判定异常。 - 不要输出密钥、token、Authorization 或完整日志原文。 """ diff --git a/pam_deploy_graph/llm/rule_based.py b/pam_deploy_graph/llm/rule_based.py index 59a7e5b..f12f9db 100644 --- a/pam_deploy_graph/llm/rule_based.py +++ b/pam_deploy_graph/llm/rule_based.py @@ -179,7 +179,6 @@ class RuleBasedLlmClient: "exit_code": result.exit_code, "tool_name": result.tool_name, "values": result.values, - "stderr": result.stderr, "error_summary": result.error_summary, }, max_text_len=1000, @@ -197,7 +196,7 @@ class RuleBasedLlmClient: if not result.ok: severity = "medium" possible_reason = result.error_summary or "action 返回失败状态。" - suggested_action = "查看 action stderr/raw_output，确认参数、网络和目标服务状态。" + suggested_action = "查看 action 诊断日志、参数、网络和目标服务状态。" notes.append("硬规则检测到 action 执行失败。") should_continue = False diff --git a/prompts/action_review.txt b/prompts/action_review.txt index a9a15fc..191f195 100644 --- a/prompts/action_review.txt +++ b/prompts/action_review.txt @@ -15,4 +15,6 @@ 要求： - 必须明确给出 `should_continue`：没有问题时为 true；存在需要人工判断的问题时为 false。 - 如果 exit_code 非 0、ok=false、verify-ip SUCCESS=false、出现 pending_confirmation，应标记异常。 +- 主要依据结构化字段 `ok`、`exit_code`、`values`、`error_summary` 判断；只有输入里存在 `diagnostic_log` 时，才把它当作异常诊断上下文。 +- 脚本正常过程日志不会作为错误依据，不能因为日志来自 stderr 就判定异常。 - 不要输出密钥、token、Authorization 或完整日志原文。 diff --git a/tests/test_llm_structured.py b/tests/test_llm_structured.py index bf72678..ca5e975 100644 --- a/tests/test_llm_structured.py +++ b/tests/test_llm_structured.py @@ -1,4 +1,5 @@ from dataclasses import asdict +import json from pam_deploy_graph.agent import PamDeployAgent from pam_deploy_graph.checkpoint_store import redact_mapping @@ -219,13 +220,66 @@ def test_openai_compatible_client_analyzes_action_result_with_redaction(): ok=False, values={"CLIENT_SECRET": "real-secret", "SUCCESS": "false"}, stderr="x" * 1200, - error_summary="failed", ), state_summary={"params": {"CLIENT_SECRET": "real-secret"}}, ) serialized_prompt = str(calls[0]) + input_payload = _llm_input_payload(calls[0]) assert analysis.has_anomaly is True assert analysis.severity == "high" assert "real-secret" not in serialized_prompt - assert "[已截断]" in serialized_prompt + assert input_payload["result"]["diagnostic_log"].startswith("[已截断]...") + + +def test_openai_compatible_client_omits_success_script_logs_from_action_review(): + calls = [] + + def transport(url, headers, payload, timeout_sec): + calls.append(payload) + return { + "choices": [ + { + "message": { + "content": ( + '{"action":"get-online-ips","has_anomaly":false,"severity":"info",' + '"possible_reason":"","suggested_action":"continue",' + '"requires_confirmation":false,"should_continue":true}' + ) + } + } + ] + } + + client = OpenAICompatibleLlmClient( + base_url="https://llm.example/v1", + api_key="secret-key", + model="model-a", + transport=transport, + ) + + client.analyze_action_result( + action="get-online-ips", + result=ActionResult( + action="get-online-ips", + backend="script", + ok=True, + values={"ACTION": "get-online-ips", "COUNT": "1", "IP": ["10.4.1.1"]}, + stdout="ACTION=get-online-ips\nCOUNT=1\nIP=10.4.1.1\n", + stderr="[INFO] [FLOW][START] get_token\n[INFO] [FLOW][DONE] get_online_ips\n", + ), + state_summary={}, + ) + + input_payload = _llm_input_payload(calls[0]) + result_payload = input_payload["result"] + assert "diagnostic_log" not in result_payload + assert "stdout" not in result_payload + assert "stderr" not in result_payload + assert "[FLOW][START]" not in json.dumps(input_payload, ensure_ascii=False) + + +def _llm_input_payload(request_payload): + content = request_payload["messages"][1]["content"] + _, _, raw_json = content.partition("输入 JSON:\n") + return json.loads(raw_json)