import { z } from "zod"; import type { JSONSchema7 } from "json-schema"; import type { ToolExecutor, ToolContext, ToolResult } from "../../registry"; import { assertSafeUrl, checkDomainAccess } from "./security"; import { parseResponse } from "./parse"; import type { FetchToolConfig } from "./config"; export const fetchInputSchema = z.object({ url: z.string().url().describe("要抓取的 URL(http/https)"), method: z.enum(["GET", "POST"]).optional().describe("HTTP 方法"), headers: z.record(z.string(), z.string()).optional().describe("自定义请求头"), body: z.string().optional().describe("POST 请求体"), }); type FetchInput = z.infer; export const fetchExecutor: ToolExecutor = { buildInputSchema(config: FetchToolConfig): JSONSchema7 { return { type: "object", properties: { url: { type: "string", description: "要抓取的 URL(http/https)" }, method: { type: "string", enum: ["GET", "POST"], default: config.defaultMethod, description: "HTTP 方法", }, headers: { type: "object", description: "自定义请求头", additionalProperties: { type: "string" }, }, body: { type: "string", description: "POST 请求体" }, }, required: ["url"], }; }, buildDescription(config: FetchToolConfig): string { const domains = config.allowedDomains.includes("*") ? "任意域名" : `仅限: ${config.allowedDomains.join(", ")}`; return `抓取网页或 API 内容。支持 ${config.parseMode} 模式。域名限制: ${domains}。超时 ${config.timeout}ms,最大响应 ${config.maxResponseSize} bytes。`; }, async execute( input: unknown, config: FetchToolConfig, ctx: ToolContext, ): Promise { const start = Date.now(); // 1. 校验 input const parsed = fetchInputSchema.safeParse(input); if (!parsed.success) { return { success: false, data: null, error: `输入参数校验失败: ${parsed.error.message}`, metadata: { durationMs: Date.now() - start }, }; } const fetchInput: FetchInput = parsed.data; const method = fetchInput.method ?? config.defaultMethod; // 2. SSRF 检查 try { await assertSafeUrl(fetchInput.url); } catch (e) { return { success: false, data: null, error: e instanceof Error ? e.message : String(e), metadata: { durationMs: Date.now() - start }, }; } // 3. 域名白/黑名单检查 const hostname = new URL(fetchInput.url).hostname; try { checkDomainAccess(hostname, config.allowedDomains, config.blockedDomains); } catch (e) { return { success: false, data: null, error: e instanceof Error ? e.message : String(e), metadata: { durationMs: Date.now() - start }, }; } // 4. 发起 fetch const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), config.timeout); try { const mergedHeaders: Record = { ...config.defaultHeaders, ...(fetchInput.headers ?? {}), }; const response = await fetch(fetchInput.url, { method, headers: mergedHeaders, body: method === "POST" ? fetchInput.body : undefined, signal: controller.signal, redirect: "follow", }); clearTimeout(timeoutId); // HTTP 非 2xx 直接返回失败,让模型知道请求无效 if (!response.ok) { const errBody = await response.text().catch(() => ""); return { success: false, data: null, error: `HTTP ${response.status} ${response.statusText}: ${errBody.slice(0, 500)}`, metadata: { statusCode: response.status, durationMs: Date.now() - start, }, }; } // 5. 大小限制检查 const contentLength = response.headers.get("content-length"); if (contentLength && parseInt(contentLength, 10) > config.maxResponseSize) { return { success: false, data: null, error: `响应过大: ${contentLength} bytes (限制 ${config.maxResponseSize})`, metadata: { statusCode: response.status, durationMs: Date.now() - start, }, }; } // 读取 body(分块检查大小) const reader = response.body?.getReader(); if (!reader) { const text = await response.text(); if (text.length > config.maxResponseSize) { return { success: false, data: null, error: `响应过大: ${text.length} bytes (限制 ${config.maxResponseSize})`, metadata: { statusCode: response.status, durationMs: Date.now() - start, }, }; } const result = parseResponse(text, config.parseMode, response.headers.get("content-type") ?? undefined); return { success: true, data: result.content, metadata: { statusCode: response.status, responseSize: text.length, durationMs: Date.now() - start, }, }; } const chunks: Uint8Array[] = []; let totalSize = 0; let oversized = false; while (true) { const { done, value } = await reader.read(); if (done) break; if (value) { totalSize += value.length; if (totalSize > config.maxResponseSize) { oversized = true; break; } chunks.push(value); } } reader.cancel(); if (oversized) { return { success: false, data: null, error: `响应过大: 超过 ${config.maxResponseSize} bytes`, metadata: { statusCode: response.status, durationMs: Date.now() - start, }, }; } const decoder = new TextDecoder("utf-8"); const bodyText = chunks.map((c) => decoder.decode(c, { stream: true })).join("") + decoder.decode(); // 6. 按 parseMode 处理 const result = parseResponse( bodyText, config.parseMode, response.headers.get("content-type") ?? undefined, ); return { success: true, data: result.content, metadata: { statusCode: response.status, responseSize: totalSize, durationMs: Date.now() - start, }, }; } catch (e) { clearTimeout(timeoutId); if (e instanceof Error && e.name === "AbortError") { return { success: false, data: null, error: `请求超时 (${config.timeout}ms)`, metadata: { durationMs: Date.now() - start }, }; } return { success: false, data: null, error: e instanceof Error ? e.message : String(e), metadata: { durationMs: Date.now() - start }, }; } }, };