import { describe, expect, test } from "bun:test"
import { Duration, Effect, Fiber, Layer, Schema } from "effect"
import * as TestClock from "effect/testing/TestClock"
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder"
import { LayerNode } from "@opencode-ai/util/effect/layer-node"
import { LayerNodePlatform } from "@opencode-ai/util/effect/app-node-platform"
import { Permission } from "@opencode-ai/core/permission"
import { Session } from "@opencode-ai/core/session"
import { Tool } from "@opencode-ai/core/tool"
import { WebFetchTool } from "@opencode-ai/core/tool/plugin/webfetch"
import { makeLocationNode } from "@opencode-ai/util/effect/app-node"
import { Image } from "@opencode-ai/core/image"
import { testEffect } from "./lib/effect"
import { imagePassthrough } from "./lib/image"
import { permissionLayer } from "./lib/permission"
import { toolIdentity, executeTool, registerToolPlugin, toolDefinitions } from "./lib/tool"
const webFetchToolNode = makeLocationNode({
name: "test/webfetch-tool-plugin",
layer: Layer.effectDiscard(registerToolPlugin(WebFetchTool.Plugin)),
deps: [Tool.node, Permission.node, LayerNodePlatform.httpClient],
})
const sessionID = Session.ID.make("ses_webfetch_test")
const requests: Array<{ readonly url: string; readonly headers: Record }> = []
const assertions: Permission.AssertInput[] = []
let respond = (_request: HttpClientRequest.HttpClientRequest) =>
Effect.succeed(new Response("hello", { headers: { "content-type": "text/plain" } }))
const http = Layer.succeed(
HttpClient.HttpClient,
HttpClient.make((request) =>
Effect.sync(() => requests.push({ url: request.url, headers: request.headers })).pipe(
Effect.andThen(respond(request)),
Effect.map((response) => HttpClientResponse.fromWeb(request, response)),
),
),
)
const permission = permissionLayer({ assert: (input) => Effect.sync(() => assertions.push(input)) })
const toolLayer = (replacements: LayerNode.Replacements = []) =>
AppNodeBuilder.build(LayerNode.group([Tool.node, webFetchToolNode]), [
[Permission.node, permission],
[Image.node, imagePassthrough],
...replacements,
])
const it = testEffect(toolLayer([[LayerNodePlatform.httpClient, http]]))
const live = testEffect(toolLayer())
const reset = () => {
requests.length = 0
assertions.length = 0
respond = () => Effect.succeed(new Response("hello", { headers: { "content-type": "text/plain" } }))
}
const call = (input: typeof WebFetchTool.Input.Type, id = "call-webfetch") => ({
sessionID,
...toolIdentity,
call: { type: "tool-call" as const, id, name: "webfetch", input },
})
describe("WebFetchTool helpers", () => {
test("defaults format and rejects invalid timeout controls", () => {
const decode = Schema.decodeUnknownSync(WebFetchTool.Input)
expect(decode({ url: "https://example.com" })).toEqual({ url: "https://example.com", format: "markdown" })
expect(() => decode({ url: "https://example.com", timeout: 0 })).toThrow()
expect(() => decode({ url: "https://example.com", timeout: WebFetchTool.MAX_TIMEOUT_SECONDS + 1 })).toThrow()
})
test("ports HTML text and markdown conversions without active content", () => {
const html =
"Hello world wide today
"
expect(WebFetchTool.extractTextFromHTML(html)).toBe("Helloworld wide today")
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("# Hello\n\nworld **wide** today")
})
test("renders headings, inline semantics, links, images, breaks, and thematic breaks", () => {
const html = `Read this docs
old
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
`## Read *this*\n\n[docs](https://example.com/a%20\\(b\\) "Example") \n![a \\] b](diagram.png)\n\n---\n\n~~old~~`,
)
})
test("preserves inline and preformatted code verbatim with safe fences", () => {
const html = `Use say(\`hello\`) now.
const fence = \`\`\`\n& stays decoded `
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
`Use \`\`say(\`hello\`)\`\` now.\n\n~~~ts\nconst fence = \`\`\`\n& stays decoded\n~~~`,
)
})
test("keeps nested ordered and unordered lists structurally readable", () => {
const html = `alpha beta first
beta second
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
`3. alpha\n\n - nested **item**\n\n4. beta first\n\n beta second`,
)
})
test("renders blockquotes and tables as readable Markdown", () => {
const html = `quoted text
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
`> quoted *text*\n\n> - point\n\n| Name | Value |\n| --- | --- |\n| one | \`1\` |`,
)
})
test("decodes entities and normalizes prose whitespace without joining words", () => {
const html = `alpha\n & beta café gamma 😀
delta
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`alpha & beta café gamma 😀\n\ndelta`)
})
test("omits active and fallback content while retaining surrounding prose", () => {
const html = `before bad bad bad after
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("before after")
})
test("is deterministic and bounded for malformed maximum-size input", () => {
const html = `${"visible & text ".repeat(250_000)}
`
const first = WebFetchTool.convertHTMLToMarkdown(html)
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(first)
expect(first.startsWith("visible & text visible & text")).toBe(true)
expect(first.length).toBeLessThanOrEqual(html.length)
})
test("bounds deeply nested list output and fragmented code fences", () => {
const lists = `${"".repeat(20_000) + "safe
tail &
",
),
).toBe("safe tail &")
})
test("escapes prose that would otherwise become Markdown structure", () => {
expect(WebFetchTool.convertHTMLToMarkdown(`
# heading
1. item
---
a | b
`)).toBe(
`\\# heading\n\n1\\. item\n\n\\---\n\na \\| b`,
)
})
test("preserves code whitespace and quotes every line of multiline blocks", () => {
const html = `
line \n\n\nnext `
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
`> \`\`\`\n> line \n> \n> \n> next\n> \`\`\`\n\n> | a\\|b | c |\n> | --- | --- |`,
)
})
test("keeps nested blockquotes inside their outer quote", () => {
const html = `
outer
inner
end
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`> outer\n>\n> > inner\n>\n> end`)
})
test("keeps visible whitespace around inline emphasis", () => {
expect(WebFetchTool.convertHTMLToMarkdown(`
a b c a b c
`)).toBe(`a **b** c a *b* c`)
expect(WebFetchTool.convertHTMLToMarkdown(`a
b a
b`)).toBe(`a b a b`)
})
test("captures formatting elements inside preformatted content as code only", () => {
expect(WebFetchTool.convertHTMLToMarkdown(`
x y z`)).toBe(`\`\`\`\nxyz\n\`\`\``)
})
test("normalizes multiline table cells without changing their columns", () => {
const html = `
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`| x y | \`a\\|b\` | first second |\n| --- | --- | --- |`)
})
test("flattens nested tables without corrupting the outer table", () => {
const html = `
Parent Sibling BeforeAfter Tail
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
`| Parent | Sibling |\n| --- | --- |\n| Before Key Value A 1 After | Tail |`,
)
})
test("preserves loose text around malformed table rows", () => {
expect(WebFetchTool.convertHTMLToMarkdown(`
`)).toBe(
`before after\n\n| cell |\n| --- |`,
)
expect(WebFetchTool.convertHTMLToMarkdown(`
`)).toBe(`alpha`)
})
test("escapes tilde fences and removes empty emphasis markers", () => {
expect(WebFetchTool.convertHTMLToMarkdown(`
~~~
content
~~~
`)).toBe(
`\\~\\~\\~\n\ncontent\n\n\\~\\~\\~`,
)
})
test("parses malformed tag prefixes in linear time without a regex prepass", () => {
const small = "
{
const prose = `${"*".repeat(WebFetchTool.MAX_RESPONSE_BYTES)}
`
const code = `${"`".repeat(WebFetchTool.MAX_RESPONSE_BYTES - 11)} `
const proseOutput = WebFetchTool.convertHTMLToMarkdown(prose)
const codeOutput = WebFetchTool.convertHTMLToMarkdown(code)
expect(Buffer.byteLength(proseOutput)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
expect(Buffer.byteLength(codeOutput)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
expect(codeOutput.startsWith("~~~\n")).toBe(true)
})
test("does not confuse source NUL text with buffered code", () => {
expect(WebFetchTool.convertHTMLToMarkdown(`before \u00000\u0000 after
code `)).toBe(
`before \u00000\u0000 after\n\n\`\`\`\ncode\n\`\`\``,
)
})
test("preserves multiline inline code verbatim", () => {
expect(WebFetchTool.convertHTMLToMarkdown(`first\n\n\nsecond
`)).toBe(
"` first\n\n\nsecond `",
)
})
test("prefixes inline code at the start of a blockquote line", () => {
expect(WebFetchTool.convertHTMLToMarkdown(`x y `)).toBe(`> \`x\` y`)
})
test("keeps links nested in inline code associated with their text", () => {
const html = `socket = new WebSocket (url)Creates one. `
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
`**\` socket = new \`[\`WebSocket\`](#constructor)\`(url)\`**\n: Creates one.`,
)
expect(WebFetchTool.convertHTMLToMarkdown(`x after`)).toBe(`[\`x\`](#x) after`)
expect(
WebFetchTool.convertHTMLToMarkdown(
`socket = new WebSocket (url )Creates one. `,
),
).toBe(`**\` socket = new \`[\`WebSocket\`](#constructor)\`(url)\`**\n: Creates one.`)
expect(WebFetchTool.convertHTMLToMarkdown(`ab c d e`)).toBe(
`\`a\`[\`b\`](\/x)[\`c\`](\/y)\`de\``,
)
expect(WebFetchTool.convertHTMLToMarkdown(`
ab c`)).toBe(`\`a\`[\`b\`](\/x)c`)
expect(WebFetchTool.convertHTMLToMarkdown(`
ab
c d`)).toBe(
`\`a\`[](\/x)\n\n\`bcd\``,
)
})
test("indents nested list continuations and preserves ordered numbering", () => {
const html = `
first
continued
next `
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
`4. first\n\n continued\n\n - nested\n\n continued nested\n\n5. next`,
)
})
test("renders block content outside link syntax", () => {
expect(WebFetchTool.convertHTMLToMarkdown(`
beforeblock
after `)).toBe(
`[before](/docs)\n\nblock\n\n[after](/docs)`,
)
})
test("recovers nested anchors without unmatched Markdown syntax", () => {
expect(WebFetchTool.convertHTMLToMarkdown(`
x y z`)).toBe(`[x](/a)[y](/b)z`)
})
test("keeps emphasis whitespace through neutral wrappers", () => {
expect(WebFetchTool.convertHTMLToMarkdown(`
a bold c
`)).toBe(`a **bold** c`)
})
test("flattens preformatted content inside table cells", () => {
const html = `
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`| a\\|b next | \`x\\|y\` |\n| --- | --- |`)
})
test("keeps each near-boundary inline construct closed and UTF-8-safe", () => {
const payload = "😀".repeat(WebFetchTool.MAX_RESPONSE_BYTES / 4)
const cases = [
[`
${payload} `, /^\*\*[\s\S]*\*\*$/],
[`
${payload} `, /^\[[\s\S]*\]\(\/docs\)$/],
[`
`, /^!\[[\s\S]*\]\(image\.png\)$/],
[`
${payload}`, /^`[\s\S]*`$/],
] as const
for (const [html, pattern] of cases) {
const output = WebFetchTool.convertHTMLToMarkdown(html)
expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
expect(output).not.toContain("�")
expect(output).toMatch(pattern)
}
})
test("keeps near-boundary block constructs syntactically complete", () => {
const payload = "x".repeat(WebFetchTool.MAX_RESPONSE_BYTES)
const table = WebFetchTool.convertHTMLToMarkdown(
`
`,
)
const list = WebFetchTool.convertHTMLToMarkdown(`
`)
const code = WebFetchTool.convertHTMLToMarkdown(`
${payload} `)
for (const output of [table, list, code]) {
expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
expect(output).not.toContain("�")
}
expect(table).toMatch(/^\| Name \|\n\| --- \|\n\| [\s\S]* \|$/)
expect(list).toMatch(/^- [\s\S]*$/)
expect(list.includes("nested")).toBe(false)
expect(code.match(/^(`{3,}|~{3,})$/gm)).toHaveLength(2)
})
test("keeps quoted code within budget with a safe closed fence", () => {
const html = `
${"`".repeat(32)}${"~".repeat(32)}${"x".repeat(WebFetchTool.MAX_RESPONSE_BYTES)} `
const output = WebFetchTool.convertHTMLToMarkdown(html)
expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
const lines = output.split("\n")
expect(lines[0]).toMatch(/^> (`{33}|~{33})$/)
expect(lines.at(-1)).toBe(lines[0])
})
test("separates reconstructed tables from adjacent inline and quoted content", () => {
const html = `intro
outro
quote `
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
`intro\n\n| x |\n| --- |\n\noutro\n\n> quote\n\n> | cell |\n> | --- |\n\n- item\n\n| cell |\n| --- |`,
)
})
test("keeps multiline quoted code closed at the content budget", () => {
const html = `
${"x\n".repeat(WebFetchTool.MAX_RESPONSE_BYTES / 2)} tail
`
const output = WebFetchTool.convertHTMLToMarkdown(html)
expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
expect((output.match(/(`{3}|~{3})/g) ?? []).length).toBe(2)
expect(output.includes("\uFFFD")).toBe(false)
expect(output.endsWith("tail")).toBe(true)
})
test("keeps active content suppressed when depth fallback begins", () => {
const html = `
${"".repeat(10_001)}LEAK${"
".repeat(10_001)} visible
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("visible")
})
test("keeps visible text after depth fallback begins inside preformatted content", () => {
const html = `
${"".repeat(10_001)}visible${" ".repeat(10_001)} after
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("visible after")
})
test("resumes links around every block structure", () => {
const html = `
beforequote
code after`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
`[before](/x)\n\n> quote\n\n- item\n\n\`\`\`\ncode\n\`\`\`\n\n| cell |\n| --- |\n\n[after](/x)`,
)
})
test("indents child lists from the actual parent marker width", () => {
expect(WebFetchTool.convertHTMLToMarkdown(`
outer `)).toBe(
`100. outer\n\n - inner`,
)
})
test("renders captions and definition lists with readable boundaries", () => {
const html = `
Cache modes Name Meaning A Local
Cache A local store Origin The remote source `
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
`Cache modes\n\n| Name | Meaning |\n| --- | --- |\n| A | Local |\n\n**Cache**\n: A local store\n\n**Origin**\n: The remote source`,
)
})
test("falls back to row-oriented text for table spans", () => {
const html = `
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`Group\n\nA | Shared\n\nB`)
})
test("suppresses head and hidden subtrees while retaining visible body content", () => {
const html = `
noise visible
hidden
aria
shown
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`visible\n\nshown`)
})
test("preserves pre breaks and normalizes multiline link titles", () => {
const html = `
first second link
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
`\`\`\`\nfirst\nsecond\n\`\`\`\n\n[link](/x "line one line two")`,
)
})
test("renders closed and open details according to visibility", () => {
const html = `
Closed secret
Open visible
`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`Closed\n\nOpen\n\nvisible`)
})
})
describe("WebFetchTool registration", () => {
it.effect("registers and fetches an ordinary hostname HTTP URL without rewriting it", () =>
Effect.gen(function* () {
reset()
const registry = yield* Tool.Service
const url = "http://example.com/public"
expect((yield* toolDefinitions(registry)).map((tool) => tool.name)).toEqual(["webfetch", "execute"])
expect(yield* executeTool(registry, call({ url, format: "text", timeout: 4 }))).toEqual({
status: "completed",
output: { url, contentType: "text/plain", format: "text", output: "hello" },
content: [{ type: "text", text: "hello" }],
metadata: { contentType: "text/plain" },
})
expect(assertions).toMatchObject([
{ sessionID, action: "webfetch", resources: [url], save: ["*"], metadata: { url, format: "text", timeout: 4 } },
])
expect(requests).toMatchObject([{ url, headers: { accept: expect.stringContaining("text/plain;q=1.0") } }])
}),
)
it.effect("accepts localhost URLs with the same requested-URL permission check", () =>
Effect.gen(function* () {
reset()
const registry = yield* Tool.Service
const url = "http://localhost/private"
expect(yield* executeTool(registry, call({ url, format: "text" }))).toMatchObject({
status: "completed",
content: [{ type: "text", text: "hello" }],
})
expect(assertions).toMatchObject([
{ sessionID, action: "webfetch", resources: [url], save: ["*"], metadata: { url, format: "text" } },
])
expect(requests.map((request) => request.url)).toEqual([url])
}),
)
live.effect("follows redirects while approving only the requested URL", () =>
Effect.acquireUseRelease(
Effect.sync(() =>
Bun.serve({
port: 0,
fetch: (request) =>
new URL(request.url).pathname === "/redirect"
? new Response("", { status: 302, headers: { location: "/target" } })
: new Response("redirected", { headers: { "content-type": "text/plain" } }),
}),
),
(server) =>
Effect.gen(function* () {
reset()
const registry = yield* Tool.Service
const url = new URL("/redirect", server.url).toString()
expect(yield* executeTool(registry, call({ url, format: "text" }))).toMatchObject({
status: "completed",
content: [{ type: "text", text: "redirected" }],
})
expect(assertions).toMatchObject([
{ sessionID, action: "webfetch", resources: [url], save: ["*"], metadata: { url, format: "text" } },
])
}),
(server) => Effect.promise(() => server.stop(true)),
),
)
it.effect("rejects non-HTTP schemes before permission or transport", () =>
Effect.gen(function* () {
reset()
const registry = yield* Tool.Service
// toSessionError unwraps the "Unable to fetch
" ToolFailure to its cause message.
expect(yield* executeTool(registry, call({ url: "file:///etc/passwd", format: "text" }))).toEqual({
status: "error",
error: { type: "unknown", message: "URL must use http:// or https://" },
})
expect(assertions).toEqual([])
expect(requests).toEqual([])
}),
)
it.effect("converts HTML to requested markdown and text", () =>
Effect.gen(function* () {
reset()
respond = () =>
Effect.succeed(
new Response("Hello world
", {
headers: { "content-type": "text/html; charset=utf-8" },
}),
)
const registry = yield* Tool.Service
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1", format: "markdown" }))).toMatchObject({
status: "completed",
content: [{ type: "text", text: "# Hello\n\nworld" }],
})
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1", format: "text" }))).toMatchObject({
status: "completed",
content: [{ type: "text", text: "Helloworld" }],
})
}),
)
it.effect("converts deeply nested HTML without overflowing", () =>
Effect.gen(function* () {
reset()
respond = () =>
Effect.succeed(
new Response("".repeat(10_000) + "content" + "
".repeat(10_000), {
headers: { "content-type": "text/html" },
}),
)
const registry = yield* Tool.Service
const url = "https://1.1.1.1/deep-html"
expect(yield* executeTool(registry, call({ url, format: "markdown" }))).toMatchObject({
status: "completed",
content: [{ type: "text", text: "content" }],
})
}),
)
it.effect("rejects declared and streamed oversized bodies", () =>
Effect.gen(function* () {
reset()
const registry = yield* Tool.Service
respond = () =>
Effect.succeed(
new Response("small", {
headers: { "content-type": "text/plain", "content-length": String(WebFetchTool.MAX_RESPONSE_BYTES + 1) },
}),
)
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/declared", format: "text" }))).toEqual({
status: "error",
error: {
type: "unknown",
message: `Response too large (exceeds ${WebFetchTool.MAX_RESPONSE_BYTES} byte limit)`,
},
})
respond = () =>
Effect.succeed(
new Response("x".repeat(WebFetchTool.MAX_RESPONSE_BYTES + 1), { headers: { "content-type": "text/plain" } }),
)
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/streamed", format: "text" }))).toEqual({
status: "error",
error: {
type: "unknown",
message: `Response too large (exceeds ${WebFetchTool.MAX_RESPONSE_BYTES} byte limit)`,
},
})
}),
)
it.effect("keeps images and files unsupported until typed outcomes can carry attachments", () =>
Effect.gen(function* () {
reset()
const registry = yield* Tool.Service
respond = () => Effect.succeed(new Response("png", { headers: { "content-type": "image/png" } }))
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/image", format: "html" }))).toEqual({
status: "error",
error: { type: "unknown", message: "Unsupported fetched image content type: image/png" },
})
respond = () => Effect.succeed(new Response("pdf", { headers: { "content-type": "application/pdf" } }))
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/file", format: "html" }))).toEqual({
status: "error",
error: { type: "unknown", message: "Unsupported fetched file content type: application/pdf" },
})
}),
)
it.effect("retries Cloudflare challenges with an honest user agent", () =>
Effect.gen(function* () {
reset()
let count = 0
respond = () =>
Effect.succeed(
++count === 1
? new Response("challenge", { status: 403, headers: { "cf-mitigated": "challenge" } })
: new Response("ok", { headers: { "content-type": "text/plain" } }),
)
const registry = yield* Tool.Service
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1", format: "text" }))).toMatchObject({
status: "completed",
content: [{ type: "text", text: "ok" }],
})
expect(requests).toHaveLength(2)
expect(requests[0]?.headers["user-agent"]).toContain("Mozilla/5.0")
expect(requests[1]?.headers["user-agent"]).toBe("opencode")
}),
)
it.effect("times out stalled requests", () =>
Effect.gen(function* () {
reset()
respond = () => Effect.never
const registry = yield* Tool.Service
const fiber = yield* executeTool(
registry,
call({ url: "https://1.1.1.1/slow", format: "text", timeout: 1 }),
).pipe(Effect.forkChild)
yield* TestClock.adjust(Duration.seconds(1))
expect(yield* Fiber.join(fiber)).toEqual({
status: "error",
error: { type: "unknown", message: "Request timed out" },
})
}),
)
})