import { describe, expect, test } from "bun:test" import { Duration, Effect, Fiber, Layer, Schema } from "effect" import * as TestClock from "effect/testing/TestClock" import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http" import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" import { LayerNode } from "@opencode-ai/util/effect/layer-node" import { LayerNodePlatform } from "@opencode-ai/util/effect/app-node-platform" import { Permission } from "@opencode-ai/core/permission" import { Session } from "@opencode-ai/core/session" import { Tool } from "@opencode-ai/core/tool" import { WebFetchTool } from "@opencode-ai/core/tool/plugin/webfetch" import { makeLocationNode } from "@opencode-ai/util/effect/app-node" import { Image } from "@opencode-ai/core/image" import { testEffect } from "./lib/effect" import { imagePassthrough } from "./lib/image" import { permissionLayer } from "./lib/permission" import { toolIdentity, executeTool, registerToolPlugin, toolDefinitions } from "./lib/tool" const webFetchToolNode = makeLocationNode({ name: "test/webfetch-tool-plugin", layer: Layer.effectDiscard(registerToolPlugin(WebFetchTool.Plugin)), deps: [Tool.node, Permission.node, LayerNodePlatform.httpClient], }) const sessionID = Session.ID.make("ses_webfetch_test") const requests: Array<{ readonly url: string; readonly headers: Record }> = [] const assertions: Permission.AssertInput[] = [] let respond = (_request: HttpClientRequest.HttpClientRequest) => Effect.succeed(new Response("hello", { headers: { "content-type": "text/plain" } })) const http = Layer.succeed( HttpClient.HttpClient, HttpClient.make((request) => Effect.sync(() => requests.push({ url: request.url, headers: request.headers })).pipe( Effect.andThen(respond(request)), Effect.map((response) => HttpClientResponse.fromWeb(request, response)), ), ), ) const permission = permissionLayer({ assert: (input) => Effect.sync(() => assertions.push(input)) }) const toolLayer = (replacements: LayerNode.Replacements = []) => AppNodeBuilder.build(LayerNode.group([Tool.node, webFetchToolNode]), [ [Permission.node, permission], [Image.node, imagePassthrough], ...replacements, ]) const it = testEffect(toolLayer([[LayerNodePlatform.httpClient, http]])) const live = testEffect(toolLayer()) const reset = () => { requests.length = 0 assertions.length = 0 respond = () => Effect.succeed(new Response("hello", { headers: { "content-type": "text/plain" } })) } const call = (input: typeof WebFetchTool.Input.Type, id = "call-webfetch") => ({ sessionID, ...toolIdentity, call: { type: "tool-call" as const, id, name: "webfetch", input }, }) describe("WebFetchTool helpers", () => { test("defaults format and rejects invalid timeout controls", () => { const decode = Schema.decodeUnknownSync(WebFetchTool.Input) expect(decode({ url: "https://example.com" })).toEqual({ url: "https://example.com", format: "markdown" }) expect(() => decode({ url: "https://example.com", timeout: 0 })).toThrow() expect(() => decode({ url: "https://example.com", timeout: WebFetchTool.MAX_TIMEOUT_SECONDS + 1 })).toThrow() }) test("ports HTML text and markdown conversions without active content", () => { const html = "

Hello

world wide today

" expect(WebFetchTool.extractTextFromHTML(html)).toBe("Helloworld wide today") expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("# Hello\n\nworld **wide** today") }) test("renders headings, inline semantics, links, images, breaks, and thematic breaks", () => { const html = `

Read this

docs
a ] b


old

` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( `## Read *this*\n\n[docs](https://example.com/a%20\\(b\\) "Example") \n![a \\] b](diagram.png)\n\n---\n\n~~old~~`, ) }) test("preserves inline and preformatted code verbatim with safe fences", () => { const html = `

Use say(\`hello\`) now.

const fence = \`\`\`\n& stays decoded
` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( `Use \`\`say(\`hello\`)\`\` now.\n\n~~~ts\nconst fence = \`\`\`\n& stays decoded\n~~~`, ) }) test("keeps nested ordered and unordered lists structurally readable", () => { const html = `
  1. alpha
    • nested item
  2. beta first

    beta second

` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( `3. alpha\n\n - nested **item**\n\n4. beta first\n\n beta second`, ) }) test("renders blockquotes and tables as readable Markdown", () => { const html = `

quoted text

NameValue
one1
` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( `> quoted *text*\n\n> - point\n\n| Name | Value |\n| --- | --- |\n| one | \`1\` |`, ) }) test("decodes entities and normalizes prose whitespace without joining words", () => { const html = `

alpha\n & beta café gamma 😀

delta

` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`alpha & beta café gamma 😀\n\ndelta`) }) test("omits active and fallback content while retaining surrounding prose", () => { const html = `

before bad after

` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("before after") }) test("is deterministic and bounded for malformed maximum-size input", () => { const html = `

${"visible & text ".repeat(250_000)}

` const first = WebFetchTool.convertHTMLToMarkdown(html) expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(first) expect(first.startsWith("visible & text visible & text")).toBe(true) expect(first.length).toBeLessThanOrEqual(html.length) }) test("bounds deeply nested list output and fragmented code fences", () => { const lists = `${"".repeat(2_000)}` const quotes = `${"

item".repeat(2_000)}${"

".repeat(2_000)}` const code = `
${"` x ".repeat(250_000)}
` expect(WebFetchTool.convertHTMLToMarkdown(lists).length).toBeLessThan(lists.length * 4) expect(WebFetchTool.convertHTMLToMarkdown(quotes).length).toBeLessThan(quotes.length * 4) expect(() => WebFetchTool.convertHTMLToMarkdown(code)).not.toThrow() expect( WebFetchTool.convertHTMLToMarkdown( "
".repeat(20_000) + "safe

tail &

", ), ).toBe("safe tail &") }) test("escapes prose that would otherwise become Markdown structure", () => { expect(WebFetchTool.convertHTMLToMarkdown(`

# heading

1. item

---

a | b

`)).toBe( `\\# heading\n\n1\\. item\n\n\\---\n\na \\| b`, ) }) test("preserves code whitespace and quotes every line of multiline blocks", () => { const html = `
line  \n\n\nnext
a|bc
` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( `> \`\`\`\n> line \n> \n> \n> next\n> \`\`\`\n\n> | a\\|b | c |\n> | --- | --- |`, ) }) test("keeps nested blockquotes inside their outer quote", () => { const html = `

outer

inner

end

` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`> outer\n>\n> > inner\n>\n> end`) }) test("keeps visible whitespace around inline emphasis", () => { expect(WebFetchTool.convertHTMLToMarkdown(`

a b c a b c

`)).toBe(`a **b** c a *b* c`) expect(WebFetchTool.convertHTMLToMarkdown(`a b a b`)).toBe(`a b a b`) }) test("captures formatting elements inside preformatted content as code only", () => { expect(WebFetchTool.convertHTMLToMarkdown(`
xyz
`)).toBe(`\`\`\`\nxyz\n\`\`\``) }) test("normalizes multiline table cells without changing their columns", () => { const html = `
x
y
a|b

first

second

` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`| x y | \`a\\|b\` | first second |\n| --- | --- | --- |`) }) test("flattens nested tables without corrupting the outer table", () => { const html = `
ParentSibling
Before
KeyValue
A1
After
Tail
` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( `| Parent | Sibling |\n| --- | --- |\n| Before Key Value A 1 After | Tail |`, ) }) test("preserves loose text around malformed table rows", () => { expect(WebFetchTool.convertHTMLToMarkdown(`beforeafter
cell
`)).toBe( `before after\n\n| cell |\n| --- |`, ) expect(WebFetchTool.convertHTMLToMarkdown(`alpha
`)).toBe(`alpha`) }) test("escapes tilde fences and removes empty emphasis markers", () => { expect(WebFetchTool.convertHTMLToMarkdown(`

~~~

content

~~~

`)).toBe( `\\~\\~\\~\n\ncontent\n\n\\~\\~\\~`, ) }) test("parses malformed tag prefixes in linear time without a regex prepass", () => { const small = " { const prose = `

${"*".repeat(WebFetchTool.MAX_RESPONSE_BYTES)}

` const code = `
${"`".repeat(WebFetchTool.MAX_RESPONSE_BYTES - 11)}
` const proseOutput = WebFetchTool.convertHTMLToMarkdown(prose) const codeOutput = WebFetchTool.convertHTMLToMarkdown(code) expect(Buffer.byteLength(proseOutput)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) expect(Buffer.byteLength(codeOutput)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) expect(codeOutput.startsWith("~~~\n")).toBe(true) }) test("does not confuse source NUL text with buffered code", () => { expect(WebFetchTool.convertHTMLToMarkdown(`

before \u00000\u0000 after

code
`)).toBe( `before \u00000\u0000 after\n\n\`\`\`\ncode\n\`\`\``, ) }) test("preserves multiline inline code verbatim", () => { expect(WebFetchTool.convertHTMLToMarkdown(`

first\n\n\nsecond

`)).toBe( "` first\n\n\nsecond `", ) }) test("prefixes inline code at the start of a blockquote line", () => { expect(WebFetchTool.convertHTMLToMarkdown(`
x y
`)).toBe(`> \`x\` y`) }) test("keeps links nested in inline code associated with their text", () => { const html = `
socket = new WebSocket(url)
Creates one.
` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( `**\` socket = new \`[\`WebSocket\`](#constructor)\`(url)\`**\n: Creates one.`, ) expect(WebFetchTool.convertHTMLToMarkdown(`x after`)).toBe(`[\`x\`](#x) after`) expect( WebFetchTool.convertHTMLToMarkdown( `
socket = new WebSocket(url)
Creates one.
`, ), ).toBe(`**\` socket = new \`[\`WebSocket\`](#constructor)\`(url)\`**\n: Creates one.`) expect(WebFetchTool.convertHTMLToMarkdown(`abcde`)).toBe( `\`a\`[\`b\`](\/x)[\`c\`](\/y)\`de\``, ) expect(WebFetchTool.convertHTMLToMarkdown(`abc`)).toBe(`\`a\`[\`b\`](\/x)c`) expect(WebFetchTool.convertHTMLToMarkdown(`a
b
c
d
`)).toBe( `\`a\`[](\/x)\n\n\`bcd\``, ) }) test("indents nested list continuations and preserves ordered numbering", () => { const html = `
  1. first

    continued

    • nested

      continued nested

  2. next
` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( `4. first\n\n continued\n\n - nested\n\n continued nested\n\n5. next`, ) }) test("renders block content outside link syntax", () => { expect(WebFetchTool.convertHTMLToMarkdown(`before
block
after
`)).toBe( `[before](/docs)\n\nblock\n\n[after](/docs)`, ) }) test("recovers nested anchors without unmatched Markdown syntax", () => { expect(WebFetchTool.convertHTMLToMarkdown(`xyz`)).toBe(`[x](/a)[y](/b)z`) }) test("keeps emphasis whitespace through neutral wrappers", () => { expect(WebFetchTool.convertHTMLToMarkdown(`

a boldc

`)).toBe(`a **bold** c`) }) test("flattens preformatted content inside table cells", () => { const html = `
a|b\nnext
x|y
` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`| a\\|b next | \`x\\|y\` |\n| --- | --- |`) }) test("keeps each near-boundary inline construct closed and UTF-8-safe", () => { const payload = "😀".repeat(WebFetchTool.MAX_RESPONSE_BYTES / 4) const cases = [ [`${payload}`, /^\*\*[\s\S]*\*\*$/], [`${payload}`, /^\[[\s\S]*\]\(\/docs\)$/], [`${payload}`, /^!\[[\s\S]*\]\(image\.png\)$/], [`${payload}`, /^`[\s\S]*`$/], ] as const for (const [html, pattern] of cases) { const output = WebFetchTool.convertHTMLToMarkdown(html) expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) expect(output).not.toContain("�") expect(output).toMatch(pattern) } }) test("keeps near-boundary block constructs syntactically complete", () => { const payload = "x".repeat(WebFetchTool.MAX_RESPONSE_BYTES) const table = WebFetchTool.convertHTMLToMarkdown( `
Name
${payload}
`, ) const list = WebFetchTool.convertHTMLToMarkdown(`
  • ${payload}
  • nested
`) const code = WebFetchTool.convertHTMLToMarkdown(`
${payload}
`) for (const output of [table, list, code]) { expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) expect(output).not.toContain("�") } expect(table).toMatch(/^\| Name \|\n\| --- \|\n\| [\s\S]* \|$/) expect(list).toMatch(/^- [\s\S]*$/) expect(list.includes("nested")).toBe(false) expect(code.match(/^(`{3,}|~{3,})$/gm)).toHaveLength(2) }) test("keeps quoted code within budget with a safe closed fence", () => { const html = `
${"`".repeat(32)}${"~".repeat(32)}${"x".repeat(WebFetchTool.MAX_RESPONSE_BYTES)}
` const output = WebFetchTool.convertHTMLToMarkdown(html) expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) const lines = output.split("\n") expect(lines[0]).toMatch(/^> (`{33}|~{33})$/) expect(lines.at(-1)).toBe(lines[0]) }) test("separates reconstructed tables from adjacent inline and quoted content", () => { const html = `intro
x
outro
quote
cell
  • item
    cell
` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( `intro\n\n| x |\n| --- |\n\noutro\n\n> quote\n\n> | cell |\n> | --- |\n\n- item\n\n| cell |\n| --- |`, ) }) test("keeps multiline quoted code closed at the content budget", () => { const html = `
${"x\n".repeat(WebFetchTool.MAX_RESPONSE_BYTES / 2)}

tail

` const output = WebFetchTool.convertHTMLToMarkdown(html) expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) expect((output.match(/(`{3}|~{3})/g) ?? []).length).toBe(2) expect(output.includes("\uFFFD")).toBe(false) expect(output.endsWith("tail")).toBe(true) }) test("keeps active content suppressed when depth fallback begins", () => { const html = `${"
".repeat(10_001)}LEAK${"
".repeat(10_001)}

visible

` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("visible") }) test("keeps visible text after depth fallback begins inside preformatted content", () => { const html = `
${"".repeat(10_001)}visible${"".repeat(10_001)}

after

` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("visible after") }) test("resumes links around every block structure", () => { const html = `before

quote

  • item
code
cell
after
` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( `[before](/x)\n\n> quote\n\n- item\n\n\`\`\`\ncode\n\`\`\`\n\n| cell |\n| --- |\n\n[after](/x)`, ) }) test("indents child lists from the actual parent marker width", () => { expect(WebFetchTool.convertHTMLToMarkdown(`
  1. outer
    • inner
`)).toBe( `100. outer\n\n - inner`, ) }) test("renders captions and definition lists with readable boundaries", () => { const html = `
Cache modes
NameMeaning
ALocal
Cache
A local store
Origin
The remote source
` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( `Cache modes\n\n| Name | Meaning |\n| --- | --- |\n| A | Local |\n\n**Cache**\n: A local store\n\n**Origin**\n: The remote source`, ) }) test("falls back to row-oriented text for table spans", () => { const html = `
Group
AShared
B
` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`Group\n\nA | Shared\n\nB`) }) test("suppresses head and hidden subtrees while retaining visible body content", () => { const html = `noise

visible

shown
` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`visible\n\nshown`) }) test("preserves pre breaks and normalizes multiline link titles", () => { const html = `
first
second

link

` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( `\`\`\`\nfirst\nsecond\n\`\`\`\n\n[link](/x "line one line two")`, ) }) test("renders closed and open details according to visibility", () => { const html = `
Closed

secret

Open

visible

` expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`Closed\n\nOpen\n\nvisible`) }) }) describe("WebFetchTool registration", () => { it.effect("registers and fetches an ordinary hostname HTTP URL without rewriting it", () => Effect.gen(function* () { reset() const registry = yield* Tool.Service const url = "http://example.com/public" expect((yield* toolDefinitions(registry)).map((tool) => tool.name)).toEqual(["webfetch", "execute"]) expect(yield* executeTool(registry, call({ url, format: "text", timeout: 4 }))).toEqual({ status: "completed", output: { url, contentType: "text/plain", format: "text", output: "hello" }, content: [{ type: "text", text: "hello" }], metadata: { contentType: "text/plain" }, }) expect(assertions).toMatchObject([ { sessionID, action: "webfetch", resources: [url], save: ["*"], metadata: { url, format: "text", timeout: 4 } }, ]) expect(requests).toMatchObject([{ url, headers: { accept: expect.stringContaining("text/plain;q=1.0") } }]) }), ) it.effect("accepts localhost URLs with the same requested-URL permission check", () => Effect.gen(function* () { reset() const registry = yield* Tool.Service const url = "http://localhost/private" expect(yield* executeTool(registry, call({ url, format: "text" }))).toMatchObject({ status: "completed", content: [{ type: "text", text: "hello" }], }) expect(assertions).toMatchObject([ { sessionID, action: "webfetch", resources: [url], save: ["*"], metadata: { url, format: "text" } }, ]) expect(requests.map((request) => request.url)).toEqual([url]) }), ) live.effect("follows redirects while approving only the requested URL", () => Effect.acquireUseRelease( Effect.sync(() => Bun.serve({ port: 0, fetch: (request) => new URL(request.url).pathname === "/redirect" ? new Response("", { status: 302, headers: { location: "/target" } }) : new Response("redirected", { headers: { "content-type": "text/plain" } }), }), ), (server) => Effect.gen(function* () { reset() const registry = yield* Tool.Service const url = new URL("/redirect", server.url).toString() expect(yield* executeTool(registry, call({ url, format: "text" }))).toMatchObject({ status: "completed", content: [{ type: "text", text: "redirected" }], }) expect(assertions).toMatchObject([ { sessionID, action: "webfetch", resources: [url], save: ["*"], metadata: { url, format: "text" } }, ]) }), (server) => Effect.promise(() => server.stop(true)), ), ) it.effect("rejects non-HTTP schemes before permission or transport", () => Effect.gen(function* () { reset() const registry = yield* Tool.Service // toSessionError unwraps the "Unable to fetch " ToolFailure to its cause message. expect(yield* executeTool(registry, call({ url: "file:///etc/passwd", format: "text" }))).toEqual({ status: "error", error: { type: "unknown", message: "URL must use http:// or https://" }, }) expect(assertions).toEqual([]) expect(requests).toEqual([]) }), ) it.effect("converts HTML to requested markdown and text", () => Effect.gen(function* () { reset() respond = () => Effect.succeed( new Response("

Hello

world

", { headers: { "content-type": "text/html; charset=utf-8" }, }), ) const registry = yield* Tool.Service expect(yield* executeTool(registry, call({ url: "https://1.1.1.1", format: "markdown" }))).toMatchObject({ status: "completed", content: [{ type: "text", text: "# Hello\n\nworld" }], }) expect(yield* executeTool(registry, call({ url: "https://1.1.1.1", format: "text" }))).toMatchObject({ status: "completed", content: [{ type: "text", text: "Helloworld" }], }) }), ) it.effect("converts deeply nested HTML without overflowing", () => Effect.gen(function* () { reset() respond = () => Effect.succeed( new Response("
".repeat(10_000) + "content" + "
".repeat(10_000), { headers: { "content-type": "text/html" }, }), ) const registry = yield* Tool.Service const url = "https://1.1.1.1/deep-html" expect(yield* executeTool(registry, call({ url, format: "markdown" }))).toMatchObject({ status: "completed", content: [{ type: "text", text: "content" }], }) }), ) it.effect("rejects declared and streamed oversized bodies", () => Effect.gen(function* () { reset() const registry = yield* Tool.Service respond = () => Effect.succeed( new Response("small", { headers: { "content-type": "text/plain", "content-length": String(WebFetchTool.MAX_RESPONSE_BYTES + 1) }, }), ) expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/declared", format: "text" }))).toEqual({ status: "error", error: { type: "unknown", message: `Response too large (exceeds ${WebFetchTool.MAX_RESPONSE_BYTES} byte limit)`, }, }) respond = () => Effect.succeed( new Response("x".repeat(WebFetchTool.MAX_RESPONSE_BYTES + 1), { headers: { "content-type": "text/plain" } }), ) expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/streamed", format: "text" }))).toEqual({ status: "error", error: { type: "unknown", message: `Response too large (exceeds ${WebFetchTool.MAX_RESPONSE_BYTES} byte limit)`, }, }) }), ) it.effect("keeps images and files unsupported until typed outcomes can carry attachments", () => Effect.gen(function* () { reset() const registry = yield* Tool.Service respond = () => Effect.succeed(new Response("png", { headers: { "content-type": "image/png" } })) expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/image", format: "html" }))).toEqual({ status: "error", error: { type: "unknown", message: "Unsupported fetched image content type: image/png" }, }) respond = () => Effect.succeed(new Response("pdf", { headers: { "content-type": "application/pdf" } })) expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/file", format: "html" }))).toEqual({ status: "error", error: { type: "unknown", message: "Unsupported fetched file content type: application/pdf" }, }) }), ) it.effect("retries Cloudflare challenges with an honest user agent", () => Effect.gen(function* () { reset() let count = 0 respond = () => Effect.succeed( ++count === 1 ? new Response("challenge", { status: 403, headers: { "cf-mitigated": "challenge" } }) : new Response("ok", { headers: { "content-type": "text/plain" } }), ) const registry = yield* Tool.Service expect(yield* executeTool(registry, call({ url: "https://1.1.1.1", format: "text" }))).toMatchObject({ status: "completed", content: [{ type: "text", text: "ok" }], }) expect(requests).toHaveLength(2) expect(requests[0]?.headers["user-agent"]).toContain("Mozilla/5.0") expect(requests[1]?.headers["user-agent"]).toBe("opencode") }), ) it.effect("times out stalled requests", () => Effect.gen(function* () { reset() respond = () => Effect.never const registry = yield* Tool.Service const fiber = yield* executeTool( registry, call({ url: "https://1.1.1.1/slow", format: "text", timeout: 1 }), ).pipe(Effect.forkChild) yield* TestClock.adjust(Duration.seconds(1)) expect(yield* Fiber.join(fiber)).toEqual({ status: "error", error: { type: "unknown", message: "Request timed out" }, }) }), ) })