|
|
@@ -592,7 +592,7 @@ describe("CodeMode public contract", () => {
|
|
|
expect(second).toStrictEqual({ ok: true, value: 1, logs: ["hi"], toolCalls: [{ name: "host.echo" }] })
|
|
|
})
|
|
|
|
|
|
- test("inlines a COMPLETE small catalog and keeps search registered but unadvertised", async () => {
|
|
|
+ test("describes the catalog and keeps the search built-in registered", async () => {
|
|
|
const runtime = CodeMode.make({ tools })
|
|
|
expect(runtime.catalog()).toStrictEqual([
|
|
|
{
|
|
|
@@ -601,16 +601,7 @@ describe("CodeMode public contract", () => {
|
|
|
signature: "tools.orders.lookup(input: {\n id: string,\n}): Promise<{\n id: string,\n status: string,\n}>",
|
|
|
},
|
|
|
])
|
|
|
- expect(runtime.instructions()).toContain("Available tools (COMPLETE list")
|
|
|
- expect(runtime.instructions()).toContain("- orders (1 tool)")
|
|
|
- expect(runtime.instructions()).toContain(
|
|
|
- " - tools.orders.lookup(input: {\n id: string,\n}): Promise<{\n id: string,\n status: string,\n}> // Look up an order by ID",
|
|
|
- )
|
|
|
- // A fully inlined catalog does not advertise search in the instructions...
|
|
|
- expect(runtime.instructions()).not.toContain("search(")
|
|
|
|
|
|
- // ...but the search built-in stays available, so a speculative call still works with the
|
|
|
- // same signature as the inline catalog.
|
|
|
const result = await Effect.runPromise(runtime.execute(`return search({ query: "order" })`))
|
|
|
expect(result.ok).toBe(true)
|
|
|
if (result.ok) {
|
|
|
@@ -646,22 +637,7 @@ describe("CodeMode public contract", () => {
|
|
|
const second = CodeMode.make({ tools: { alpha: { alpha, zeta }, zeta: { alpha, zeta } } })
|
|
|
|
|
|
expect(first.catalog()).toStrictEqual(second.catalog())
|
|
|
- expect(first.instructions()).toBe(second.instructions())
|
|
|
expect(first.catalog().map((tool) => tool.path)).toEqual(["alpha.alpha", "alpha.zeta", "zeta.alpha", "zeta.zeta"])
|
|
|
-
|
|
|
- for (const catalogBudget of [0, 10, 20, 40]) {
|
|
|
- expect(
|
|
|
- CodeMode.make({
|
|
|
- tools: { zeta: { zeta, alpha }, alpha: { zeta, alpha } },
|
|
|
- discovery: { catalogBudget },
|
|
|
- }).instructions(),
|
|
|
- ).toBe(
|
|
|
- CodeMode.make({
|
|
|
- tools: { alpha: { alpha, zeta }, zeta: { alpha, zeta } },
|
|
|
- discovery: { catalogBudget },
|
|
|
- }).instructions(),
|
|
|
- )
|
|
|
- }
|
|
|
})
|
|
|
|
|
|
test("renders bracket notation for tool names that are not JavaScript identifiers", async () => {
|
|
|
@@ -680,9 +656,6 @@ describe("CodeMode public contract", () => {
|
|
|
signature: 'tools.context7["resolve-library-id"](input: {\n libraryName: string,\n}): Promise<string>',
|
|
|
},
|
|
|
])
|
|
|
- expect(runtime.instructions()).toContain(
|
|
|
- 'tools.context7["resolve-library-id"](input: {\n libraryName: string,\n}): Promise<string>',
|
|
|
- )
|
|
|
|
|
|
const search = await Effect.runPromise(runtime.execute(`return search({ query: "resolve library id" })`))
|
|
|
expect(search.ok).toBe(true)
|
|
|
@@ -713,88 +686,6 @@ describe("CodeMode public contract", () => {
|
|
|
if (exact.ok) expect(exact.value).toMatchObject({ remaining: 0, next: null })
|
|
|
})
|
|
|
|
|
|
- test("instructions use markdown sections with placeholder-only call forms", () => {
|
|
|
- const runtime = CodeMode.make({ tools })
|
|
|
- const instructions = runtime.instructions()
|
|
|
- // Sections in order: workflow at the top, catalog at the bottom.
|
|
|
- expect(instructions).toContain("## Workflow")
|
|
|
- expect(instructions).toContain("## Rules")
|
|
|
- expect(instructions).toContain("## Language")
|
|
|
- expect(instructions.indexOf("## Workflow")).toBeLessThan(instructions.indexOf("## Rules"))
|
|
|
- expect(instructions.indexOf("## Rules")).toBeLessThan(instructions.indexOf("## Language"))
|
|
|
- expect(instructions.indexOf("## Language")).toBeLessThan(
|
|
|
- instructions.indexOf("\n## Available tools (COMPLETE list"),
|
|
|
- )
|
|
|
- expect(instructions).not.toContain("JSON.parse(res)")
|
|
|
- expect(instructions).toContain("Return only the fields you need")
|
|
|
- expect(instructions).toContain("avoid returning large raw payloads")
|
|
|
- expect(instructions).toContain("Do not infer or normalize tool names")
|
|
|
- expect(instructions).toContain("bracket notation and quotes are part of the path")
|
|
|
- expect(instructions).toContain("surrounding agent tools are not available")
|
|
|
- expect(instructions).toContain("Only tools listed here are available")
|
|
|
- // Placeholders use generic namespace/tool/field names only - no fabricated real tools
|
|
|
- // and no real catalog tools cherry-picked into example lines.
|
|
|
- expect(instructions).toContain("`const result = await tools.<namespace>.<tool>(input)`")
|
|
|
- expect(instructions).toContain("Return only the fields you need from structured results")
|
|
|
- expect(instructions).toContain("check that it is a non-null object and not an array")
|
|
|
- expect(instructions).not.toContain("result.<field>")
|
|
|
- expect(instructions).not.toContain("data.<field>")
|
|
|
- expect(instructions).not.toContain("total_count")
|
|
|
- expect(instructions).not.toContain("list_issues")
|
|
|
- expect(instructions).not.toContain("tools.orders.lookup({")
|
|
|
- // COMPLETE: step 1 picks from the inlined list; search is not advertised.
|
|
|
- expect(instructions).toContain("1. Pick a tool from the list under `## Available tools`")
|
|
|
- expect(instructions).not.toContain("Browse one namespace")
|
|
|
-
|
|
|
- const partial = CodeMode.make({ tools, discovery: { catalogBudget: 0 } }).instructions()
|
|
|
- // PARTIAL: the workflow starts with search (with query-style guidance that is clearly
|
|
|
- // a query string, never a tool name) and the browse-namespace rule appears.
|
|
|
- expect(partial).toContain(
|
|
|
- '1. If needed, discover tools with the built-in search function: `return search({ query: "<intent + key nouns>" })`.',
|
|
|
- )
|
|
|
- expect(partial).toContain("In the next execution, copy a returned path exactly")
|
|
|
- expect(partial).toContain("Only tools listed here or returned by the built-in `search` function")
|
|
|
- expect(partial).toContain('- Browse one namespace: `search({ query: "", namespace: "<name>" })`.')
|
|
|
- expect(partial).toContain("repeat the same search with `offset: next.offset`")
|
|
|
- expect(partial).toContain(" limit?: number,\n offset?: number,")
|
|
|
- expect(partial).not.toContain("total_count")
|
|
|
- expect(partial).not.toContain("tools.orders.lookup({")
|
|
|
- })
|
|
|
-
|
|
|
- test("the language section describes the restricted runtime without overclaiming", () => {
|
|
|
- const instructions = CodeMode.make({ tools }).instructions()
|
|
|
- expect(instructions).toContain("restricted JavaScript language for calling tools")
|
|
|
- expect(instructions).toContain("not a general-purpose runtime")
|
|
|
- expect(instructions).not.toContain("Standard modern JavaScript works")
|
|
|
- expect(instructions).not.toContain("TypeScript type annotations")
|
|
|
- for (const missing of ["Modules/imports", "classes", "fetch"]) {
|
|
|
- expect(instructions).toContain(missing)
|
|
|
- }
|
|
|
- expect(instructions).not.toContain("generators")
|
|
|
- expect(instructions).not.toContain("new Promise(...) are unavailable")
|
|
|
- expect(instructions).not.toContain("promise chaining")
|
|
|
- expect(instructions).toContain("URL, URLSearchParams, and URI encoding helpers")
|
|
|
- expect(instructions).not.toContain("host globals")
|
|
|
- expect(instructions).toContain("Use tools for external operations")
|
|
|
- expect(instructions).toContain(
|
|
|
- "Prefer explicit `return`; otherwise only the final top-level expression becomes the result.",
|
|
|
- )
|
|
|
- expect(instructions).toContain(
|
|
|
- "Dates and URLs serialize to strings at data boundaries; Map/Set/RegExp/URLSearchParams serialize to `{}`.",
|
|
|
- )
|
|
|
- })
|
|
|
-
|
|
|
- test("zero tools keep minimal sections and the no-tools notice", () => {
|
|
|
- const runtime = CodeMode.make({})
|
|
|
- const instructions = runtime.instructions()
|
|
|
- expect(instructions).toContain("No tools are currently available.")
|
|
|
- expect(instructions).toContain("## Language")
|
|
|
- expect(instructions).toContain("## Available tools")
|
|
|
- expect(instructions).not.toContain("## Workflow")
|
|
|
- expect(instructions).not.toContain("## Rules")
|
|
|
- expect(instructions).not.toContain("search(")
|
|
|
- })
|
|
|
-
|
|
|
test("uses one ranked search returning complete tools for large catalogs", async () => {
|
|
|
const upload = Tool.make({
|
|
|
description: "Upload one readable local file to the current Discord thread",
|
|
|
@@ -810,13 +701,7 @@ describe("CodeMode public contract", () => {
|
|
|
})
|
|
|
const runtime = CodeMode.make({
|
|
|
tools: { thread: { uploadFile: upload, generateImage: generate }, orders: { lookup } },
|
|
|
- discovery: { catalogBudget: 0 },
|
|
|
})
|
|
|
- expect(runtime.instructions()).toContain("Available tools (PARTIAL - 0 of 3 shown; find the rest with search(...))")
|
|
|
- expect(runtime.instructions()).toContain("- thread (2 tools, none shown)")
|
|
|
- expect(runtime.instructions()).toContain("- orders (1 tool, none shown)")
|
|
|
- expect(runtime.instructions()).toContain("Search returns complete callable signatures:\n- search(input: {")
|
|
|
- expect(runtime.instructions()).not.toMatch(/tools\.thread\.uploadFile\(input/)
|
|
|
|
|
|
const result = await Effect.runPromise(
|
|
|
runtime.execute(`
|
|
|
@@ -1101,64 +986,6 @@ describe("CodeMode public contract", () => {
|
|
|
if (exhausted.ok) expect(exhausted.value).toStrictEqual({ items: [], remaining: 0, next: null })
|
|
|
})
|
|
|
|
|
|
- test("inlines round-robin across namespaces so one expensive namespace cannot starve the rest", () => {
|
|
|
- const cheap = Tool.make({
|
|
|
- description: "Cheap",
|
|
|
- input: Schema.Struct({ q: Schema.String }),
|
|
|
- output: Schema.String,
|
|
|
- execute: () => Effect.succeed("ok"),
|
|
|
- })
|
|
|
- const expensive = Tool.make({
|
|
|
- description:
|
|
|
- "An expensive tool whose description alone consumes far more than the remaining inline catalog byte budget for this runtime",
|
|
|
- input: Schema.Struct({
|
|
|
- someRatherLongParameterName: Schema.String,
|
|
|
- anotherEvenLongerParameterName: Schema.Number,
|
|
|
- }),
|
|
|
- output: Schema.String,
|
|
|
- execute: () => Effect.succeed("ok"),
|
|
|
- })
|
|
|
- // Round 1 places alpha.cheap (~17 estimated tokens) and beta.cheap (~17); in round 2
|
|
|
- // alpha.expensive does not fit, which marks only alpha done - it must NOT prevent
|
|
|
- // other namespaces from inlining (beta already got its line in the same round).
|
|
|
- const runtime = CodeMode.make({
|
|
|
- tools: { alpha: { cheap, expensive }, beta: { cheap } },
|
|
|
- discovery: { catalogBudget: 40 },
|
|
|
- })
|
|
|
-
|
|
|
- const instructions = runtime.instructions()
|
|
|
- expect(instructions).toContain("Available tools (PARTIAL - 2 of 3 shown; find the rest with search(...))")
|
|
|
- expect(instructions).toContain("- alpha (2 tools, 1 shown)")
|
|
|
- expect(instructions).toContain(" - tools.alpha.cheap(input: {\n q: string,\n}): Promise<string> // Cheap")
|
|
|
- expect(instructions).not.toContain("tools.alpha.expensive(")
|
|
|
- // Fully shown namespaces read cleanly (no "shown" annotation).
|
|
|
- expect(instructions).toContain("- beta (1 tool)")
|
|
|
- expect(instructions).toContain(" - tools.beta.cheap(input: {\n q: string,\n}): Promise<string> // Cheap")
|
|
|
- expect(instructions).toContain("Search returns complete callable signatures:\n- search(input: {")
|
|
|
- })
|
|
|
-
|
|
|
- test("charges inline JSDoc against the catalog token budget", () => {
|
|
|
- const documented = Tool.make({
|
|
|
- description: "Look up a record",
|
|
|
- input: {
|
|
|
- type: "object",
|
|
|
- properties: {
|
|
|
- id: { type: "string", description: "A detailed identifier description. ".repeat(20) },
|
|
|
- },
|
|
|
- required: ["id"],
|
|
|
- } as const,
|
|
|
- execute: () => Effect.succeed("ok"),
|
|
|
- })
|
|
|
- const runtime = CodeMode.make({
|
|
|
- tools: { records: { lookup: documented } },
|
|
|
- discovery: { catalogBudget: 40 },
|
|
|
- })
|
|
|
-
|
|
|
- expect(runtime.catalog()[0]?.signature).toContain("/** A detailed identifier description.")
|
|
|
- expect(runtime.instructions()).toContain("Available tools (PARTIAL - 0 of 1 shown; find the rest with search(...))")
|
|
|
- expect(runtime.instructions()).not.toContain("tools.records.lookup(input:")
|
|
|
- })
|
|
|
-
|
|
|
test("decodes tool input and output before exposing either side", async () => {
|
|
|
const observed: Array<unknown> = []
|
|
|
const transformed = Tool.make({
|
|
|
@@ -1219,7 +1046,7 @@ describe("CodeMode public contract", () => {
|
|
|
expect(result).toStrictEqual({ ok: true, value: null, toolCalls: [] })
|
|
|
})
|
|
|
|
|
|
- test("rejects invalid configuration and discovery limits", async () => {
|
|
|
+ test("rejects invalid configuration and search limits", async () => {
|
|
|
expect(() => CodeMode.execute({ code: "return 1", limits: { timeoutMs: 0 } })).toThrow(RangeError)
|
|
|
expect(() => CodeMode.execute({ code: "return 1", limits: { timeoutMs: Number.POSITIVE_INFINITY } })).toThrow(
|
|
|
RangeError,
|
|
|
@@ -1227,13 +1054,8 @@ describe("CodeMode public contract", () => {
|
|
|
expect(() => CodeMode.execute({ code: "return 1", limits: { maxToolCalls: -1 } })).toThrow(RangeError)
|
|
|
expect(() => CodeMode.execute({ code: "return 1", limits: { maxOutputBytes: -1 } })).toThrow(RangeError)
|
|
|
|
|
|
- expect(() => CodeMode.make({ tools, discovery: { catalogBudget: -1 } })).toThrow(RangeError)
|
|
|
-
|
|
|
const result = await Effect.runPromise(
|
|
|
- CodeMode.make({
|
|
|
- tools,
|
|
|
- discovery: { catalogBudget: 0 },
|
|
|
- }).execute(`return search({ query: "order", limit: 0.5 })`),
|
|
|
+ CodeMode.make({ tools }).execute(`return search({ query: "order", limit: 0.5 })`),
|
|
|
)
|
|
|
expect(result.ok).toBe(false)
|
|
|
if (result.ok) return
|