session-runner.test.ts 176 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834
  1. import { describe, expect, test } from "bun:test"
  2. import {
  3. AIError,
  4. LLMEvent,
  5. LLMRequest,
  6. Message,
  7. LanguageModel,
  8. SystemPart,
  9. ToolFailure,
  10. TransportReason,
  11. InvalidProviderOutputReason,
  12. InvalidRequestReason,
  13. RateLimitReason,
  14. } from "@opencode-ai/ai"
  15. import * as OpenAIChat from "@opencode-ai/ai/protocols/openai-chat"
  16. import { TestLLM } from "@opencode-ai/ai/testing"
  17. import { Catalog } from "@opencode-ai/core/catalog"
  18. import { Database } from "@opencode-ai/core/database/database"
  19. import { makeLocationNode } from "@opencode-ai/util/effect/app-node"
  20. import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder"
  21. import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform"
  22. import { LayerNode } from "@opencode-ai/util/effect/layer-node"
  23. import { Bus } from "@opencode-ai/core/bus"
  24. import { Event } from "@opencode-ai/schema/event"
  25. import { App } from "@opencode-ai/core/app"
  26. import { Permission } from "@opencode-ai/core/permission"
  27. import { EventTable } from "@opencode-ai/core/event/sql"
  28. import { Project } from "@opencode-ai/core/project"
  29. import { ProjectTable } from "@opencode-ai/core/project/sql"
  30. import { Form } from "@opencode-ai/core/form"
  31. import { AbsolutePath } from "@opencode-ai/core/schema"
  32. import { Session } from "@opencode-ai/core/session"
  33. import { Snapshot } from "@opencode-ai/core/snapshot"
  34. import { SessionEvent } from "@opencode-ai/core/session/event"
  35. import { SessionPending } from "@opencode-ai/core/session/pending"
  36. import { SessionMessage } from "@opencode-ai/core/session/message"
  37. import { Money } from "@opencode-ai/schema/money"
  38. import { SessionProjector } from "@opencode-ai/core/session/projector"
  39. import { SessionExecution } from "@opencode-ai/core/session/execution"
  40. import { SessionRunCoordinator } from "@opencode-ai/core/session/run-coordinator"
  41. import { SessionRunner } from "@opencode-ai/core/session/runner"
  42. import * as SessionRunnerLLM from "@opencode-ai/core/session/runner/llm"
  43. import { SessionRunnerModel } from "@opencode-ai/core/session/runner/model"
  44. import { PromptCacheDiagnostics } from "@opencode-ai/core/session/prompt-cache-diagnostics"
  45. import { SessionUsage } from "@opencode-ai/core/session/usage"
  46. import { PluginSupervisor } from "@opencode-ai/core/plugin/supervisor"
  47. import { PluginHooks } from "@opencode-ai/core/plugin/hooks"
  48. import { SystemPromptPlugin } from "@opencode-ai/core/plugin/system-prompt"
  49. import { QuestionTool } from "@opencode-ai/core/tool/plugin/question"
  50. import { Agent } from "@opencode-ai/core/agent"
  51. import { Config } from "@opencode-ai/core/config"
  52. import { ConfigCompaction } from "@opencode-ai/core/config/compaction"
  53. import { Tool } from "@opencode-ai/core/tool"
  54. import type { Info } from "@opencode-ai/schema/tool"
  55. import {
  56. InstructionStateTable,
  57. SessionPendingTable,
  58. SessionMessageTable,
  59. SessionTable,
  60. } from "@opencode-ai/core/session/sql"
  61. import { InstructionEntry } from "@opencode-ai/core/session/instruction-entry"
  62. import { SessionStore } from "@opencode-ai/core/session/store"
  63. import { Instructions } from "@opencode-ai/core/instructions"
  64. import { InstructionBuiltIns } from "@opencode-ai/core/instructions/builtins"
  65. import { InstructionDiscovery } from "@opencode-ai/core/instruction-discovery"
  66. import { SkillInstructions } from "@opencode-ai/core/skill/instructions"
  67. import { ReferenceInstructions } from "@opencode-ai/core/reference/instructions"
  68. import { McpInstructions } from "@opencode-ai/core/mcp/instructions"
  69. import { ID } from "@opencode-ai/core/model"
  70. import { Location } from "@opencode-ai/core/location"
  71. import { Provider } from "@opencode-ai/core/provider"
  72. import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Schema, Scope, Stream } from "effect"
  73. import { TestClock } from "effect/testing"
  74. import { asc, eq } from "drizzle-orm"
  75. import { testEffect } from "./lib/effect"
  76. import { agentHost, catalogHost, host } from "./plugin/host"
  77. import PROMPT_DEFAULT from "../src/session/runner/prompt/base.txt"
  78. import { CodeModeInstructions } from "@opencode-ai/core/codemode/instructions"
  79. let requests: LLMRequest[] = []
  80. const emptyCodeMode = `\n\n${CodeModeInstructions.render({ total: 0, shown: 0, namespaces: [] })}`
  81. type ToolBarrier = {
  82. readonly count: number
  83. readonly started: Deferred.Deferred<void>
  84. readonly release: Deferred.Deferred<void>
  85. active: number
  86. maxActive: number
  87. }
  88. let toolBarrier: ToolBarrier | undefined
  89. const releaseTools = (barrier: ToolBarrier) =>
  90. Effect.sync(() => {
  91. if (toolBarrier === barrier) toolBarrier = undefined
  92. }).pipe(Effect.andThen(Deferred.succeed(barrier.release, undefined)), Effect.asVoid)
  93. const blockTools = (count = 1) =>
  94. Effect.acquireRelease(
  95. Effect.all({ started: Deferred.make<void>(), release: Deferred.make<void>() }).pipe(
  96. Effect.map((deferreds) => {
  97. const barrier = { count, ...deferreds, active: 0, maxActive: 0 }
  98. toolBarrier = barrier
  99. return barrier
  100. }),
  101. ),
  102. releaseTools,
  103. ).pipe(
  104. Effect.map((barrier) => ({
  105. started: Deferred.await(barrier.started),
  106. release: releaseTools(barrier),
  107. maxActive: Effect.sync(() => barrier.maxActive),
  108. })),
  109. )
  110. const awaitToolBarrier = Effect.suspend(() => {
  111. const barrier = toolBarrier
  112. if (!barrier) return Effect.void
  113. barrier.active++
  114. barrier.maxActive = Math.max(barrier.maxActive, barrier.active)
  115. return (barrier.active === barrier.count ? Deferred.succeed(barrier.started, undefined) : Effect.void).pipe(
  116. Effect.andThen(Deferred.await(barrier.release)),
  117. Effect.ensuring(Effect.sync(() => barrier.active--)),
  118. )
  119. })
  120. const testLLM = TestLLM.layer({
  121. fallback: [],
  122. transformRequest: (request) =>
  123. LLMRequest.update(request, {
  124. system: request.system.map((part) => ({
  125. ...part,
  126. text: part.text.replace(emptyCodeMode, ""),
  127. })),
  128. tools: request.tools.filter((tool) => tool.name !== "execute"),
  129. }),
  130. })
  131. const client = TestLLM.clientLayer
  132. const model = LanguageModel.make({ id: "fake-model", provider: "fake", route: OpenAIChat.route })
  133. const defaultSystem = PROMPT_DEFAULT
  134. const replacementModel = LanguageModel.make({ id: "replacement", provider: "fake", route: OpenAIChat.route })
  135. const compactModel = LanguageModel.make({
  136. id: "compact",
  137. provider: "fake",
  138. route: OpenAIChat.route.with({ limits: { context: 4_000, output: 50 } }),
  139. })
  140. const fullOutputModel = LanguageModel.make({
  141. id: "full-output",
  142. provider: "fake",
  143. route: OpenAIChat.route.with({ limits: { context: 262_144, output: 262_144 } }),
  144. })
  145. const undersizedContextModel = LanguageModel.make({
  146. id: "undersized-context",
  147. provider: "fake",
  148. route: OpenAIChat.route.with({ limits: { context: 1, output: 1_000 } }),
  149. })
  150. const recoveryModel = LanguageModel.make({
  151. id: "recovery",
  152. provider: "fake",
  153. route: OpenAIChat.route.with({ limits: { context: 20_000, output: 1_000 } }),
  154. })
  155. test("calculates step cost using the matching context tier", () => {
  156. expect(
  157. SessionUsage.calculateCost(
  158. [
  159. {
  160. input: Money.USDPerMillionTokens.make(1),
  161. output: Money.USDPerMillionTokens.make(2),
  162. cache: {
  163. read: Money.USDPerMillionTokens.make(0.1),
  164. write: Money.USDPerMillionTokens.make(0.5),
  165. },
  166. },
  167. {
  168. tier: { type: "context", size: 100 },
  169. input: Money.USDPerMillionTokens.make(3),
  170. output: Money.USDPerMillionTokens.make(4),
  171. cache: {
  172. read: Money.USDPerMillionTokens.make(0.2),
  173. write: Money.USDPerMillionTokens.make(0.6),
  174. },
  175. },
  176. ],
  177. { input: 80, output: 10, reasoning: 2, cache: { read: 20, write: 1 } },
  178. ),
  179. ).toBeCloseTo(0.0002926)
  180. })
  181. test("does not apply an ineligible tier without base pricing", () => {
  182. expect(
  183. SessionUsage.calculateCost(
  184. [
  185. {
  186. tier: { type: "context", size: 100 },
  187. input: Money.USDPerMillionTokens.make(3),
  188. output: Money.USDPerMillionTokens.make(4),
  189. cache: {
  190. read: Money.USDPerMillionTokens.make(0.2),
  191. write: Money.USDPerMillionTokens.make(0.6),
  192. },
  193. },
  194. ],
  195. { input: 80, output: 10, reasoning: 2, cache: { read: 20, write: 0 } },
  196. ),
  197. ).toBe(Money.USD.zero)
  198. })
  199. const authorizations: Tool.Context[] = []
  200. const executions: string[] = []
  201. const permissionFail = {
  202. name: "permission_fail",
  203. description: "Reject a permission",
  204. input: Schema.Struct({}),
  205. output: Schema.Struct({}),
  206. execute: () =>
  207. new ToolFailure({
  208. message: "Permission denied: edit",
  209. error: new Permission.BlockedError({
  210. rules: [],
  211. permission: "edit",
  212. resources: ["src/index.ts"],
  213. }),
  214. }),
  215. }
  216. const permission = Layer.succeed(
  217. Permission.Service,
  218. Permission.Service.of({
  219. assert: () => Effect.die("unused"),
  220. ask: () => Effect.die("unused"),
  221. reply: () => Effect.die("unused"),
  222. get: () => Effect.die("unused"),
  223. forSession: () => Effect.die("unused"),
  224. list: () => Effect.die("unused"),
  225. }),
  226. )
  227. const transformTools = (registry: Tool.Interface, tools: Readonly<Record<string, Info>>, options?: Tool.Options) =>
  228. registry.transform((draft) =>
  229. Object.entries(tools).forEach(([name, tool]) => draft.add({ ...tool, name, options: options ?? tool.options })),
  230. )
  231. const echo = Layer.effectDiscard(
  232. Tool.Service.use((registry) =>
  233. transformTools(
  234. registry,
  235. {
  236. echo: {
  237. name: "echo",
  238. description: "Echo text",
  239. input: Schema.Struct({ text: Schema.String }),
  240. output: Schema.Struct({ text: Schema.String }),
  241. execute: ({ text }, context) =>
  242. Effect.gen(function* () {
  243. authorizations.push(context)
  244. executions.push(text)
  245. yield* awaitToolBarrier
  246. return { output: { text }, content: text }
  247. }),
  248. },
  249. defect: {
  250. name: "defect",
  251. description: "Fail unexpectedly",
  252. input: Schema.Struct({}),
  253. output: Schema.Struct({}),
  254. execute: () => awaitToolBarrier.pipe(Effect.andThen(Effect.die("unexpected tool defect"))),
  255. },
  256. storefail: {
  257. name: "storefail",
  258. description: "Produce output that cannot be persisted",
  259. input: Schema.Struct({}),
  260. output: Schema.Struct({}),
  261. execute: () => Effect.succeed({ output: {} }),
  262. },
  263. },
  264. { codemode: false },
  265. ),
  266. ),
  267. )
  268. const echoNode = makeLocationNode({ name: "test/session-runner-tools", layer: echo, deps: [Tool.node] })
  269. let modelResolveHook = Effect.void
  270. let currentModel = model
  271. const models = Layer.mock(SessionRunnerModel.Service)({
  272. resolve: (session) =>
  273. modelResolveHook.pipe(
  274. Effect.as(
  275. SessionRunnerModel.resolved(session.model?.id === "replacement" ? replacementModel : currentModel, {
  276. capabilities: { tools: true, input: ["text", "image"], output: ["text"] },
  277. cost: [],
  278. variant: session.model?.variant,
  279. }),
  280. ),
  281. ),
  282. })
  283. const systemContextKey = Instructions.Key.make("test/context")
  284. let systemBaseline = "Initial context"
  285. let systemRemoved = false
  286. let systemUnavailable = false
  287. let systemLoadHook = Effect.void
  288. const skillBaselines = new Map<Agent.ID, string>()
  289. const systemContext = Layer.mock(InstructionBuiltIns.Service, {
  290. load: () =>
  291. Effect.sync(() =>
  292. Instructions.make({
  293. key: systemContextKey,
  294. codec: Schema.toCodecJson(Schema.String),
  295. read: systemLoadHook.pipe(
  296. Effect.andThen(
  297. Effect.sync(() =>
  298. systemUnavailable ? Instructions.unavailable : systemRemoved ? Instructions.removed : systemBaseline,
  299. ),
  300. ),
  301. ),
  302. render: {
  303. initial: String,
  304. changed: (_previous, current) => current,
  305. removed: () => "System context source removed: test/context",
  306. },
  307. }),
  308. ),
  309. })
  310. const instructionContext = Layer.mock(InstructionDiscovery.Service, { load: () => Effect.succeed(Instructions.empty) })
  311. const skillInstructions = Layer.mock(SkillInstructions.Service, {
  312. load: (agent) =>
  313. Effect.succeed(
  314. skillBaselines.has(agent.id)
  315. ? Instructions.make({
  316. key: Instructions.Key.make("test/skill-guidance"),
  317. codec: Schema.toCodecJson(Schema.String),
  318. read: Effect.succeed(skillBaselines.get(agent.id)!),
  319. render: {
  320. initial: String,
  321. changed: (_previous, current) => current,
  322. removed: () => "Skill guidance removed",
  323. },
  324. })
  325. : Instructions.empty,
  326. ),
  327. })
  328. const referenceInstructions = Layer.mock(ReferenceInstructions.Service, {
  329. load: () => Effect.succeed(Instructions.empty),
  330. })
  331. const mcpInstructions = Layer.mock(McpInstructions.Service, { load: () => Effect.succeed(Instructions.empty) })
  332. const config = Config.testLayer([
  333. new Config.Document({
  334. type: "document",
  335. info: new Config.Info({
  336. compaction: new ConfigCompaction.Info({
  337. buffer: 3_000,
  338. keep: new ConfigCompaction.Keep({ tokens: 1_000 }),
  339. }),
  340. }),
  341. }),
  342. ])
  343. let pluginFlushHook = Effect.void
  344. const pluginSupervisor = Layer.succeed(
  345. PluginSupervisor.Service,
  346. PluginSupervisor.Service.of({
  347. flush: Effect.suspend(() => pluginFlushHook),
  348. }),
  349. )
  350. const promptCatalog = Layer.mock(Catalog.Service, {
  351. provider: {
  352. get: () => Effect.succeed(undefined),
  353. all: () => Effect.succeed([]),
  354. available: () => Effect.succeed([]),
  355. },
  356. model: {
  357. get: () => Effect.succeed(undefined),
  358. all: () => Effect.succeed([]),
  359. available: () => Effect.succeed([]),
  360. default: () => Effect.succeed(undefined),
  361. small: () => Effect.succeed(undefined),
  362. },
  363. })
  364. const runnerLayer = AppNodeBuilder.build(SessionRunnerLLM.node, [
  365. [Snapshot.node, Snapshot.noopLayer],
  366. [LayerNodePlatform.llmClient, client],
  367. [SessionRunnerModel.node, models],
  368. [InstructionBuiltIns.node, systemContext],
  369. [InstructionDiscovery.node, instructionContext],
  370. [Location.node, Location.boundNode({ directory: AbsolutePath.make("/project") })],
  371. [SkillInstructions.node, skillInstructions],
  372. [ReferenceInstructions.node, referenceInstructions],
  373. [Permission.node, permission],
  374. [Config.node, config],
  375. [McpInstructions.node, mcpInstructions],
  376. [PluginSupervisor.node, pluginSupervisor],
  377. ])
  378. const execution = Layer.effect(
  379. SessionExecution.Service,
  380. Effect.gen(function* () {
  381. const sessionRunner = yield* SessionRunner.Service
  382. const coordinator = yield* SessionRunCoordinator.make<Session.ID, SessionRunner.RunError>({
  383. drain: (sessionID, force) => sessionRunner.drain({ sessionID, force }),
  384. })
  385. return SessionExecution.Service.of({
  386. active: coordinator.active,
  387. resume: coordinator.run,
  388. wake: coordinator.wake,
  389. interrupt: coordinator.interrupt,
  390. awaitIdle: coordinator.awaitIdle,
  391. })
  392. }),
  393. ).pipe(Layer.provide(runnerLayer))
  394. const it = testEffect(
  395. AppNodeBuilder.build(
  396. LayerNode.group([
  397. Database.node,
  398. Bus.node,
  399. Form.node,
  400. SessionProjector.node,
  401. SessionStore.node,
  402. Agent.node,
  403. Catalog.node,
  404. Tool.node,
  405. Tool.node,
  406. PluginHooks.node,
  407. PluginHooks.node,
  408. echoNode,
  409. SessionRunnerModel.node,
  410. InstructionBuiltIns.node,
  411. InstructionDiscovery.node,
  412. InstructionEntry.node,
  413. SkillInstructions.node,
  414. ReferenceInstructions.node,
  415. Config.node,
  416. Snapshot.node,
  417. SessionRunnerLLM.node,
  418. SessionExecution.node,
  419. Session.node,
  420. ]),
  421. [
  422. [LayerNodePlatform.llmClient, client],
  423. [Permission.node, permission],
  424. [Catalog.node, promptCatalog],
  425. [SessionRunnerModel.node, models],
  426. [InstructionBuiltIns.node, systemContext],
  427. [InstructionDiscovery.node, instructionContext],
  428. [Location.node, Location.boundNode({ directory: AbsolutePath.make("/project") })],
  429. [SkillInstructions.node, skillInstructions],
  430. [ReferenceInstructions.node, referenceInstructions],
  431. [Snapshot.node, Snapshot.noopLayer],
  432. [SessionExecution.node, execution],
  433. [Config.node, config],
  434. [PluginSupervisor.node, pluginSupervisor],
  435. ],
  436. ).pipe(Layer.provideMerge(testLLM)),
  437. )
  438. const sessionID = Session.ID.make("ses_runner_test")
  439. const otherSessionID = Session.ID.make("ses_runner_other")
  440. const admit = (session: Session.Interface, text: string) => session.prompt({ sessionID, text, resume: false })
  441. const runPrompt = Effect.fnUntraced(function* (session: Session.Interface, text: string) {
  442. const message = yield* admit(session, text)
  443. yield* session.resume(sessionID)
  444. return message
  445. })
  446. const insertSession = (id: Session.ID) =>
  447. Effect.gen(function* () {
  448. const { db } = yield* Database.Service
  449. yield* db
  450. .insert(SessionTable)
  451. .values({
  452. id,
  453. project_id: Project.ID.global,
  454. slug: id,
  455. directory: "/project",
  456. title: "test",
  457. version: "test",
  458. })
  459. .onConflictDoNothing()
  460. .run()
  461. .pipe(Effect.orDie)
  462. })
  463. const setup = Effect.gen(function* () {
  464. const { db } = yield* Database.Service
  465. const agents = yield* Agent.Service
  466. const catalog = yield* Catalog.Service
  467. const hooks = yield* PluginHooks.Service
  468. const pluginHost = host({
  469. agent: agentHost(agents),
  470. catalog: catalogHost(catalog),
  471. session: { hook: (name, callback) => hooks.register("session", name, callback) },
  472. })
  473. yield* Effect.forEach(SystemPromptPlugin.Plugins, (plugin) => plugin.effect(pluginHost), {
  474. discard: true,
  475. })
  476. requests = (yield* TestLLM.Service).requests
  477. authorizations.length = 0
  478. executions.length = 0
  479. systemBaseline = "Initial context"
  480. systemRemoved = false
  481. systemUnavailable = false
  482. systemLoadHook = Effect.void
  483. modelResolveHook = Effect.void
  484. pluginFlushHook = Effect.void
  485. currentModel = model
  486. skillBaselines.clear()
  487. toolBarrier = undefined
  488. yield* agents.transform((draft) =>
  489. draft.update(Agent.ID.make("build"), (agent) => {
  490. agent.mode = "primary"
  491. }),
  492. )
  493. yield* db
  494. .insert(ProjectTable)
  495. .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] })
  496. .onConflictDoNothing()
  497. .run()
  498. .pipe(Effect.orDie)
  499. yield* insertSession(sessionID)
  500. return yield* Session.Service
  501. })
  502. const providerUnavailable = () =>
  503. new AIError({
  504. module: "test",
  505. method: "stream",
  506. reason: new TransportReason({ message: "Provider unavailable" }),
  507. })
  508. const incompleteStream = () =>
  509. new AIError({
  510. module: "test",
  511. method: "stream",
  512. reason: new InvalidProviderOutputReason({
  513. classification: "incomplete-stream",
  514. message: "The provider response ended unexpectedly.",
  515. }),
  516. })
  517. const invalidRequest = () =>
  518. new AIError({
  519. module: "test",
  520. method: "stream",
  521. reason: new InvalidRequestReason({ message: "Invalid request" }),
  522. })
  523. const rateLimited = (retryAfterMs?: number) =>
  524. new AIError({
  525. module: "test",
  526. method: "stream",
  527. reason: new RateLimitReason({ message: "Rate limited", retryAfterMs }),
  528. })
  529. const setupOverflowRecovery = Effect.gen(function* () {
  530. const session = yield* setup
  531. yield* TestLLM.push(TestLLM.text("Earlier answer", "text-earlier"))
  532. yield* runPrompt(session, "Earlier question ".repeat(700))
  533. currentModel = recoveryModel
  534. requests.length = 0
  535. return session
  536. })
  537. const messageTexts = (request: LLMRequest, role: "user" | "system") =>
  538. request.messages.flatMap((message) =>
  539. message.role === role ? message.content.flatMap((content) => (content.type === "text" ? [content.text] : [])) : [],
  540. )
  541. const userTexts = (request: LLMRequest) => messageTexts(request, "user")
  542. const systemTexts = (request: LLMRequest) => messageTexts(request, "system")
  543. const messageRoles = (request: LLMRequest | undefined) => request?.messages.map((message) => message.role)
  544. const recordedEventTypes = (id: Session.ID) =>
  545. Effect.gen(function* () {
  546. const { db } = yield* Database.Service
  547. return yield* db
  548. .select({ type: EventTable.type })
  549. .from(EventTable)
  550. .where(eq(EventTable.aggregate_id, id))
  551. .orderBy(asc(EventTable.seq))
  552. .all()
  553. .pipe(
  554. Effect.orDie,
  555. Effect.map((rows) => rows.map((row) => row.type)),
  556. )
  557. })
  558. const recordedStepSettlementEvents = (id: Session.ID, assistantMessageID: SessionMessage.ID) =>
  559. Effect.gen(function* () {
  560. const { db } = yield* Database.Service
  561. const settlementTypes = new Set([
  562. "session.step.started.1",
  563. "session.tool.called.1",
  564. "session.tool.success.2",
  565. "session.tool.failed.2",
  566. "session.step.ended.1",
  567. "session.step.failed.1",
  568. ])
  569. return (yield* db
  570. .select({ type: EventTable.type, data: EventTable.data })
  571. .from(EventTable)
  572. .where(eq(EventTable.aggregate_id, id))
  573. .orderBy(asc(EventTable.seq))
  574. .all()
  575. .pipe(Effect.orDie)).filter(
  576. (event) => settlementTypes.has(event.type) && event.data.assistantMessageID === assistantMessageID,
  577. )
  578. })
  579. const recordedStepSettlementTypes = (id: Session.ID, assistantMessageID: SessionMessage.ID) =>
  580. recordedStepSettlementEvents(id, assistantMessageID).pipe(Effect.map((events) => events.map((event) => event.type)))
  581. const hostedCall = (id: string, query: string) =>
  582. LLMEvent.toolCall({ id, name: "web_search", input: { query }, providerExecuted: true })
  583. const requireAssistant = (messages: readonly SessionMessage.Info[]) => {
  584. const assistant = messages.find((message) => message.type === "assistant")
  585. if (!assistant) throw new Error("Assistant message missing")
  586. return assistant
  587. }
  588. const replaySessionProjection = (id: Session.ID) =>
  589. Effect.gen(function* () {
  590. const { db } = yield* Database.Service
  591. const bus = yield* Bus.Service
  592. const recorded = yield* db
  593. .select()
  594. .from(EventTable)
  595. .where(eq(EventTable.aggregate_id, id))
  596. .orderBy(asc(EventTable.seq))
  597. .all()
  598. .pipe(Effect.orDie)
  599. yield* bus.remove(id)
  600. yield* db.delete(InstructionStateTable).where(eq(InstructionStateTable.session_id, id)).run().pipe(Effect.orDie)
  601. yield* db.delete(SessionPendingTable).where(eq(SessionPendingTable.session_id, id)).run().pipe(Effect.orDie)
  602. yield* db.delete(SessionMessageTable).where(eq(SessionMessageTable.session_id, id)).run().pipe(Effect.orDie)
  603. yield* bus.replayAll(
  604. recorded.map((event) => ({
  605. id: event.id,
  606. created: DateTime.makeUnsafe(event.created),
  607. aggregateID: event.aggregate_id,
  608. seq: event.seq,
  609. type: event.type,
  610. data: event.data,
  611. })),
  612. )
  613. })
  614. type FragmentKind = "text" | "reasoning" | "tool input"
  615. type FragmentFixture = {
  616. readonly delta: Event.Definition
  617. readonly completeEvents: LLMEvent[]
  618. readonly partialEvents: LLMEvent[]
  619. readonly expectedAssistant: unknown
  620. readonly expectedContent: unknown
  621. }
  622. const fragmentKinds: readonly FragmentKind[] = ["text", "reasoning", "tool input"]
  623. const fragmentID = (kind: FragmentKind, suffix: string) => `${kind === "tool input" ? "call" : kind}-${suffix}`
  624. const fragmentFixture = (kind: FragmentKind, id: string, chunks: readonly string[]): FragmentFixture => {
  625. const text = chunks.join("")
  626. switch (kind) {
  627. case "text": {
  628. const partialEvents = [
  629. LLMEvent.stepStart({ index: 0 }),
  630. LLMEvent.textStart({ id }),
  631. ...chunks.map((text) => LLMEvent.textDelta({ id, text })),
  632. ]
  633. const expectedContent = { type: "text", text }
  634. return {
  635. delta: SessionEvent.Text.Delta,
  636. partialEvents,
  637. completeEvents: [
  638. ...partialEvents,
  639. LLMEvent.textEnd({ id }),
  640. LLMEvent.stepFinish({ index: 0, reason: { normalized: "stop" } }),
  641. LLMEvent.finish({ reason: { normalized: "stop" } }),
  642. ],
  643. expectedAssistant: { type: "assistant", finish: "stop", content: [expectedContent] },
  644. expectedContent,
  645. }
  646. }
  647. case "reasoning": {
  648. const partialEvents = [
  649. LLMEvent.stepStart({ index: 0 }),
  650. LLMEvent.reasoningStart({ id }),
  651. ...chunks.map((text) => LLMEvent.reasoningDelta({ id, text })),
  652. ]
  653. const expectedContent = { type: "reasoning", text }
  654. return {
  655. delta: SessionEvent.Reasoning.Delta,
  656. partialEvents,
  657. completeEvents: [
  658. ...partialEvents,
  659. LLMEvent.reasoningEnd({ id }),
  660. LLMEvent.stepFinish({ index: 0, reason: { normalized: "stop" } }),
  661. LLMEvent.finish({ reason: { normalized: "stop" } }),
  662. ],
  663. expectedAssistant: { type: "assistant", finish: "stop", content: [expectedContent] },
  664. expectedContent,
  665. }
  666. }
  667. case "tool input": {
  668. const partialEvents = [
  669. LLMEvent.stepStart({ index: 0 }),
  670. LLMEvent.toolInputStart({ id, name: "echo" }),
  671. ...chunks.map((text) => LLMEvent.toolInputDelta({ id, name: "echo", text })),
  672. ]
  673. const expectedContent = { type: "tool", id, state: { status: "streaming", input: text } }
  674. return {
  675. delta: SessionEvent.Tool.Input.Delta,
  676. partialEvents,
  677. completeEvents: [...partialEvents, LLMEvent.toolInputEnd({ id, name: "echo" })],
  678. expectedAssistant: { type: "assistant", content: [expectedContent] },
  679. expectedContent,
  680. }
  681. }
  682. }
  683. }
  684. const verifyEphemeralDeltas = (kind: FragmentKind) =>
  685. Effect.gen(function* () {
  686. const session = yield* setup
  687. const prompt = `Stream ${kind}`
  688. const chunks = Array.from({ length: 32 }, (_, index) => `${index},`)
  689. const fixture = fragmentFixture(kind, fragmentID(kind, "many"), chunks)
  690. const expectedContext = [{ type: "user", text: prompt }, fixture.expectedAssistant]
  691. yield* admit(session, prompt)
  692. const bus = yield* Bus.Service
  693. const live = yield* bus.subscribe(fixture.delta).pipe(Stream.take(32), Stream.runCollect, Effect.forkScoped)
  694. yield* Effect.yieldNow
  695. yield* TestLLM.push(fixture.completeEvents)
  696. yield* session.resume(sessionID)
  697. const { db } = yield* Database.Service
  698. const deltas = yield* db
  699. .select({ type: EventTable.type })
  700. .from(EventTable)
  701. .where(eq(EventTable.type, Bus.versionedType(fixture.delta.type, 1)))
  702. .all()
  703. .pipe(Effect.orDie)
  704. expect(Array.from(yield* Fiber.join(live))).toHaveLength(32)
  705. expect(deltas).toHaveLength(0)
  706. expect(yield* session.context(sessionID)).toMatchObject(expectedContext)
  707. yield* replaySessionProjection(sessionID)
  708. expect(yield* session.context(sessionID)).toMatchObject(expectedContext)
  709. })
  710. const verifyPartialFlushOnFailure = (kind: FragmentKind) =>
  711. Effect.gen(function* () {
  712. const session = yield* setup
  713. const prompt = `Fail after ${kind}`
  714. const fixture = fragmentFixture(kind, fragmentID(kind, "partial"), ["Partial"])
  715. const failure = providerUnavailable()
  716. yield* admit(session, prompt)
  717. yield* TestLLM.push(TestLLM.failAfter(failure, ...fixture.partialEvents))
  718. expect(yield* session.resume(sessionID).pipe(Effect.flip)).toBe(failure)
  719. expect(yield* session.context(sessionID)).toMatchObject([
  720. { type: "user", text: prompt },
  721. {
  722. type: "assistant",
  723. finish: "error",
  724. error: { type: "provider.transport", message: "Provider unavailable" },
  725. content: [
  726. kind === "tool input"
  727. ? {
  728. type: "tool",
  729. id: fragmentID(kind, "partial"),
  730. state: {
  731. status: "error",
  732. error: { type: "provider.transport", message: "Provider unavailable" },
  733. },
  734. }
  735. : fixture.expectedContent,
  736. ],
  737. },
  738. ])
  739. expect(requests).toHaveLength(1)
  740. })
  741. const verifyPartialFlushOnInterruption = (kind: FragmentKind) =>
  742. Effect.gen(function* () {
  743. const session = yield* setup
  744. const prompt = `Interrupt after ${kind}`
  745. const fixture = fragmentFixture(kind, fragmentID(kind, "interrupted"), ["Partial"])
  746. const streamed = yield* Deferred.make<void>()
  747. yield* admit(session, prompt)
  748. yield* TestLLM.push(
  749. Stream.concat(
  750. Stream.fromIterable(fixture.partialEvents),
  751. Stream.fromEffect(Deferred.succeed(streamed, undefined)).pipe(Stream.flatMap(() => Stream.never)),
  752. ),
  753. )
  754. const runner = yield* SessionRunner.Service
  755. const fiber = yield* runner.drain({ sessionID, force: true }).pipe(Effect.forkChild)
  756. yield* Deferred.await(streamed)
  757. yield* Fiber.interrupt(fiber)
  758. expect(yield* session.context(sessionID)).toMatchObject([
  759. { type: "user", text: prompt },
  760. {
  761. type: "assistant",
  762. finish: "error",
  763. error: { type: "aborted", message: "Step interrupted" },
  764. content: [
  765. kind === "tool input"
  766. ? { type: "tool", id: fragmentID(kind, "interrupted"), state: { status: "error" } }
  767. : fixture.expectedContent,
  768. ],
  769. },
  770. ])
  771. })
  772. describe("SessionRunnerLLM", () => {
  773. it.effect("retries title generation from the first prompt after execution and title failures", () =>
  774. Effect.gen(function* () {
  775. const session = yield* setup
  776. const agents = yield* Agent.Service
  777. const { db } = yield* Database.Service
  778. yield* db.update(SessionTable).set({ title: null }).where(eq(SessionTable.id, sessionID)).run().pipe(Effect.orDie)
  779. yield* agents.transform((draft) =>
  780. draft.update(Agent.ID.make("title"), (agent) => {
  781. agent.mode = "primary"
  782. agent.hidden = true
  783. agent.system = "Generate a title."
  784. }),
  785. )
  786. yield* admit(session, "First prompt")
  787. yield* TestLLM.push(Stream.fail(invalidRequest()))
  788. expect((yield* session.resume(sessionID).pipe(Effect.exit))._tag).toBe("Failure")
  789. yield* admit(session, "Second prompt")
  790. const titleFailed = yield* Deferred.make<void>()
  791. yield* TestLLM.push(
  792. TestLLM.text("Recovered", "text-recovered"),
  793. Stream.make(LLMEvent.providerError({ message: "Title provider unavailable" })).pipe(
  794. Stream.ensuring(Deferred.succeed(titleFailed, undefined)),
  795. ),
  796. )
  797. yield* session.resume(sessionID)
  798. yield* Deferred.await(titleFailed)
  799. yield* Effect.yieldNow
  800. expect((yield* session.get(sessionID)).title).toBeUndefined()
  801. const bus = yield* Bus.Service
  802. const renamed = yield* bus.subscribe(SessionEvent.Renamed).pipe(
  803. Stream.filter((event) => event.data.sessionID === sessionID),
  804. Stream.take(1),
  805. Stream.runCollect,
  806. Effect.forkScoped({ startImmediately: true }),
  807. )
  808. yield* admit(session, "Third prompt")
  809. yield* TestLLM.push(
  810. TestLLM.text("Recovered again", "text-recovered-again"),
  811. TestLLM.text("Generated title", "text-title"),
  812. )
  813. yield* session.resume(sessionID)
  814. yield* Fiber.join(renamed)
  815. expect(requests).toHaveLength(5)
  816. expect(requests[2]?.messages).toContainEqual(Message.user("First prompt"))
  817. expect(requests[4]?.messages).toContainEqual(Message.user("First prompt"))
  818. expect((yield* session.get(sessionID)).title).toBe("Generated title")
  819. }),
  820. )
  821. it.effect("applies session context hooks without exposing unavailable tools", () =>
  822. Effect.gen(function* () {
  823. const session = yield* setup
  824. const hooks = yield* PluginHooks.Service
  825. yield* hooks.register("session", "context", (event) =>
  826. Effect.sync(() => {
  827. event.system = [SystemPart.make("Hooked system")]
  828. event.messages = [Message.user("Hooked message")]
  829. delete event.tools.echo
  830. event.tools.unregistered = { description: "Unavailable", input: { type: "object" } }
  831. }),
  832. )
  833. yield* admit(session, "Original message")
  834. yield* TestLLM.push(TestLLM.tool("call-removed", "echo", { text: "blocked" }))
  835. yield* session.resume(sessionID)
  836. // A hook-removed call fails independently and continues while step allowance remains.
  837. expect(requests).toHaveLength(2)
  838. expect(requests[0]?.system.map((part) => part.text)).toEqual(["Hooked system"])
  839. expect(requests[0]?.messages).toEqual([Message.user("Hooked message")])
  840. expect(requests[0]?.tools.map((tool) => tool.name)).not.toContain("echo")
  841. expect(requests[0]?.tools.map((tool) => tool.name)).not.toContain("unregistered")
  842. expect(executions).toEqual([])
  843. expect(yield* session.context(sessionID)).toMatchObject([
  844. { type: "user", text: "Original message" },
  845. {
  846. type: "assistant",
  847. content: [
  848. {
  849. type: "tool",
  850. id: "call-removed",
  851. state: { status: "error", error: { type: "tool.execution" } },
  852. },
  853. ],
  854. },
  855. ])
  856. }),
  857. )
  858. it.effect("collects session HTTP middleware once per prepared request", () =>
  859. Effect.gen(function* () {
  860. const session = yield* setup
  861. const hooks = yield* PluginHooks.Service
  862. let triggers = 0
  863. yield* hooks.register("session", "http", (event) =>
  864. Effect.gen(function* () {
  865. triggers++
  866. yield* event.use((request, next) => next(request))
  867. }),
  868. )
  869. yield* admit(session, "Use HTTP middleware")
  870. yield* TestLLM.push(TestLLM.text("Done", "text-http-middleware"))
  871. yield* session.resume(sessionID)
  872. expect(triggers).toBe(1)
  873. }),
  874. )
  875. it.effect("executes a tool renamed by a session context hook", () =>
  876. Effect.gen(function* () {
  877. const session = yield* setup
  878. const hooks = yield* PluginHooks.Service
  879. yield* hooks.register("session", "context", (event) =>
  880. Effect.sync(() => {
  881. event.tools.renamed_echo = event.tools.echo!
  882. delete event.tools.echo
  883. }),
  884. )
  885. yield* admit(session, "Use the renamed tool")
  886. yield* TestLLM.push(TestLLM.tool("call-renamed", "renamed_echo", { text: "renamed" }), [])
  887. yield* session.resume(sessionID)
  888. expect(requests[0]?.tools.map((tool) => tool.name)).toContain("renamed_echo")
  889. expect(requests[0]?.tools.map((tool) => tool.name)).not.toContain("echo")
  890. expect(executions).toEqual(["renamed"])
  891. }),
  892. )
  893. it.effect("advertises and executes a location registered tool", () =>
  894. Effect.gen(function* () {
  895. const session = yield* setup
  896. const registry = yield* Tool.Service
  897. const contexts: Tool.Context[] = []
  898. yield* transformTools(
  899. registry,
  900. {
  901. location_context: {
  902. name: "location_context",
  903. description: "Read application context",
  904. input: Schema.Struct({ query: Schema.String }),
  905. output: Schema.Struct({ answer: Schema.String }),
  906. execute: ({ query }, context) =>
  907. Effect.gen(function* () {
  908. contexts.push(context)
  909. yield* context.progress({ phase: "reading" })
  910. return { output: { answer: query.toUpperCase() } }
  911. }),
  912. },
  913. },
  914. { codemode: false },
  915. )
  916. yield* admit(session, "Use application context")
  917. yield* TestLLM.push(TestLLM.tool("call-location", "location_context", { query: "hello" }), [])
  918. const bus = yield* Bus.Service
  919. const progressFiber = yield* bus.subscribe(SessionEvent.Tool.Progress).pipe(
  920. Stream.filter((event) => event.data.sessionID === sessionID && event.data.id === "call-location"),
  921. Stream.take(1),
  922. Stream.runCollect,
  923. Effect.forkScoped({ startImmediately: true }),
  924. )
  925. yield* session.resume(sessionID)
  926. expect(requests[0]?.tools.map((tool) => tool.name)).toContain("location_context")
  927. expect(contexts).toEqual([
  928. {
  929. sessionID,
  930. agent: Agent.ID.make("build"),
  931. messageID: expect.stringMatching(/^msg_/),
  932. id: Tool.CallID.make("call-location"),
  933. progress: expect.any(Function),
  934. },
  935. ])
  936. expect(Array.from(yield* Fiber.join(progressFiber))[0]?.data.metadata).toEqual({ phase: "reading" })
  937. expect(yield* session.context(sessionID)).toMatchObject([
  938. { type: "user", text: "Use application context" },
  939. {
  940. type: "assistant",
  941. content: [
  942. {
  943. type: "tool",
  944. id: "call-location",
  945. state: { status: "completed", content: [{ type: "text", text: '{"answer":"HELLO"}' }] },
  946. },
  947. ],
  948. },
  949. ])
  950. }),
  951. )
  952. it.effect("executes the tool advertised before a registry reload", () =>
  953. Effect.gen(function* () {
  954. const session = yield* setup
  955. const registry = yield* Tool.Service
  956. const scope = yield* Scope.make()
  957. const executions: string[] = []
  958. yield* transformTools(
  959. registry,
  960. {
  961. reloaded: {
  962. name: "reloaded",
  963. description: "Record the advertised tool",
  964. input: Schema.Struct({}),
  965. output: Schema.Struct({ value: Schema.String }),
  966. execute: () =>
  967. Effect.sync(() => executions.push("advertised")).pipe(Effect.as({ output: { value: "advertised" } })),
  968. },
  969. },
  970. { codemode: false },
  971. ).pipe(Scope.provide(scope))
  972. yield* admit(session, "Use the reloaded tool")
  973. yield* TestLLM.push(TestLLM.tool("call-reloaded", "reloaded", {}), [])
  974. const stream = yield* TestLLM.gate
  975. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  976. yield* stream.started
  977. yield* Scope.close(scope, Exit.void)
  978. yield* transformTools(
  979. registry,
  980. {
  981. reloaded: {
  982. name: "reloaded",
  983. description: "Record the replacement tool",
  984. input: Schema.Struct({}),
  985. output: Schema.Struct({ value: Schema.String }),
  986. execute: () =>
  987. Effect.sync(() => executions.push("replacement")).pipe(Effect.as({ output: { value: "replacement" } })),
  988. },
  989. },
  990. { codemode: false },
  991. )
  992. yield* stream.release
  993. yield* Fiber.join(run)
  994. expect(executions).toEqual(["advertised"])
  995. expect(yield* session.context(sessionID)).toMatchObject([
  996. { type: "user", text: "Use the reloaded tool" },
  997. {
  998. type: "assistant",
  999. content: [
  1000. {
  1001. type: "tool",
  1002. id: "call-reloaded",
  1003. state: { status: "completed", content: [{ type: "text", text: '{"value":"advertised"}' }] },
  1004. },
  1005. ],
  1006. },
  1007. ])
  1008. }),
  1009. )
  1010. it.effect("starts a real runner step after default prompt recording", () =>
  1011. Effect.gen(function* () {
  1012. const session = yield* setup
  1013. const message = yield* session.prompt({
  1014. sessionID,
  1015. text: "Run automatically",
  1016. })
  1017. yield* session.wait(sessionID)
  1018. expect(requests).toHaveLength(1)
  1019. expect(yield* session.messages({ sessionID })).toMatchObject([
  1020. { id: message.id, type: "user", text: "Run automatically" },
  1021. ])
  1022. }),
  1023. )
  1024. it.effect("runs a follow-up when synthetic input arrives during an active continuation", () =>
  1025. Effect.gen(function* () {
  1026. const session = yield* setup
  1027. const secondStarted = yield* Deferred.make<void>()
  1028. const releaseSecond = yield* Deferred.make<void>()
  1029. yield* TestLLM.push(
  1030. Stream.fromIterable(TestLLM.tool("call-echo", "echo", { text: "background started" })),
  1031. Stream.unwrap(
  1032. Deferred.succeed(secondStarted, undefined).pipe(
  1033. Effect.andThen(Deferred.await(releaseSecond)),
  1034. Effect.as(Stream.fromIterable(TestLLM.stop())),
  1035. ),
  1036. ),
  1037. Stream.fromIterable(TestLLM.text("Handled completion", "text-completion")),
  1038. )
  1039. yield* admit(session, "Start background work")
  1040. const running = yield* session.resume(sessionID).pipe(Effect.forkChild({ startImmediately: true }))
  1041. yield* Deferred.await(secondStarted)
  1042. yield* session.synthetic({ sessionID, text: "Background work completed" })
  1043. yield* Deferred.succeed(releaseSecond, undefined)
  1044. yield* Fiber.join(running)
  1045. expect(requests).toHaveLength(3)
  1046. expect(userTexts(requests[2])).toContain("Background work completed")
  1047. }),
  1048. )
  1049. it.effect("streams one request with registry definitions from chronological user history", () =>
  1050. Effect.gen(function* () {
  1051. const session = yield* setup
  1052. yield* admit(session, "First")
  1053. yield* runPrompt(session, "Second")
  1054. expect(requests).toHaveLength(1)
  1055. expect(requests[0]?.model).toBe(model)
  1056. expect(requests[0]?.tools.map((tool) => tool.name)).toEqual(["defect", "echo", "storefail"])
  1057. expect(requests[0]?.messages.map((message) => ({ role: message.role, content: message.content }))).toEqual([
  1058. { role: "user", content: [{ type: "text", text: "First" }] },
  1059. { role: "user", content: [{ type: "text", text: "Second" }] },
  1060. ])
  1061. expect(yield* session.messages({ sessionID })).toHaveLength(2)
  1062. }),
  1063. )
  1064. it.effect("marks the initial instruction sync as baseline metadata", () =>
  1065. Effect.gen(function* () {
  1066. const session = yield* setup
  1067. const bus = yield* Bus.Service
  1068. const instructionEvents: Event.Payload[] = []
  1069. const unsubscribe = yield* bus.listen((event) =>
  1070. Effect.sync(() => {
  1071. if (event.type === "session.instructions.updated") instructionEvents.push(event)
  1072. }),
  1073. )
  1074. yield* runPrompt(session, "First")
  1075. systemBaseline = "Changed context"
  1076. yield* runPrompt(session, "Second")
  1077. yield* unsubscribe
  1078. expect(instructionEvents).toHaveLength(2)
  1079. expect(instructionEvents[0]?.metadata).toEqual({ instructions: { initial: true } })
  1080. expect(instructionEvents[1]?.metadata).toBeUndefined()
  1081. }),
  1082. )
  1083. it.effect("retries the first request after system context becomes available", () =>
  1084. Effect.gen(function* () {
  1085. const session = yield* setup
  1086. const { db } = yield* Database.Service
  1087. const messageID = SessionMessage.ID.create()
  1088. systemUnavailable = true
  1089. yield* session.prompt({
  1090. id: messageID,
  1091. sessionID,
  1092. text: "First",
  1093. resume: false,
  1094. })
  1095. const exit = yield* session.resume(sessionID).pipe(Effect.exit)
  1096. expect(Exit.isFailure(exit)).toBe(true)
  1097. if (Exit.isFailure(exit)) expect(Cause.squash(exit.cause)).toBeInstanceOf(Instructions.InitializationBlocked)
  1098. expect(requests).toHaveLength(0)
  1099. expect(yield* SessionPending.has(db, sessionID, "steer")).toBe(true)
  1100. expect(
  1101. yield* db.select().from(InstructionStateTable).where(eq(InstructionStateTable.session_id, sessionID)).get(),
  1102. ).toBeUndefined()
  1103. systemUnavailable = false
  1104. yield* session.prompt({ id: messageID, sessionID, text: "First" })
  1105. yield* session.wait(sessionID)
  1106. expect(requests).toHaveLength(1)
  1107. expect(messageRoles(requests[0])).toEqual(["user"])
  1108. }),
  1109. )
  1110. it.effect("interrupts a source Location runner after a Session moves", () =>
  1111. Effect.gen(function* () {
  1112. const session = yield* setup
  1113. const bus = yield* Bus.Service
  1114. const { db } = yield* Database.Service
  1115. yield* runPrompt(session, "First")
  1116. yield* bus.publish(SessionEvent.Moved, {
  1117. sessionID,
  1118. location: Location.Ref.make({ directory: AbsolutePath.make("/moved") }),
  1119. })
  1120. expect(
  1121. yield* db.select().from(InstructionStateTable).where(eq(InstructionStateTable.session_id, sessionID)).get(),
  1122. ).toBeUndefined()
  1123. yield* admit(session, "Second")
  1124. const exit = yield* session.resume(sessionID).pipe(Effect.exit)
  1125. expect(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)).toBe(true)
  1126. expect(requests).toHaveLength(1)
  1127. expect(yield* SessionPending.has(db, sessionID, "steer")).toBe(true)
  1128. }),
  1129. )
  1130. it.effect("seeds a fork with the parent's newest instruction values", () =>
  1131. Effect.gen(function* () {
  1132. const session = yield* setup
  1133. yield* runPrompt(session, "First")
  1134. systemBaseline = "Changed context"
  1135. const second = yield* runPrompt(session, "Second")
  1136. systemBaseline = "Latest context"
  1137. yield* runPrompt(session, "Third")
  1138. const forked = yield* session.fork({ sessionID, boundary: { type: "before", messageID: second.id } })
  1139. expect(
  1140. yield* (yield* Database.Service).db
  1141. .select()
  1142. .from(InstructionStateTable)
  1143. .where(eq(InstructionStateTable.session_id, forked.id))
  1144. .get(),
  1145. ).toMatchObject({
  1146. initial_values: { "test/context": Instructions.hash("Latest context") },
  1147. current_values: { "test/context": Instructions.hash("Latest context") },
  1148. })
  1149. yield* session.prompt({ sessionID: forked.id, text: "Forked", resume: false })
  1150. yield* session.resume(forked.id)
  1151. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual([defaultSystem, "Latest context"])
  1152. // Copied history keeps the frozen chronological update; no new update is emitted.
  1153. expect(systemTexts(requests.at(-1)!)).toContain("Changed context")
  1154. expect(systemTexts(requests.at(-1)!)).not.toContain("Latest context")
  1155. const { db } = yield* Database.Service
  1156. const bus = yield* Bus.Service
  1157. const recorded = yield* db
  1158. .select()
  1159. .from(EventTable)
  1160. .where(eq(EventTable.aggregate_id, forked.id))
  1161. .orderBy(asc(EventTable.seq))
  1162. .all()
  1163. yield* bus.remove(forked.id)
  1164. yield* db.delete(SessionTable).where(eq(SessionTable.id, forked.id)).run()
  1165. yield* bus.replayAll(
  1166. recorded.map((event) => ({
  1167. id: event.id,
  1168. created: DateTime.makeUnsafe(event.created),
  1169. aggregateID: event.aggregate_id,
  1170. seq: event.seq,
  1171. type: event.type,
  1172. data: event.data,
  1173. })),
  1174. )
  1175. expect(
  1176. yield* db.select().from(InstructionStateTable).where(eq(InstructionStateTable.session_id, forked.id)).get(),
  1177. ).toMatchObject({ current_values: { "test/context": Instructions.hash("Latest context") } })
  1178. }),
  1179. )
  1180. it.effect("keeps nested forks self-contained", () =>
  1181. Effect.gen(function* () {
  1182. const session = yield* setup
  1183. yield* runPrompt(session, "First")
  1184. systemBaseline = "Changed context"
  1185. const second = yield* runPrompt(session, "Second")
  1186. const child = yield* session.fork({ sessionID, boundary: { type: "before", messageID: second.id } })
  1187. const inheritedFirst = (yield* session.messages({ sessionID: child.id })).find(
  1188. (message) => message.type === "user" && message.text === "First",
  1189. )
  1190. if (!inheritedFirst) return yield* Effect.die(new Error("Nested fork boundary message not found"))
  1191. const grandchild = yield* session.fork({
  1192. sessionID: child.id,
  1193. boundary: { type: "before", messageID: inheritedFirst.id },
  1194. })
  1195. expect(
  1196. yield* (yield* Database.Service).db
  1197. .select()
  1198. .from(InstructionStateTable)
  1199. .where(eq(InstructionStateTable.session_id, grandchild.id))
  1200. .get(),
  1201. ).toMatchObject({
  1202. initial_values: { "test/context": Instructions.hash("Changed context") },
  1203. current_values: { "test/context": Instructions.hash("Changed context") },
  1204. })
  1205. return undefined
  1206. }),
  1207. )
  1208. it.effect("re-establishes a fresh baseline when instruction state is missing", () =>
  1209. Effect.gen(function* () {
  1210. const session = yield* setup
  1211. const { db } = yield* Database.Service
  1212. yield* runPrompt(session, "First")
  1213. yield* db.delete(InstructionStateTable).where(eq(InstructionStateTable.session_id, sessionID)).run()
  1214. yield* admit(session, "Second")
  1215. requests.length = 0
  1216. yield* session.resume(sessionID)
  1217. expect(requests).toHaveLength(1)
  1218. expect(requests[0]?.system.map((part) => part.text)).toEqual([defaultSystem, "Initial context"])
  1219. expect(messageRoles(requests[0])).toEqual(["user", "user"])
  1220. // The projected row is authoritative: a missing row admits a fresh baseline
  1221. // instead of rebuilding from durable events.
  1222. expect(
  1223. yield* db
  1224. .select({ data: EventTable.data })
  1225. .from(EventTable)
  1226. .where(eq(EventTable.type, "session.instructions.updated.2"))
  1227. .all(),
  1228. ).toHaveLength(2)
  1229. expect(yield* db.select().from(InstructionStateTable).get()).toMatchObject({
  1230. initial_values: { "test/context": Instructions.hash("Initial context") },
  1231. current_values: { "test/context": Instructions.hash("Initial context") },
  1232. })
  1233. }),
  1234. )
  1235. it.effect("keeps the initial instructions stable and derives a chronological update from values", () =>
  1236. Effect.gen(function* () {
  1237. const session = yield* setup
  1238. yield* runPrompt(session, "First")
  1239. systemBaseline = "Changed context"
  1240. yield* runPrompt(session, "Second")
  1241. expect(
  1242. PromptCacheDiagnostics.compare(
  1243. PromptCacheDiagnostics.snapshot(requests[0]),
  1244. PromptCacheDiagnostics.snapshot(requests[1]),
  1245. ),
  1246. ).toEqual({ status: "append-only", previousMessages: 1, currentMessages: 3 })
  1247. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  1248. [defaultSystem, "Initial context"],
  1249. [defaultSystem, "Initial context"],
  1250. ])
  1251. expect(messageRoles(requests[1])).toEqual(["user", "system", "user"])
  1252. expect(requests[1]?.messages.at(1)?.content).toEqual([{ type: "text", text: "Changed context" }])
  1253. // The chronological update is a durable client-visible system message.
  1254. const messages = yield* session.messages({ sessionID })
  1255. expect(messages).toHaveLength(3)
  1256. expect(messages[1]).toMatchObject({ type: "system", text: "Changed context" })
  1257. const { db } = yield* Database.Service
  1258. const updates = yield* db
  1259. .select({ data: EventTable.data })
  1260. .from(EventTable)
  1261. .where(eq(EventTable.type, "session.instructions.updated.2"))
  1262. .orderBy(asc(EventTable.seq))
  1263. .all()
  1264. .pipe(Effect.orDie)
  1265. expect(updates).toHaveLength(2)
  1266. expect(updates[0]?.data).toMatchObject({
  1267. sessionID,
  1268. delta: { "test/context": Instructions.hash("Initial context") },
  1269. })
  1270. expect(updates[1]?.data).toEqual({
  1271. sessionID,
  1272. delta: { "test/context": Instructions.hash("Changed context") },
  1273. text: "Changed context",
  1274. })
  1275. yield* replaySessionProjection(sessionID)
  1276. expect(yield* session.messages({ sessionID })).toHaveLength(3)
  1277. }),
  1278. )
  1279. it.effect("uses the selected model family prompt when the agent does not override it", () =>
  1280. Effect.gen(function* () {
  1281. const session = yield* setup
  1282. currentModel = LanguageModel.make({ id: "gpt-5", provider: "openai", route: OpenAIChat.route })
  1283. yield* admit(session, "First")
  1284. yield* TestLLM.push(TestLLM.text("Done", "text-provider-prompt"))
  1285. yield* session.resume(sessionID)
  1286. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual([
  1287. expect.stringContaining("You are OpenCode, You and the user share the same workspace"),
  1288. "Initial context",
  1289. ])
  1290. }),
  1291. )
  1292. it.effect("uses the selected model family prompt when the agent system override is empty", () =>
  1293. Effect.gen(function* () {
  1294. const session = yield* setup
  1295. currentModel = LanguageModel.make({ id: "gpt-5", provider: "openai", route: OpenAIChat.route })
  1296. const agent = yield* Agent.Service
  1297. yield* agent.transform((editor) =>
  1298. editor.update(Agent.ID.make("build"), (agent) => {
  1299. agent.system = ""
  1300. agent.mode = "primary"
  1301. }),
  1302. )
  1303. yield* admit(session, "First")
  1304. yield* TestLLM.push(TestLLM.text("Done", "text-empty-agent-system"))
  1305. yield* session.resume(sessionID)
  1306. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual([
  1307. expect.stringContaining("You are OpenCode, You and the user share the same workspace"),
  1308. "Initial context",
  1309. ])
  1310. }),
  1311. )
  1312. it.effect("includes the effective default agent system before durable context", () =>
  1313. Effect.gen(function* () {
  1314. const session = yield* setup
  1315. const agent = yield* Agent.Service
  1316. yield* agent.transform((editor) =>
  1317. editor.update(Agent.ID.make("build"), (agent) => {
  1318. agent.system = "Build agent instructions"
  1319. agent.mode = "primary"
  1320. }),
  1321. )
  1322. yield* admit(session, "First")
  1323. yield* TestLLM.push(TestLLM.text("Done", "text-build"))
  1324. yield* session.resume(sessionID)
  1325. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual(["Build agent instructions", "Initial context"])
  1326. }),
  1327. )
  1328. it.effect("uses the configured default agent system for omitted-agent sessions", () =>
  1329. Effect.gen(function* () {
  1330. const session = yield* setup
  1331. const agent = yield* Agent.Service
  1332. yield* agent.transform((editor) => {
  1333. editor.update(Agent.ID.make("build"), (agent) => {
  1334. agent.system = "Build agent instructions"
  1335. agent.mode = "primary"
  1336. })
  1337. editor.update(Agent.ID.make("reviewer"), (agent) => {
  1338. agent.system = "Reviewer instructions"
  1339. agent.mode = "primary"
  1340. })
  1341. editor.default(Agent.ID.make("reviewer"))
  1342. })
  1343. yield* admit(session, "First")
  1344. yield* TestLLM.push(TestLLM.text("Done", "text-reviewer"))
  1345. yield* session.resume(sessionID)
  1346. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual(["Reviewer instructions", "Initial context"])
  1347. expect((yield* session.messages({ sessionID }))[0]).toMatchObject({ type: "assistant", agent: "reviewer" })
  1348. }),
  1349. )
  1350. it.effect("uses only the agent prompt and initial instructions as system parts", () =>
  1351. Effect.gen(function* () {
  1352. const session = yield* setup
  1353. const agent = yield* Agent.Service
  1354. yield* agent.transform((editor) =>
  1355. editor.update(Agent.ID.make("build"), (agent) => {
  1356. agent.system = "Build agent instructions"
  1357. agent.mode = "primary"
  1358. }),
  1359. )
  1360. yield* admit(session, "First")
  1361. yield* TestLLM.push(TestLLM.text("Done", "text-no-system"))
  1362. yield* session.resume(sessionID)
  1363. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual(["Build agent instructions", "Initial context"])
  1364. }),
  1365. )
  1366. it.effect("uses an explicitly selected non-build agent system", () =>
  1367. Effect.gen(function* () {
  1368. const session = yield* setup
  1369. const { db } = yield* Database.Service
  1370. const agent = yield* Agent.Service
  1371. yield* agent.transform((editor) =>
  1372. editor.update(Agent.ID.make("reviewer"), (agent) => {
  1373. agent.system = "Reviewer instructions"
  1374. agent.mode = "primary"
  1375. }),
  1376. )
  1377. yield* db
  1378. .update(SessionTable)
  1379. .set({ agent: "reviewer" })
  1380. .where(eq(SessionTable.id, sessionID))
  1381. .run()
  1382. .pipe(Effect.orDie)
  1383. yield* admit(session, "First")
  1384. yield* TestLLM.push(TestLLM.text("Done", "text-selected"))
  1385. yield* session.resume(sessionID)
  1386. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual(["Reviewer instructions", "Initial context"])
  1387. expect((yield* session.messages({ sessionID }))[0]).toMatchObject({ type: "assistant", agent: "reviewer" })
  1388. }),
  1389. )
  1390. it.effect("fails before the model request when the selected agent is unavailable", () =>
  1391. Effect.gen(function* () {
  1392. yield* setup
  1393. const { db } = yield* Database.Service
  1394. yield* db
  1395. .update(SessionTable)
  1396. .set({ agent: "explore" })
  1397. .where(eq(SessionTable.id, sessionID))
  1398. .run()
  1399. .pipe(Effect.orDie)
  1400. const session = yield* Session.Service
  1401. yield* session.prompt({ sessionID, text: "Inspect files", resume: false })
  1402. requests.length = 0
  1403. yield* TestLLM.push([])
  1404. const failure = yield* session.resume(sessionID).pipe(Effect.flip)
  1405. expect(failure).toMatchObject({
  1406. _tag: "Session.AgentNotFoundError",
  1407. sessionID,
  1408. agent: "explore",
  1409. })
  1410. expect(requests).toHaveLength(0)
  1411. }),
  1412. )
  1413. it.effect("waits for initial plugin readiness before constructing the model request", () =>
  1414. Effect.gen(function* () {
  1415. yield* setup
  1416. const release = yield* Deferred.make<void>()
  1417. pluginFlushHook = Deferred.await(release)
  1418. const session = yield* Session.Service
  1419. yield* session.prompt({ sessionID, text: "Wait for plugins", resume: false })
  1420. requests.length = 0
  1421. yield* TestLLM.push([])
  1422. const running = yield* session.resume(sessionID).pipe(Effect.forkChild({ startImmediately: true }))
  1423. yield* Effect.yieldNow
  1424. expect(requests).toHaveLength(0)
  1425. expect(running.pollUnsafe()).toBeUndefined()
  1426. yield* Deferred.succeed(release, undefined)
  1427. yield* Fiber.join(running)
  1428. expect(requests).toHaveLength(1)
  1429. }),
  1430. )
  1431. it.effect("updates selected-agent skill instructions after an agent switch", () =>
  1432. Effect.gen(function* () {
  1433. const session = yield* setup
  1434. const bus = yield* Bus.Service
  1435. const agents = yield* Agent.Service
  1436. yield* agents.transform((draft) =>
  1437. draft.update(Agent.ID.make("reviewer"), (agent) => {
  1438. agent.mode = "primary"
  1439. }),
  1440. )
  1441. skillBaselines.set(Agent.ID.make("build"), "Build skills")
  1442. yield* runPrompt(session, "First")
  1443. skillBaselines.set(Agent.ID.make("reviewer"), "Reviewer skills")
  1444. yield* bus.publish(SessionEvent.AgentSelected, {
  1445. sessionID,
  1446. agent: Agent.ID.make("reviewer"),
  1447. })
  1448. yield* runPrompt(session, "Second")
  1449. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  1450. [defaultSystem, "Initial context\n\nBuild skills"],
  1451. [defaultSystem, "Initial context\n\nBuild skills"],
  1452. ])
  1453. expect(systemTexts(requests[1])).toContainEqual(expect.stringContaining("Reviewer skills"))
  1454. }),
  1455. )
  1456. it.effect("keeps the sampled agent when selection changes during observation", () =>
  1457. Effect.gen(function* () {
  1458. const session = yield* setup
  1459. const bus = yield* Bus.Service
  1460. skillBaselines.set(Agent.ID.make("build"), "Build skills")
  1461. skillBaselines.set(Agent.ID.make("reviewer"), "Reviewer skills")
  1462. let switched = false
  1463. systemLoadHook = Effect.suspend(() => {
  1464. if (switched) return Effect.void
  1465. switched = true
  1466. return bus
  1467. .publish(SessionEvent.AgentSelected, {
  1468. sessionID,
  1469. agent: Agent.ID.make("reviewer"),
  1470. })
  1471. .pipe(Effect.asVoid)
  1472. })
  1473. yield* runPrompt(session, "First")
  1474. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  1475. [defaultSystem, "Initial context\n\nBuild skills"],
  1476. ])
  1477. }),
  1478. )
  1479. it.effect("keeps the sampled model when selection changes during model resolution", () =>
  1480. Effect.gen(function* () {
  1481. const session = yield* setup
  1482. const bus = yield* Bus.Service
  1483. let switched = false
  1484. modelResolveHook = Effect.suspend(() => {
  1485. if (switched) return Effect.void
  1486. switched = true
  1487. return bus
  1488. .publish(SessionEvent.ModelSelected, {
  1489. sessionID,
  1490. model: { id: ID.make("replacement"), providerID: Provider.ID.make("fake") },
  1491. })
  1492. .pipe(Effect.asVoid)
  1493. })
  1494. yield* runPrompt(session, "First")
  1495. expect(requests.map((request) => request.model)).toEqual([model])
  1496. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  1497. [defaultSystem, "Initial context"],
  1498. ])
  1499. }),
  1500. )
  1501. it.effect("admits removed context as a chronological System message", () =>
  1502. Effect.gen(function* () {
  1503. const session = yield* setup
  1504. yield* runPrompt(session, "First")
  1505. systemRemoved = true
  1506. yield* runPrompt(session, "Second")
  1507. expect(messageRoles(requests[1])).toEqual(["user", "system", "user"])
  1508. expect(requests[1]?.messages.at(1)?.content).toEqual([
  1509. { type: "text", text: "System context source removed: test/context" },
  1510. ])
  1511. expect(yield* session.messages({ sessionID })).toHaveLength(3)
  1512. }),
  1513. )
  1514. it.effect("renders API context entries through add, change, and removal", () =>
  1515. Effect.gen(function* () {
  1516. const session = yield* setup
  1517. const contextEntries = yield* InstructionEntry.Service
  1518. yield* contextEntries.put({ sessionID, key: "deploy-target", value: "production" })
  1519. yield* runPrompt(session, "First")
  1520. // String values render verbatim inside the initial tagged block.
  1521. expect(requests[0]?.system.map((part) => part.text)).toEqual([
  1522. defaultSystem,
  1523. ["Initial context", "", '<context key="deploy-target">', "production", "</context>"].join("\n"),
  1524. ])
  1525. // Non-string JSON pretty-prints; the change narrates as a System update.
  1526. yield* contextEntries.put({ sessionID, key: "deploy-target", value: { region: "us-east-1" } })
  1527. yield* runPrompt(session, "Second")
  1528. expect(messageRoles(requests[1])).toEqual(["user", "system", "user"])
  1529. expect(requests[1]?.messages.at(1)?.content).toEqual([
  1530. {
  1531. type: "text",
  1532. text: [
  1533. 'The context under "deploy-target" changed and supersedes the previous value:',
  1534. '<context key="deploy-target">',
  1535. "{",
  1536. ' "region": "us-east-1"',
  1537. "}",
  1538. "</context>",
  1539. ].join("\n"),
  1540. },
  1541. ])
  1542. expect(yield* contextEntries.list(sessionID)).toEqual([{ key: "deploy-target", value: { region: "us-east-1" } }])
  1543. // Deleting the row announces removal through the stored removal text.
  1544. yield* contextEntries.remove({ sessionID, key: "deploy-target" })
  1545. yield* runPrompt(session, "Third")
  1546. expect(messageRoles(requests[2])).toEqual(["user", "system", "user", "system", "user"])
  1547. expect(requests[2]?.messages.at(-2)?.content).toEqual([
  1548. { type: "text", text: 'The context under "deploy-target" no longer applies. Disregard it.' },
  1549. ])
  1550. expect(yield* contextEntries.list(sessionID)).toEqual([])
  1551. }),
  1552. )
  1553. it.effect("retains JSON null API entries as values", () =>
  1554. Effect.gen(function* () {
  1555. const session = yield* setup
  1556. const entries = yield* InstructionEntry.Service
  1557. yield* entries.put({ sessionID, key: "nullable", value: "present" })
  1558. yield* runPrompt(session, "First")
  1559. yield* entries.put({ sessionID, key: "nullable", value: null })
  1560. yield* runPrompt(session, "Second")
  1561. expect(requests[1]?.messages.at(1)?.content).toEqual([
  1562. {
  1563. type: "text",
  1564. text: [
  1565. 'The context under "nullable" changed and supersedes the previous value:',
  1566. '<context key="nullable">',
  1567. "null",
  1568. "</context>",
  1569. ].join("\n"),
  1570. },
  1571. ])
  1572. expect(yield* entries.list(sessionID)).toEqual([{ key: "nullable", value: null }])
  1573. }),
  1574. )
  1575. it.effect("rejects API instruction entries larger than 8KB", () =>
  1576. Effect.gen(function* () {
  1577. yield* setup
  1578. const entries = yield* InstructionEntry.Service
  1579. const exit = yield* entries
  1580. .put({ sessionID, key: "oversized", value: "x".repeat(InstructionEntry.MaxValueBytes) })
  1581. .pipe(Effect.exit)
  1582. expect(Exit.isFailure(exit)).toBe(true)
  1583. if (Exit.isFailure(exit)) expect(Cause.squash(exit.cause)).toBeInstanceOf(InstructionEntry.ValueTooLargeError)
  1584. expect(yield* entries.list(sessionID)).toEqual([])
  1585. }),
  1586. )
  1587. it.effect("keeps initial instructions and chronological updates after a model switch", () =>
  1588. Effect.gen(function* () {
  1589. const session = yield* setup
  1590. const bus = yield* Bus.Service
  1591. yield* runPrompt(session, "First")
  1592. systemBaseline = "Changed context"
  1593. yield* runPrompt(session, "Second")
  1594. yield* bus.publish(SessionEvent.ModelSelected, {
  1595. sessionID,
  1596. model: { id: ID.make("replacement"), providerID: Provider.ID.make("fake") },
  1597. })
  1598. systemBaseline = "Replacement context"
  1599. yield* runPrompt(session, "Third")
  1600. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  1601. [defaultSystem, "Initial context"],
  1602. [defaultSystem, "Initial context"],
  1603. [defaultSystem, "Initial context"],
  1604. ])
  1605. expect(messageRoles(requests[1])).toEqual(["user", "system", "user"])
  1606. expect(requests[2]?.messages.filter((message) => message.role === "system")).toHaveLength(2)
  1607. expect((yield* session.context(sessionID)).map((message) => message.type)).toEqual([
  1608. "user",
  1609. "system",
  1610. "user",
  1611. "model-switched",
  1612. "system",
  1613. "user",
  1614. ])
  1615. yield* replaySessionProjection(sessionID)
  1616. expect(yield* session.messages({ sessionID })).toHaveLength(6)
  1617. yield* runPrompt(session, "Fourth")
  1618. }),
  1619. )
  1620. it.effect("preserves instruction values while a source is temporarily unavailable", () =>
  1621. Effect.gen(function* () {
  1622. const session = yield* setup
  1623. const bus = yield* Bus.Service
  1624. yield* runPrompt(session, "First")
  1625. yield* bus.publish(SessionEvent.ModelSelected, {
  1626. sessionID,
  1627. model: { id: ID.make("replacement"), providerID: Provider.ID.make("fake") },
  1628. })
  1629. systemUnavailable = true
  1630. yield* runPrompt(session, "Second")
  1631. systemUnavailable = false
  1632. systemBaseline = "Replacement context"
  1633. yield* runPrompt(session, "Third")
  1634. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  1635. [defaultSystem, "Initial context"],
  1636. [defaultSystem, "Initial context"],
  1637. [defaultSystem, "Initial context"],
  1638. ])
  1639. }),
  1640. )
  1641. it.effect("moves the epoch at compaction and narrates later changes", () =>
  1642. Effect.gen(function* () {
  1643. const session = yield* setup
  1644. const bus = yield* Bus.Service
  1645. yield* runPrompt(session, "First")
  1646. yield* bus.publish(SessionEvent.Compaction.Started, {
  1647. sessionID,
  1648. reason: "manual",
  1649. recent: "",
  1650. })
  1651. yield* bus.publish(SessionEvent.Compaction.Ended, {
  1652. sessionID,
  1653. reason: "manual",
  1654. text: "summary",
  1655. recent: "",
  1656. })
  1657. systemBaseline = "Replacement context"
  1658. yield* runPrompt(session, "Second")
  1659. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  1660. [defaultSystem, "Initial context"],
  1661. [defaultSystem, "Initial context"],
  1662. ])
  1663. expect(messageRoles(requests[1])).toEqual(["user", "system", "user"])
  1664. expect(requests[1]?.messages.at(1)?.content).toEqual([{ type: "text", text: "Replacement context" }])
  1665. yield* replaySessionProjection(sessionID)
  1666. yield* runPrompt(session, "Third")
  1667. }),
  1668. )
  1669. it.effect("runs one durable compaction barrier after tool settlement and before later inputs", () =>
  1670. Effect.gen(function* () {
  1671. const session = yield* setup
  1672. currentModel = recoveryModel
  1673. const stream = yield* TestLLM.gate
  1674. yield* TestLLM.push(
  1675. TestLLM.tool("call-active", "echo", { text: "active" }),
  1676. [LLMEvent.textDelta({ id: "summary", text: "durable summary" })],
  1677. TestLLM.text("Steer complete", "text-steer"),
  1678. TestLLM.text("Queue complete", "text-queue"),
  1679. )
  1680. yield* admit(session, "Active work")
  1681. const active = yield* session.resume(sessionID).pipe(Effect.forkChild)
  1682. yield* stream.started
  1683. const first = yield* session.compact({ sessionID })
  1684. const second = yield* session.compact({ sessionID })
  1685. expect(second.id).toBe(first.id)
  1686. expect(yield* SessionPending.compaction((yield* Database.Service).db, sessionID)).toMatchObject({
  1687. id: first.id,
  1688. })
  1689. expect((yield* session.messages({ sessionID })).find((message) => message.id === first.id)).toBeUndefined()
  1690. yield* admit(session, "Steer after compaction")
  1691. yield* session.synthetic({ sessionID, text: "Completion after compaction", resume: false })
  1692. yield* session.prompt({
  1693. sessionID,
  1694. text: "Queue after compaction",
  1695. delivery: "queue",
  1696. resume: false,
  1697. })
  1698. expect(yield* SessionPending.has((yield* Database.Service).db, sessionID, "steer")).toBe(false)
  1699. yield* stream.release
  1700. yield* Fiber.join(active)
  1701. expect(requests).toHaveLength(4)
  1702. expect(userTexts(requests[1])[0]).toContain("Create a new anchored summary")
  1703. expect(userTexts(requests[2])).toContain("Steer after compaction")
  1704. expect(userTexts(requests[2])).toContain("Completion after compaction")
  1705. expect(userTexts(requests[3])).toContain("Queue after compaction")
  1706. expect(yield* SessionPending.compaction((yield* Database.Service).db, sessionID)).toBeUndefined()
  1707. expect((yield* session.messages({ sessionID })).find((message) => message.id === first.id)).toMatchObject({
  1708. type: "compaction",
  1709. status: "completed",
  1710. summary: "durable summary",
  1711. })
  1712. }),
  1713. )
  1714. it.effect("releases queued prompts when durable compaction fails", () =>
  1715. Effect.gen(function* () {
  1716. const session = yield* setup
  1717. currentModel = recoveryModel
  1718. const stream = yield* TestLLM.gate
  1719. yield* TestLLM.push(
  1720. TestLLM.text("Active complete", "text-active-failure"),
  1721. [],
  1722. TestLLM.text("Continued", "text-after-failure"),
  1723. )
  1724. yield* admit(session, "Active work")
  1725. const active = yield* session.resume(sessionID).pipe(Effect.forkChild)
  1726. yield* stream.started
  1727. const compaction = yield* session.compact({ sessionID })
  1728. yield* session.prompt({
  1729. sessionID,
  1730. text: "Continue after failure",
  1731. delivery: "queue",
  1732. resume: false,
  1733. })
  1734. yield* stream.release
  1735. yield* Fiber.join(active)
  1736. expect(requests).toHaveLength(3)
  1737. expect(userTexts(requests[2])).toContain("Continue after failure")
  1738. expect(yield* SessionPending.compaction((yield* Database.Service).db, sessionID)).toBeUndefined()
  1739. expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
  1740. type: "compaction",
  1741. status: "failed",
  1742. })
  1743. expect(
  1744. (yield* recordedEventTypes(sessionID)).filter(
  1745. (type) => type === Bus.versionedType(SessionEvent.Compaction.Failed.type, 1),
  1746. ),
  1747. ).toHaveLength(1)
  1748. }),
  1749. )
  1750. it.effect("explains when manual compaction has no history", () =>
  1751. Effect.gen(function* () {
  1752. yield* setup
  1753. const session = yield* Session.Service
  1754. const compaction = yield* session.compact({ sessionID })
  1755. modelResolveHook = Effect.die("model resolution should not run")
  1756. yield* session.resume(sessionID)
  1757. expect(yield* SessionPending.compaction((yield* Database.Service).db, sessionID)).toBeUndefined()
  1758. expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
  1759. type: "compaction",
  1760. status: "failed",
  1761. reason: "manual",
  1762. error: { type: "compaction.unavailable", message: "Nothing to compact yet" },
  1763. })
  1764. expect(
  1765. (yield* recordedEventTypes(sessionID)).filter(
  1766. (type) => type === Bus.versionedType(SessionEvent.Compaction.Failed.type, 1),
  1767. ),
  1768. ).toHaveLength(1)
  1769. }),
  1770. )
  1771. it.effect("manually compacts when the model has no context limit", () =>
  1772. Effect.gen(function* () {
  1773. const session = yield* setup
  1774. yield* TestLLM.push(TestLLM.text("Earlier answer", "text-manual-unknown-history"))
  1775. yield* runPrompt(session, "Earlier question")
  1776. requests.length = 0
  1777. yield* TestLLM.push(TestLLM.text("Manual summary", "text-manual-unknown-summary"))
  1778. const compaction = yield* session.compact({ sessionID })
  1779. yield* session.resume(sessionID)
  1780. expect(requests).toHaveLength(1)
  1781. expect(userTexts(requests[0])[0]).toContain("Earlier question")
  1782. expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
  1783. type: "compaction",
  1784. status: "completed",
  1785. summary: "Manual summary",
  1786. })
  1787. }),
  1788. )
  1789. it.effect("preserves provider errors from manual compaction", () =>
  1790. Effect.gen(function* () {
  1791. const session = yield* setup
  1792. yield* TestLLM.push(TestLLM.text("Earlier answer", "text-manual-provider-history"))
  1793. yield* runPrompt(session, "Earlier question")
  1794. yield* TestLLM.push([LLMEvent.providerError({ message: "summary unavailable" })])
  1795. const compaction = yield* session.compact({ sessionID })
  1796. yield* session.resume(sessionID)
  1797. expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
  1798. type: "compaction",
  1799. status: "failed",
  1800. error: { type: "provider.error", message: "summary unavailable" },
  1801. })
  1802. }),
  1803. )
  1804. it.effect("preserves typed provider failures from manual compaction", () =>
  1805. Effect.gen(function* () {
  1806. const session = yield* setup
  1807. yield* TestLLM.push(TestLLM.text("Earlier answer", "text-manual-failure-history"))
  1808. yield* runPrompt(session, "Earlier question")
  1809. yield* TestLLM.push(Stream.fail(providerUnavailable()))
  1810. const compaction = yield* session.compact({ sessionID })
  1811. yield* session.resume(sessionID)
  1812. expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
  1813. type: "compaction",
  1814. status: "failed",
  1815. error: { type: "provider.transport", message: "Provider unavailable" },
  1816. })
  1817. }),
  1818. )
  1819. it.effect("records cancelled manual compaction without surfacing an internal failure", () =>
  1820. Effect.gen(function* () {
  1821. const session = yield* setup
  1822. yield* TestLLM.push(TestLLM.text("Earlier answer", "text-manual-interrupt-history"))
  1823. yield* runPrompt(session, "Earlier question")
  1824. const streamed = yield* Deferred.make<void>()
  1825. const partial = fragmentFixture("text", "text-manual-interrupt-summary", ["Partial summary"])
  1826. yield* TestLLM.push(
  1827. Stream.concat(
  1828. Stream.fromIterable(partial.partialEvents),
  1829. Stream.fromEffect(Deferred.succeed(streamed, undefined)).pipe(Stream.flatMap(() => Stream.never)),
  1830. ),
  1831. )
  1832. const compaction = yield* session.compact({ sessionID })
  1833. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  1834. yield* Deferred.await(streamed)
  1835. yield* session.interrupt(sessionID)
  1836. yield* Fiber.await(run)
  1837. expect(yield* SessionPending.compaction((yield* Database.Service).db, sessionID)).toBeUndefined()
  1838. expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
  1839. type: "compaction",
  1840. status: "failed",
  1841. reason: "manual",
  1842. error: { type: "aborted", message: "Compaction cancelled" },
  1843. })
  1844. }),
  1845. )
  1846. it.effect("settles an admitted manual compaction when pre-start resolution throws", () =>
  1847. Effect.gen(function* () {
  1848. const session = yield* setup
  1849. yield* TestLLM.push(TestLLM.text("Earlier answer", "text-manual-resolution-history"))
  1850. yield* runPrompt(session, "Earlier question")
  1851. const compaction = yield* session.compact({ sessionID })
  1852. modelResolveHook = Effect.die("model resolution failed")
  1853. expect(yield* Effect.exit(session.resume(sessionID))).toMatchObject({ _tag: "Failure" })
  1854. expect(yield* SessionPending.compaction((yield* Database.Service).db, sessionID)).toBeUndefined()
  1855. expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
  1856. type: "compaction",
  1857. status: "failed",
  1858. reason: "manual",
  1859. })
  1860. expect(
  1861. (yield* recordedEventTypes(sessionID)).filter(
  1862. (type) => type === Bus.versionedType(SessionEvent.Compaction.Failed.type, 1),
  1863. ),
  1864. ).toHaveLength(1)
  1865. }),
  1866. )
  1867. it.effect("automatically compacts into a completed summary and retained recent turn", () =>
  1868. Effect.gen(function* () {
  1869. const session = yield* setup
  1870. yield* TestLLM.push(TestLLM.textWithUsage("Earlier answer", "text-first", 3_950))
  1871. yield* runPrompt(session, "Earlier question ".repeat(180))
  1872. currentModel = compactModel
  1873. requests.length = 0
  1874. yield* TestLLM.push(
  1875. TestLLM.text("## Objective\n- Preserve the task", "text-summary"),
  1876. TestLLM.textWithUsage("Continued", "text-final", 3_950),
  1877. )
  1878. yield* runPrompt(session, "Recent exact request ".repeat(180))
  1879. expect(requests).toHaveLength(2)
  1880. expect(userTexts(requests[0])[0]).toContain("## Objective")
  1881. expect(userTexts(requests[1])).toHaveLength(1)
  1882. expect(userTexts(requests[1])[0]).toContain("<summary>\n## Objective\n- Preserve the task\n</summary>")
  1883. expect(userTexts(requests[1])[0]).toContain(`[User]: ${"Recent exact request ".repeat(180)}`)
  1884. const context = yield* (yield* SessionStore.Service).context(sessionID)
  1885. expect(context.map((message) => message.type)).toEqual(["compaction", "assistant"])
  1886. expect(context[0]).toMatchObject({
  1887. type: "compaction",
  1888. summary: "## Objective\n- Preserve the task",
  1889. })
  1890. requests.length = 0
  1891. executions.length = 0
  1892. yield* TestLLM.push(
  1893. TestLLM.text("## Objective\n- Preserve the updated task", "text-summary-2"),
  1894. TestLLM.text("Continued again", "text-final-2"),
  1895. )
  1896. yield* runPrompt(session, "Newest exact request ".repeat(180))
  1897. expect(requests).toHaveLength(2)
  1898. expect(userTexts(requests[0])[0]).toContain(
  1899. "<previous-summary>\n## Objective\n- Preserve the task\n</previous-summary>",
  1900. )
  1901. expect(userTexts(requests[0])[0]).toContain("Recent exact request")
  1902. expect((yield* (yield* SessionStore.Service).context(sessionID))[0]).toMatchObject({
  1903. type: "compaction",
  1904. summary: "## Objective\n- Preserve the updated task",
  1905. })
  1906. }),
  1907. )
  1908. it.effect("does not compact immediately when the advertised output limit fills the context", () =>
  1909. Effect.gen(function* () {
  1910. const session = yield* setup
  1911. currentModel = fullOutputModel
  1912. yield* TestLLM.push(TestLLM.textWithUsage("Earlier answer", "text-full-output-first", 9_500))
  1913. yield* runPrompt(session, "Earlier question")
  1914. requests.length = 0
  1915. yield* TestLLM.push(TestLLM.text("Continued", "text-full-output-final"))
  1916. yield* runPrompt(session, "Continue")
  1917. expect(requests).toHaveLength(1)
  1918. expect(userTexts(requests[0])).toContain("Continue")
  1919. expect(yield* session.context(sessionID)).not.toContainEqual(expect.objectContaining({ type: "compaction" }))
  1920. }),
  1921. )
  1922. it.effect("stops after required automatic compaction fails", () =>
  1923. Effect.gen(function* () {
  1924. const session = yield* setup
  1925. yield* TestLLM.push(TestLLM.textWithUsage("Earlier answer", "text-before-failed-compaction", 3_950))
  1926. yield* runPrompt(session, "Earlier question ".repeat(180))
  1927. currentModel = compactModel
  1928. requests.length = 0
  1929. yield* TestLLM.push(
  1930. [LLMEvent.providerError({ message: "Unsupported parameter: max_output_tokens" })],
  1931. TestLLM.text("Must not run", "text-after-failed-compaction"),
  1932. )
  1933. yield* admit(session, "Recent exact request ".repeat(180))
  1934. expect(yield* Effect.exit(session.resume(sessionID))).toMatchObject({ _tag: "Failure" })
  1935. expect(requests).toHaveLength(1)
  1936. expect(requests[0]?.generation).toBeUndefined()
  1937. expect(yield* session.context(sessionID)).toContainEqual(
  1938. expect.objectContaining({
  1939. type: "compaction",
  1940. status: "failed",
  1941. reason: "auto",
  1942. error: expect.objectContaining({ message: "Unsupported parameter: max_output_tokens" }),
  1943. }),
  1944. )
  1945. }),
  1946. )
  1947. it.effect("forces one compaction and retries after provider context overflow", () =>
  1948. Effect.gen(function* () {
  1949. const session = yield* setupOverflowRecovery
  1950. yield* TestLLM.push(
  1951. [
  1952. LLMEvent.stepStart({ index: 0 }),
  1953. LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }),
  1954. ],
  1955. TestLLM.text("## Objective\n- Recover overflow", "text-summary"),
  1956. TestLLM.text("Recovered", "text-final"),
  1957. )
  1958. yield* runPrompt(session, "Continue")
  1959. expect(requests).toHaveLength(3)
  1960. expect(userTexts(requests[1])[0]).toContain("## Objective")
  1961. expect(userTexts(requests[2])[0]).toContain("<summary>\n## Objective\n- Recover overflow\n</summary>")
  1962. expect(yield* session.context(sessionID)).toMatchObject([
  1963. { type: "compaction", summary: "## Objective\n- Recover overflow" },
  1964. { type: "assistant", finish: "stop" },
  1965. ])
  1966. yield* replaySessionProjection(sessionID)
  1967. expect(yield* session.context(sessionID)).toMatchObject([
  1968. { type: "compaction" },
  1969. { type: "assistant", finish: "stop" },
  1970. ])
  1971. }),
  1972. )
  1973. it.effect("recovers from provider context overflow without a configured context limit", () =>
  1974. Effect.gen(function* () {
  1975. const session = yield* setupOverflowRecovery
  1976. currentModel = model
  1977. yield* TestLLM.push(
  1978. [LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" })],
  1979. TestLLM.text("## Objective\n- Recover unknown limit", "text-summary-unknown-limit"),
  1980. TestLLM.text("Recovered", "text-final-unknown-limit"),
  1981. )
  1982. yield* runPrompt(session, "Continue")
  1983. expect(requests).toHaveLength(3)
  1984. expect(yield* session.context(sessionID)).toMatchObject([
  1985. { type: "compaction", summary: "## Objective\n- Recover unknown limit" },
  1986. { type: "assistant", finish: "stop" },
  1987. ])
  1988. }),
  1989. )
  1990. it.effect("recovers from provider context overflow despite an undersized configured context limit", () =>
  1991. Effect.gen(function* () {
  1992. const session = yield* setupOverflowRecovery
  1993. currentModel = undersizedContextModel
  1994. yield* TestLLM.push(
  1995. [LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" })],
  1996. TestLLM.text("## Objective\n- Recover undersized limit", "text-summary-undersized-limit"),
  1997. TestLLM.text("Recovered", "text-final-undersized-limit"),
  1998. )
  1999. yield* runPrompt(session, "Continue")
  2000. expect(requests).toHaveLength(3)
  2001. expect(yield* session.context(sessionID)).toMatchObject([
  2002. { type: "compaction", summary: "## Objective\n- Recover undersized limit" },
  2003. { type: "assistant", finish: "stop" },
  2004. ])
  2005. }),
  2006. )
  2007. it.effect("persists a second context overflow after one recovery", () =>
  2008. Effect.gen(function* () {
  2009. const session = yield* setupOverflowRecovery
  2010. const overflow = () => [
  2011. LLMEvent.stepStart({ index: 0 }),
  2012. LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }),
  2013. ]
  2014. yield* TestLLM.push(overflow(), TestLLM.text("## Objective\n- Recover once", "text-summary"), overflow())
  2015. yield* admit(session, "Continue")
  2016. expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("prompt too long")
  2017. expect(requests).toHaveLength(3)
  2018. expect(yield* session.context(sessionID)).toMatchObject([
  2019. { type: "compaction" },
  2020. { type: "assistant", finish: "error", error: { message: "prompt too long" } },
  2021. ])
  2022. }),
  2023. )
  2024. it.effect("recovers once from a raw context overflow failure", () =>
  2025. Effect.gen(function* () {
  2026. const session = yield* setupOverflowRecovery
  2027. yield* TestLLM.push(
  2028. Stream.fail(
  2029. new AIError({
  2030. module: "test",
  2031. method: "stream",
  2032. reason: new InvalidRequestReason({
  2033. message: "prompt too long",
  2034. classification: "context-overflow",
  2035. }),
  2036. }),
  2037. ),
  2038. )
  2039. yield* TestLLM.push(
  2040. TestLLM.text("## Objective\n- Recover raw overflow", "text-summary"),
  2041. TestLLM.text("Recovered", "text-final"),
  2042. )
  2043. yield* runPrompt(session, "Continue")
  2044. expect(requests).toHaveLength(3)
  2045. expect(yield* session.context(sessionID)).toMatchObject([
  2046. { type: "compaction", summary: "## Objective\n- Recover raw overflow" },
  2047. { type: "assistant", finish: "stop" },
  2048. ])
  2049. }),
  2050. )
  2051. it.effect("publishes the original overflow when recovery summarization fails", () =>
  2052. Effect.gen(function* () {
  2053. const session = yield* setupOverflowRecovery
  2054. yield* TestLLM.push(
  2055. [LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" })],
  2056. [LLMEvent.providerError({ message: "summary unavailable" })],
  2057. )
  2058. yield* admit(session, "Continue")
  2059. expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("prompt too long")
  2060. expect(requests).toHaveLength(2)
  2061. const context = yield* session.context(sessionID)
  2062. expect(context).toContainEqual(
  2063. expect.objectContaining({
  2064. type: "compaction",
  2065. status: "failed",
  2066. reason: "auto",
  2067. error: { type: "provider.error", message: "summary unavailable" },
  2068. }),
  2069. )
  2070. expect(context.slice(-3)).toMatchObject([
  2071. { type: "user", text: "Continue" },
  2072. { type: "compaction", status: "failed", reason: "auto" },
  2073. { type: "assistant", finish: "error", error: { message: "prompt too long" } },
  2074. ])
  2075. }),
  2076. )
  2077. it.effect("interrupts overflow recovery while the summary provider is running", () =>
  2078. Effect.gen(function* () {
  2079. const session = yield* setupOverflowRecovery
  2080. yield* TestLLM.push(
  2081. [LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" })],
  2082. TestLLM.text("## Objective\n- Interrupted", "text-summary"),
  2083. )
  2084. const first = yield* TestLLM.gate
  2085. yield* admit(session, "Continue")
  2086. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2087. yield* first.started
  2088. const summary = yield* TestLLM.gate
  2089. yield* first.release
  2090. yield* summary.started
  2091. yield* session.interrupt(sessionID)
  2092. const exit = yield* Fiber.await(run)
  2093. expect(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)).toBeTrue()
  2094. expect(yield* session.context(sessionID)).toContainEqual(
  2095. expect.objectContaining({
  2096. type: "compaction",
  2097. status: "failed",
  2098. reason: "auto",
  2099. error: { type: "compaction.interrupted", message: "Compaction was interrupted" },
  2100. }),
  2101. )
  2102. }),
  2103. )
  2104. it.effect("uses epoch values after compaction while a source is unavailable", () =>
  2105. Effect.gen(function* () {
  2106. const session = yield* setup
  2107. const bus = yield* Bus.Service
  2108. yield* runPrompt(session, "First")
  2109. systemBaseline = "Changed context"
  2110. yield* runPrompt(session, "Second")
  2111. yield* bus.publish(SessionEvent.Compaction.Started, {
  2112. sessionID,
  2113. reason: "manual",
  2114. recent: "",
  2115. })
  2116. yield* bus.publish(SessionEvent.Compaction.Ended, {
  2117. sessionID,
  2118. reason: "manual",
  2119. text: "summary",
  2120. recent: "",
  2121. })
  2122. systemUnavailable = true
  2123. yield* runPrompt(session, "Third")
  2124. // Compaction already moved current values into the new epoch before the unavailable read.
  2125. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual([defaultSystem, "Changed context"])
  2126. expect(systemTexts(requests.at(-1)!)).not.toContain("Changed context")
  2127. }),
  2128. )
  2129. it.effect("projects reasoning and tool events without executing or continuing tools", () =>
  2130. Effect.gen(function* () {
  2131. const session = yield* setup
  2132. yield* admit(session, "Use tools")
  2133. yield* TestLLM.push(
  2134. TestLLM.complete(
  2135. {
  2136. reason: { normalized: "tool-calls" },
  2137. usage: {
  2138. inputTokens: 10,
  2139. nonCachedInputTokens: 8,
  2140. outputTokens: 4,
  2141. reasoningTokens: 1,
  2142. cacheReadInputTokens: 2,
  2143. },
  2144. },
  2145. LLMEvent.reasoningStart({ id: "reasoning-1" }),
  2146. LLMEvent.reasoningDelta({ id: "reasoning-1", text: "Think" }),
  2147. LLMEvent.reasoningEnd({ id: "reasoning-1" }),
  2148. LLMEvent.toolInputStart({ id: "call-error", name: "write" }),
  2149. LLMEvent.toolInputDelta({ id: "call-error", name: "write", text: '{"path":"README.md"}' }),
  2150. LLMEvent.toolInputEnd({ id: "call-error", name: "write" }),
  2151. LLMEvent.toolCall({ id: "call-error", name: "write", input: { path: "README.md" }, providerExecuted: true }),
  2152. LLMEvent.toolError({ id: "call-error", name: "write", message: "Denied" }),
  2153. LLMEvent.toolResult({ id: "call-error", name: "write", result: { type: "error", value: "Denied" } }),
  2154. LLMEvent.toolCall({
  2155. id: "call-provider",
  2156. name: "web_search",
  2157. input: { query: "hello" },
  2158. providerExecuted: true,
  2159. providerMetadata: { openai: { source: "provider" } },
  2160. }),
  2161. LLMEvent.toolResult({
  2162. id: "call-provider",
  2163. name: "web_search",
  2164. result: {
  2165. type: "content",
  2166. value: [
  2167. { type: "text", text: "Hello" },
  2168. { type: "file", uri: "data:image/png;base64,aGVsbG8=", mime: "image/png", name: "hello.png" },
  2169. ],
  2170. },
  2171. providerExecuted: true,
  2172. providerMetadata: { openai: { source: "provider" } },
  2173. }),
  2174. ),
  2175. )
  2176. yield* session.resume(sessionID)
  2177. expect(requests).toHaveLength(1)
  2178. expect(requests[0]?.tools.map((tool) => tool.name)).toEqual(["defect", "echo", "storefail"])
  2179. expect(yield* session.context(sessionID)).toMatchObject([
  2180. { type: "user", text: "Use tools" },
  2181. {
  2182. type: "assistant",
  2183. finish: "tool-calls",
  2184. cost: 0,
  2185. tokens: { input: 8, output: 3, reasoning: 1, cache: { read: 2, write: 0 } },
  2186. content: [
  2187. { type: "reasoning", text: "Think" },
  2188. {
  2189. type: "tool",
  2190. id: "call-error",
  2191. name: "write",
  2192. state: {
  2193. status: "error",
  2194. input: { path: "README.md" },
  2195. error: { type: "tool.execution", message: "Denied" },
  2196. },
  2197. },
  2198. {
  2199. type: "tool",
  2200. id: "call-provider",
  2201. name: "web_search",
  2202. executed: true,
  2203. providerState: { source: "provider" },
  2204. providerResultState: { source: "provider" },
  2205. state: {
  2206. status: "completed",
  2207. input: { query: "hello" },
  2208. content: [
  2209. { type: "text", text: "Hello" },
  2210. { type: "file", mime: "image/png", uri: "data:image/png;base64,aGVsbG8=", name: "hello.png" },
  2211. ],
  2212. },
  2213. },
  2214. ],
  2215. },
  2216. ])
  2217. }),
  2218. )
  2219. it.effect("continues with reloaded history after durably settling one local tool call", () =>
  2220. Effect.gen(function* () {
  2221. const session = yield* setup
  2222. yield* admit(session, "Echo this")
  2223. yield* TestLLM.push(TestLLM.tool("call-echo", "echo", { text: "hello" }), TestLLM.text("Done", "text-final"))
  2224. yield* session.resume(sessionID)
  2225. expect(requests).toHaveLength(2)
  2226. expect(messageRoles(requests[1])).toEqual(["user", "assistant", "tool"])
  2227. expect(authorizations).toMatchObject([{ sessionID, id: "call-echo" }])
  2228. expect(executions).toEqual(["hello"])
  2229. const context = yield* session.context(sessionID)
  2230. expect(context).toMatchObject([
  2231. { type: "user", text: "Echo this" },
  2232. {
  2233. type: "assistant",
  2234. finish: "tool-calls",
  2235. content: [
  2236. {
  2237. type: "tool",
  2238. id: "call-echo",
  2239. name: "echo",
  2240. state: {
  2241. status: "completed",
  2242. input: { text: "hello" },
  2243. content: [{ type: "text", text: "hello" }],
  2244. },
  2245. },
  2246. ],
  2247. },
  2248. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Done" }] },
  2249. ])
  2250. const assistant = requireAssistant(context)
  2251. expect(yield* recordedStepSettlementTypes(sessionID, assistant.id)).toEqual([
  2252. "session.step.started.1",
  2253. "session.tool.called.1",
  2254. "session.tool.success.2",
  2255. "session.step.ended.1",
  2256. ])
  2257. }),
  2258. )
  2259. it.effect("reloads a model switch before a tool-driven continuation step", () =>
  2260. Effect.gen(function* () {
  2261. const session = yield* setup
  2262. const bus = yield* Bus.Service
  2263. yield* admit(session, "Echo this")
  2264. yield* TestLLM.push(TestLLM.tool("call-echo", "echo", { text: "hello" }), TestLLM.stop())
  2265. const tools = yield* blockTools()
  2266. const run = yield* Effect.forkChild(session.resume(sessionID))
  2267. yield* tools.started
  2268. yield* bus.publish(SessionEvent.ModelSelected, {
  2269. sessionID,
  2270. model: { id: ID.make("replacement"), providerID: Provider.ID.make("fake") },
  2271. })
  2272. systemBaseline = "Replacement context"
  2273. yield* tools.release
  2274. yield* Fiber.join(run)
  2275. expect(requests.map((request) => request.model)).toEqual([model, replacementModel])
  2276. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  2277. [defaultSystem, "Initial context"],
  2278. [defaultSystem, "Initial context"],
  2279. ])
  2280. expect(systemTexts(requests[1])).toContain("Replacement context")
  2281. }),
  2282. )
  2283. it.effect("restores durable reasoning provider metadata in the next request", () =>
  2284. Effect.gen(function* () {
  2285. const session = yield* setup
  2286. yield* admit(session, "Think first")
  2287. yield* TestLLM.push(
  2288. TestLLM.stop(
  2289. LLMEvent.reasoningStart({ id: "reasoning-anthropic" }),
  2290. LLMEvent.reasoningDelta({ id: "reasoning-anthropic", text: "Signed thought" }),
  2291. LLMEvent.reasoningEnd({
  2292. id: "reasoning-anthropic",
  2293. providerMetadata: { openai: { signature: "sig_1" }, anthropic: { ignored: true } },
  2294. }),
  2295. LLMEvent.reasoningStart({
  2296. id: "reasoning-openai",
  2297. providerMetadata: {
  2298. openai: { itemId: "rs_1", reasoningEncryptedContent: null },
  2299. anthropic: { ignored: true },
  2300. },
  2301. }),
  2302. LLMEvent.reasoningDelta({ id: "reasoning-openai", text: "Encrypted thought" }),
  2303. LLMEvent.reasoningEnd({
  2304. id: "reasoning-openai",
  2305. providerMetadata: {
  2306. openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" },
  2307. anthropic: { ignored: true },
  2308. },
  2309. }),
  2310. ),
  2311. )
  2312. yield* session.resume(sessionID)
  2313. yield* replaySessionProjection(sessionID)
  2314. expect(yield* session.context(sessionID)).toMatchObject([
  2315. { type: "user", text: "Think first" },
  2316. {
  2317. type: "assistant",
  2318. content: [
  2319. {
  2320. type: "reasoning",
  2321. text: "Signed thought",
  2322. state: { signature: "sig_1" },
  2323. },
  2324. {
  2325. type: "reasoning",
  2326. text: "Encrypted thought",
  2327. state: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" },
  2328. },
  2329. ],
  2330. },
  2331. ])
  2332. yield* admit(session, "Continue")
  2333. yield* TestLLM.push([])
  2334. yield* session.resume(sessionID)
  2335. expect(requests[1]?.messages[1]?.content).toEqual([
  2336. {
  2337. type: "reasoning",
  2338. text: "Signed thought",
  2339. providerMetadata: { openai: { signature: "sig_1" } },
  2340. },
  2341. {
  2342. type: "reasoning",
  2343. text: "Encrypted thought",
  2344. providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } },
  2345. },
  2346. ])
  2347. }),
  2348. )
  2349. it.effect("restores durable text provider metadata in the next request", () =>
  2350. Effect.gen(function* () {
  2351. const session = yield* setup
  2352. yield* admit(session, "Check first")
  2353. yield* TestLLM.push(
  2354. TestLLM.stop(
  2355. LLMEvent.textStart({ id: "commentary", providerMetadata: { openai: { phase: "commentary" } } }),
  2356. LLMEvent.textDelta({ id: "commentary", text: "Checking." }),
  2357. LLMEvent.textEnd({
  2358. id: "commentary",
  2359. providerMetadata: { openai: { phase: "commentary" }, anthropic: { ignored: true } },
  2360. }),
  2361. ),
  2362. )
  2363. yield* session.resume(sessionID)
  2364. yield* replaySessionProjection(sessionID)
  2365. expect(yield* session.context(sessionID)).toMatchObject([
  2366. { type: "user", text: "Check first" },
  2367. {
  2368. type: "assistant",
  2369. content: [{ type: "text", text: "Checking.", state: { phase: "commentary" } }],
  2370. },
  2371. ])
  2372. yield* admit(session, "Continue")
  2373. yield* TestLLM.push([])
  2374. yield* session.resume(sessionID)
  2375. expect(requests[1]?.messages[1]?.content).toEqual([
  2376. {
  2377. type: "text",
  2378. text: "Checking.",
  2379. providerMetadata: { openai: { phase: "commentary" } },
  2380. },
  2381. ])
  2382. }),
  2383. )
  2384. it.effect("replays durable provider-executed tool results inline in the next request", () =>
  2385. Effect.gen(function* () {
  2386. const session = yield* setup
  2387. yield* admit(session, "Search first")
  2388. yield* TestLLM.push(
  2389. TestLLM.stop(
  2390. LLMEvent.toolCall({
  2391. id: "hosted-search",
  2392. name: "web_search",
  2393. input: { query: "Effect" },
  2394. providerExecuted: true,
  2395. providerMetadata: { openai: { itemId: "hosted-search" }, fake: { ignored: true } },
  2396. }),
  2397. LLMEvent.toolResult({
  2398. id: "hosted-search",
  2399. name: "web_search",
  2400. result: { type: "json", value: [{ title: "Effect" }] },
  2401. providerExecuted: true,
  2402. providerMetadata: { openai: { blockType: "web_search_tool_result" }, anthropic: { ignored: true } },
  2403. }),
  2404. ),
  2405. )
  2406. yield* session.resume(sessionID)
  2407. yield* replaySessionProjection(sessionID)
  2408. yield* admit(session, "Continue")
  2409. yield* TestLLM.push([])
  2410. yield* session.resume(sessionID)
  2411. expect(messageRoles(requests[1])).toEqual(["user", "assistant", "user"])
  2412. expect(requests[1]?.messages[1]?.content).toMatchObject([
  2413. {
  2414. type: "tool-call",
  2415. id: "hosted-search",
  2416. name: "web_search",
  2417. input: { query: "Effect" },
  2418. providerExecuted: true,
  2419. providerMetadata: { openai: { itemId: "hosted-search" } },
  2420. },
  2421. {
  2422. type: "tool-result",
  2423. id: "hosted-search",
  2424. name: "web_search",
  2425. // The generic replay result derives from canonical stored content.
  2426. result: { type: "text", value: '[{"title":"Effect"}]' },
  2427. providerExecuted: true,
  2428. providerMetadata: { openai: { blockType: "web_search_tool_result" } },
  2429. },
  2430. ])
  2431. }),
  2432. )
  2433. it.effect("starts recorded local tools eagerly and awaits settlement before continuing", () =>
  2434. Effect.gen(function* () {
  2435. const session = yield* setup
  2436. yield* admit(session, "Echo five times")
  2437. const tools = yield* blockTools(5)
  2438. const providerGate = yield* Deferred.make<void>()
  2439. const initial = Stream.fromIterable([
  2440. LLMEvent.stepStart({ index: 0 }),
  2441. ...Array.from({ length: 5 }, (_, index) =>
  2442. LLMEvent.toolCall({ id: `call-echo-${index}`, name: "echo", input: { text: `${index}` } }),
  2443. ),
  2444. ])
  2445. const final = Stream.fromIterable([
  2446. LLMEvent.stepFinish({ index: 0, reason: { normalized: "tool-calls" } }),
  2447. LLMEvent.finish({ reason: { normalized: "tool-calls" } }),
  2448. ])
  2449. yield* TestLLM.push(
  2450. Stream.concat(initial, Stream.fromEffect(Deferred.await(providerGate)).pipe(Stream.flatMap(() => final))),
  2451. )
  2452. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2453. yield* tools.started
  2454. expect(executions).toHaveLength(5)
  2455. expect(yield* tools.maxActive).toBe(5)
  2456. expect(yield* session.context(sessionID)).toMatchObject([
  2457. { type: "user", text: "Echo five times" },
  2458. {
  2459. type: "assistant",
  2460. content: Array.from({ length: 5 }, (_, index) => ({
  2461. type: "tool",
  2462. id: `call-echo-${index}`,
  2463. state: { status: "running", input: { text: `${index}` } },
  2464. })),
  2465. },
  2466. ])
  2467. yield* Deferred.succeed(providerGate, undefined)
  2468. yield* Effect.yieldNow
  2469. expect(requests).toHaveLength(1)
  2470. yield* tools.release
  2471. yield* Fiber.join(run)
  2472. expect(executions).toHaveLength(5)
  2473. expect(yield* tools.maxActive).toBe(5)
  2474. expect(requests).toHaveLength(2)
  2475. }),
  2476. )
  2477. it.effect("settles repeated provider-local tool call IDs against their owning assistant messages", () =>
  2478. Effect.gen(function* () {
  2479. const session = yield* setup
  2480. yield* admit(session, "Echo twice")
  2481. yield* TestLLM.push(
  2482. TestLLM.tool("tool_0", "echo", { text: "first" }),
  2483. TestLLM.tool("tool_0", "echo", { text: "second" }),
  2484. [],
  2485. )
  2486. yield* session.resume(sessionID)
  2487. const expected = [
  2488. { type: "user", text: "Echo twice" },
  2489. {
  2490. type: "assistant",
  2491. content: [
  2492. {
  2493. type: "tool",
  2494. id: "tool_0",
  2495. state: { status: "completed", content: [{ type: "text", text: "first" }] },
  2496. },
  2497. ],
  2498. },
  2499. {
  2500. type: "assistant",
  2501. content: [
  2502. {
  2503. type: "tool",
  2504. id: "tool_0",
  2505. state: { status: "completed", content: [{ type: "text", text: "second" }] },
  2506. },
  2507. ],
  2508. },
  2509. ]
  2510. expect(executions).toEqual(["first", "second"])
  2511. expect(requests).toHaveLength(3)
  2512. expect(yield* session.context(sessionID)).toMatchObject(expected)
  2513. yield* replaySessionProjection(sessionID)
  2514. expect(yield* session.context(sessionID)).toMatchObject(expected)
  2515. }),
  2516. )
  2517. it.effect("joins concurrent resume calls into one active provider run", () =>
  2518. Effect.gen(function* () {
  2519. const session = yield* setup
  2520. yield* admit(session, "Run once")
  2521. yield* TestLLM.push(TestLLM.text("Once", "text-once"))
  2522. const stream = yield* TestLLM.gate
  2523. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2524. yield* stream.started
  2525. const second = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2526. yield* Effect.yieldNow
  2527. expect(requests).toHaveLength(1)
  2528. yield* stream.release
  2529. yield* Fiber.join(first)
  2530. yield* Fiber.join(second)
  2531. expect(requests).toHaveLength(1)
  2532. expect(yield* session.context(sessionID)).toMatchObject([
  2533. { type: "user", text: "Run once" },
  2534. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Once" }] },
  2535. ])
  2536. }),
  2537. )
  2538. it.effect("steers an active step with newly recorded prompts", () =>
  2539. Effect.gen(function* () {
  2540. const session = yield* setup
  2541. yield* admit(session, "Start working")
  2542. yield* TestLLM.push(TestLLM.stop(), TestLLM.stop())
  2543. const stream = yield* TestLLM.gate
  2544. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2545. yield* stream.started
  2546. yield* session.prompt({ sessionID, text: "Change direction" })
  2547. yield* stream.release
  2548. yield* Fiber.join(first)
  2549. yield* Effect.yieldNow
  2550. expect(requests).toHaveLength(2)
  2551. expect(userTexts(requests[0])).toEqual(["Start working"])
  2552. expect(userTexts(requests[1])).toEqual(["Start working", "Change direction"])
  2553. expect((yield* session.context(sessionID)).map((message) => message.type)).toEqual([
  2554. "user",
  2555. "assistant",
  2556. "user",
  2557. "assistant",
  2558. ])
  2559. }),
  2560. )
  2561. it.effect("promotes queued input after continuation ends", () =>
  2562. Effect.gen(function* () {
  2563. const session = yield* setup
  2564. yield* admit(session, "Start working")
  2565. yield* TestLLM.push(TestLLM.tool("call-echo", "echo", { text: "hello" }), TestLLM.stop(), TestLLM.stop())
  2566. const stream = yield* TestLLM.gate
  2567. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2568. yield* stream.started
  2569. yield* session.prompt({
  2570. sessionID,
  2571. text: "Wait until continuation ends",
  2572. delivery: "queue",
  2573. })
  2574. yield* stream.release
  2575. yield* Fiber.join(first)
  2576. expect(requests).toHaveLength(3)
  2577. expect(userTexts(requests[0])).toEqual(["Start working"])
  2578. expect(userTexts(requests[1])).toEqual(["Start working"])
  2579. expect(userTexts(requests[2])).toEqual(["Start working", "Wait until continuation ends"])
  2580. }),
  2581. )
  2582. it.effect("preserves durable queued input for a later wake after interruption", () =>
  2583. Effect.gen(function* () {
  2584. const session = yield* setup
  2585. const { db } = yield* Database.Service
  2586. yield* admit(session, "Interrupt current work")
  2587. yield* TestLLM.push([], TestLLM.stop())
  2588. const stream = yield* TestLLM.gate
  2589. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2590. yield* stream.started
  2591. yield* session.prompt({
  2592. sessionID,
  2593. text: "Run after interrupt",
  2594. delivery: "queue",
  2595. })
  2596. yield* session.interrupt(sessionID)
  2597. expect(yield* Fiber.await(run)).toMatchObject({ _tag: "Failure" })
  2598. expect(requests).toHaveLength(1)
  2599. expect(yield* SessionPending.has(db, sessionID, "queue")).toBe(true)
  2600. const resumed = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2601. yield* stream.started
  2602. yield* stream.release
  2603. yield* Fiber.join(resumed)
  2604. expect(requests).toHaveLength(2)
  2605. expect(userTexts(requests[0])).toEqual(["Interrupt current work"])
  2606. expect(userTexts(requests[1])).toEqual(["Interrupt current work", "Run after interrupt"])
  2607. }),
  2608. )
  2609. it.effect("preserves durable steering input for a later resume after interruption", () =>
  2610. Effect.gen(function* () {
  2611. const session = yield* setup
  2612. const { db } = yield* Database.Service
  2613. yield* admit(session, "Interrupt current work")
  2614. yield* TestLLM.push([], TestLLM.stop())
  2615. const stream = yield* TestLLM.gate
  2616. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2617. yield* stream.started
  2618. yield* session.prompt({
  2619. sessionID,
  2620. text: "Steer after interrupt",
  2621. })
  2622. yield* session.interrupt(sessionID)
  2623. expect(yield* Fiber.await(run)).toMatchObject({ _tag: "Failure" })
  2624. expect(requests).toHaveLength(1)
  2625. expect(yield* SessionPending.has(db, sessionID, "steer")).toBe(true)
  2626. const resumed = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2627. yield* stream.started
  2628. yield* stream.release
  2629. yield* Fiber.join(resumed)
  2630. expect(requests).toHaveLength(2)
  2631. expect(userTexts(requests[0])).toEqual(["Interrupt current work"])
  2632. expect(userTexts(requests[1])).toEqual(["Interrupt current work", "Steer after interrupt"])
  2633. }),
  2634. )
  2635. it.effect("promotes queued inputs one at a time in FIFO order", () =>
  2636. Effect.gen(function* () {
  2637. const session = yield* setup
  2638. yield* admit(session, "Start working")
  2639. yield* TestLLM.push(TestLLM.stop(), TestLLM.stop(), TestLLM.stop())
  2640. const stream = yield* TestLLM.gate
  2641. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2642. yield* stream.started
  2643. yield* session.prompt({ sessionID, text: "Queue first", delivery: "queue" })
  2644. yield* session.prompt({ sessionID, text: "Queue second", delivery: "queue" })
  2645. yield* stream.release
  2646. yield* Fiber.join(first)
  2647. expect(requests).toHaveLength(3)
  2648. expect(userTexts(requests[0])).toEqual(["Start working"])
  2649. expect(userTexts(requests[1])).toEqual(["Start working", "Queue first"])
  2650. expect(userTexts(requests[2])).toEqual(["Start working", "Queue first", "Queue second"])
  2651. }),
  2652. )
  2653. it.effect("promotes queued input after steering continuation ends", () =>
  2654. Effect.gen(function* () {
  2655. const session = yield* setup
  2656. yield* admit(session, "Start steering")
  2657. yield* session.prompt({
  2658. sessionID,
  2659. text: "Queue for later",
  2660. delivery: "queue",
  2661. resume: false,
  2662. })
  2663. yield* TestLLM.push(TestLLM.stop(), TestLLM.stop())
  2664. yield* session.resume(sessionID)
  2665. expect(requests).toHaveLength(2)
  2666. expect(userTexts(requests[0])).toEqual(["Start steering"])
  2667. expect(userTexts(requests[1])).toEqual(["Start steering", "Queue for later"])
  2668. }),
  2669. )
  2670. it.effect("promotes steers before the next queued input", () =>
  2671. Effect.gen(function* () {
  2672. const session = yield* setup
  2673. yield* admit(session, "Start working")
  2674. yield* TestLLM.push(TestLLM.stop(), TestLLM.stop(), TestLLM.stop(), TestLLM.stop())
  2675. const firstStream = yield* TestLLM.gate
  2676. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2677. yield* firstStream.started
  2678. yield* session.prompt({ sessionID, text: "Queue first", delivery: "queue" })
  2679. yield* session.prompt({ sessionID, text: "Queue second", delivery: "queue" })
  2680. const secondStream = yield* TestLLM.gate
  2681. yield* firstStream.release
  2682. yield* secondStream.started
  2683. yield* session.prompt({ sessionID, text: "Steer before next queued input" })
  2684. yield* session.prompt({
  2685. sessionID,
  2686. text: "Also steer before next queued input",
  2687. })
  2688. yield* session.synthetic({ sessionID, text: "Background completion before next queued input" })
  2689. yield* secondStream.release
  2690. yield* Fiber.join(first)
  2691. expect(requests).toHaveLength(4)
  2692. expect(userTexts(requests[0])).toEqual(["Start working"])
  2693. expect(userTexts(requests[1])).toEqual(["Start working", "Queue first"])
  2694. expect(userTexts(requests[2])).toEqual([
  2695. "Start working",
  2696. "Queue first",
  2697. "Steer before next queued input",
  2698. "Also steer before next queued input",
  2699. "Background completion before next queued input",
  2700. ])
  2701. expect(userTexts(requests[3])).toEqual([
  2702. "Start working",
  2703. "Queue first",
  2704. "Steer before next queued input",
  2705. "Also steer before next queued input",
  2706. "Background completion before next queued input",
  2707. "Queue second",
  2708. ])
  2709. }),
  2710. )
  2711. it.effect("coalesces multiple active steering prompts into one continuation step", () =>
  2712. Effect.gen(function* () {
  2713. const session = yield* setup
  2714. yield* admit(session, "Start working")
  2715. yield* TestLLM.push(TestLLM.stop(), TestLLM.stop())
  2716. const stream = yield* TestLLM.gate
  2717. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2718. yield* stream.started
  2719. yield* session.prompt({ sessionID, text: "First steer" })
  2720. yield* session.prompt({ sessionID, text: "Second steer" })
  2721. yield* stream.release
  2722. yield* Fiber.join(first)
  2723. yield* Effect.yieldNow
  2724. expect(requests).toHaveLength(2)
  2725. expect(userTexts(requests[1])).toEqual(["Start working", "First steer", "Second steer"])
  2726. yield* (yield* SessionExecution.Service).wake(sessionID)
  2727. yield* Effect.yieldNow
  2728. expect(requests).toHaveLength(2)
  2729. }),
  2730. )
  2731. it.effect("runs steering input accepted while the active step fails", () =>
  2732. Effect.gen(function* () {
  2733. const session = yield* setup
  2734. yield* admit(session, "Start working")
  2735. const failure = invalidRequest()
  2736. yield* TestLLM.push(Stream.fail(failure))
  2737. const stream = yield* TestLLM.gate
  2738. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2739. yield* stream.started
  2740. yield* session.prompt({ sessionID, text: "Recover with this" })
  2741. yield* stream.release
  2742. expect(yield* Fiber.join(first).pipe(Effect.flip)).toBe(failure)
  2743. yield* TestLLM.push([])
  2744. yield* session.wait(sessionID)
  2745. expect(requests).toHaveLength(2)
  2746. expect(userTexts(requests[1])).toEqual(["Start working", "Recover with this"])
  2747. }),
  2748. )
  2749. it.effect("durably fails local tools left running by a prior process before continuing", () =>
  2750. Effect.gen(function* () {
  2751. const session = yield* setup
  2752. const bus = yield* Bus.Service
  2753. yield* admit(session, "Recover interrupted tool")
  2754. yield* SessionPending.promote((yield* Database.Service).db, bus, sessionID, "steer")
  2755. const assistantMessageID = SessionMessage.ID.create()
  2756. yield* bus.publish(SessionEvent.Step.Started, {
  2757. sessionID,
  2758. assistantMessageID,
  2759. agent: Agent.ID.make("build"),
  2760. model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") },
  2761. })
  2762. yield* bus.publish(SessionEvent.Tool.Input.Started, {
  2763. sessionID,
  2764. assistantMessageID,
  2765. id: "call-interrupted",
  2766. name: "echo",
  2767. })
  2768. yield* bus.publish(SessionEvent.Tool.Input.Ended, {
  2769. sessionID,
  2770. assistantMessageID,
  2771. id: "call-interrupted",
  2772. text: '{"text":"stale"}',
  2773. })
  2774. yield* bus.publish(SessionEvent.Tool.Called, {
  2775. sessionID,
  2776. assistantMessageID,
  2777. id: "call-interrupted",
  2778. input: { text: "stale" },
  2779. executed: false,
  2780. })
  2781. requests.length = 0
  2782. yield* TestLLM.push([])
  2783. yield* session.resume(sessionID)
  2784. expect(requests).toHaveLength(1)
  2785. expect(messageRoles(requests[0])).toEqual(["user", "assistant", "tool"])
  2786. expect(yield* session.context(sessionID)).toMatchObject([
  2787. { type: "user", text: "Recover interrupted tool" },
  2788. {
  2789. type: "assistant",
  2790. content: [
  2791. {
  2792. type: "tool",
  2793. id: "call-interrupted",
  2794. state: {
  2795. status: "error",
  2796. error: { type: "aborted", message: "Tool execution interrupted: echo" },
  2797. },
  2798. },
  2799. ],
  2800. },
  2801. ])
  2802. }),
  2803. )
  2804. it.effect("durably fails hosted tools left running by a prior process before continuing inline", () =>
  2805. Effect.gen(function* () {
  2806. const session = yield* setup
  2807. const bus = yield* Bus.Service
  2808. yield* admit(session, "Recover interrupted hosted tool")
  2809. yield* SessionPending.promote((yield* Database.Service).db, bus, sessionID, "steer")
  2810. const assistantMessageID = SessionMessage.ID.create()
  2811. yield* bus.publish(SessionEvent.Step.Started, {
  2812. sessionID,
  2813. assistantMessageID,
  2814. agent: Agent.ID.make("build"),
  2815. model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") },
  2816. })
  2817. yield* bus.publish(SessionEvent.Tool.Input.Started, {
  2818. sessionID,
  2819. assistantMessageID,
  2820. id: "call-hosted-interrupted",
  2821. name: "web_search",
  2822. })
  2823. yield* bus.publish(SessionEvent.Tool.Input.Ended, {
  2824. sessionID,
  2825. assistantMessageID,
  2826. id: "call-hosted-interrupted",
  2827. text: '{"query":"stale"}',
  2828. })
  2829. yield* bus.publish(SessionEvent.Tool.Called, {
  2830. sessionID,
  2831. assistantMessageID,
  2832. id: "call-hosted-interrupted",
  2833. input: { query: "stale" },
  2834. executed: true,
  2835. state: { itemId: "call-hosted-interrupted" },
  2836. })
  2837. requests.length = 0
  2838. yield* TestLLM.push([])
  2839. yield* session.resume(sessionID)
  2840. expect(requests).toHaveLength(1)
  2841. expect(messageRoles(requests[0])).toEqual(["user", "assistant"])
  2842. expect(requests[0]?.messages[1]?.content).toMatchObject([
  2843. {
  2844. type: "tool-call",
  2845. id: "call-hosted-interrupted",
  2846. providerExecuted: true,
  2847. providerMetadata: { openai: { itemId: "call-hosted-interrupted" } },
  2848. },
  2849. { type: "tool-result", id: "call-hosted-interrupted", providerExecuted: true, result: { type: "error" } },
  2850. ])
  2851. }),
  2852. )
  2853. it.effect("durably fails pending tool input left by a prior process before continuing", () =>
  2854. Effect.gen(function* () {
  2855. const session = yield* setup
  2856. const bus = yield* Bus.Service
  2857. yield* admit(session, "Recover interrupted tool input")
  2858. yield* SessionPending.promote((yield* Database.Service).db, bus, sessionID, "steer")
  2859. const assistantMessageID = SessionMessage.ID.create()
  2860. yield* bus.publish(SessionEvent.Step.Started, {
  2861. sessionID,
  2862. assistantMessageID,
  2863. agent: Agent.ID.make("build"),
  2864. model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") },
  2865. })
  2866. yield* bus.publish(SessionEvent.Tool.Input.Started, {
  2867. sessionID,
  2868. assistantMessageID,
  2869. id: "call-pending-interrupted",
  2870. name: "echo",
  2871. })
  2872. requests.length = 0
  2873. yield* TestLLM.push([])
  2874. yield* session.resume(sessionID)
  2875. expect(requests).toHaveLength(1)
  2876. expect(messageRoles(requests[0])).toEqual(["user", "assistant", "tool"])
  2877. expect(yield* session.context(sessionID)).toMatchObject([
  2878. { type: "user", text: "Recover interrupted tool input" },
  2879. { type: "assistant", content: [{ type: "tool", id: "call-pending-interrupted", state: { status: "error" } }] },
  2880. ])
  2881. }),
  2882. )
  2883. it.effect("promotes the first queued input when woken while idle", () =>
  2884. Effect.gen(function* () {
  2885. const session = yield* setup
  2886. yield* session.prompt({
  2887. sessionID,
  2888. text: "Wait in queue",
  2889. delivery: "queue",
  2890. resume: false,
  2891. })
  2892. const stream = yield* TestLLM.gate
  2893. yield* (yield* SessionExecution.Service).wake(sessionID)
  2894. yield* stream.started
  2895. yield* stream.release
  2896. expect(requests).toHaveLength(1)
  2897. expect(userTexts(requests[0])).toEqual(["Wait in queue"])
  2898. }),
  2899. )
  2900. it.effect("retries inbox input after prompt projection rolls back", () =>
  2901. Effect.gen(function* () {
  2902. const session = yield* setup
  2903. const bus = yield* Bus.Service
  2904. const defect = new Error("fail after prompt promotion")
  2905. let fail = true
  2906. yield* bus.project(SessionEvent.InputPromoted, () => (fail ? Effect.die(defect) : Effect.void))
  2907. yield* admit(session, "Recover promoted input")
  2908. expect(yield* session.resume(sessionID).pipe(Effect.catchDefect(Effect.succeed))).toBe(defect)
  2909. fail = false
  2910. requests.length = 0
  2911. yield* TestLLM.push(TestLLM.stop())
  2912. const stream = yield* TestLLM.gate
  2913. yield* (yield* SessionExecution.Service).wake(sessionID)
  2914. yield* stream.started
  2915. yield* stream.release
  2916. expect(userTexts(requests[0])).toEqual(["Recover promoted input"])
  2917. }),
  2918. )
  2919. it.effect("does not strand a committed promotion when a post-commit listener defects", () =>
  2920. Effect.gen(function* () {
  2921. const session = yield* setup
  2922. const bus = yield* Bus.Service
  2923. yield* bus.listen((event) =>
  2924. event.type === SessionEvent.InputPromoted.type
  2925. ? Effect.die("fail after prompt promotion commits")
  2926. : Effect.void,
  2927. )
  2928. yield* runPrompt(session, "Run committed promotion")
  2929. expect(requests).toHaveLength(1)
  2930. expect(userTexts(requests[0])).toEqual(["Run committed promotion"])
  2931. }),
  2932. )
  2933. it.effect("adds session correlation headers to model requests", () =>
  2934. Effect.gen(function* () {
  2935. const session = yield* setup
  2936. yield* runPrompt(session, "Run correlated request")
  2937. expect(requests[0]?.http?.headers).toEqual({
  2938. "x-session-affinity": sessionID,
  2939. "X-Session-Id": sessionID,
  2940. "User-Agent": App.useragent(App.make()),
  2941. "x-opencode-project": Project.ID.global,
  2942. "x-opencode-session": sessionID,
  2943. "x-opencode-client": "opencode",
  2944. })
  2945. }),
  2946. )
  2947. it.effect("adds the parent session header to child model requests", () =>
  2948. Effect.gen(function* () {
  2949. const session = yield* setup
  2950. const parentID = Session.ID.make("ses_runner_parent")
  2951. const { db } = yield* Database.Service
  2952. yield* db
  2953. .update(SessionTable)
  2954. .set({ parent_id: parentID })
  2955. .where(eq(SessionTable.id, sessionID))
  2956. .run()
  2957. .pipe(Effect.orDie)
  2958. yield* runPrompt(session, "Run child request")
  2959. expect(requests[0]?.http?.headers?.["x-parent-session-id"]).toBe(parentID)
  2960. }),
  2961. )
  2962. it.effect("runs different sessions concurrently", () =>
  2963. Effect.gen(function* () {
  2964. const session = yield* setup
  2965. yield* insertSession(otherSessionID)
  2966. yield* admit(session, "Run first")
  2967. yield* session.prompt({
  2968. sessionID: otherSessionID,
  2969. text: "Run second",
  2970. resume: false,
  2971. })
  2972. const stream = yield* TestLLM.gate
  2973. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2974. yield* stream.started
  2975. const second = yield* session.resume(otherSessionID).pipe(Effect.forkChild)
  2976. yield* stream.started
  2977. expect(requests).toHaveLength(2)
  2978. expect(requests.map((request) => request.providerOptions?.openai?.promptCacheKey)).toEqual([
  2979. sessionID,
  2980. otherSessionID,
  2981. ])
  2982. yield* stream.release
  2983. yield* Fiber.join(first)
  2984. yield* Fiber.join(second)
  2985. }),
  2986. )
  2987. it.effect("bounds 64-character session prompt cache keys", () =>
  2988. Effect.gen(function* () {
  2989. const session = yield* setup
  2990. const longSessionID = Session.ID.make(`ses_${"a".repeat(64)}`)
  2991. const otherLongSessionID = Session.ID.make(`ses_${"b".repeat(64)}`)
  2992. yield* insertSession(longSessionID)
  2993. yield* insertSession(otherLongSessionID)
  2994. yield* session.prompt({
  2995. sessionID: longSessionID,
  2996. text: "Run long session",
  2997. resume: false,
  2998. })
  2999. yield* session.prompt({
  3000. sessionID: otherLongSessionID,
  3001. text: "Run other long session",
  3002. resume: false,
  3003. })
  3004. yield* session.resume(longSessionID)
  3005. yield* session.resume(otherLongSessionID)
  3006. const keys = requests.map((request) => request.providerOptions?.openai?.promptCacheKey)
  3007. expect(keys).toEqual([longSessionID.slice(4), otherLongSessionID.slice(4)])
  3008. expect(keys.every((key) => typeof key === "string" && key.length === 64)).toBe(true)
  3009. expect(keys[0]).not.toBe(keys[1])
  3010. }),
  3011. )
  3012. it.effect("fans out one failed run and allows a later retry", () =>
  3013. Effect.gen(function* () {
  3014. const session = yield* setup
  3015. yield* admit(session, "Retry after failure")
  3016. yield* TestLLM.push(Stream.fail(invalidRequest()))
  3017. const stream = yield* TestLLM.gate
  3018. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3019. yield* stream.started
  3020. const second = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3021. yield* Effect.yieldNow
  3022. expect(requests).toHaveLength(1)
  3023. yield* stream.release
  3024. const [firstExit, secondExit] = yield* Effect.all([Fiber.await(first), Fiber.await(second)])
  3025. expect(secondExit).toEqual(firstExit)
  3026. yield* TestLLM.push([])
  3027. yield* session.resume(sessionID)
  3028. expect(requests).toHaveLength(2)
  3029. }),
  3030. )
  3031. it.effect("durably settles local tool failures before continuing", () =>
  3032. Effect.gen(function* () {
  3033. const session = yield* setup
  3034. yield* admit(session, "Call missing")
  3035. yield* TestLLM.push(TestLLM.tool("call-missing", "missing", {}), TestLLM.text("Recovered", "text-after-error"))
  3036. yield* session.resume(sessionID)
  3037. expect(requests).toHaveLength(2)
  3038. expect(yield* session.context(sessionID)).toMatchObject([
  3039. { type: "user", text: "Call missing" },
  3040. {
  3041. type: "assistant",
  3042. content: [
  3043. {
  3044. type: "tool",
  3045. id: "call-missing",
  3046. state: {
  3047. status: "error",
  3048. error: { type: "tool.execution", message: "Unknown tool: missing" },
  3049. },
  3050. },
  3051. ],
  3052. },
  3053. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Recovered" }] },
  3054. ])
  3055. }),
  3056. )
  3057. it.effect("returns unexpected local tool defects to the model and continues", () =>
  3058. Effect.gen(function* () {
  3059. const session = yield* setup
  3060. yield* admit(session, "Call defect")
  3061. yield* TestLLM.push(TestLLM.tool("call-defect", "defect", {}), TestLLM.text("Recovered", "text-after-defect"))
  3062. yield* session.resume(sessionID)
  3063. expect(requests).toHaveLength(2)
  3064. expect(messageRoles(requests[1])).toEqual(["user", "assistant", "tool"])
  3065. const context = yield* session.context(sessionID)
  3066. expect(context).toMatchObject([
  3067. { type: "user", text: "Call defect" },
  3068. {
  3069. type: "assistant",
  3070. content: [
  3071. {
  3072. type: "tool",
  3073. id: "call-defect",
  3074. state: {
  3075. status: "error",
  3076. error: { type: "unknown", message: "unexpected tool defect" },
  3077. },
  3078. },
  3079. ],
  3080. },
  3081. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Recovered" }] },
  3082. ])
  3083. const assistant = requireAssistant(context)
  3084. expect(yield* recordedStepSettlementTypes(sessionID, assistant.id)).toEqual([
  3085. "session.step.started.1",
  3086. "session.tool.called.1",
  3087. "session.tool.failed.2",
  3088. "session.step.ended.1",
  3089. ])
  3090. }),
  3091. )
  3092. it.effect("returns tool-wrapped policy blocks to the model and continues", () =>
  3093. Effect.gen(function* () {
  3094. const session = yield* setup
  3095. const registry = yield* Tool.Service
  3096. yield* transformTools(
  3097. registry,
  3098. {
  3099. blocked: {
  3100. name: "blocked",
  3101. description: "Fail because policy blocked execution",
  3102. input: Schema.Struct({}),
  3103. output: Schema.Struct({}),
  3104. execute: () =>
  3105. Effect.fail(new Permission.BlockedError({ rules: [], permission: "blocked", resources: ["*"] })).pipe(
  3106. Effect.mapError(() => new Tool.Error({ message: "Permission blocked" })),
  3107. ),
  3108. },
  3109. },
  3110. { codemode: false },
  3111. )
  3112. yield* admit(session, "Call blocked")
  3113. yield* TestLLM.push(TestLLM.tool("call-blocked", "blocked", {}), TestLLM.stop())
  3114. yield* session.resume(sessionID)
  3115. expect(requests).toHaveLength(2)
  3116. expect(yield* session.context(sessionID)).toMatchObject([
  3117. { type: "user", text: "Call blocked" },
  3118. {
  3119. type: "assistant",
  3120. content: [
  3121. { type: "tool", id: "call-blocked", state: { status: "error", error: { message: "Permission blocked" } } },
  3122. ],
  3123. },
  3124. { type: "assistant", finish: "stop" },
  3125. ])
  3126. }),
  3127. )
  3128. it.effect("interrupts runner continuation when permission approval is declined", () =>
  3129. Effect.gen(function* () {
  3130. const session = yield* setup
  3131. const registry = yield* Tool.Service
  3132. yield* transformTools(
  3133. registry,
  3134. {
  3135. declined: {
  3136. name: "declined",
  3137. description: "Fail because the user declined approval",
  3138. input: Schema.Struct({}),
  3139. output: Schema.Struct({}),
  3140. execute: () => Effect.die(new Permission.DeclinedError()),
  3141. },
  3142. },
  3143. { codemode: false },
  3144. )
  3145. yield* admit(session, "Call declined")
  3146. yield* TestLLM.push(TestLLM.tool("call-declined", "declined", {}))
  3147. const exit = yield* session.resume(sessionID).pipe(Effect.exit)
  3148. expect(exit._tag).toBe("Failure")
  3149. if (exit._tag === "Failure") expect(Cause.hasInterruptsOnly(exit.cause)).toBe(true)
  3150. expect(requests).toHaveLength(1)
  3151. expect(yield* session.context(sessionID)).toMatchObject([
  3152. { type: "user", text: "Call declined" },
  3153. {
  3154. type: "assistant",
  3155. content: [
  3156. {
  3157. type: "tool",
  3158. id: "call-declined",
  3159. state: { status: "error", error: { type: "aborted", message: "The user declined this tool call" } },
  3160. },
  3161. ],
  3162. },
  3163. ])
  3164. }),
  3165. )
  3166. it.effect("returns permission corrections to the model and continues", () =>
  3167. Effect.gen(function* () {
  3168. const session = yield* setup
  3169. const registry = yield* Tool.Service
  3170. yield* transformTools(
  3171. registry,
  3172. {
  3173. corrected: {
  3174. name: "corrected",
  3175. description: "Fail with user correction feedback",
  3176. input: Schema.Struct({}),
  3177. output: Schema.Struct({}),
  3178. execute: () =>
  3179. Effect.fail(new Permission.CorrectedError({ feedback: "Use another tool" })).pipe(
  3180. Effect.mapError(() => new Tool.Error({ message: "Use another tool" })),
  3181. ),
  3182. },
  3183. },
  3184. { codemode: false },
  3185. )
  3186. yield* admit(session, "Call corrected")
  3187. yield* TestLLM.push(TestLLM.tool("call-corrected", "corrected", {}), TestLLM.stop())
  3188. yield* session.resume(sessionID)
  3189. expect(requests).toHaveLength(2)
  3190. expect(yield* session.context(sessionID)).toMatchObject([
  3191. { type: "user", text: "Call corrected" },
  3192. {
  3193. type: "assistant",
  3194. content: [
  3195. { type: "tool", id: "call-corrected", state: { status: "error", error: { message: "Use another tool" } } },
  3196. ],
  3197. },
  3198. { type: "assistant", finish: "stop" },
  3199. ])
  3200. }),
  3201. )
  3202. it.effect("returns configured permission denials to the model and continues", () =>
  3203. Effect.gen(function* () {
  3204. const session = yield* setup
  3205. const registry = yield* Tool.Service
  3206. yield* transformTools(registry, { permissionfail: permissionFail }, { codemode: false })
  3207. yield* admit(session, "Reject permission")
  3208. yield* TestLLM.push(TestLLM.tool("call-permission", "permissionfail", {}), [
  3209. LLMEvent.stepStart({ index: 0 }),
  3210. LLMEvent.stepFinish({ index: 0, reason: { normalized: "stop" } }),
  3211. ])
  3212. yield* session.resume(sessionID)
  3213. expect(requests).toHaveLength(2)
  3214. expect(yield* session.context(sessionID)).toMatchObject([
  3215. { type: "user" },
  3216. {
  3217. type: "assistant",
  3218. content: [
  3219. {
  3220. type: "tool",
  3221. id: "call-permission",
  3222. state: {
  3223. status: "error",
  3224. error: {
  3225. type: "permission.rejected",
  3226. message: "Permission denied: edit",
  3227. },
  3228. },
  3229. },
  3230. ],
  3231. },
  3232. { type: "assistant", finish: "stop" },
  3233. ])
  3234. expect(yield* recordedEventTypes(sessionID)).not.toContain("session.step.failed.1")
  3235. }),
  3236. )
  3237. it.effect("interrupts runner continuation when a question is cancelled", () =>
  3238. Effect.gen(function* () {
  3239. const session = yield* setup
  3240. const registry = yield* Tool.Service
  3241. yield* transformTools(
  3242. registry,
  3243. {
  3244. question: {
  3245. name: "question",
  3246. description: "Ask the user",
  3247. input: Schema.Struct({}),
  3248. output: Schema.Struct({}),
  3249. execute: () => Effect.die(new QuestionTool.CancelledError()),
  3250. },
  3251. },
  3252. { codemode: false },
  3253. )
  3254. yield* admit(session, "Ask then stop")
  3255. yield* TestLLM.push(TestLLM.tool("call-question", "question", {}), [])
  3256. const run = yield* session.resume(sessionID).pipe(Effect.exit, Effect.forkChild)
  3257. const exit = yield* Fiber.join(run)
  3258. expect(exit._tag).toBe("Failure")
  3259. if (exit._tag === "Failure") expect(Cause.hasInterruptsOnly(exit.cause)).toBe(true)
  3260. expect(requests).toHaveLength(1)
  3261. expect(yield* session.context(sessionID)).toMatchObject([
  3262. { type: "user", text: "Ask then stop" },
  3263. {
  3264. type: "assistant",
  3265. content: [
  3266. {
  3267. type: "tool",
  3268. id: "call-question",
  3269. state: { status: "error", error: { type: "aborted", message: "The user dismissed this question" } },
  3270. },
  3271. ],
  3272. },
  3273. ])
  3274. }),
  3275. )
  3276. it.effect("awaits started local tools before surfacing provider stream failure", () =>
  3277. Effect.gen(function* () {
  3278. const session = yield* setup
  3279. yield* admit(session, "Settle before failing")
  3280. const failure = providerUnavailable()
  3281. const tools = yield* blockTools()
  3282. yield* TestLLM.push(
  3283. TestLLM.failAfter(
  3284. failure,
  3285. LLMEvent.stepStart({ index: 0 }),
  3286. LLMEvent.toolCall({ id: "call-before-failure", name: "echo", input: { text: "settle" } }),
  3287. ),
  3288. )
  3289. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3290. yield* tools.started
  3291. yield* tools.release
  3292. expect(yield* Fiber.join(run).pipe(Effect.flip)).toBe(failure)
  3293. const context = yield* session.context(sessionID)
  3294. expect(context).toMatchObject([
  3295. { type: "user", text: "Settle before failing" },
  3296. {
  3297. type: "assistant",
  3298. content: [
  3299. {
  3300. type: "tool",
  3301. id: "call-before-failure",
  3302. state: { status: "completed", content: [{ type: "text", text: "settle" }] },
  3303. },
  3304. ],
  3305. },
  3306. ])
  3307. const assistant = requireAssistant(context)
  3308. expect(yield* recordedStepSettlementTypes(sessionID, assistant.id)).toEqual([
  3309. "session.step.started.1",
  3310. "session.tool.called.1",
  3311. "session.tool.success.2",
  3312. "session.step.failed.1",
  3313. ])
  3314. }),
  3315. )
  3316. it.effect("durably fails blocked local tools when a step is interrupted", () =>
  3317. Effect.gen(function* () {
  3318. const session = yield* setup
  3319. yield* admit(session, "Interrupt blocked tool")
  3320. const tools = yield* blockTools()
  3321. yield* TestLLM.push(
  3322. TestLLM.hangAfter(
  3323. LLMEvent.stepStart({ index: 0 }),
  3324. LLMEvent.toolCall({ id: "call-before-interrupt", name: "echo", input: { text: "blocked" } }),
  3325. ),
  3326. )
  3327. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3328. yield* tools.started
  3329. yield* session.interrupt(sessionID)
  3330. expect(yield* Fiber.await(run)).toMatchObject({ _tag: "Failure" })
  3331. yield* session.interrupt(sessionID)
  3332. const context = yield* session.context(sessionID)
  3333. expect(context).toMatchObject([
  3334. { type: "user", text: "Interrupt blocked tool" },
  3335. {
  3336. type: "assistant",
  3337. content: [
  3338. {
  3339. type: "tool",
  3340. id: "call-before-interrupt",
  3341. state: { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } },
  3342. },
  3343. ],
  3344. },
  3345. ])
  3346. const assistant = requireAssistant(context)
  3347. expect(yield* recordedStepSettlementTypes(sessionID, assistant.id)).toEqual([
  3348. "session.step.started.1",
  3349. "session.tool.called.1",
  3350. "session.tool.failed.2",
  3351. "session.step.failed.1",
  3352. ])
  3353. yield* replaySessionProjection(sessionID)
  3354. expect(yield* session.context(sessionID)).toMatchObject([
  3355. { type: "user", text: "Interrupt blocked tool" },
  3356. { type: "assistant", content: [{ type: "tool", id: "call-before-interrupt", state: { status: "error" } }] },
  3357. ])
  3358. requests.length = 0
  3359. yield* TestLLM.push([])
  3360. yield* session.resume(sessionID)
  3361. expect(messageRoles(requests[0])).toEqual(["user", "assistant", "tool"])
  3362. }),
  3363. )
  3364. it.effect("interrupts a blocked step without local tool execution", () =>
  3365. Effect.gen(function* () {
  3366. const session = yield* setup
  3367. yield* admit(session, "Interrupt provider")
  3368. const stream = yield* TestLLM.gate
  3369. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3370. yield* stream.started
  3371. yield* session.interrupt(sessionID)
  3372. const exit = yield* Fiber.await(run)
  3373. expect(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)).toBeTrue()
  3374. expect(requests).toHaveLength(1)
  3375. expect(yield* session.context(sessionID)).toMatchObject([
  3376. { type: "user", text: "Interrupt provider" },
  3377. { type: "assistant", finish: "error", error: { type: "aborted", message: "Step interrupted" } },
  3378. ])
  3379. expect(yield* recordedEventTypes(sessionID)).toContain("session.step.failed.1")
  3380. yield* session.interrupt(sessionID)
  3381. }),
  3382. )
  3383. it.effect("durably fails blocked local tools when interrupted while awaiting settlement", () =>
  3384. Effect.gen(function* () {
  3385. const session = yield* setup
  3386. yield* admit(session, "Interrupt tool settlement")
  3387. const tools = yield* blockTools()
  3388. yield* TestLLM.push(TestLLM.tool("call-await-interrupt", "echo", { text: "blocked" }))
  3389. const runner = yield* SessionRunner.Service
  3390. const run = yield* runner.drain({ sessionID, force: true }).pipe(Effect.forkChild)
  3391. yield* tools.started
  3392. yield* Fiber.interrupt(run)
  3393. expect(yield* Fiber.await(run)).toMatchObject({ _tag: "Failure" })
  3394. expect(yield* session.context(sessionID)).toMatchObject([
  3395. { type: "user", text: "Interrupt tool settlement" },
  3396. {
  3397. type: "assistant",
  3398. finish: "error",
  3399. error: { type: "aborted", message: "Step interrupted" },
  3400. content: [
  3401. {
  3402. type: "tool",
  3403. id: "call-await-interrupt",
  3404. state: { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } },
  3405. },
  3406. ],
  3407. },
  3408. ])
  3409. const eventTypes = yield* recordedEventTypes(sessionID)
  3410. expect(eventTypes).toContain("session.step.failed.1")
  3411. expect(eventTypes).not.toContain("session.step.ended.1")
  3412. }),
  3413. )
  3414. it.effect("forces a text response on an agent's configured final step", () =>
  3415. Effect.gen(function* () {
  3416. const session = yield* setup
  3417. const agents = yield* Agent.Service
  3418. yield* agents.transform((editor) =>
  3419. editor.update(Agent.ID.make("build"), (agent) => {
  3420. agent.steps = 2
  3421. }),
  3422. )
  3423. yield* admit(session, "Finish at the limit")
  3424. yield* TestLLM.push(
  3425. TestLLM.tool("call-terminal", "echo", { text: "done" }),
  3426. TestLLM.tool("call-forbidden", "echo", { text: "forbidden" }),
  3427. )
  3428. yield* session.resume(sessionID)
  3429. expect(requests).toHaveLength(2)
  3430. expect(requests[0]?.toolChoice).toBeUndefined()
  3431. expect(requests[1]?.toolChoice).toMatchObject({ type: "none" })
  3432. // Protocols with native "none" keep these definitions for prompt caching.
  3433. expect(requests[1]?.tools.map((tool) => tool.name)).toContain("echo")
  3434. expect(requests[1]?.messages.at(-1)).toMatchObject({
  3435. role: "assistant",
  3436. content: [{ type: "text", text: expect.stringContaining("MAXIMUM STEPS REACHED") }],
  3437. })
  3438. expect(executions).toEqual(["done"])
  3439. expect(yield* session.context(sessionID)).toMatchObject([
  3440. { type: "user", text: "Finish at the limit" },
  3441. { type: "assistant", content: [{ type: "tool", id: "call-terminal", state: { status: "completed" } }] },
  3442. { type: "assistant", content: [{ type: "tool", id: "call-forbidden", state: { status: "error" } }] },
  3443. ])
  3444. }),
  3445. )
  3446. it.effect("resets the configured step allowance when steering input promotes", () =>
  3447. Effect.gen(function* () {
  3448. const session = yield* setup
  3449. const agents = yield* Agent.Service
  3450. yield* agents.transform((editor) =>
  3451. editor.update(Agent.ID.make("build"), (agent) => {
  3452. agent.steps = 2
  3453. }),
  3454. )
  3455. yield* admit(session, "Start work")
  3456. yield* TestLLM.push(
  3457. TestLLM.tool("call-before-steer", "echo", { text: "before" }),
  3458. TestLLM.tool("call-after-steer", "echo", { text: "after" }),
  3459. TestLLM.stop(),
  3460. )
  3461. const stream = yield* TestLLM.gate
  3462. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3463. yield* stream.started
  3464. yield* session.prompt({ sessionID, text: "Change direction" })
  3465. yield* stream.release
  3466. yield* Fiber.join(run)
  3467. expect(requests).toHaveLength(3)
  3468. expect(requests[1]?.toolChoice).toBeUndefined()
  3469. expect(requests[1]?.tools).not.toEqual([])
  3470. expect(requests[2]?.toolChoice).toMatchObject({ type: "none" })
  3471. expect(executions).toEqual(["before", "after"])
  3472. }),
  3473. )
  3474. it.effect("projects provider errors as terminal assistant step failures", () =>
  3475. Effect.gen(function* () {
  3476. const session = yield* setup
  3477. yield* TestLLM.push([
  3478. LLMEvent.stepStart({ index: 0 }),
  3479. LLMEvent.providerError({ message: "Provider unavailable" }),
  3480. ])
  3481. expect((yield* runPrompt(session, "Fail durably").pipe(Effect.flip)).message).toBe("Provider unavailable")
  3482. expect(requests).toHaveLength(1)
  3483. expect(yield* session.context(sessionID)).toMatchObject([
  3484. { type: "user", text: "Fail durably" },
  3485. { type: "assistant", finish: "error", error: { type: "provider.unknown", message: "Provider unavailable" } },
  3486. ])
  3487. }),
  3488. )
  3489. it.effect("projects provider errors emitted before assistant step start", () =>
  3490. Effect.gen(function* () {
  3491. const session = yield* setup
  3492. yield* TestLLM.push([LLMEvent.providerError({ message: "Provider unavailable" })])
  3493. expect((yield* runPrompt(session, "Fail before step").pipe(Effect.flip)).message).toBe("Provider unavailable")
  3494. expect(requests).toHaveLength(1)
  3495. expect(yield* session.context(sessionID)).toMatchObject([
  3496. { type: "user", text: "Fail before step" },
  3497. { type: "assistant", finish: "error", error: { type: "provider.unknown", message: "Provider unavailable" } },
  3498. ])
  3499. }),
  3500. )
  3501. it.effect("projects content-filter finishes as visible terminal failures", () =>
  3502. Effect.gen(function* () {
  3503. const session = yield* setup
  3504. yield* TestLLM.push(
  3505. TestLLM.complete(
  3506. {
  3507. reason: { normalized: "content-filter" },
  3508. usage: { nonCachedInputTokens: 8, outputTokens: 3, reasoningTokens: 1 },
  3509. },
  3510. LLMEvent.textStart({ id: "partial" }),
  3511. LLMEvent.textDelta({ id: "partial", text: "Partial" }),
  3512. ),
  3513. )
  3514. expect((yield* runPrompt(session, "Blocked response").pipe(Effect.flip)).message).toBe(
  3515. "Provider blocked the response",
  3516. )
  3517. expect(yield* session.context(sessionID)).toMatchObject([
  3518. { type: "user" },
  3519. {
  3520. type: "assistant",
  3521. finish: "error",
  3522. error: { type: "provider.content-filter" },
  3523. cost: 0,
  3524. tokens: { input: 8, output: 2, reasoning: 1, cache: { read: 0, write: 0 } },
  3525. content: [{ type: "text", text: "Partial" }],
  3526. },
  3527. ])
  3528. expect(yield* session.get(sessionID)).toMatchObject({
  3529. cost: 0,
  3530. tokens: { input: 8, output: 2, reasoning: 1, cache: { read: 0, write: 0 } },
  3531. })
  3532. expect(yield* recordedEventTypes(sessionID)).not.toContain("session.step.ended.1")
  3533. }),
  3534. )
  3535. it.effect("settles a local tool before one content-filter step failure", () =>
  3536. Effect.gen(function* () {
  3537. const session = yield* setup
  3538. yield* admit(session, "Tool before blocked response")
  3539. const tools = yield* blockTools()
  3540. yield* TestLLM.push(
  3541. TestLLM.complete(
  3542. { reason: { normalized: "content-filter" } },
  3543. LLMEvent.toolCall({ id: "call-before-content-filter", name: "echo", input: { text: "settled" } }),
  3544. ),
  3545. )
  3546. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3547. yield* tools.started
  3548. yield* tools.release
  3549. expect((yield* Fiber.join(run).pipe(Effect.flip)).message).toBe("Provider blocked the response")
  3550. const assistant = requireAssistant(yield* session.context(sessionID))
  3551. const bus = yield* recordedStepSettlementEvents(sessionID, assistant.id)
  3552. expect(bus.map((event) => event.type)).toEqual([
  3553. "session.step.started.1",
  3554. "session.tool.called.1",
  3555. "session.tool.success.2",
  3556. "session.step.failed.1",
  3557. ])
  3558. expect(
  3559. bus.filter((event) => event.type.startsWith("session.step.") && event.type !== "session.step.started.1"),
  3560. ).toHaveLength(1)
  3561. }),
  3562. )
  3563. it.effect("does not recover context overflow after durable assistant output", () =>
  3564. Effect.gen(function* () {
  3565. const session = yield* setup
  3566. yield* TestLLM.push([
  3567. LLMEvent.stepStart({ index: 0 }),
  3568. LLMEvent.textStart({ id: "text-partial" }),
  3569. LLMEvent.textDelta({ id: "text-partial", text: "Partial" }),
  3570. LLMEvent.textEnd({ id: "text-partial" }),
  3571. LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }),
  3572. ])
  3573. expect((yield* runPrompt(session, "Fail after output").pipe(Effect.flip)).message).toBe("prompt too long")
  3574. expect(requests).toHaveLength(1)
  3575. expect(yield* session.context(sessionID)).toMatchObject([
  3576. { type: "user", text: "Fail after output" },
  3577. {
  3578. type: "assistant",
  3579. finish: "error",
  3580. error: { message: "prompt too long" },
  3581. content: [{ type: "text", text: "Partial" }],
  3582. },
  3583. ])
  3584. }),
  3585. )
  3586. it.effect("projects raw provider stream failures as terminal assistant step failures", () =>
  3587. Effect.gen(function* () {
  3588. const session = yield* setup
  3589. const failure = invalidRequest()
  3590. yield* TestLLM.push(Stream.fail(failure))
  3591. expect(yield* runPrompt(session, "Fail raw stream durably").pipe(Effect.flip)).toBe(failure)
  3592. yield* replaySessionProjection(sessionID)
  3593. expect(yield* session.context(sessionID)).toMatchObject([
  3594. { type: "user", text: "Fail raw stream durably" },
  3595. { type: "assistant", finish: "error", error: { type: "provider.invalid-request", message: "Invalid request" } },
  3596. ])
  3597. }),
  3598. )
  3599. it.effect("retries eligible pre-output failures after exponential backoff", () =>
  3600. Effect.gen(function* () {
  3601. const session = yield* setup
  3602. yield* admit(session, "Retry transport")
  3603. yield* TestLLM.push(Stream.fail(providerUnavailable()))
  3604. yield* TestLLM.push(TestLLM.text("Recovered", "retry-success"))
  3605. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3606. yield* TestLLM.wait(1)
  3607. yield* TestClock.adjust("1999 millis")
  3608. expect(requests).toHaveLength(1)
  3609. yield* TestClock.adjust("1 millis")
  3610. yield* Fiber.join(run)
  3611. expect(requests).toHaveLength(2)
  3612. const eventTypes = yield* recordedEventTypes(sessionID)
  3613. expect(eventTypes).toContain("session.retry.scheduled.1")
  3614. expect(eventTypes.filter((type) => type === "session.step.started.1")).toHaveLength(2)
  3615. expect(yield* session.context(sessionID)).toMatchObject([
  3616. { type: "user" },
  3617. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Recovered" }] },
  3618. ])
  3619. yield* replaySessionProjection(sessionID)
  3620. expect((yield* session.context(sessionID)).filter((message) => message.type === "assistant")).toHaveLength(1)
  3621. }),
  3622. )
  3623. it.effect("retries an incomplete stream before output", () =>
  3624. Effect.gen(function* () {
  3625. const session = yield* setup
  3626. yield* admit(session, "Retry incomplete stream")
  3627. yield* TestLLM.push(Stream.fail(incompleteStream()))
  3628. yield* TestLLM.push(TestLLM.text("Recovered", "incomplete-stream-success"))
  3629. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3630. yield* TestLLM.wait(1)
  3631. yield* TestClock.adjust("2 seconds")
  3632. yield* Fiber.join(run)
  3633. expect(requests).toHaveLength(2)
  3634. expect(yield* session.context(sessionID)).toMatchObject([
  3635. { type: "user" },
  3636. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Recovered" }] },
  3637. ])
  3638. }),
  3639. )
  3640. it.effect("uses a larger provider retry-after delay", () =>
  3641. Effect.gen(function* () {
  3642. const session = yield* setup
  3643. yield* admit(session, "Retry rate limit")
  3644. yield* TestLLM.push(Stream.fail(rateLimited(5_000)))
  3645. yield* TestLLM.push(TestLLM.text("Recovered", "retry-after-success"))
  3646. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3647. yield* TestLLM.wait(1)
  3648. yield* TestClock.adjust("4999 millis")
  3649. expect(requests).toHaveLength(1)
  3650. yield* TestClock.adjust("1 millis")
  3651. yield* Fiber.join(run)
  3652. expect(requests).toHaveLength(2)
  3653. }),
  3654. )
  3655. it.effect("does not retry eligible failures after observable output", () =>
  3656. Effect.gen(function* () {
  3657. const session = yield* setup
  3658. const failure = incompleteStream()
  3659. yield* TestLLM.push(
  3660. TestLLM.failAfter(
  3661. failure,
  3662. LLMEvent.stepStart({ index: 0 }),
  3663. LLMEvent.textStart({ id: "partial-rate-limit" }),
  3664. LLMEvent.textDelta({ id: "partial-rate-limit", text: "Partial" }),
  3665. ),
  3666. )
  3667. expect(yield* runPrompt(session, "Do not replay partial output").pipe(Effect.flip)).toBe(failure)
  3668. expect(requests).toHaveLength(1)
  3669. expect(yield* recordedEventTypes(sessionID)).not.toContain("session.retry.scheduled.1")
  3670. expect(yield* session.context(sessionID)).toMatchObject([
  3671. { type: "user" },
  3672. {
  3673. type: "assistant",
  3674. finish: "error",
  3675. error: { type: "provider.invalid-output" },
  3676. content: [{ type: "text", text: "Partial" }],
  3677. },
  3678. ])
  3679. }),
  3680. )
  3681. it.effect("stops after five total retry attempts", () =>
  3682. Effect.gen(function* () {
  3683. const session = yield* setup
  3684. yield* admit(session, "Exhaust retries")
  3685. const failure = providerUnavailable()
  3686. yield* TestLLM.always(Stream.fail(failure))
  3687. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3688. yield* TestLLM.wait(1)
  3689. for (const [index, delay] of [2_000, 4_000, 8_000, 16_000].entries()) {
  3690. yield* TestClock.adjust(delay)
  3691. yield* TestLLM.wait(index + 2)
  3692. }
  3693. expect(yield* Fiber.join(run).pipe(Effect.flip)).toBe(failure)
  3694. expect(requests).toHaveLength(5)
  3695. const database = (yield* Database.Service).db
  3696. const retries = yield* database
  3697. .select({ data: EventTable.data })
  3698. .from(EventTable)
  3699. .where(eq(EventTable.type, "session.retry.scheduled.1"))
  3700. .orderBy(asc(EventTable.seq))
  3701. .all()
  3702. .pipe(Effect.orDie)
  3703. expect(retries.map((event) => event.data)).toMatchObject([
  3704. { attempt: 2, at: 2_000 },
  3705. { attempt: 3, at: 6_000 },
  3706. { attempt: 4, at: 14_000 },
  3707. { attempt: 5, at: 30_000 },
  3708. ])
  3709. expect((yield* recordedEventTypes(sessionID)).filter((type) => type === "session.step.started.1")).toHaveLength(5)
  3710. const assistant = requireAssistant(yield* session.context(sessionID))
  3711. expect(yield* recordedStepSettlementEvents(sessionID, assistant.id)).toMatchObject([
  3712. { type: "session.step.started.1" },
  3713. { type: "session.step.started.1" },
  3714. { type: "session.step.started.1" },
  3715. { type: "session.step.started.1" },
  3716. { type: "session.step.started.1" },
  3717. { type: "session.step.failed.1" },
  3718. ])
  3719. }),
  3720. )
  3721. it.effect("retries a model call without consuming the logical agent step", () =>
  3722. Effect.gen(function* () {
  3723. const session = yield* setup
  3724. const agents = yield* Agent.Service
  3725. yield* agents.transform((editor) =>
  3726. editor.update(Agent.ID.make("build"), (agent) => {
  3727. agent.steps = 2
  3728. }),
  3729. )
  3730. yield* admit(session, "Retry without consuming a step")
  3731. const failure = providerUnavailable()
  3732. yield* TestLLM.push(Stream.fail(failure))
  3733. yield* TestLLM.push(TestLLM.tool("call-after-retry", "echo", { text: "recovered" }), TestLLM.stop())
  3734. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3735. yield* TestLLM.wait(1)
  3736. yield* TestClock.adjust("2 seconds")
  3737. yield* Fiber.join(run)
  3738. expect(requests).toHaveLength(3)
  3739. expect(requests[0]?.toolChoice).toBeUndefined()
  3740. expect(requests[0]?.tools.map((tool) => tool.name)).toContain("echo")
  3741. expect(requests[1]?.toolChoice).toBeUndefined()
  3742. expect(requests[1]?.tools.map((tool) => tool.name)).toContain("echo")
  3743. expect(requests[1]?.messages.at(-1)).not.toMatchObject({
  3744. role: "assistant",
  3745. content: [{ type: "text", text: expect.stringContaining("MAXIMUM STEPS REACHED") }],
  3746. })
  3747. expect(requests[2]?.toolChoice).toMatchObject({ type: "none" })
  3748. // The final step keeps tool definitions to preserve provider prompt caching.
  3749. expect(requests[2]?.tools.map((tool) => tool.name)).toContain("echo")
  3750. expect(requests[2]?.messages.at(-1)).toMatchObject({
  3751. role: "assistant",
  3752. content: [{ type: "text", text: expect.stringContaining("MAXIMUM STEPS REACHED") }],
  3753. })
  3754. expect(executions).toEqual(["recovered"])
  3755. const eventTypes = yield* recordedEventTypes(sessionID)
  3756. expect(eventTypes.filter((type) => type === "session.step.started.1")).toHaveLength(3)
  3757. expect(eventTypes.filter((type) => type === "session.retry.scheduled.1")).toHaveLength(1)
  3758. expect((yield* session.context(sessionID)).filter((message) => message.type === "assistant")).toHaveLength(2)
  3759. }),
  3760. )
  3761. it.effect("does not retry non-eligible provider failures", () =>
  3762. Effect.gen(function* () {
  3763. const session = yield* setup
  3764. const failure = invalidRequest()
  3765. yield* TestLLM.push(Stream.fail(failure))
  3766. expect(yield* runPrompt(session, "Do not retry").pipe(Effect.flip)).toBe(failure)
  3767. expect(requests).toHaveLength(1)
  3768. expect(yield* recordedEventTypes(sessionID)).not.toContain("session.retry.scheduled.1")
  3769. }),
  3770. )
  3771. it.effect("settles malformed streamed tool input before the provider failure", () =>
  3772. Effect.gen(function* () {
  3773. const session = yield* setup
  3774. const failure = new AIError({
  3775. module: "test",
  3776. method: "stream",
  3777. reason: new InvalidProviderOutputReason({ message: "Invalid JSON input for tool call echo" }),
  3778. })
  3779. yield* TestLLM.push(
  3780. TestLLM.failAfter(
  3781. failure,
  3782. LLMEvent.stepStart({ index: 0 }),
  3783. LLMEvent.toolInputStart({ id: "call-malformed", name: "echo" }),
  3784. LLMEvent.toolInputDelta({ id: "call-malformed", name: "echo", text: '{"text":"partial' }),
  3785. ),
  3786. )
  3787. expect(yield* runPrompt(session, "Call a malformed tool").pipe(Effect.flip)).toBe(failure)
  3788. const assistant = requireAssistant(yield* session.context(sessionID))
  3789. yield* TestLLM.push(TestLLM.stop())
  3790. yield* runPrompt(session, "Continue")
  3791. expect(yield* recordedStepSettlementEvents(sessionID, assistant.id)).toMatchObject([
  3792. { type: "session.step.started.1" },
  3793. {
  3794. type: "session.tool.failed.2",
  3795. data: {
  3796. id: "call-malformed",
  3797. error: { type: "provider.invalid-output", message: "Invalid JSON input for tool call echo" },
  3798. },
  3799. },
  3800. {
  3801. type: "session.step.failed.1",
  3802. data: { error: { type: "provider.invalid-output", message: "Invalid JSON input for tool call echo" } },
  3803. },
  3804. ])
  3805. }),
  3806. )
  3807. it.effect("continues after malformed local tool input without exposing raw arguments", () =>
  3808. Effect.gen(function* () {
  3809. const session = yield* setup
  3810. const marker = "raw-malformed-marker"
  3811. const raw = `{"text":"${marker}`
  3812. yield* TestLLM.push(
  3813. TestLLM.toolCalls(
  3814. LLMEvent.toolInputStart({ id: "call-malformed", name: "echo" }),
  3815. LLMEvent.toolInputDelta({ id: "call-malformed", name: "echo", text: raw }),
  3816. LLMEvent.toolInputEnd({ id: "call-malformed", name: "echo" }),
  3817. LLMEvent.toolInputError({
  3818. id: "call-malformed",
  3819. name: "echo",
  3820. raw,
  3821. }),
  3822. ),
  3823. TestLLM.stop(),
  3824. )
  3825. yield* runPrompt(session, "Recover malformed tool input")
  3826. expect(requests).toHaveLength(2)
  3827. expect(executions).toEqual([])
  3828. expect(JSON.stringify(requests[1])).not.toContain(marker)
  3829. expect(requests[1]?.messages).toEqual(
  3830. expect.arrayContaining([
  3831. expect.objectContaining({
  3832. role: "assistant",
  3833. content: expect.arrayContaining([
  3834. expect.objectContaining({ type: "tool-call", id: "call-malformed", name: "echo", input: {} }),
  3835. ]),
  3836. }),
  3837. expect.objectContaining({
  3838. role: "tool",
  3839. content: expect.arrayContaining([
  3840. expect.objectContaining({
  3841. type: "tool-result",
  3842. id: "call-malformed",
  3843. result: expect.objectContaining({
  3844. type: "error",
  3845. value: expect.objectContaining({
  3846. error: expect.objectContaining({
  3847. message: "Tool call arguments were malformed JSON and were not executed. Retry with valid JSON.",
  3848. }),
  3849. }),
  3850. }),
  3851. }),
  3852. ]),
  3853. }),
  3854. ]),
  3855. )
  3856. const context = yield* session.context(sessionID)
  3857. const failed = context.find(
  3858. (message): message is SessionMessage.Assistant =>
  3859. message.type === "assistant" && message.content.some((item) => item.type === "tool"),
  3860. )
  3861. expect(failed).toMatchObject({
  3862. content: [
  3863. {
  3864. type: "tool",
  3865. id: "call-malformed",
  3866. executed: false,
  3867. state: {
  3868. status: "error",
  3869. input: {},
  3870. error: {
  3871. type: "tool.input-json",
  3872. message: "Tool call arguments were malformed JSON and were not executed. Retry with valid JSON.",
  3873. },
  3874. },
  3875. },
  3876. ],
  3877. })
  3878. if (!failed) throw new Error("Malformed tool assistant missing")
  3879. expect(failed.error).toBeUndefined()
  3880. expect(yield* recordedStepSettlementTypes(sessionID, failed.id)).toEqual([
  3881. "session.step.started.1",
  3882. "session.tool.failed.2",
  3883. "session.step.ended.1",
  3884. ])
  3885. const database = (yield* Database.Service).db
  3886. const durable = yield* database
  3887. .select({ type: EventTable.type, data: EventTable.data })
  3888. .from(EventTable)
  3889. .where(eq(EventTable.aggregate_id, sessionID))
  3890. .all()
  3891. .pipe(Effect.orDie)
  3892. expect(durable.find((event) => event.type === "session.tool.input.ended.1")?.data).toMatchObject({
  3893. id: "call-malformed",
  3894. text: raw,
  3895. })
  3896. }),
  3897. )
  3898. it.effect("settles a valid sibling before recovering malformed tool input", () =>
  3899. Effect.gen(function* () {
  3900. const session = yield* setup
  3901. yield* admit(session, "Run parallel tools")
  3902. const tools = yield* blockTools()
  3903. yield* TestLLM.push(
  3904. TestLLM.toolCalls(
  3905. LLMEvent.toolCall({ id: "call-valid", name: "echo", input: { text: "valid" } }),
  3906. LLMEvent.toolInputError({
  3907. id: "call-malformed",
  3908. name: "echo",
  3909. raw: '{"text":"partial',
  3910. }),
  3911. ),
  3912. TestLLM.stop(),
  3913. )
  3914. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3915. yield* tools.started
  3916. expect(requests).toHaveLength(1)
  3917. yield* tools.release
  3918. yield* Fiber.join(run)
  3919. expect(requests).toHaveLength(2)
  3920. expect(executions).toEqual(["valid"])
  3921. const request = requests[1]
  3922. if (!request) throw new Error("Malformed recovery request missing")
  3923. expect(request.messages.flatMap((message) => (message.role === "tool" ? message.content : []))).toEqual(
  3924. expect.arrayContaining([
  3925. expect.objectContaining({ id: "call-valid", type: "tool-result" }),
  3926. expect.objectContaining({ id: "call-malformed", type: "tool-result" }),
  3927. ]),
  3928. )
  3929. }),
  3930. )
  3931. it.effect("does not recover malformed input after sibling execution is interrupted", () =>
  3932. Effect.gen(function* () {
  3933. const session = yield* setup
  3934. yield* admit(session, "Interrupt malformed recovery")
  3935. const tools = yield* blockTools()
  3936. yield* TestLLM.push(
  3937. TestLLM.toolCalls(
  3938. LLMEvent.toolCall({ id: "call-valid", name: "echo", input: { text: "blocked" } }),
  3939. LLMEvent.toolInputError({
  3940. id: "call-malformed",
  3941. name: "echo",
  3942. raw: '{"text":"partial',
  3943. }),
  3944. ),
  3945. )
  3946. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3947. yield* tools.started
  3948. while (
  3949. !(yield* session.context(sessionID)).some(
  3950. (message) =>
  3951. message.type === "assistant" &&
  3952. message.content.some((item) => item.type === "tool" && item.id === "call-malformed"),
  3953. )
  3954. )
  3955. yield* Effect.yieldNow
  3956. yield* session.interrupt(sessionID)
  3957. expect(yield* Fiber.await(run)).toMatchObject({ _tag: "Failure" })
  3958. expect(requests).toHaveLength(1)
  3959. expect(yield* session.context(sessionID)).toMatchObject([
  3960. { type: "user", text: "Interrupt malformed recovery" },
  3961. {
  3962. type: "assistant",
  3963. error: { type: "aborted", message: "Step interrupted" },
  3964. content: [
  3965. { type: "tool", id: "call-valid", state: { status: "error", error: { type: "aborted" } } },
  3966. { type: "tool", id: "call-malformed", state: { status: "error" } },
  3967. ],
  3968. },
  3969. ])
  3970. }),
  3971. )
  3972. it.effect("records malformed provider-executed input as executed", () =>
  3973. Effect.gen(function* () {
  3974. const session = yield* setup
  3975. const failure = new AIError({
  3976. module: "test",
  3977. method: "stream",
  3978. reason: new InvalidProviderOutputReason({ message: "Invalid hosted tool input" }),
  3979. })
  3980. yield* TestLLM.push(
  3981. TestLLM.failAfter(
  3982. failure,
  3983. LLMEvent.stepStart({ index: 0 }),
  3984. LLMEvent.toolInputStart({ id: "call-hosted", name: "web_search", providerExecuted: true }),
  3985. LLMEvent.toolInputDelta({ id: "call-hosted", name: "web_search", text: '{"query":"partial' }),
  3986. ),
  3987. )
  3988. expect(yield* runPrompt(session, "Fail malformed hosted input").pipe(Effect.flip)).toBe(failure)
  3989. expect(requireAssistant(yield* session.context(sessionID))).toMatchObject({
  3990. error: { type: "provider.invalid-output", message: "Invalid hosted tool input" },
  3991. content: [
  3992. {
  3993. type: "tool",
  3994. id: "call-hosted",
  3995. executed: true,
  3996. state: { status: "error", error: { type: "provider.invalid-output" } },
  3997. },
  3998. ],
  3999. })
  4000. }),
  4001. )
  4002. it.effect("records a provider failure after malformed input", () =>
  4003. Effect.gen(function* () {
  4004. const session = yield* setup
  4005. const failure = new AIError({
  4006. module: "test",
  4007. method: "stream",
  4008. reason: new InvalidProviderOutputReason({ message: "Provider failed after malformed input" }),
  4009. })
  4010. yield* TestLLM.push(
  4011. TestLLM.failAfter(
  4012. failure,
  4013. LLMEvent.stepStart({ index: 0 }),
  4014. LLMEvent.toolInputError({
  4015. id: "call-malformed",
  4016. name: "echo",
  4017. raw: '{"text":"partial',
  4018. }),
  4019. ),
  4020. )
  4021. expect(yield* runPrompt(session, "Fail after malformed input").pipe(Effect.flip)).toBe(failure)
  4022. expect(requireAssistant(yield* session.context(sessionID))).toMatchObject({
  4023. error: { type: "provider.invalid-output", message: "Provider failed after malformed input" },
  4024. content: [
  4025. {
  4026. type: "tool",
  4027. id: "call-malformed",
  4028. executed: false,
  4029. state: { status: "error", error: { type: "tool.input-json" } },
  4030. },
  4031. ],
  4032. })
  4033. expect(requests).toHaveLength(1)
  4034. }),
  4035. )
  4036. it.effect("continues after repeated malformed tool input", () =>
  4037. Effect.gen(function* () {
  4038. const session = yield* setup
  4039. const malformed = (id: string) =>
  4040. TestLLM.toolCalls(
  4041. LLMEvent.toolInputError({
  4042. id,
  4043. name: "echo",
  4044. raw: '{"text":"partial',
  4045. }),
  4046. )
  4047. yield* TestLLM.push(
  4048. malformed("call-first"),
  4049. TestLLM.tool("call-valid-between", "echo", { text: "valid" }),
  4050. malformed("call-second"),
  4051. TestLLM.stop(),
  4052. )
  4053. yield* runPrompt(session, "Keep producing malformed tools")
  4054. expect(requests).toHaveLength(4)
  4055. expect(executions).toEqual(["valid"])
  4056. expect((yield* recordedEventTypes(sessionID)).filter((type) => type === "session.step.failed.1")).toHaveLength(0)
  4057. }),
  4058. )
  4059. it.effect("does not continue malformed tool input past the agent step limit", () =>
  4060. Effect.gen(function* () {
  4061. const session = yield* setup
  4062. const agents = yield* Agent.Service
  4063. yield* agents.transform((editor) =>
  4064. editor.update(Agent.ID.make("build"), (agent) => {
  4065. agent.steps = 2
  4066. }),
  4067. )
  4068. const malformed = (id: string) =>
  4069. TestLLM.toolCalls(
  4070. LLMEvent.toolInputError({
  4071. id,
  4072. name: "echo",
  4073. raw: '{"text":"partial',
  4074. }),
  4075. )
  4076. yield* TestLLM.push(malformed("call-first"), malformed("call-at-limit"))
  4077. yield* runPrompt(session, "Stop malformed tools at the step limit")
  4078. expect(requests).toHaveLength(2)
  4079. expect(requests[0]?.toolChoice).toBeUndefined()
  4080. expect(requests[1]?.toolChoice).toMatchObject({ type: "none" })
  4081. expect((yield* recordedEventTypes(sessionID)).filter((type) => type === "session.tool.failed.2")).toHaveLength(2)
  4082. }),
  4083. )
  4084. it.effect("does not continue automatically after a provider error follows a local tool call", () =>
  4085. Effect.gen(function* () {
  4086. const session = yield* setup
  4087. yield* admit(session, "Do not continue failed provider")
  4088. const tools = yield* blockTools()
  4089. yield* TestLLM.push([
  4090. LLMEvent.stepStart({ index: 0 }),
  4091. LLMEvent.toolCall({ id: "call-before-provider-error", name: "echo", input: { text: "settled" } }),
  4092. LLMEvent.providerError({ message: "Provider unavailable" }),
  4093. ])
  4094. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  4095. yield* tools.started
  4096. yield* tools.release
  4097. expect((yield* Fiber.join(run).pipe(Effect.flip)).message).toBe("Provider unavailable")
  4098. expect(requests).toHaveLength(1)
  4099. expect(executions).toEqual(["settled"])
  4100. const context = yield* session.context(sessionID)
  4101. const assistant = requireAssistant(context)
  4102. expect(yield* recordedStepSettlementTypes(sessionID, assistant.id)).toEqual([
  4103. "session.step.started.1",
  4104. "session.tool.called.1",
  4105. "session.tool.success.2",
  4106. "session.step.failed.1",
  4107. ])
  4108. }),
  4109. )
  4110. it.effect("durably fails a hosted tool when its provider errors before returning a result", () =>
  4111. Effect.gen(function* () {
  4112. const session = yield* setup
  4113. yield* TestLLM.push([
  4114. LLMEvent.stepStart({ index: 0 }),
  4115. hostedCall("call-hosted-provider-error", "effect"),
  4116. LLMEvent.providerError({ message: "Provider unavailable" }),
  4117. ])
  4118. expect((yield* runPrompt(session, "Fail hosted tool durably").pipe(Effect.flip)).message).toBe(
  4119. "Provider unavailable",
  4120. )
  4121. expect(requests).toHaveLength(1)
  4122. const context = yield* session.context(sessionID)
  4123. expect(context).toMatchObject([
  4124. { type: "user", text: "Fail hosted tool durably" },
  4125. {
  4126. type: "assistant",
  4127. content: [{ type: "tool", id: "call-hosted-provider-error", state: { status: "error" } }],
  4128. },
  4129. ])
  4130. const assistant = requireAssistant(context)
  4131. expect(yield* recordedStepSettlementTypes(sessionID, assistant.id)).toEqual([
  4132. "session.step.started.1",
  4133. "session.tool.called.1",
  4134. "session.tool.failed.2",
  4135. "session.step.failed.1",
  4136. ])
  4137. }),
  4138. )
  4139. it.effect("preserves a tool defect before provider failure settlement", () =>
  4140. Effect.gen(function* () {
  4141. const session = yield* setup
  4142. yield* TestLLM.push([
  4143. LLMEvent.stepStart({ index: 0 }),
  4144. LLMEvent.toolCall({ id: "call-defect-provider-error", name: "defect", input: {} }),
  4145. LLMEvent.providerError({ message: "Provider unavailable" }),
  4146. ])
  4147. expect((yield* runPrompt(session, "Defect while provider fails").pipe(Effect.flip)).message).toBe(
  4148. "Provider unavailable",
  4149. )
  4150. const context = yield* session.context(sessionID)
  4151. const assistant = requireAssistant(context)
  4152. const bus = yield* recordedStepSettlementEvents(sessionID, assistant.id)
  4153. expect(bus.map((event) => event.type)).toEqual([
  4154. "session.step.started.1",
  4155. "session.tool.called.1",
  4156. "session.tool.failed.2",
  4157. "session.step.failed.1",
  4158. ])
  4159. expect(bus[2]?.data.error).toMatchObject({ type: "unknown", message: "unexpected tool defect" })
  4160. }),
  4161. )
  4162. it.effect("preserves the provider failure when tool output persistence also fails", () =>
  4163. Effect.gen(function* () {
  4164. const session = yield* setup
  4165. yield* admit(session, "Storage fails while provider fails")
  4166. yield* TestLLM.push([
  4167. LLMEvent.stepStart({ index: 0 }),
  4168. LLMEvent.toolCall({ id: "call-store-provider-error", name: "storefail", input: {} }),
  4169. LLMEvent.providerError({ message: "Provider unavailable" }),
  4170. ])
  4171. expect(yield* session.resume(sessionID).pipe(Effect.exit)).toMatchObject({
  4172. _tag: "Failure",
  4173. })
  4174. expect(requireAssistant(yield* session.context(sessionID))).toMatchObject({
  4175. error: { type: "provider.unknown", message: "Provider unavailable" },
  4176. })
  4177. }),
  4178. )
  4179. it.effect("durably fails a hosted tool left unresolved at normal provider EOF", () =>
  4180. Effect.gen(function* () {
  4181. const session = yield* setup
  4182. yield* TestLLM.push([LLMEvent.stepStart({ index: 0 }), hostedCall("call-hosted-eof", "effect")])
  4183. expect((yield* runPrompt(session, "Fail hosted tool at EOF").pipe(Effect.flip)).message).toBe(
  4184. "Provider did not return a tool result",
  4185. )
  4186. const assistant = requireAssistant(yield* session.context(sessionID))
  4187. const bus = yield* recordedStepSettlementEvents(sessionID, assistant.id)
  4188. expect(bus.map((event) => event.type)).toEqual([
  4189. "session.step.started.1",
  4190. "session.tool.called.1",
  4191. "session.tool.failed.2",
  4192. "session.step.failed.1",
  4193. ])
  4194. expect(
  4195. bus.filter((event) => event.type.startsWith("session.step.") && event.type !== "session.step.started.1"),
  4196. ).toHaveLength(1)
  4197. yield* replaySessionProjection(sessionID)
  4198. expect(yield* session.context(sessionID)).toMatchObject([
  4199. { type: "user", text: "Fail hosted tool at EOF" },
  4200. {
  4201. type: "assistant",
  4202. finish: "error",
  4203. error: { type: "tool.result-missing" },
  4204. content: [{ type: "tool", id: "call-hosted-eof", state: { status: "error" } }],
  4205. },
  4206. ])
  4207. }),
  4208. )
  4209. it.effect("fails an unresolved hosted tool before one clean step end", () =>
  4210. Effect.gen(function* () {
  4211. const session = yield* setup
  4212. yield* TestLLM.push(TestLLM.stop(hostedCall("call-hosted-clean-end", "effect")))
  4213. yield* runPrompt(session, "Settle hosted tool before ending")
  4214. const assistant = requireAssistant(yield* session.context(sessionID))
  4215. const bus = yield* recordedStepSettlementEvents(sessionID, assistant.id)
  4216. expect(bus.map((event) => event.type)).toEqual([
  4217. "session.step.started.1",
  4218. "session.tool.called.1",
  4219. "session.tool.failed.2",
  4220. "session.step.ended.1",
  4221. ])
  4222. expect(
  4223. bus.filter((event) => event.type.startsWith("session.step.") && event.type !== "session.step.started.1"),
  4224. ).toHaveLength(1)
  4225. }),
  4226. )
  4227. it.effect("settles unresolved local and hosted tools before one raw provider failure", () =>
  4228. Effect.gen(function* () {
  4229. const session = yield* setup
  4230. yield* admit(session, "Fail unresolved tools")
  4231. const failure = invalidRequest()
  4232. const providerFailed = yield* Deferred.make<void>()
  4233. const tools = yield* blockTools()
  4234. yield* TestLLM.push(
  4235. Stream.concat(
  4236. Stream.fromIterable([
  4237. LLMEvent.stepStart({ index: 0 }),
  4238. LLMEvent.toolCall({ id: "call-local-raw-failure", name: "defect", input: {} }),
  4239. hostedCall("call-hosted-raw-failure-pair", "effect"),
  4240. ]),
  4241. Stream.fromEffect(Deferred.succeed(providerFailed, undefined)).pipe(
  4242. Stream.flatMap(() => Stream.fail(failure)),
  4243. ),
  4244. ),
  4245. )
  4246. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  4247. yield* Deferred.await(providerFailed)
  4248. yield* tools.release
  4249. expect(yield* Fiber.join(run).pipe(Effect.flip)).toBe(failure)
  4250. const assistant = requireAssistant(yield* session.context(sessionID))
  4251. const bus = yield* recordedStepSettlementEvents(sessionID, assistant.id)
  4252. expect(bus.map((event) => ({ type: event.type, id: event.data.id }))).toEqual([
  4253. { type: "session.step.started.1", id: undefined },
  4254. { type: "session.tool.called.1", id: "call-local-raw-failure" },
  4255. { type: "session.tool.called.1", id: "call-hosted-raw-failure-pair" },
  4256. { type: "session.tool.failed.2", id: "call-local-raw-failure" },
  4257. { type: "session.tool.failed.2", id: "call-hosted-raw-failure-pair" },
  4258. { type: "session.step.failed.1", id: undefined },
  4259. ])
  4260. expect(
  4261. bus.filter((event) => event.type.startsWith("session.step.") && event.type !== "session.step.started.1"),
  4262. ).toHaveLength(1)
  4263. }),
  4264. )
  4265. it.effect("durably fails a hosted tool left unresolved by a raw provider stream failure", () =>
  4266. Effect.gen(function* () {
  4267. const session = yield* setup
  4268. const failure = providerUnavailable()
  4269. yield* TestLLM.push(
  4270. Stream.concat(
  4271. Stream.fromIterable([LLMEvent.stepStart({ index: 0 }), hostedCall("call-hosted-raw-failure", "effect")]),
  4272. Stream.fail(failure),
  4273. ),
  4274. )
  4275. expect(yield* runPrompt(session, "Fail hosted tool on raw failure").pipe(Effect.flip)).toBe(failure)
  4276. expect(requests).toHaveLength(1)
  4277. const assistant = requireAssistant(yield* session.context(sessionID))
  4278. const bus = yield* recordedStepSettlementEvents(sessionID, assistant.id)
  4279. expect(bus.map((event) => event.type)).toEqual([
  4280. "session.step.started.1",
  4281. "session.tool.called.1",
  4282. "session.tool.failed.2",
  4283. "session.step.failed.1",
  4284. ])
  4285. expect(
  4286. bus.filter((event) => event.type.startsWith("session.step.") && event.type !== "session.step.started.1"),
  4287. ).toHaveLength(1)
  4288. yield* replaySessionProjection(sessionID)
  4289. expect(yield* session.context(sessionID)).toMatchObject([
  4290. { type: "user", text: "Fail hosted tool on raw failure" },
  4291. {
  4292. type: "assistant",
  4293. finish: "error",
  4294. error: { type: "provider.transport", message: "Provider unavailable" },
  4295. content: [{ type: "tool", id: "call-hosted-raw-failure", state: { status: "error" } }],
  4296. },
  4297. ])
  4298. }),
  4299. )
  4300. it.effect("rejects a second text start before the open fragment ends", () =>
  4301. Effect.gen(function* () {
  4302. const session = yield* setup
  4303. yield* TestLLM.push([
  4304. LLMEvent.stepStart({ index: 0 }),
  4305. LLMEvent.textStart({ id: "text-1" }),
  4306. LLMEvent.textStart({ id: "text-2" }),
  4307. ])
  4308. const defect = yield* runPrompt(session, "Two blocks").pipe(Effect.catchDefect(Effect.succeed))
  4309. expect(defect).toBeInstanceOf(Error)
  4310. if (!(defect instanceof Error)) return
  4311. expect(defect.message).toBe("text start before end: text-2")
  4312. }),
  4313. )
  4314. it.effect("projects sequential text fragments as separate content parts", () =>
  4315. Effect.gen(function* () {
  4316. const session = yield* setup
  4317. yield* TestLLM.push(
  4318. TestLLM.stop(
  4319. LLMEvent.textStart({ id: "text-1" }),
  4320. LLMEvent.textDelta({ id: "text-1", text: "First" }),
  4321. LLMEvent.textEnd({ id: "text-1" }),
  4322. LLMEvent.textStart({ id: "text-2" }),
  4323. LLMEvent.textDelta({ id: "text-2", text: "Second" }),
  4324. LLMEvent.textEnd({ id: "text-2" }),
  4325. ),
  4326. )
  4327. yield* runPrompt(session, "Two blocks")
  4328. expect(yield* session.context(sessionID)).toMatchObject([
  4329. { type: "user", text: "Two blocks" },
  4330. {
  4331. type: "assistant",
  4332. content: [
  4333. { type: "text", text: "First" },
  4334. { type: "text", text: "Second" },
  4335. ],
  4336. },
  4337. ])
  4338. }),
  4339. )
  4340. for (const kind of fragmentKinds) {
  4341. it.effect(`broadcasts provider ${kind} deltas without storing projection rewrites`, () =>
  4342. verifyEphemeralDeltas(kind),
  4343. )
  4344. it.effect(`durably closes partial ${kind} when the provider stream fails`, () => verifyPartialFlushOnFailure(kind))
  4345. it.effect(`durably closes partial ${kind} when the provider stream is interrupted`, () =>
  4346. verifyPartialFlushOnInterruption(kind),
  4347. )
  4348. }
  4349. it.effect("rejects duplicate streamed text starts", () =>
  4350. Effect.gen(function* () {
  4351. const session = yield* setup
  4352. yield* TestLLM.push([LLMEvent.textStart({ id: "text-1" }), LLMEvent.textStart({ id: "text-1" })])
  4353. const defect = yield* session.resume(sessionID).pipe(Effect.catchDefect(Effect.succeed))
  4354. expect(defect).toBeInstanceOf(Error)
  4355. if (!(defect instanceof Error)) return
  4356. expect(defect.message).toBe("Duplicate text start: text-1")
  4357. }),
  4358. )
  4359. it.effect("transitions streamed raw tool input to parsed called input", () =>
  4360. Effect.gen(function* () {
  4361. const session = yield* setup
  4362. yield* TestLLM.push(
  4363. TestLLM.stop(
  4364. LLMEvent.toolInputStart({ id: "call-parsed", name: "web_search" }),
  4365. LLMEvent.toolInputDelta({ id: "call-parsed", name: "web_search", text: '{"query":"hello"}' }),
  4366. LLMEvent.toolInputEnd({ id: "call-parsed", name: "web_search" }),
  4367. hostedCall("call-parsed", "hello"),
  4368. ),
  4369. )
  4370. yield* runPrompt(session, "Call provider tool")
  4371. expect(yield* session.context(sessionID)).toMatchObject([
  4372. { type: "user", text: "Call provider tool" },
  4373. {
  4374. type: "assistant",
  4375. content: [{ type: "tool", id: "call-parsed", state: { status: "error", input: { query: "hello" } } }],
  4376. },
  4377. ])
  4378. }),
  4379. )
  4380. it.effect("rejects malformed streamed tool input ordering", () =>
  4381. Effect.gen(function* () {
  4382. const session = yield* setup
  4383. yield* TestLLM.push([LLMEvent.toolInputDelta({ id: "call-1", name: "read", text: "{}" })])
  4384. const defect = yield* session.resume(sessionID).pipe(Effect.catchDefect(Effect.succeed))
  4385. expect(defect).toBeInstanceOf(Error)
  4386. if (!(defect instanceof Error)) return
  4387. expect(defect.message).toBe("Tool input delta before start: call-1")
  4388. }),
  4389. )
  4390. })