1
0

session-runner.test.ts 193 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299
  1. import { describe, expect, test } from "bun:test"
  2. import {
  3. AIError,
  4. LLMEvent,
  5. LLMRequest,
  6. Message,
  7. LanguageModel,
  8. SystemPart,
  9. ToolFailure,
  10. TransportReason,
  11. InvalidProviderOutputReason,
  12. InvalidRequestReason,
  13. RateLimitReason,
  14. } from "@opencode-ai/ai"
  15. import * as OpenAIChat from "@opencode-ai/ai/protocols/openai-chat"
  16. import { TestLLM } from "@opencode-ai/ai/testing"
  17. import { Catalog } from "@opencode-ai/core/catalog"
  18. import { Database } from "@opencode-ai/core/database/database"
  19. import { makeLocationNode } from "@opencode-ai/util/effect/app-node"
  20. import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder"
  21. import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform"
  22. import { LayerNode } from "@opencode-ai/util/effect/layer-node"
  23. import { Bus } from "@opencode-ai/core/bus"
  24. import { Event } from "@opencode-ai/schema/event"
  25. import { App } from "@opencode-ai/core/app"
  26. import { Permission } from "@opencode-ai/core/permission"
  27. import { EventTable } from "@opencode-ai/core/event/sql"
  28. import { Project } from "@opencode-ai/core/project"
  29. import { ProjectTable } from "@opencode-ai/core/project/sql"
  30. import { Form } from "@opencode-ai/core/form"
  31. import { AbsolutePath } from "@opencode-ai/core/schema"
  32. import { Session } from "@opencode-ai/core/session"
  33. import { Snapshot } from "@opencode-ai/core/snapshot"
  34. import { SessionEvent } from "@opencode-ai/core/session/event"
  35. import { SessionContext } from "@opencode-ai/core/session/context"
  36. import { SessionInbox } from "@opencode-ai/core/session/inbox"
  37. import { SessionMessage } from "@opencode-ai/core/session/message"
  38. import { SessionModelRequest } from "@opencode-ai/core/session/model-request"
  39. import { SessionModelTransport } from "@opencode-ai/core/session/model-transport"
  40. import { Money } from "@opencode-ai/schema/money"
  41. import { SessionProjector } from "@opencode-ai/core/session/projector"
  42. import { SessionExecution } from "@opencode-ai/core/session/execution"
  43. import { SessionRunCoordinator } from "@opencode-ai/core/session/run-coordinator"
  44. import { SessionRunner } from "@opencode-ai/core/session/runner/index"
  45. import * as SessionRunnerLLM from "@opencode-ai/core/session/runner/llm"
  46. import { SessionRunnerModel } from "@opencode-ai/core/session/runner/model"
  47. import { PromptCacheDiagnostics } from "@opencode-ai/core/session/prompt-cache-diagnostics"
  48. import { SessionUsage } from "@opencode-ai/core/session/usage"
  49. import { PluginSupervisor } from "@opencode-ai/core/plugin/supervisor"
  50. import { PluginHooks } from "@opencode-ai/core/plugin/hooks"
  51. import { SystemPromptPlugin } from "@opencode-ai/core/plugin/system-prompt"
  52. import { QuestionTool } from "@opencode-ai/core/tool/plugin/question"
  53. import { Agent } from "@opencode-ai/core/agent"
  54. import { Config } from "@opencode-ai/core/config"
  55. import { Document, Info } from "@opencode-ai/schema/config"
  56. import { ConfigCompaction } from "@opencode-ai/schema/config/compaction"
  57. import { Tool } from "@opencode-ai/core/tool"
  58. import type { Info as ToolInfo } from "@opencode-ai/schema/tool"
  59. import {
  60. InstructionStateTable,
  61. SessionInboxTable,
  62. SessionMessageTable,
  63. SessionTable,
  64. } from "@opencode-ai/core/session/sql"
  65. import { InstructionEntry } from "@opencode-ai/core/session/instruction-entry"
  66. import { InstructionState } from "@opencode-ai/core/session/instruction-state"
  67. import { SessionStore } from "@opencode-ai/core/session/store"
  68. import { Instructions } from "@opencode-ai/core/instructions/index"
  69. import { InstructionBuiltIns } from "@opencode-ai/core/instructions/builtins"
  70. import { InstructionDiscovery } from "@opencode-ai/core/instruction-discovery"
  71. import { SkillInstructions } from "@opencode-ai/core/skill/instructions"
  72. import { ReferenceInstructions } from "@opencode-ai/core/reference/instructions"
  73. import { McpInstructions } from "@opencode-ai/core/mcp/instructions"
  74. import { SessionSystemPrompt } from "@opencode-ai/core/session/system-prompt"
  75. import { ID } from "@opencode-ai/core/model"
  76. import { Location } from "@opencode-ai/core/location"
  77. import { Provider } from "@opencode-ai/core/provider"
  78. import { Cause, Deferred, Effect, Exit, Fiber, Layer, Schema, Scope, Stream } from "effect"
  79. import { TestClock } from "effect/testing"
  80. import { HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
  81. import { asc, desc, eq } from "drizzle-orm"
  82. import { testEffect } from "./lib/effect"
  83. import { permissionLayer } from "./lib/permission"
  84. import { agentHost, catalogHost, host } from "./plugin/host"
  85. import { CodeModeInstructions } from "@opencode-ai/core/codemode/instructions"
  86. let requests: LLMRequest[] = []
  87. const emptyCodeMode = `\n\n${CodeModeInstructions.render({ total: 0, shown: 0, namespaces: [] })}`
  88. type ToolBarrier = {
  89. readonly count: number
  90. readonly started: Deferred.Deferred<void>
  91. readonly release: Deferred.Deferred<void>
  92. active: number
  93. maxActive: number
  94. }
  95. let toolBarrier: ToolBarrier | undefined
  96. const releaseTools = (barrier: ToolBarrier) =>
  97. Effect.sync(() => {
  98. if (toolBarrier === barrier) toolBarrier = undefined
  99. }).pipe(Effect.andThen(Deferred.succeed(barrier.release, undefined)), Effect.asVoid)
  100. const blockTools = (count = 1) =>
  101. Effect.acquireRelease(
  102. Effect.all({ started: Deferred.make<void>(), release: Deferred.make<void>() }).pipe(
  103. Effect.map((deferreds) => {
  104. const barrier = { count, ...deferreds, active: 0, maxActive: 0 }
  105. toolBarrier = barrier
  106. return barrier
  107. }),
  108. ),
  109. releaseTools,
  110. ).pipe(
  111. Effect.map((barrier) => ({
  112. started: Deferred.await(barrier.started),
  113. release: releaseTools(barrier),
  114. maxActive: Effect.sync(() => barrier.maxActive),
  115. })),
  116. )
  117. const awaitToolBarrier = Effect.suspend(() => {
  118. const barrier = toolBarrier
  119. if (!barrier) return Effect.void
  120. barrier.active++
  121. barrier.maxActive = Math.max(barrier.maxActive, barrier.active)
  122. return (barrier.active === barrier.count ? Deferred.succeed(barrier.started, undefined) : Effect.void).pipe(
  123. Effect.andThen(Deferred.await(barrier.release)),
  124. Effect.ensuring(Effect.sync(() => barrier.active--)),
  125. )
  126. })
  127. const testLLM = TestLLM.layer({
  128. fallback: [],
  129. transformRequest: (request) =>
  130. LLMRequest.update(request, {
  131. system: request.system.map((part) => ({
  132. ...part,
  133. text: part.text.replace(emptyCodeMode, ""),
  134. })),
  135. tools: request.tools.filter((tool) => tool.name !== "execute"),
  136. }),
  137. })
  138. const client = TestLLM.clientLayer
  139. const closedTransports: Session.ID[] = []
  140. const modelTransport = Layer.succeed(
  141. SessionModelTransport.Service,
  142. SessionModelTransport.Service.of({
  143. bind: () => ({ execute: () => Effect.die("Unexpected WebSocket execution") }),
  144. close: (sessionID) => Effect.sync(() => closedTransports.push(sessionID)),
  145. closeAll: Effect.void,
  146. }),
  147. )
  148. const model = LanguageModel.make({ id: "fake-model", provider: "fake", route: OpenAIChat.route })
  149. const defaultSystem = SessionSystemPrompt.make([])
  150. const replacementModel = LanguageModel.make({ id: "replacement", provider: "fake", route: OpenAIChat.route })
  151. const compactModel = LanguageModel.make({
  152. id: "compact",
  153. provider: "fake",
  154. route: OpenAIChat.route.with({ limits: { context: 4_000, output: 50 } }),
  155. })
  156. const fullOutputModel = LanguageModel.make({
  157. id: "full-output",
  158. provider: "fake",
  159. route: OpenAIChat.route.with({ limits: { context: 262_144, output: 262_144 } }),
  160. })
  161. const undersizedContextModel = LanguageModel.make({
  162. id: "undersized-context",
  163. provider: "fake",
  164. route: OpenAIChat.route.with({ limits: { context: 1, output: 1_000 } }),
  165. })
  166. const recoveryModel = LanguageModel.make({
  167. id: "recovery",
  168. provider: "fake",
  169. route: OpenAIChat.route.with({ limits: { context: 20_000, output: 1_000 } }),
  170. })
  171. test("calculates step cost using the matching context tier", () => {
  172. expect(
  173. SessionUsage.calculateCost(
  174. [
  175. {
  176. input: Money.USDPerMillionTokens.make(1),
  177. output: Money.USDPerMillionTokens.make(2),
  178. cache: {
  179. read: Money.USDPerMillionTokens.make(0.1),
  180. write: Money.USDPerMillionTokens.make(0.5),
  181. },
  182. },
  183. {
  184. tier: { type: "context", size: 100 },
  185. input: Money.USDPerMillionTokens.make(3),
  186. output: Money.USDPerMillionTokens.make(4),
  187. cache: {
  188. read: Money.USDPerMillionTokens.make(0.2),
  189. write: Money.USDPerMillionTokens.make(0.6),
  190. },
  191. },
  192. ],
  193. { input: 80, output: 10, reasoning: 2, cache: { read: 20, write: 1 } },
  194. ),
  195. ).toBeCloseTo(0.0002926)
  196. })
  197. test("does not apply an ineligible tier without base pricing", () => {
  198. expect(
  199. SessionUsage.calculateCost(
  200. [
  201. {
  202. tier: { type: "context", size: 100 },
  203. input: Money.USDPerMillionTokens.make(3),
  204. output: Money.USDPerMillionTokens.make(4),
  205. cache: {
  206. read: Money.USDPerMillionTokens.make(0.2),
  207. write: Money.USDPerMillionTokens.make(0.6),
  208. },
  209. },
  210. ],
  211. { input: 80, output: 10, reasoning: 2, cache: { read: 20, write: 0 } },
  212. ),
  213. ).toBe(Money.USD.zero)
  214. })
  215. const authorizations: Tool.Context[] = []
  216. const executions: string[] = []
  217. const permissionFail = {
  218. name: "permission_fail",
  219. description: "Reject a permission",
  220. input: Schema.Struct({}),
  221. output: Schema.Struct({}),
  222. execute: () =>
  223. new ToolFailure({
  224. message: "Permission denied: edit",
  225. error: new Permission.BlockedError({
  226. rules: [],
  227. permission: "edit",
  228. resources: ["src/index.ts"],
  229. }),
  230. }),
  231. }
  232. const permission = permissionLayer()
  233. const transformTools = (registry: Tool.Interface, tools: Readonly<Record<string, ToolInfo>>, options?: Tool.Options) =>
  234. registry.transform((draft) =>
  235. Object.entries(tools).forEach(([name, tool]) => draft.add({ ...tool, name, options: options ?? tool.options })),
  236. )
  237. const echo = Layer.effectDiscard(
  238. Tool.Service.use((registry) =>
  239. transformTools(
  240. registry,
  241. {
  242. echo: {
  243. name: "echo",
  244. description: "Echo text",
  245. input: Schema.Struct({ text: Schema.String }),
  246. output: Schema.Struct({ text: Schema.String }),
  247. execute: ({ text }, context) =>
  248. Effect.gen(function* () {
  249. authorizations.push(context)
  250. executions.push(text)
  251. yield* awaitToolBarrier
  252. return { output: { text }, content: text }
  253. }),
  254. },
  255. defect: {
  256. name: "defect",
  257. description: "Fail unexpectedly",
  258. input: Schema.Struct({}),
  259. output: Schema.Struct({}),
  260. execute: () => awaitToolBarrier.pipe(Effect.andThen(Effect.die("unexpected tool defect"))),
  261. },
  262. storefail: {
  263. name: "storefail",
  264. description: "Produce output that cannot be persisted",
  265. input: Schema.Struct({}),
  266. output: Schema.Struct({}),
  267. execute: () => Effect.succeed({ output: {} }),
  268. },
  269. },
  270. { codemode: false },
  271. ),
  272. ),
  273. )
  274. const echoNode = makeLocationNode({ name: "test/session-runner-tools", layer: echo, deps: [Tool.node] })
  275. let modelResolveHook = Effect.void
  276. let currentModel = model
  277. const models = Layer.mock(SessionRunnerModel.Service)({
  278. resolve: (session) =>
  279. modelResolveHook.pipe(
  280. Effect.as(
  281. SessionRunnerModel.resolved(session.model?.id === "replacement" ? replacementModel : currentModel, {
  282. capabilities: { tools: true, input: ["text", "image"], output: ["text"] },
  283. cost: [],
  284. variant: session.model?.variant,
  285. }),
  286. ),
  287. ),
  288. })
  289. const systemContextKey = Instructions.Key.make("test/context")
  290. let systemBaseline = "Initial context"
  291. let systemRemoved = false
  292. let systemUnavailable = false
  293. let systemLoadHook = Effect.void
  294. const skillBaselines = new Map<Agent.ID, string>()
  295. const systemContext = Layer.mock(InstructionBuiltIns.Service, {
  296. load: () =>
  297. Effect.sync(() =>
  298. Instructions.make({
  299. key: systemContextKey,
  300. codec: Schema.toCodecJson(Schema.String),
  301. read: systemLoadHook.pipe(
  302. Effect.andThen(
  303. Effect.sync(() =>
  304. systemUnavailable ? Instructions.unavailable : systemRemoved ? Instructions.removed : systemBaseline,
  305. ),
  306. ),
  307. ),
  308. render: {
  309. initial: String,
  310. changed: (_previous, current) => current,
  311. removed: () => "System context source removed: test/context",
  312. },
  313. }),
  314. ),
  315. })
  316. const instructionContext = Layer.mock(InstructionDiscovery.Service, {
  317. project: true,
  318. load: () => Effect.succeed(Instructions.empty),
  319. })
  320. const skillInstructions = Layer.mock(SkillInstructions.Service, {
  321. load: (agent) =>
  322. Effect.succeed(
  323. skillBaselines.has(agent.id)
  324. ? Instructions.make({
  325. key: Instructions.Key.make("test/skill-guidance"),
  326. codec: Schema.toCodecJson(Schema.String),
  327. read: Effect.succeed(skillBaselines.get(agent.id)!),
  328. render: {
  329. initial: String,
  330. changed: (_previous, current) => current,
  331. removed: () => "Skill guidance removed",
  332. },
  333. })
  334. : Instructions.empty,
  335. ),
  336. })
  337. const referenceInstructions = Layer.mock(ReferenceInstructions.Service, {
  338. load: () => Effect.succeed(Instructions.empty),
  339. })
  340. const mcpInstructions = Layer.mock(McpInstructions.Service, { load: () => Effect.succeed(Instructions.empty) })
  341. const config = Config.testLayer([
  342. new Document({
  343. type: "document",
  344. info: new Info({
  345. compaction: new ConfigCompaction.Info({
  346. buffer: 3_000,
  347. keep: new ConfigCompaction.Keep({ tokens: 1_000 }),
  348. }),
  349. }),
  350. }),
  351. ])
  352. let pluginFlushHook = Effect.void
  353. const pluginSupervisor = Layer.succeed(
  354. PluginSupervisor.Service,
  355. PluginSupervisor.Service.of({
  356. flush: Effect.suspend(() => pluginFlushHook),
  357. }),
  358. )
  359. const promptCatalog = Layer.mock(Catalog.Service, {
  360. provider: {
  361. get: () => Effect.succeed(undefined),
  362. all: () => Effect.succeed([]),
  363. available: () => Effect.succeed([]),
  364. },
  365. model: {
  366. get: () => Effect.succeed(undefined),
  367. all: () => Effect.succeed([]),
  368. available: () => Effect.succeed([]),
  369. default: () => Effect.succeed(undefined),
  370. small: () => Effect.succeed(undefined),
  371. },
  372. })
  373. const runnerLayer = AppNodeBuilder.build(SessionRunnerLLM.node, [
  374. [Snapshot.node, Snapshot.noopLayer],
  375. [LayerNodePlatform.llmClient, client],
  376. [SessionRunnerModel.node, models],
  377. [InstructionBuiltIns.node, systemContext],
  378. [InstructionDiscovery.node, instructionContext],
  379. [Location.node, Location.boundNode({ directory: AbsolutePath.make("/project") })],
  380. [SkillInstructions.node, skillInstructions],
  381. [ReferenceInstructions.node, referenceInstructions],
  382. [Permission.node, permission],
  383. [Config.node, config],
  384. [McpInstructions.node, mcpInstructions],
  385. [PluginSupervisor.node, pluginSupervisor],
  386. [SessionModelTransport.node, modelTransport],
  387. ])
  388. const execution = Layer.effect(
  389. SessionExecution.Service,
  390. Effect.gen(function* () {
  391. const sessionRunner = yield* SessionRunner.Service
  392. function drain(
  393. sessionID: Session.ID,
  394. force: boolean,
  395. continuation?: SessionRunner.Continuation,
  396. ): Effect.Effect<void, SessionRunner.RunError> {
  397. return sessionRunner
  398. .drain({ sessionID, force, continuation })
  399. .pipe(
  400. Effect.flatMap((result) =>
  401. result.type === "complete" ? Effect.void : drain(sessionID, false, result.continuation),
  402. ),
  403. )
  404. }
  405. const coordinator = yield* SessionRunCoordinator.make<Session.ID, SessionRunner.RunError>({
  406. drain: (sessionID, force) => drain(sessionID, force),
  407. })
  408. return SessionExecution.Service.of({
  409. active: coordinator.active,
  410. resume: coordinator.run,
  411. wake: coordinator.wake,
  412. wakeActive: coordinator.wakeActive,
  413. interrupt: (sessionID) => coordinator.interrupt(sessionID),
  414. awaitIdle: coordinator.awaitIdle,
  415. })
  416. }),
  417. ).pipe(Layer.provide(runnerLayer))
  418. const it = testEffect(
  419. AppNodeBuilder.build(
  420. LayerNode.group([
  421. Database.node,
  422. Bus.node,
  423. Form.node,
  424. SessionProjector.node,
  425. SessionStore.node,
  426. Agent.node,
  427. Catalog.node,
  428. Tool.node,
  429. Tool.node,
  430. PluginHooks.node,
  431. PluginHooks.node,
  432. echoNode,
  433. SessionRunnerModel.node,
  434. InstructionBuiltIns.node,
  435. InstructionDiscovery.node,
  436. InstructionEntry.node,
  437. SkillInstructions.node,
  438. ReferenceInstructions.node,
  439. Config.node,
  440. Snapshot.node,
  441. SessionContext.node,
  442. SessionModelRequest.node,
  443. SessionRunnerLLM.node,
  444. SessionExecution.node,
  445. Session.node,
  446. ]),
  447. [
  448. [Bus.node, Bus.configured({ persist: true })],
  449. [LayerNodePlatform.llmClient, client],
  450. [Permission.node, permission],
  451. [Catalog.node, promptCatalog],
  452. [SessionRunnerModel.node, models],
  453. [InstructionBuiltIns.node, systemContext],
  454. [InstructionDiscovery.node, instructionContext],
  455. [Location.node, Location.boundNode({ directory: AbsolutePath.make("/project") })],
  456. [SkillInstructions.node, skillInstructions],
  457. [ReferenceInstructions.node, referenceInstructions],
  458. [Snapshot.node, Snapshot.noopLayer],
  459. [SessionExecution.node, execution],
  460. [Config.node, config],
  461. [PluginSupervisor.node, pluginSupervisor],
  462. [SessionModelTransport.node, modelTransport],
  463. ],
  464. ).pipe(Layer.provideMerge(testLLM)),
  465. )
  466. const sessionID = Session.ID.make("ses_runner_test")
  467. const otherSessionID = Session.ID.make("ses_runner_other")
  468. const admit = (session: Session.Interface, text: string) => session.prompt({ sessionID, text, resume: false })
  469. const runPrompt = Effect.fnUntraced(function* (session: Session.Interface, text: string) {
  470. const message = yield* admit(session, text)
  471. yield* session.resume(sessionID)
  472. return message
  473. })
  474. const insertSession = (id: Session.ID) =>
  475. Effect.gen(function* () {
  476. const { db } = yield* Database.Service
  477. yield* db
  478. .insert(SessionTable)
  479. .values({
  480. id,
  481. project_id: Project.ID.global,
  482. slug: id,
  483. directory: "/project",
  484. title: "test",
  485. version: "test",
  486. })
  487. .onConflictDoNothing()
  488. .run()
  489. .pipe(Effect.orDie)
  490. })
  491. const setup = Effect.gen(function* () {
  492. const { db } = yield* Database.Service
  493. const agents = yield* Agent.Service
  494. const catalog = yield* Catalog.Service
  495. const hooks = yield* PluginHooks.Service
  496. const pluginHost = host({
  497. agent: agentHost(agents),
  498. catalog: catalogHost(catalog),
  499. session: { hook: (name, callback) => hooks.register("session", name, callback) },
  500. })
  501. yield* Effect.forEach(SystemPromptPlugin.Plugins, (plugin) => plugin.effect(pluginHost), {
  502. discard: true,
  503. })
  504. requests = (yield* TestLLM.Service).requests
  505. authorizations.length = 0
  506. executions.length = 0
  507. closedTransports.length = 0
  508. systemBaseline = "Initial context"
  509. systemRemoved = false
  510. systemUnavailable = false
  511. systemLoadHook = Effect.void
  512. modelResolveHook = Effect.void
  513. pluginFlushHook = Effect.void
  514. currentModel = model
  515. skillBaselines.clear()
  516. toolBarrier = undefined
  517. yield* agents.transform((draft) =>
  518. draft.update(Agent.ID.make("build"), (agent) => {
  519. agent.mode = "primary"
  520. }),
  521. )
  522. yield* db
  523. .insert(ProjectTable)
  524. .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] })
  525. .onConflictDoNothing()
  526. .run()
  527. .pipe(Effect.orDie)
  528. yield* insertSession(sessionID)
  529. return yield* Session.Service
  530. })
  531. const providerUnavailable = () =>
  532. new AIError({
  533. module: "test",
  534. method: "stream",
  535. reason: new TransportReason({
  536. message: "Provider unavailable",
  537. transport: "http",
  538. operation: "request",
  539. }),
  540. })
  541. const continuationRejected = (recovery: "retry-full" | "rotate-and-retry-full") =>
  542. new AIError({
  543. module: "test",
  544. method: "stream",
  545. reason: new TransportReason({
  546. message: "Continuation rejected",
  547. transport: "websocket",
  548. operation: "read",
  549. phase: "receive",
  550. delivery: "rejected",
  551. recovery,
  552. }),
  553. })
  554. const incompleteStream = () =>
  555. new AIError({
  556. module: "test",
  557. method: "stream",
  558. reason: new InvalidProviderOutputReason({
  559. classification: "incomplete-stream",
  560. message: "The provider response ended unexpectedly.",
  561. }),
  562. })
  563. const INCOMPLETE_STREAM_CONTINUATION =
  564. "The previous response was interrupted. Continue from where you left off without repeating completed content."
  565. const invalidRequest = () =>
  566. new AIError({
  567. module: "test",
  568. method: "stream",
  569. reason: new InvalidRequestReason({ message: "Invalid request" }),
  570. })
  571. const rateLimited = (retryAfterMs?: number) =>
  572. new AIError({
  573. module: "test",
  574. method: "stream",
  575. reason: new RateLimitReason({ message: "Rate limited", retryAfterMs }),
  576. })
  577. const setupOverflowRecovery = Effect.gen(function* () {
  578. const session = yield* setup
  579. yield* TestLLM.push(TestLLM.text("Earlier answer", "text-earlier"))
  580. yield* runPrompt(session, "Earlier question ".repeat(700))
  581. currentModel = recoveryModel
  582. requests.length = 0
  583. return session
  584. })
  585. const messageTexts = (request: LLMRequest, role: "user" | "system") =>
  586. request.messages.flatMap((message) =>
  587. message.role === role ? message.content.flatMap((content) => (content.type === "text" ? [content.text] : [])) : [],
  588. )
  589. const userTexts = (request: LLMRequest) => messageTexts(request, "user")
  590. const systemTexts = (request: LLMRequest) => messageTexts(request, "system")
  591. const messageRoles = (request: LLMRequest | undefined) => request?.messages.map((message) => message.role)
  592. const recordedEventTypes = (id: Session.ID) =>
  593. Effect.gen(function* () {
  594. const { db } = yield* Database.Service
  595. return yield* db
  596. .select({ type: EventTable.type })
  597. .from(EventTable)
  598. .where(eq(EventTable.aggregate_id, id))
  599. .orderBy(asc(EventTable.seq))
  600. .all()
  601. .pipe(
  602. Effect.orDie,
  603. Effect.map((rows) => rows.map((row) => row.type)),
  604. )
  605. })
  606. const recordedStepSettlementEvents = (id: Session.ID, assistantMessageID: SessionMessage.ID) =>
  607. Effect.gen(function* () {
  608. const { db } = yield* Database.Service
  609. const settlementTypes = new Set([
  610. "session.step.started.1",
  611. "session.tool.called.1",
  612. "session.tool.success.2",
  613. "session.tool.failed.2",
  614. "session.step.ended.1",
  615. "session.step.failed.1",
  616. ])
  617. return (yield* db
  618. .select({ type: EventTable.type, data: EventTable.data })
  619. .from(EventTable)
  620. .where(eq(EventTable.aggregate_id, id))
  621. .orderBy(asc(EventTable.seq))
  622. .all()
  623. .pipe(Effect.orDie)).filter(
  624. (event) => settlementTypes.has(event.type) && event.data.assistantMessageID === assistantMessageID,
  625. )
  626. })
  627. const recordedStepSettlementTypes = (id: Session.ID, assistantMessageID: SessionMessage.ID) =>
  628. recordedStepSettlementEvents(id, assistantMessageID).pipe(Effect.map((events) => events.map((event) => event.type)))
  629. const hostedCall = (id: string, query: string) =>
  630. LLMEvent.toolCall({ id, name: "web_search", input: { query }, providerExecuted: true })
  631. const requireAssistant = (messages: readonly SessionMessage.Info[]) => {
  632. const assistant = messages.find((message) => message.type === "assistant")
  633. if (!assistant) throw new Error("Assistant message missing")
  634. return assistant
  635. }
  636. const replaySessionProjection = (id: Session.ID) =>
  637. Effect.gen(function* () {
  638. const { db } = yield* Database.Service
  639. const bus = yield* Bus.Service
  640. const recorded = yield* db
  641. .select()
  642. .from(EventTable)
  643. .where(eq(EventTable.aggregate_id, id))
  644. .orderBy(asc(EventTable.seq))
  645. .all()
  646. .pipe(Effect.orDie)
  647. yield* bus.remove(id)
  648. yield* db.delete(InstructionStateTable).where(eq(InstructionStateTable.session_id, id)).run().pipe(Effect.orDie)
  649. yield* db.delete(SessionInboxTable).where(eq(SessionInboxTable.session_id, id)).run().pipe(Effect.orDie)
  650. yield* db.delete(SessionMessageTable).where(eq(SessionMessageTable.session_id, id)).run().pipe(Effect.orDie)
  651. yield* Effect.forEach(
  652. recorded.map((event) => ({
  653. id: event.id,
  654. created: event.created,
  655. aggregateID: event.aggregate_id,
  656. seq: event.seq,
  657. type: event.type,
  658. data: event.data,
  659. })),
  660. (event) => bus.replay(event),
  661. { discard: true },
  662. )
  663. })
  664. type FragmentKind = "text" | "reasoning" | "tool input"
  665. type FragmentFixture = {
  666. readonly delta?: Event.Definition
  667. readonly completeEvents: LLMEvent[]
  668. readonly partialEvents: LLMEvent[]
  669. readonly expectedAssistant: unknown
  670. readonly expectedContent: unknown
  671. }
  672. const fragmentKinds: readonly FragmentKind[] = ["text", "reasoning", "tool input"]
  673. const fragmentID = (kind: FragmentKind, suffix: string) => `${kind === "tool input" ? "call" : kind}-${suffix}`
  674. const fragmentFixture = (kind: FragmentKind, id: string, chunks: readonly string[]): FragmentFixture => {
  675. const text = chunks.join("")
  676. switch (kind) {
  677. case "text": {
  678. const partialEvents = [
  679. LLMEvent.stepStart({ index: 0 }),
  680. LLMEvent.textStart({ id }),
  681. ...chunks.map((text) => LLMEvent.textDelta({ id, text })),
  682. ]
  683. const expectedContent = { type: "text", text }
  684. return {
  685. delta: SessionEvent.Text.Delta,
  686. partialEvents,
  687. completeEvents: [
  688. ...partialEvents,
  689. LLMEvent.textEnd({ id }),
  690. LLMEvent.stepFinish({ index: 0, reason: { normalized: "stop" } }),
  691. LLMEvent.finish({ reason: { normalized: "stop" } }),
  692. ],
  693. expectedAssistant: { type: "assistant", finish: "stop", content: [expectedContent] },
  694. expectedContent,
  695. }
  696. }
  697. case "reasoning": {
  698. const partialEvents = [
  699. LLMEvent.stepStart({ index: 0 }),
  700. LLMEvent.reasoningStart({ id }),
  701. ...chunks.map((text) => LLMEvent.reasoningDelta({ id, text })),
  702. ]
  703. const expectedContent = { type: "reasoning", text }
  704. return {
  705. delta: SessionEvent.Reasoning.Delta,
  706. partialEvents,
  707. completeEvents: [
  708. ...partialEvents,
  709. LLMEvent.reasoningEnd({ id }),
  710. LLMEvent.stepFinish({ index: 0, reason: { normalized: "stop" } }),
  711. LLMEvent.finish({ reason: { normalized: "stop" } }),
  712. ],
  713. expectedAssistant: { type: "assistant", finish: "stop", content: [expectedContent] },
  714. expectedContent,
  715. }
  716. }
  717. case "tool input": {
  718. const partialEvents = [
  719. LLMEvent.stepStart({ index: 0 }),
  720. LLMEvent.toolInputStart({ id, name: "echo" }),
  721. ...chunks.map((text) => LLMEvent.toolInputDelta({ id, name: "echo", text })),
  722. ]
  723. const expectedContent = { type: "tool", id, state: { status: "streaming", input: text } }
  724. return {
  725. partialEvents,
  726. completeEvents: [...partialEvents, LLMEvent.toolInputEnd({ id, name: "echo" })],
  727. expectedAssistant: { type: "assistant", content: [expectedContent] },
  728. expectedContent,
  729. }
  730. }
  731. }
  732. }
  733. const verifyEphemeralDeltas = (kind: FragmentKind) =>
  734. Effect.gen(function* () {
  735. const session = yield* setup
  736. const prompt = `Stream ${kind}`
  737. const chunks = Array.from({ length: 32 }, (_, index) => `${index},`)
  738. const fixture = fragmentFixture(kind, fragmentID(kind, "many"), chunks)
  739. const expectedContext = [{ type: "user", text: prompt }, fixture.expectedAssistant]
  740. yield* admit(session, prompt)
  741. const bus = yield* Bus.Service
  742. const live = fixture.delta
  743. ? yield* bus.subscribe(fixture.delta).pipe(Stream.take(1), Stream.runCollect, Effect.forkScoped)
  744. : undefined
  745. yield* Effect.yieldNow
  746. yield* TestLLM.push(fixture.completeEvents)
  747. yield* session.resume(sessionID)
  748. const { db } = yield* Database.Service
  749. const deltas = fixture.delta
  750. ? yield* db
  751. .select({ type: EventTable.type })
  752. .from(EventTable)
  753. .where(eq(EventTable.type, Bus.versionedType(fixture.delta.type, 1)))
  754. .all()
  755. .pipe(Effect.orDie)
  756. : []
  757. if (live) {
  758. const streamed = Array.from(yield* Fiber.join(live))
  759. expect(streamed).toHaveLength(1)
  760. expect(
  761. streamed
  762. .map((event) => {
  763. if (!event.data || typeof event.data !== "object" || !("delta" in event.data))
  764. throw new Error("Expected delta event")
  765. if (typeof event.data.delta !== "string") throw new Error("Expected string delta")
  766. return event.data.delta
  767. })
  768. .join(""),
  769. ).toBe(chunks.join(""))
  770. }
  771. expect(deltas).toHaveLength(0)
  772. expect(yield* session.context(sessionID)).toMatchObject(expectedContext)
  773. yield* replaySessionProjection(sessionID)
  774. expect(yield* session.context(sessionID)).toMatchObject(expectedContext)
  775. })
  776. const verifyPartialFlushOnFailure = (kind: FragmentKind) =>
  777. Effect.gen(function* () {
  778. const session = yield* setup
  779. const prompt = `Fail after ${kind}`
  780. const fixture = fragmentFixture(kind, fragmentID(kind, "partial"), ["Partial"])
  781. const failure = providerUnavailable()
  782. yield* admit(session, prompt)
  783. yield* TestLLM.push(TestLLM.failAfter(failure, ...fixture.partialEvents))
  784. expect(yield* session.resume(sessionID).pipe(Effect.flip)).toBe(failure)
  785. expect(yield* session.context(sessionID)).toMatchObject([
  786. { type: "user", text: prompt },
  787. {
  788. type: "assistant",
  789. finish: "error",
  790. error: { type: "provider.transport", message: "Provider unavailable" },
  791. content: [
  792. kind === "tool input"
  793. ? {
  794. type: "tool",
  795. id: fragmentID(kind, "partial"),
  796. state: {
  797. status: "error",
  798. error: { type: "provider.transport", message: "Provider unavailable" },
  799. },
  800. }
  801. : fixture.expectedContent,
  802. ],
  803. },
  804. ])
  805. expect(requests).toHaveLength(1)
  806. })
  807. const verifyPartialFlushOnInterruption = (kind: FragmentKind) =>
  808. Effect.gen(function* () {
  809. const session = yield* setup
  810. const prompt = `Interrupt after ${kind}`
  811. const fixture = fragmentFixture(kind, fragmentID(kind, "interrupted"), ["Partial"])
  812. const streamed = yield* Deferred.make<void>()
  813. yield* admit(session, prompt)
  814. yield* TestLLM.push(
  815. Stream.concat(
  816. Stream.fromIterable(fixture.partialEvents),
  817. Stream.fromEffect(Deferred.succeed(streamed, undefined)).pipe(Stream.flatMap(() => Stream.never)),
  818. ),
  819. )
  820. const runner = yield* SessionRunner.Service
  821. const fiber = yield* runner.drain({ sessionID, force: true }).pipe(Effect.forkChild)
  822. yield* Deferred.await(streamed)
  823. yield* Fiber.interrupt(fiber)
  824. expect(yield* session.context(sessionID)).toMatchObject([
  825. { type: "user", text: prompt },
  826. {
  827. type: "assistant",
  828. finish: "error",
  829. error: { type: "aborted", message: "Step interrupted" },
  830. content: [
  831. kind === "tool input"
  832. ? { type: "tool", id: fragmentID(kind, "interrupted"), state: { status: "error" } }
  833. : fixture.expectedContent,
  834. ],
  835. },
  836. ])
  837. })
  838. describe("SessionRunnerLLM", () => {
  839. it.effect("generates the title while the first model step is still running", () =>
  840. Effect.gen(function* () {
  841. const session = yield* setup
  842. const agents = yield* Agent.Service
  843. const { db } = yield* Database.Service
  844. yield* db.update(SessionTable).set({ title: null }).where(eq(SessionTable.id, sessionID)).run().pipe(Effect.orDie)
  845. yield* agents.transform((draft) =>
  846. draft.update(Agent.ID.make("title"), (agent) => {
  847. agent.mode = "primary"
  848. agent.hidden = true
  849. agent.system = "Generate a title."
  850. }),
  851. )
  852. yield* admit(session, "First prompt")
  853. yield* TestLLM.push(TestLLM.text("Generated title", "text-title"), Stream.never)
  854. const bus = yield* Bus.Service
  855. const renamed = yield* bus.subscribe(SessionEvent.Renamed).pipe(
  856. Stream.filter((event) => event.data.sessionID === sessionID),
  857. Stream.take(1),
  858. Stream.runDrain,
  859. Effect.forkScoped({ startImmediately: true }),
  860. )
  861. const runner = yield* SessionRunner.Service
  862. const fiber = yield* runner.drain({ sessionID, force: true }).pipe(Effect.forkChild)
  863. yield* Fiber.join(renamed)
  864. expect((yield* session.get(sessionID)).title).toBe("Generated title")
  865. yield* Fiber.interrupt(fiber)
  866. }),
  867. )
  868. it.effect("retries title generation from the first prompt after title and execution failures", () =>
  869. Effect.gen(function* () {
  870. const session = yield* setup
  871. const agents = yield* Agent.Service
  872. const { db } = yield* Database.Service
  873. yield* db.update(SessionTable).set({ title: null }).where(eq(SessionTable.id, sessionID)).run().pipe(Effect.orDie)
  874. yield* agents.transform((draft) =>
  875. draft.update(Agent.ID.make("title"), (agent) => {
  876. agent.mode = "primary"
  877. agent.hidden = true
  878. agent.system = "Generate a title."
  879. }),
  880. )
  881. yield* admit(session, "First prompt")
  882. yield* TestLLM.push(Stream.fail(invalidRequest()), Stream.fail(invalidRequest()))
  883. expect((yield* session.resume(sessionID).pipe(Effect.exit))._tag).toBe("Failure")
  884. yield* admit(session, "Second prompt")
  885. const titleFailed = yield* Deferred.make<void>()
  886. yield* TestLLM.push(
  887. Stream.make(LLMEvent.providerError({ message: "Title provider unavailable" })).pipe(
  888. Stream.ensuring(Deferred.succeed(titleFailed, undefined)),
  889. ),
  890. TestLLM.text("Recovered", "text-recovered"),
  891. )
  892. yield* session.resume(sessionID)
  893. yield* Deferred.await(titleFailed)
  894. yield* Effect.yieldNow
  895. expect((yield* session.get(sessionID)).title).toBeUndefined()
  896. const bus = yield* Bus.Service
  897. const renamed = yield* bus.subscribe(SessionEvent.Renamed).pipe(
  898. Stream.filter((event) => event.data.sessionID === sessionID),
  899. Stream.take(1),
  900. Stream.runCollect,
  901. Effect.forkScoped({ startImmediately: true }),
  902. )
  903. yield* admit(session, "Third prompt")
  904. yield* TestLLM.push(
  905. TestLLM.text("Generated title", "text-title"),
  906. TestLLM.text("Recovered again", "text-recovered-again"),
  907. )
  908. yield* session.resume(sessionID)
  909. yield* Fiber.join(renamed)
  910. expect(requests).toHaveLength(6)
  911. expect(requests[2]?.messages).toContainEqual(Message.user("First prompt"))
  912. expect(requests[4]?.messages).toContainEqual(Message.user("First prompt"))
  913. expect((yield* session.get(sessionID)).title).toBe("Generated title")
  914. }),
  915. )
  916. it.effect("applies session context hooks without exposing unavailable tools", () =>
  917. Effect.gen(function* () {
  918. const session = yield* setup
  919. const hooks = yield* PluginHooks.Service
  920. yield* hooks.register("session", "context", (event) =>
  921. Effect.sync(() => {
  922. event.system = [SystemPart.make("Hooked system")]
  923. event.messages = [Message.user("Hooked message")]
  924. delete event.tools.echo
  925. event.tools.unregistered = { description: "Unavailable", input: { type: "object" } }
  926. }),
  927. )
  928. yield* admit(session, "Original message")
  929. yield* TestLLM.push(TestLLM.tool("call-removed", "echo", { text: "blocked" }))
  930. yield* session.resume(sessionID)
  931. // A hook-removed call fails independently and continues while step allowance remains.
  932. expect(requests).toHaveLength(2)
  933. expect(requests[0]?.system.map((part) => part.text)).toEqual(["Hooked system"])
  934. expect(requests[0]?.messages).toEqual([Message.user("Hooked message")])
  935. expect(requests[0]?.tools.map((tool) => tool.name)).not.toContain("echo")
  936. expect(requests[0]?.tools.map((tool) => tool.name)).not.toContain("unregistered")
  937. expect(executions).toEqual([])
  938. expect(yield* session.context(sessionID)).toMatchObject([
  939. { type: "user", text: "Original message" },
  940. {
  941. type: "assistant",
  942. content: [
  943. {
  944. type: "tool",
  945. id: "call-removed",
  946. state: { status: "error", error: { type: "tool.execution" } },
  947. },
  948. ],
  949. },
  950. ])
  951. }),
  952. )
  953. it.effect("forces HTTP and triggers active request and response hooks once", () =>
  954. Effect.gen(function* () {
  955. yield* setup
  956. const hooks = yield* PluginHooks.Service
  957. let requestTriggers = 0
  958. let responseTriggers = 0
  959. yield* hooks.register("session", "http.request", (event) =>
  960. Effect.sync(() => {
  961. requestTriggers++
  962. event.request.headers.set("x-request-hook", "active")
  963. }),
  964. )
  965. yield* hooks.register("session", "http.response", (event) =>
  966. Effect.sync(() => {
  967. responseTriggers++
  968. event.response.headers.set("x-response-hook", "active")
  969. }),
  970. )
  971. const context = yield* SessionContext.Service
  972. const modelRequests = yield* SessionModelRequest.Service
  973. const selected = yield* context.select(sessionID)
  974. const database = yield* Database.Service
  975. const bus = yield* Bus.Service
  976. yield* InstructionState.prepare(database.db, bus, selected.instructions, sessionID)
  977. const prepared = yield* modelRequests.prepare({
  978. context: yield* context.load(selected),
  979. step: 1,
  980. })
  981. const http = prepared.options.http ?? (yield* Effect.die("Expected Session HTTP middleware"))
  982. const response = yield* http(HttpClientRequest.post("https://provider.test/responses"), (request) => {
  983. expect(request.headers["x-request-hook"]).toBe("active")
  984. return Effect.succeed(HttpClientResponse.fromWeb(request, new Response("network")))
  985. })
  986. expect(prepared.webSocketEligible).toBe(false)
  987. expect(response.headers["x-response-hook"]).toBe("active")
  988. expect(requestTriggers).toBe(1)
  989. expect(responseTriggers).toBe(1)
  990. }),
  991. )
  992. it.effect("executes a tool renamed by a session context hook", () =>
  993. Effect.gen(function* () {
  994. const session = yield* setup
  995. const hooks = yield* PluginHooks.Service
  996. yield* hooks.register("session", "context", (event) =>
  997. Effect.sync(() => {
  998. event.tools.renamed_echo = event.tools.echo!
  999. delete event.tools.echo
  1000. }),
  1001. )
  1002. yield* admit(session, "Use the renamed tool")
  1003. yield* TestLLM.push(TestLLM.tool("call-renamed", "renamed_echo", { text: "renamed" }), [])
  1004. yield* session.resume(sessionID)
  1005. expect(requests[0]?.tools.map((tool) => tool.name)).toContain("renamed_echo")
  1006. expect(requests[0]?.tools.map((tool) => tool.name)).not.toContain("echo")
  1007. expect(executions).toEqual(["renamed"])
  1008. }),
  1009. )
  1010. it.effect("advertises and executes a location registered tool", () =>
  1011. Effect.gen(function* () {
  1012. const session = yield* setup
  1013. const registry = yield* Tool.Service
  1014. const contexts: Tool.Context[] = []
  1015. yield* transformTools(
  1016. registry,
  1017. {
  1018. location_context: {
  1019. name: "location_context",
  1020. description: "Read application context",
  1021. input: Schema.Struct({ query: Schema.String }),
  1022. output: Schema.Struct({ answer: Schema.String }),
  1023. execute: ({ query }, context) =>
  1024. Effect.gen(function* () {
  1025. contexts.push(context)
  1026. yield* context.progress({ phase: "reading" })
  1027. return { output: { answer: query.toUpperCase() } }
  1028. }),
  1029. },
  1030. },
  1031. { codemode: false },
  1032. )
  1033. yield* admit(session, "Use application context")
  1034. yield* TestLLM.push(TestLLM.tool("call-location", "location_context", { query: "hello" }), [])
  1035. const bus = yield* Bus.Service
  1036. const progressFiber = yield* bus.subscribe(SessionEvent.Tool.Progress).pipe(
  1037. Stream.filter((event) => event.data.sessionID === sessionID && event.data.id === "call-location"),
  1038. Stream.take(1),
  1039. Stream.runCollect,
  1040. Effect.forkScoped({ startImmediately: true }),
  1041. )
  1042. yield* session.resume(sessionID)
  1043. expect(requests[0]?.tools.map((tool) => tool.name)).toContain("location_context")
  1044. expect(contexts).toEqual([
  1045. {
  1046. sessionID,
  1047. agent: Agent.ID.make("build"),
  1048. messageID: expect.stringMatching(/^msg_/),
  1049. id: Tool.CallID.make("call-location"),
  1050. progress: expect.any(Function),
  1051. },
  1052. ])
  1053. expect(Array.from(yield* Fiber.join(progressFiber))[0]?.data.metadata).toEqual({ phase: "reading" })
  1054. expect(yield* session.context(sessionID)).toMatchObject([
  1055. { type: "user", text: "Use application context" },
  1056. {
  1057. type: "assistant",
  1058. content: [
  1059. {
  1060. type: "tool",
  1061. id: "call-location",
  1062. state: { status: "completed", content: [{ type: "text", text: '{"answer":"HELLO"}' }] },
  1063. },
  1064. ],
  1065. },
  1066. ])
  1067. }),
  1068. )
  1069. it.effect("executes the tool advertised before a registry reload", () =>
  1070. Effect.gen(function* () {
  1071. const session = yield* setup
  1072. const registry = yield* Tool.Service
  1073. const scope = yield* Scope.make()
  1074. const executions: string[] = []
  1075. yield* transformTools(
  1076. registry,
  1077. {
  1078. reloaded: {
  1079. name: "reloaded",
  1080. description: "Record the advertised tool",
  1081. input: Schema.Struct({}),
  1082. output: Schema.Struct({ value: Schema.String }),
  1083. execute: () =>
  1084. Effect.sync(() => executions.push("advertised")).pipe(Effect.as({ output: { value: "advertised" } })),
  1085. },
  1086. },
  1087. { codemode: false },
  1088. ).pipe(Scope.provide(scope))
  1089. yield* admit(session, "Use the reloaded tool")
  1090. yield* TestLLM.push(TestLLM.tool("call-reloaded", "reloaded", {}), [])
  1091. const stream = yield* TestLLM.gate
  1092. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  1093. yield* stream.started
  1094. yield* Scope.close(scope, Exit.void)
  1095. yield* transformTools(
  1096. registry,
  1097. {
  1098. reloaded: {
  1099. name: "reloaded",
  1100. description: "Record the replacement tool",
  1101. input: Schema.Struct({}),
  1102. output: Schema.Struct({ value: Schema.String }),
  1103. execute: () =>
  1104. Effect.sync(() => executions.push("replacement")).pipe(Effect.as({ output: { value: "replacement" } })),
  1105. },
  1106. },
  1107. { codemode: false },
  1108. )
  1109. yield* stream.release
  1110. yield* Fiber.join(run)
  1111. expect(executions).toEqual(["advertised"])
  1112. expect(yield* session.context(sessionID)).toMatchObject([
  1113. { type: "user", text: "Use the reloaded tool" },
  1114. {
  1115. type: "assistant",
  1116. content: [
  1117. {
  1118. type: "tool",
  1119. id: "call-reloaded",
  1120. state: { status: "completed", content: [{ type: "text", text: '{"value":"advertised"}' }] },
  1121. },
  1122. ],
  1123. },
  1124. ])
  1125. }),
  1126. )
  1127. it.effect("starts a real runner step after default prompt recording", () =>
  1128. Effect.gen(function* () {
  1129. const session = yield* setup
  1130. const message = yield* session.prompt({
  1131. sessionID,
  1132. text: "Run automatically",
  1133. })
  1134. yield* session.wait(sessionID)
  1135. expect(requests).toHaveLength(1)
  1136. expect(yield* session.messages({ sessionID })).toMatchObject([
  1137. { id: message.id, type: "user", text: "Run automatically" },
  1138. ])
  1139. }),
  1140. )
  1141. it.effect("runs a follow-up when synthetic input arrives during an active continuation", () =>
  1142. Effect.gen(function* () {
  1143. const session = yield* setup
  1144. const secondStarted = yield* Deferred.make<void>()
  1145. const releaseSecond = yield* Deferred.make<void>()
  1146. yield* TestLLM.push(
  1147. Stream.fromIterable(TestLLM.tool("call-echo", "echo", { text: "background started" })),
  1148. Stream.unwrap(
  1149. Deferred.succeed(secondStarted, undefined).pipe(
  1150. Effect.andThen(Deferred.await(releaseSecond)),
  1151. Effect.as(Stream.fromIterable(TestLLM.stop())),
  1152. ),
  1153. ),
  1154. Stream.fromIterable(TestLLM.text("Handled completion", "text-completion")),
  1155. )
  1156. yield* admit(session, "Start background work")
  1157. const running = yield* session.resume(sessionID).pipe(Effect.forkChild({ startImmediately: true }))
  1158. yield* Deferred.await(secondStarted)
  1159. yield* session.synthetic({ sessionID, text: "Background work completed" })
  1160. yield* Deferred.succeed(releaseSecond, undefined)
  1161. yield* Fiber.join(running)
  1162. expect(requests).toHaveLength(3)
  1163. expect(userTexts(requests[2])).toContain("Background work completed")
  1164. }),
  1165. )
  1166. it.effect("streams one request with registry definitions from chronological user history", () =>
  1167. Effect.gen(function* () {
  1168. const session = yield* setup
  1169. yield* admit(session, "First")
  1170. yield* runPrompt(session, "Second")
  1171. expect(requests).toHaveLength(1)
  1172. expect(requests[0]?.model).toBe(model)
  1173. expect(requests[0]?.tools.map((tool) => tool.name)).toEqual(["defect", "echo", "storefail"])
  1174. expect(requests[0]?.messages.map((message) => ({ role: message.role, content: message.content }))).toEqual([
  1175. { role: "user", content: [{ type: "text", text: "First" }] },
  1176. { role: "user", content: [{ type: "text", text: "Second" }] },
  1177. ])
  1178. expect(yield* session.messages({ sessionID })).toHaveLength(2)
  1179. }),
  1180. )
  1181. it.effect("marks the initial instruction sync as baseline metadata", () =>
  1182. Effect.gen(function* () {
  1183. const session = yield* setup
  1184. const bus = yield* Bus.Service
  1185. const instructionEvents: Event.Payload[] = []
  1186. const unsubscribe = yield* bus.listen((event) =>
  1187. Effect.sync(() => {
  1188. if (event.type === "session.instructions.updated") instructionEvents.push(event)
  1189. }),
  1190. )
  1191. yield* runPrompt(session, "First")
  1192. systemBaseline = "Changed context"
  1193. yield* runPrompt(session, "Second")
  1194. yield* unsubscribe
  1195. expect(instructionEvents).toHaveLength(2)
  1196. expect(instructionEvents[0]?.metadata).toEqual({ instructions: { initial: true } })
  1197. expect(instructionEvents[1]?.metadata).toBeUndefined()
  1198. }),
  1199. )
  1200. it.effect("retries the first request after system context becomes available", () =>
  1201. Effect.gen(function* () {
  1202. const session = yield* setup
  1203. const { db } = yield* Database.Service
  1204. const messageID = SessionMessage.ID.create()
  1205. systemUnavailable = true
  1206. yield* session.prompt({
  1207. id: messageID,
  1208. sessionID,
  1209. text: "First",
  1210. resume: false,
  1211. })
  1212. const exit = yield* session.resume(sessionID).pipe(Effect.exit)
  1213. expect(Exit.isFailure(exit)).toBe(true)
  1214. if (Exit.isFailure(exit)) expect(Cause.squash(exit.cause)).toBeInstanceOf(Instructions.InitializationBlocked)
  1215. expect(requests).toHaveLength(0)
  1216. expect(yield* SessionInbox.has(db, sessionID, "steer")).toBe(true)
  1217. expect(
  1218. yield* db.select().from(InstructionStateTable).where(eq(InstructionStateTable.session_id, sessionID)).get(),
  1219. ).toBeUndefined()
  1220. systemUnavailable = false
  1221. yield* session.prompt({ id: messageID, sessionID, text: "First" })
  1222. yield* session.wait(sessionID)
  1223. expect(requests).toHaveLength(1)
  1224. expect(messageRoles(requests[0])).toEqual(["user"])
  1225. }),
  1226. )
  1227. it.effect("preserves instruction state and interrupts the source Location runner after a Session moves", () =>
  1228. Effect.gen(function* () {
  1229. const session = yield* setup
  1230. const bus = yield* Bus.Service
  1231. const { db } = yield* Database.Service
  1232. yield* runPrompt(session, "First")
  1233. const instructionState = yield* db
  1234. .select()
  1235. .from(InstructionStateTable)
  1236. .where(eq(InstructionStateTable.session_id, sessionID))
  1237. .get()
  1238. yield* bus.publish(SessionEvent.Moved, {
  1239. sessionID,
  1240. location: Location.Ref.make({ directory: AbsolutePath.make("/moved") }),
  1241. projectID: Project.ID.global,
  1242. })
  1243. expect(
  1244. yield* db.select().from(InstructionStateTable).where(eq(InstructionStateTable.session_id, sessionID)).get(),
  1245. ).toEqual(instructionState)
  1246. yield* admit(session, "Second")
  1247. const exit = yield* session.resume(sessionID).pipe(Effect.exit)
  1248. expect(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)).toBe(true)
  1249. expect(requests).toHaveLength(1)
  1250. expect(yield* SessionInbox.has(db, sessionID, "steer")).toBe(true)
  1251. }),
  1252. )
  1253. it.effect("delivers a queued move atomically at the idle boundary", () =>
  1254. Effect.gen(function* () {
  1255. const session = yield* setup
  1256. const bus = yield* Bus.Service
  1257. const { db } = yield* Database.Service
  1258. const inboxID = SessionMessage.ID.create()
  1259. yield* SessionInbox.admit(db, bus, {
  1260. id: inboxID,
  1261. sessionID,
  1262. item: {
  1263. type: "move",
  1264. payload: {
  1265. location: Location.Ref.make({ directory: AbsolutePath.make("/moved") }),
  1266. projectID: Project.ID.global,
  1267. },
  1268. delivery: "queue",
  1269. },
  1270. })
  1271. yield* session.resume(sessionID)
  1272. expect((yield* session.get(sessionID)).location.directory).toBe(AbsolutePath.make("/moved"))
  1273. expect(yield* session.inbox(sessionID)).toEqual([])
  1274. expect(requests).toEqual([])
  1275. expect(closedTransports).toEqual([sessionID])
  1276. expect(
  1277. (yield* db
  1278. .select({ type: EventTable.type })
  1279. .from(EventTable)
  1280. .where(eq(EventTable.aggregate_id, sessionID))
  1281. .orderBy(desc(EventTable.seq))
  1282. .limit(2)
  1283. .all()).map((event) => event.type),
  1284. ).toEqual([Bus.versionedType(SessionEvent.Moved.type, 1), Bus.versionedType(SessionEvent.InboxDelivered.type, 1)])
  1285. }),
  1286. )
  1287. it.effect("preserves a tool continuation across a steered move", () =>
  1288. Effect.gen(function* () {
  1289. const session = yield* setup
  1290. const bus = yield* Bus.Service
  1291. const { db } = yield* Database.Service
  1292. yield* admit(session, "Echo before moving")
  1293. yield* TestLLM.push(
  1294. TestLLM.tool("call-move", "echo", { text: "moving" }),
  1295. TestLLM.text("Done", "text-after-move"),
  1296. )
  1297. const tools = yield* blockTools()
  1298. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  1299. yield* tools.started
  1300. yield* SessionInbox.admit(db, bus, {
  1301. id: SessionMessage.ID.create(),
  1302. sessionID,
  1303. item: {
  1304. type: "move",
  1305. payload: {
  1306. location: Location.Ref.make({ directory: AbsolutePath.make("/project") }),
  1307. projectID: Project.ID.global,
  1308. },
  1309. delivery: "steer",
  1310. },
  1311. })
  1312. yield* tools.release
  1313. yield* Fiber.join(run)
  1314. expect(requests).toHaveLength(2)
  1315. expect(requests.map(messageRoles).at(1)?.slice(0, 3)).toEqual(["user", "assistant", "tool"])
  1316. expect(yield* session.inbox(sessionID)).toEqual([])
  1317. }),
  1318. )
  1319. it.effect("keeps queued input parked across a mid-turn move", () =>
  1320. Effect.gen(function* () {
  1321. const session = yield* setup
  1322. const bus = yield* Bus.Service
  1323. const { db } = yield* Database.Service
  1324. yield* admit(session, "Echo before moving")
  1325. yield* TestLLM.push(
  1326. TestLLM.tool("call-move", "echo", { text: "moving" }),
  1327. TestLLM.text("Done", "text-after-move"),
  1328. TestLLM.text("Handled queue", "text-after-queue"),
  1329. )
  1330. const tools = yield* blockTools()
  1331. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  1332. yield* tools.started
  1333. yield* session.prompt({ sessionID, text: "Queued for later", delivery: "queue", resume: false })
  1334. yield* SessionInbox.admit(db, bus, {
  1335. id: SessionMessage.ID.create(),
  1336. sessionID,
  1337. item: {
  1338. type: "move",
  1339. payload: {
  1340. location: Location.Ref.make({ directory: AbsolutePath.make("/project") }),
  1341. projectID: Project.ID.global,
  1342. },
  1343. delivery: "steer",
  1344. },
  1345. })
  1346. yield* tools.release
  1347. yield* Fiber.join(run)
  1348. // The resumed turn absorbs steers only; queued input waits for the turn to end.
  1349. expect(requests).toHaveLength(3)
  1350. expect(userTexts(requests[1])).not.toContain("Queued for later")
  1351. expect(userTexts(requests[2])).toContain("Queued for later")
  1352. }),
  1353. )
  1354. it.effect("seeds a fork with the parent's newest instruction values", () =>
  1355. Effect.gen(function* () {
  1356. const session = yield* setup
  1357. yield* runPrompt(session, "First")
  1358. systemBaseline = "Changed context"
  1359. const second = yield* runPrompt(session, "Second")
  1360. systemBaseline = "Latest context"
  1361. yield* runPrompt(session, "Third")
  1362. const forked = yield* session.fork({ sessionID, boundary: { type: "before", messageID: second.id } })
  1363. expect(
  1364. yield* (yield* Database.Service).db
  1365. .select()
  1366. .from(InstructionStateTable)
  1367. .where(eq(InstructionStateTable.session_id, forked.id))
  1368. .get(),
  1369. ).toMatchObject({
  1370. initial_values: { "test/context": Instructions.hash("Latest context") },
  1371. current_values: { "test/context": Instructions.hash("Latest context") },
  1372. })
  1373. yield* session.prompt({ sessionID: forked.id, text: "Forked", resume: false })
  1374. yield* session.resume(forked.id)
  1375. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual([defaultSystem, "Latest context"])
  1376. // Copied history keeps the frozen chronological update; no new update is emitted.
  1377. expect(systemTexts(requests.at(-1)!)).toContain("Changed context")
  1378. expect(systemTexts(requests.at(-1)!)).not.toContain("Latest context")
  1379. const { db } = yield* Database.Service
  1380. const bus = yield* Bus.Service
  1381. const recorded = yield* db
  1382. .select()
  1383. .from(EventTable)
  1384. .where(eq(EventTable.aggregate_id, forked.id))
  1385. .orderBy(asc(EventTable.seq))
  1386. .all()
  1387. yield* bus.remove(forked.id)
  1388. yield* db.delete(SessionTable).where(eq(SessionTable.id, forked.id)).run()
  1389. yield* Effect.forEach(
  1390. recorded.map((event) => ({
  1391. id: event.id,
  1392. created: event.created,
  1393. aggregateID: event.aggregate_id,
  1394. seq: event.seq,
  1395. type: event.type,
  1396. data: event.data,
  1397. })),
  1398. (event) => bus.replay(event),
  1399. { discard: true },
  1400. )
  1401. expect(
  1402. yield* db.select().from(InstructionStateTable).where(eq(InstructionStateTable.session_id, forked.id)).get(),
  1403. ).toMatchObject({ current_values: { "test/context": Instructions.hash("Latest context") } })
  1404. }),
  1405. )
  1406. it.effect("keeps nested forks self-contained", () =>
  1407. Effect.gen(function* () {
  1408. const session = yield* setup
  1409. yield* runPrompt(session, "First")
  1410. systemBaseline = "Changed context"
  1411. const second = yield* runPrompt(session, "Second")
  1412. const child = yield* session.fork({ sessionID, boundary: { type: "before", messageID: second.id } })
  1413. const inheritedFirst = (yield* session.messages({ sessionID: child.id })).find(
  1414. (message) => message.type === "user" && message.text === "First",
  1415. )
  1416. if (!inheritedFirst) return yield* Effect.die(new Error("Nested fork boundary message not found"))
  1417. const grandchild = yield* session.fork({
  1418. sessionID: child.id,
  1419. boundary: { type: "before", messageID: inheritedFirst.id },
  1420. })
  1421. expect(
  1422. yield* (yield* Database.Service).db
  1423. .select()
  1424. .from(InstructionStateTable)
  1425. .where(eq(InstructionStateTable.session_id, grandchild.id))
  1426. .get(),
  1427. ).toMatchObject({
  1428. initial_values: { "test/context": Instructions.hash("Changed context") },
  1429. current_values: { "test/context": Instructions.hash("Changed context") },
  1430. })
  1431. return undefined
  1432. }),
  1433. )
  1434. it.effect("re-establishes a fresh baseline when instruction state is missing", () =>
  1435. Effect.gen(function* () {
  1436. const session = yield* setup
  1437. const { db } = yield* Database.Service
  1438. yield* runPrompt(session, "First")
  1439. yield* db.delete(InstructionStateTable).where(eq(InstructionStateTable.session_id, sessionID)).run()
  1440. yield* admit(session, "Second")
  1441. requests.length = 0
  1442. yield* session.resume(sessionID)
  1443. expect(requests).toHaveLength(1)
  1444. expect(requests[0]?.system.map((part) => part.text)).toEqual([defaultSystem, "Initial context"])
  1445. expect(messageRoles(requests[0])).toEqual(["user", "user"])
  1446. // The projected row is authoritative: a missing row admits a fresh baseline
  1447. // instead of rebuilding from durable events.
  1448. expect(
  1449. yield* db
  1450. .select({ data: EventTable.data })
  1451. .from(EventTable)
  1452. .where(eq(EventTable.type, "session.instructions.updated.2"))
  1453. .all(),
  1454. ).toHaveLength(2)
  1455. expect(yield* db.select().from(InstructionStateTable).get()).toMatchObject({
  1456. initial_values: { "test/context": Instructions.hash("Initial context") },
  1457. current_values: { "test/context": Instructions.hash("Initial context") },
  1458. })
  1459. }),
  1460. )
  1461. it.effect("keeps the initial instructions stable and derives a chronological update from values", () =>
  1462. Effect.gen(function* () {
  1463. const session = yield* setup
  1464. yield* runPrompt(session, "First")
  1465. systemBaseline = "Changed context"
  1466. yield* runPrompt(session, "Second")
  1467. expect(
  1468. PromptCacheDiagnostics.compare(
  1469. PromptCacheDiagnostics.snapshot(requests[0]),
  1470. PromptCacheDiagnostics.snapshot(requests[1]),
  1471. ),
  1472. ).toEqual({ status: "append-only", previousMessages: 1, currentMessages: 3 })
  1473. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  1474. [defaultSystem, "Initial context"],
  1475. [defaultSystem, "Initial context"],
  1476. ])
  1477. expect(messageRoles(requests[1])).toEqual(["user", "system", "user"])
  1478. expect(requests[1]?.messages.at(1)?.content).toEqual([{ type: "text", text: "Changed context" }])
  1479. // The chronological update is a durable client-visible system message.
  1480. const messages = yield* session.messages({ sessionID })
  1481. expect(messages).toHaveLength(3)
  1482. expect(messages[1]).toMatchObject({ type: "system", text: "Changed context" })
  1483. const { db } = yield* Database.Service
  1484. const updates = yield* db
  1485. .select({ data: EventTable.data })
  1486. .from(EventTable)
  1487. .where(eq(EventTable.type, "session.instructions.updated.2"))
  1488. .orderBy(asc(EventTable.seq))
  1489. .all()
  1490. .pipe(Effect.orDie)
  1491. expect(updates).toHaveLength(2)
  1492. expect(updates[0]?.data).toMatchObject({
  1493. sessionID,
  1494. delta: { "test/context": Instructions.hash("Initial context") },
  1495. })
  1496. expect(updates[1]?.data).toEqual({
  1497. sessionID,
  1498. delta: { "test/context": Instructions.hash("Changed context") },
  1499. text: "Changed context",
  1500. })
  1501. yield* replaySessionProjection(sessionID)
  1502. expect(yield* session.messages({ sessionID })).toHaveLength(3)
  1503. }),
  1504. )
  1505. it.effect("uses the selected model family prompt when the agent does not override it", () =>
  1506. Effect.gen(function* () {
  1507. const session = yield* setup
  1508. currentModel = LanguageModel.make({ id: "gpt-5", provider: "openai", route: OpenAIChat.route })
  1509. yield* admit(session, "First")
  1510. yield* TestLLM.push(TestLLM.text("Done", "text-provider-prompt"))
  1511. yield* session.resume(sessionID)
  1512. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual([
  1513. expect.stringContaining("You are OpenCode, You and the user share the same workspace"),
  1514. "Initial context",
  1515. ])
  1516. }),
  1517. )
  1518. it.effect("uses the selected model family prompt when the agent system override is empty", () =>
  1519. Effect.gen(function* () {
  1520. const session = yield* setup
  1521. currentModel = LanguageModel.make({ id: "gpt-5", provider: "openai", route: OpenAIChat.route })
  1522. const agent = yield* Agent.Service
  1523. yield* agent.transform((editor) =>
  1524. editor.update(Agent.ID.make("build"), (agent) => {
  1525. agent.system = ""
  1526. agent.mode = "primary"
  1527. }),
  1528. )
  1529. yield* admit(session, "First")
  1530. yield* TestLLM.push(TestLLM.text("Done", "text-empty-agent-system"))
  1531. yield* session.resume(sessionID)
  1532. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual([
  1533. expect.stringContaining("You are OpenCode, You and the user share the same workspace"),
  1534. "Initial context",
  1535. ])
  1536. }),
  1537. )
  1538. it.effect("includes the effective default agent system before durable context", () =>
  1539. Effect.gen(function* () {
  1540. const session = yield* setup
  1541. const agent = yield* Agent.Service
  1542. yield* agent.transform((editor) =>
  1543. editor.update(Agent.ID.make("build"), (agent) => {
  1544. agent.system = "Build agent instructions"
  1545. agent.mode = "primary"
  1546. }),
  1547. )
  1548. yield* admit(session, "First")
  1549. yield* TestLLM.push(TestLLM.text("Done", "text-build"))
  1550. yield* session.resume(sessionID)
  1551. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual(["Build agent instructions", "Initial context"])
  1552. }),
  1553. )
  1554. it.effect("uses the configured default agent system for omitted-agent sessions", () =>
  1555. Effect.gen(function* () {
  1556. const session = yield* setup
  1557. const agent = yield* Agent.Service
  1558. yield* agent.transform((editor) => {
  1559. editor.update(Agent.ID.make("build"), (agent) => {
  1560. agent.system = "Build agent instructions"
  1561. agent.mode = "primary"
  1562. })
  1563. editor.update(Agent.ID.make("reviewer"), (agent) => {
  1564. agent.system = "Reviewer instructions"
  1565. agent.mode = "primary"
  1566. })
  1567. editor.default(Agent.ID.make("reviewer"))
  1568. })
  1569. yield* admit(session, "First")
  1570. yield* TestLLM.push(TestLLM.text("Done", "text-reviewer"))
  1571. yield* session.resume(sessionID)
  1572. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual(["Reviewer instructions", "Initial context"])
  1573. expect((yield* session.messages({ sessionID }))[0]).toMatchObject({ type: "assistant", agent: "reviewer" })
  1574. }),
  1575. )
  1576. it.effect("uses only the agent prompt and initial instructions as system parts", () =>
  1577. Effect.gen(function* () {
  1578. const session = yield* setup
  1579. const agent = yield* Agent.Service
  1580. yield* agent.transform((editor) =>
  1581. editor.update(Agent.ID.make("build"), (agent) => {
  1582. agent.system = "Build agent instructions"
  1583. agent.mode = "primary"
  1584. }),
  1585. )
  1586. yield* admit(session, "First")
  1587. yield* TestLLM.push(TestLLM.text("Done", "text-no-system"))
  1588. yield* session.resume(sessionID)
  1589. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual(["Build agent instructions", "Initial context"])
  1590. }),
  1591. )
  1592. it.effect("uses an explicitly selected non-build agent system", () =>
  1593. Effect.gen(function* () {
  1594. const session = yield* setup
  1595. const { db } = yield* Database.Service
  1596. const agent = yield* Agent.Service
  1597. yield* agent.transform((editor) =>
  1598. editor.update(Agent.ID.make("reviewer"), (agent) => {
  1599. agent.system = "Reviewer instructions"
  1600. agent.mode = "primary"
  1601. }),
  1602. )
  1603. yield* db
  1604. .update(SessionTable)
  1605. .set({ agent: "reviewer" })
  1606. .where(eq(SessionTable.id, sessionID))
  1607. .run()
  1608. .pipe(Effect.orDie)
  1609. yield* admit(session, "First")
  1610. yield* TestLLM.push(TestLLM.text("Done", "text-selected"))
  1611. yield* session.resume(sessionID)
  1612. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual(["Reviewer instructions", "Initial context"])
  1613. expect((yield* session.messages({ sessionID }))[0]).toMatchObject({ type: "assistant", agent: "reviewer" })
  1614. }),
  1615. )
  1616. it.effect("fails before the model request when the selected agent is unavailable", () =>
  1617. Effect.gen(function* () {
  1618. yield* setup
  1619. const { db } = yield* Database.Service
  1620. yield* db
  1621. .update(SessionTable)
  1622. .set({ agent: "explore" })
  1623. .where(eq(SessionTable.id, sessionID))
  1624. .run()
  1625. .pipe(Effect.orDie)
  1626. const session = yield* Session.Service
  1627. yield* session.prompt({ sessionID, text: "Inspect files", resume: false })
  1628. requests.length = 0
  1629. yield* TestLLM.push([])
  1630. const failure = yield* session.resume(sessionID).pipe(Effect.flip)
  1631. expect(failure).toMatchObject({
  1632. _tag: "Session.AgentNotFoundError",
  1633. sessionID,
  1634. agent: "explore",
  1635. })
  1636. expect(requests).toHaveLength(0)
  1637. }),
  1638. )
  1639. it.effect("waits for initial plugin readiness before constructing the model request", () =>
  1640. Effect.gen(function* () {
  1641. yield* setup
  1642. const release = yield* Deferred.make<void>()
  1643. pluginFlushHook = Deferred.await(release)
  1644. const session = yield* Session.Service
  1645. yield* session.prompt({ sessionID, text: "Wait for plugins", resume: false })
  1646. requests.length = 0
  1647. yield* TestLLM.push([])
  1648. const running = yield* session.resume(sessionID).pipe(Effect.forkChild({ startImmediately: true }))
  1649. yield* Effect.yieldNow
  1650. expect(requests).toHaveLength(0)
  1651. expect(running.pollUnsafe()).toBeUndefined()
  1652. yield* Deferred.succeed(release, undefined)
  1653. yield* Fiber.join(running)
  1654. expect(requests).toHaveLength(1)
  1655. }),
  1656. )
  1657. it.effect("updates selected-agent skill instructions after an agent switch", () =>
  1658. Effect.gen(function* () {
  1659. const session = yield* setup
  1660. const bus = yield* Bus.Service
  1661. const agents = yield* Agent.Service
  1662. yield* agents.transform((draft) =>
  1663. draft.update(Agent.ID.make("reviewer"), (agent) => {
  1664. agent.mode = "primary"
  1665. }),
  1666. )
  1667. skillBaselines.set(Agent.ID.make("build"), "Build skills")
  1668. yield* runPrompt(session, "First")
  1669. skillBaselines.set(Agent.ID.make("reviewer"), "Reviewer skills")
  1670. yield* bus.publish(SessionEvent.AgentSelected, {
  1671. sessionID,
  1672. agent: Agent.ID.make("reviewer"),
  1673. })
  1674. yield* runPrompt(session, "Second")
  1675. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  1676. [defaultSystem, "Initial context\n\nBuild skills"],
  1677. [defaultSystem, "Initial context\n\nBuild skills"],
  1678. ])
  1679. expect(systemTexts(requests[1])).toContainEqual(expect.stringContaining("Reviewer skills"))
  1680. }),
  1681. )
  1682. it.effect("keeps the sampled agent when selection changes during observation", () =>
  1683. Effect.gen(function* () {
  1684. const session = yield* setup
  1685. const bus = yield* Bus.Service
  1686. skillBaselines.set(Agent.ID.make("build"), "Build skills")
  1687. skillBaselines.set(Agent.ID.make("reviewer"), "Reviewer skills")
  1688. let switched = false
  1689. systemLoadHook = Effect.suspend(() => {
  1690. if (switched) return Effect.void
  1691. switched = true
  1692. return bus
  1693. .publish(SessionEvent.AgentSelected, {
  1694. sessionID,
  1695. agent: Agent.ID.make("reviewer"),
  1696. })
  1697. .pipe(Effect.asVoid)
  1698. })
  1699. yield* runPrompt(session, "First")
  1700. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  1701. [defaultSystem, "Initial context\n\nBuild skills"],
  1702. ])
  1703. }),
  1704. )
  1705. it.effect("keeps the sampled model when selection changes during model resolution", () =>
  1706. Effect.gen(function* () {
  1707. const session = yield* setup
  1708. const bus = yield* Bus.Service
  1709. let switched = false
  1710. modelResolveHook = Effect.suspend(() => {
  1711. if (switched) return Effect.void
  1712. switched = true
  1713. return bus
  1714. .publish(SessionEvent.ModelSelected, {
  1715. sessionID,
  1716. model: { id: ID.make("replacement"), providerID: Provider.ID.make("fake") },
  1717. })
  1718. .pipe(Effect.asVoid)
  1719. })
  1720. yield* runPrompt(session, "First")
  1721. expect(requests.map((request) => request.model)).toEqual([model])
  1722. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  1723. [defaultSystem, "Initial context"],
  1724. ])
  1725. }),
  1726. )
  1727. it.effect("admits removed context as a chronological System message", () =>
  1728. Effect.gen(function* () {
  1729. const session = yield* setup
  1730. yield* runPrompt(session, "First")
  1731. systemRemoved = true
  1732. yield* runPrompt(session, "Second")
  1733. expect(messageRoles(requests[1])).toEqual(["user", "system", "user"])
  1734. expect(requests[1]?.messages.at(1)?.content).toEqual([
  1735. { type: "text", text: "System context source removed: test/context" },
  1736. ])
  1737. expect(yield* session.messages({ sessionID })).toHaveLength(3)
  1738. }),
  1739. )
  1740. it.effect("renders API context entries through add, change, and removal", () =>
  1741. Effect.gen(function* () {
  1742. const session = yield* setup
  1743. const contextEntries = yield* InstructionEntry.Service
  1744. yield* contextEntries.put({ sessionID, key: "deploy-target", value: "production" })
  1745. yield* runPrompt(session, "First")
  1746. // String values render verbatim inside the initial tagged block.
  1747. expect(requests[0]?.system.map((part) => part.text)).toEqual([
  1748. defaultSystem,
  1749. ["Initial context", "", '<context key="deploy-target">', "production", "</context>"].join("\n"),
  1750. ])
  1751. // Non-string JSON pretty-prints; the change narrates as a System update.
  1752. yield* contextEntries.put({ sessionID, key: "deploy-target", value: { region: "us-east-1" } })
  1753. yield* runPrompt(session, "Second")
  1754. expect(messageRoles(requests[1])).toEqual(["user", "system", "user"])
  1755. expect(requests[1]?.messages.at(1)?.content).toEqual([
  1756. {
  1757. type: "text",
  1758. text: [
  1759. 'The context under "deploy-target" changed and supersedes the previous value:',
  1760. '<context key="deploy-target">',
  1761. "{",
  1762. ' "region": "us-east-1"',
  1763. "}",
  1764. "</context>",
  1765. ].join("\n"),
  1766. },
  1767. ])
  1768. expect(yield* contextEntries.list(sessionID)).toEqual([{ key: "deploy-target", value: { region: "us-east-1" } }])
  1769. // Deleting the row announces removal through the stored removal text.
  1770. yield* contextEntries.remove({ sessionID, key: "deploy-target" })
  1771. yield* runPrompt(session, "Third")
  1772. expect(messageRoles(requests[2])).toEqual(["user", "system", "user", "system", "user"])
  1773. expect(requests[2]?.messages.at(-2)?.content).toEqual([
  1774. { type: "text", text: 'The context under "deploy-target" no longer applies. Disregard it.' },
  1775. ])
  1776. expect(yield* contextEntries.list(sessionID)).toEqual([])
  1777. }),
  1778. )
  1779. it.effect("retains JSON null API entries as values", () =>
  1780. Effect.gen(function* () {
  1781. const session = yield* setup
  1782. const entries = yield* InstructionEntry.Service
  1783. yield* entries.put({ sessionID, key: "nullable", value: "present" })
  1784. yield* runPrompt(session, "First")
  1785. yield* entries.put({ sessionID, key: "nullable", value: null })
  1786. yield* runPrompt(session, "Second")
  1787. expect(requests[1]?.messages.at(1)?.content).toEqual([
  1788. {
  1789. type: "text",
  1790. text: [
  1791. 'The context under "nullable" changed and supersedes the previous value:',
  1792. '<context key="nullable">',
  1793. "null",
  1794. "</context>",
  1795. ].join("\n"),
  1796. },
  1797. ])
  1798. expect(yield* entries.list(sessionID)).toEqual([{ key: "nullable", value: null }])
  1799. }),
  1800. )
  1801. it.effect("rejects API instruction entries larger than 8KB", () =>
  1802. Effect.gen(function* () {
  1803. yield* setup
  1804. const entries = yield* InstructionEntry.Service
  1805. const exit = yield* entries
  1806. .put({ sessionID, key: "oversized", value: "x".repeat(InstructionEntry.MaxValueBytes) })
  1807. .pipe(Effect.exit)
  1808. expect(Exit.isFailure(exit)).toBe(true)
  1809. if (Exit.isFailure(exit)) expect(Cause.squash(exit.cause)).toBeInstanceOf(InstructionEntry.ValueTooLargeError)
  1810. expect(yield* entries.list(sessionID)).toEqual([])
  1811. }),
  1812. )
  1813. it.effect("keeps initial instructions and chronological updates after a model switch", () =>
  1814. Effect.gen(function* () {
  1815. const session = yield* setup
  1816. const bus = yield* Bus.Service
  1817. yield* runPrompt(session, "First")
  1818. systemBaseline = "Changed context"
  1819. yield* runPrompt(session, "Second")
  1820. yield* bus.publish(SessionEvent.ModelSelected, {
  1821. sessionID,
  1822. model: { id: ID.make("replacement"), providerID: Provider.ID.make("fake") },
  1823. })
  1824. systemBaseline = "Replacement context"
  1825. yield* runPrompt(session, "Third")
  1826. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  1827. [defaultSystem, "Initial context"],
  1828. [defaultSystem, "Initial context"],
  1829. [defaultSystem, "Initial context"],
  1830. ])
  1831. expect(messageRoles(requests[1])).toEqual(["user", "system", "user"])
  1832. expect(requests[2]?.messages.filter((message) => message.role === "system")).toHaveLength(2)
  1833. expect((yield* session.context(sessionID)).map((message) => message.type)).toEqual([
  1834. "user",
  1835. "system",
  1836. "user",
  1837. "model-switched",
  1838. "system",
  1839. "user",
  1840. ])
  1841. yield* replaySessionProjection(sessionID)
  1842. expect(yield* session.messages({ sessionID })).toHaveLength(6)
  1843. yield* runPrompt(session, "Fourth")
  1844. }),
  1845. )
  1846. it.effect("preserves instruction values while a source is temporarily unavailable", () =>
  1847. Effect.gen(function* () {
  1848. const session = yield* setup
  1849. const bus = yield* Bus.Service
  1850. yield* runPrompt(session, "First")
  1851. yield* bus.publish(SessionEvent.ModelSelected, {
  1852. sessionID,
  1853. model: { id: ID.make("replacement"), providerID: Provider.ID.make("fake") },
  1854. })
  1855. systemUnavailable = true
  1856. yield* runPrompt(session, "Second")
  1857. systemUnavailable = false
  1858. systemBaseline = "Replacement context"
  1859. yield* runPrompt(session, "Third")
  1860. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  1861. [defaultSystem, "Initial context"],
  1862. [defaultSystem, "Initial context"],
  1863. [defaultSystem, "Initial context"],
  1864. ])
  1865. }),
  1866. )
  1867. it.effect("moves the epoch at compaction and narrates later changes", () =>
  1868. Effect.gen(function* () {
  1869. const session = yield* setup
  1870. const bus = yield* Bus.Service
  1871. yield* runPrompt(session, "First")
  1872. yield* bus.publish(SessionEvent.Compaction.Started, {
  1873. sessionID,
  1874. reason: "manual",
  1875. recent: "",
  1876. })
  1877. yield* bus.publish(SessionEvent.Compaction.Ended, {
  1878. sessionID,
  1879. reason: "manual",
  1880. text: "summary",
  1881. recent: "",
  1882. })
  1883. systemBaseline = "Replacement context"
  1884. yield* runPrompt(session, "Second")
  1885. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  1886. [defaultSystem, "Initial context"],
  1887. [defaultSystem, "Initial context"],
  1888. ])
  1889. expect(messageRoles(requests[1])).toEqual(["user", "system", "user"])
  1890. expect(requests[1]?.messages.at(1)?.content).toEqual([{ type: "text", text: "Replacement context" }])
  1891. yield* replaySessionProjection(sessionID)
  1892. yield* runPrompt(session, "Third")
  1893. }),
  1894. )
  1895. it.effect("runs steers before queued compaction and later queued input", () =>
  1896. Effect.gen(function* () {
  1897. const session = yield* setup
  1898. currentModel = recoveryModel
  1899. const stream = yield* TestLLM.gate
  1900. yield* TestLLM.push(
  1901. TestLLM.tool("call-active", "echo", { text: "active" }),
  1902. TestLLM.text("Steer complete", "text-steer"),
  1903. [LLMEvent.textDelta({ id: "summary", text: "durable summary" })],
  1904. TestLLM.text("Queue complete", "text-queue"),
  1905. )
  1906. yield* admit(session, "Active work")
  1907. const active = yield* session.resume(sessionID).pipe(Effect.forkChild)
  1908. yield* stream.started
  1909. const first = yield* session.compact({ sessionID })
  1910. expect(yield* SessionInbox.find((yield* Database.Service).db, first.id)).toMatchObject({
  1911. id: first.id,
  1912. })
  1913. expect((yield* session.messages({ sessionID })).find((message) => message.id === first.id)).toBeUndefined()
  1914. yield* admit(session, "Steer after compaction")
  1915. yield* session.synthetic({ sessionID, text: "Completion after compaction", resume: false })
  1916. yield* session.prompt({
  1917. sessionID,
  1918. text: "Queue after compaction",
  1919. delivery: "queue",
  1920. resume: false,
  1921. })
  1922. expect(yield* SessionInbox.has((yield* Database.Service).db, sessionID, "steer")).toBe(true)
  1923. yield* stream.release
  1924. yield* Fiber.join(active)
  1925. expect(requests).toHaveLength(4)
  1926. expect(userTexts(requests[1])).toContain("Steer after compaction")
  1927. expect(userTexts(requests[1])).toContain("Completion after compaction")
  1928. expect(userTexts(requests[2])[0]).toContain("Create a new anchored summary")
  1929. expect(userTexts(requests[3])).toContain("Queue after compaction")
  1930. expect(yield* SessionInbox.find((yield* Database.Service).db, first.id)).toBeUndefined()
  1931. expect((yield* session.messages({ sessionID })).find((message) => message.id === first.id)).toMatchObject({
  1932. type: "compaction",
  1933. status: "completed",
  1934. summary: "durable summary",
  1935. })
  1936. }),
  1937. )
  1938. it.effect("releases queued prompts when durable compaction fails", () =>
  1939. Effect.gen(function* () {
  1940. const session = yield* setup
  1941. currentModel = recoveryModel
  1942. const stream = yield* TestLLM.gate
  1943. yield* TestLLM.push(
  1944. TestLLM.text("Active complete", "text-active-failure"),
  1945. [],
  1946. TestLLM.text("Continued", "text-after-failure"),
  1947. )
  1948. yield* admit(session, "Active work")
  1949. const active = yield* session.resume(sessionID).pipe(Effect.forkChild)
  1950. yield* stream.started
  1951. const compaction = yield* session.compact({ sessionID })
  1952. yield* session.prompt({
  1953. sessionID,
  1954. text: "Continue after failure",
  1955. delivery: "queue",
  1956. resume: false,
  1957. })
  1958. yield* stream.release
  1959. yield* Fiber.join(active)
  1960. expect(requests).toHaveLength(3)
  1961. expect(userTexts(requests[2])).toContain("Continue after failure")
  1962. expect(yield* SessionInbox.find((yield* Database.Service).db, compaction.id)).toBeUndefined()
  1963. expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
  1964. type: "compaction",
  1965. status: "failed",
  1966. })
  1967. expect(
  1968. (yield* recordedEventTypes(sessionID)).filter(
  1969. (type) => type === Bus.versionedType(SessionEvent.Compaction.Failed.type, 1),
  1970. ),
  1971. ).toHaveLength(1)
  1972. }),
  1973. )
  1974. it.effect("explains when manual compaction has no history", () =>
  1975. Effect.gen(function* () {
  1976. yield* setup
  1977. const session = yield* Session.Service
  1978. const compaction = yield* session.compact({ sessionID })
  1979. modelResolveHook = Effect.die("model resolution should not run")
  1980. yield* session.resume(sessionID)
  1981. expect(yield* SessionInbox.find((yield* Database.Service).db, compaction.id)).toBeUndefined()
  1982. expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
  1983. type: "compaction",
  1984. status: "failed",
  1985. reason: "manual",
  1986. error: { type: "compaction.unavailable", message: "Nothing to compact yet" },
  1987. })
  1988. expect(
  1989. (yield* recordedEventTypes(sessionID)).filter(
  1990. (type) => type === Bus.versionedType(SessionEvent.Compaction.Failed.type, 1),
  1991. ),
  1992. ).toHaveLength(1)
  1993. }),
  1994. )
  1995. it.effect("delivers steered manual compaction when the model has no context limit", () =>
  1996. Effect.gen(function* () {
  1997. const session = yield* setup
  1998. yield* TestLLM.push(TestLLM.text("Earlier answer", "text-manual-unknown-history"))
  1999. yield* runPrompt(session, "Earlier question")
  2000. requests.length = 0
  2001. yield* TestLLM.push(TestLLM.text("Manual summary", "text-manual-unknown-summary"))
  2002. const compaction = yield* session.compact({ sessionID, delivery: "steer" })
  2003. yield* session.resume(sessionID)
  2004. expect(requests).toHaveLength(1)
  2005. expect(userTexts(requests[0])[0]).toContain("Earlier question")
  2006. expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
  2007. type: "compaction",
  2008. status: "completed",
  2009. summary: "Manual summary",
  2010. })
  2011. }),
  2012. )
  2013. it.effect("preserves provider errors from manual compaction", () =>
  2014. Effect.gen(function* () {
  2015. const session = yield* setup
  2016. yield* TestLLM.push(TestLLM.text("Earlier answer", "text-manual-provider-history"))
  2017. yield* runPrompt(session, "Earlier question")
  2018. yield* TestLLM.push([LLMEvent.providerError({ message: "summary unavailable" })])
  2019. const compaction = yield* session.compact({ sessionID })
  2020. yield* session.resume(sessionID)
  2021. expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
  2022. type: "compaction",
  2023. status: "failed",
  2024. error: { type: "provider.error", message: "summary unavailable" },
  2025. })
  2026. }),
  2027. )
  2028. it.effect("preserves typed provider failures from manual compaction", () =>
  2029. Effect.gen(function* () {
  2030. const session = yield* setup
  2031. yield* TestLLM.push(TestLLM.text("Earlier answer", "text-manual-failure-history"))
  2032. yield* runPrompt(session, "Earlier question")
  2033. yield* TestLLM.push(Stream.fail(providerUnavailable()))
  2034. const compaction = yield* session.compact({ sessionID })
  2035. yield* session.resume(sessionID)
  2036. expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
  2037. type: "compaction",
  2038. status: "failed",
  2039. error: { type: "provider.transport", message: "Provider unavailable" },
  2040. })
  2041. }),
  2042. )
  2043. it.effect("records cancelled manual compaction without surfacing an internal failure", () =>
  2044. Effect.gen(function* () {
  2045. const session = yield* setup
  2046. yield* TestLLM.push(TestLLM.text("Earlier answer", "text-manual-interrupt-history"))
  2047. yield* runPrompt(session, "Earlier question")
  2048. const streamed = yield* Deferred.make<void>()
  2049. const partial = fragmentFixture("text", "text-manual-interrupt-summary", ["Partial summary"])
  2050. yield* TestLLM.push(
  2051. Stream.concat(
  2052. Stream.fromIterable(partial.partialEvents),
  2053. Stream.fromEffect(Deferred.succeed(streamed, undefined)).pipe(Stream.flatMap(() => Stream.never)),
  2054. ),
  2055. )
  2056. const compaction = yield* session.compact({ sessionID })
  2057. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2058. yield* Deferred.await(streamed)
  2059. yield* session.interrupt(sessionID)
  2060. yield* Fiber.await(run)
  2061. expect(yield* SessionInbox.find((yield* Database.Service).db, compaction.id)).toBeUndefined()
  2062. expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
  2063. type: "compaction",
  2064. status: "failed",
  2065. reason: "manual",
  2066. error: { type: "aborted", message: "Compaction cancelled" },
  2067. })
  2068. }),
  2069. )
  2070. it.effect("settles an admitted manual compaction when pre-start resolution throws", () =>
  2071. Effect.gen(function* () {
  2072. const session = yield* setup
  2073. yield* TestLLM.push(TestLLM.text("Earlier answer", "text-manual-resolution-history"))
  2074. yield* runPrompt(session, "Earlier question")
  2075. const compaction = yield* session.compact({ sessionID })
  2076. modelResolveHook = Effect.die("model resolution failed")
  2077. expect(yield* Effect.exit(session.resume(sessionID))).toMatchObject({ _tag: "Failure" })
  2078. expect(yield* SessionInbox.find((yield* Database.Service).db, compaction.id)).toBeUndefined()
  2079. expect((yield* session.messages({ sessionID })).find((message) => message.id === compaction.id)).toMatchObject({
  2080. type: "compaction",
  2081. status: "failed",
  2082. reason: "manual",
  2083. })
  2084. expect(
  2085. (yield* recordedEventTypes(sessionID)).filter(
  2086. (type) => type === Bus.versionedType(SessionEvent.Compaction.Failed.type, 1),
  2087. ),
  2088. ).toHaveLength(1)
  2089. }),
  2090. )
  2091. it.effect("automatically compacts into a completed summary and retained recent turn", () =>
  2092. Effect.gen(function* () {
  2093. const session = yield* setup
  2094. yield* TestLLM.push(TestLLM.textWithUsage("Earlier answer", "text-first", 3_950))
  2095. yield* runPrompt(session, "Earlier question ".repeat(180))
  2096. currentModel = compactModel
  2097. requests.length = 0
  2098. yield* TestLLM.push(
  2099. TestLLM.text("## Objective\n- Preserve the task", "text-summary"),
  2100. TestLLM.textWithUsage("Continued", "text-final", 3_950),
  2101. )
  2102. yield* runPrompt(session, "Recent exact request ".repeat(180))
  2103. expect(requests).toHaveLength(2)
  2104. expect(userTexts(requests[0])[0]).toContain("## Objective")
  2105. expect(userTexts(requests[1])).toHaveLength(1)
  2106. expect(userTexts(requests[1])[0]).toContain("<summary>\n## Objective\n- Preserve the task\n</summary>")
  2107. expect(userTexts(requests[1])[0]).toContain(`[User]: ${"Recent exact request ".repeat(180)}`)
  2108. const context = yield* (yield* SessionStore.Service).context(sessionID)
  2109. expect(context.map((message) => message.type)).toEqual(["compaction", "assistant"])
  2110. expect(context[0]).toMatchObject({
  2111. type: "compaction",
  2112. summary: "## Objective\n- Preserve the task",
  2113. })
  2114. requests.length = 0
  2115. executions.length = 0
  2116. yield* TestLLM.push(
  2117. TestLLM.text("## Objective\n- Preserve the updated task", "text-summary-2"),
  2118. TestLLM.text("Continued again", "text-final-2"),
  2119. )
  2120. yield* runPrompt(session, "Newest exact request ".repeat(180))
  2121. expect(requests).toHaveLength(2)
  2122. expect(userTexts(requests[0])[0]).toContain(
  2123. "<previous-summary>\n## Objective\n- Preserve the task\n</previous-summary>",
  2124. )
  2125. expect(userTexts(requests[0])[0]).toContain("Recent exact request")
  2126. expect((yield* (yield* SessionStore.Service).context(sessionID))[0]).toMatchObject({
  2127. type: "compaction",
  2128. summary: "## Objective\n- Preserve the updated task",
  2129. })
  2130. }),
  2131. )
  2132. it.effect("does not compact immediately when the advertised output limit fills the context", () =>
  2133. Effect.gen(function* () {
  2134. const session = yield* setup
  2135. currentModel = fullOutputModel
  2136. yield* TestLLM.push(TestLLM.textWithUsage("Earlier answer", "text-full-output-first", 9_500))
  2137. yield* runPrompt(session, "Earlier question")
  2138. requests.length = 0
  2139. yield* TestLLM.push(TestLLM.text("Continued", "text-full-output-final"))
  2140. yield* runPrompt(session, "Continue")
  2141. expect(requests).toHaveLength(1)
  2142. expect(userTexts(requests[0])).toContain("Continue")
  2143. expect(yield* session.context(sessionID)).not.toContainEqual(expect.objectContaining({ type: "compaction" }))
  2144. }),
  2145. )
  2146. it.effect("stops after required automatic compaction fails", () =>
  2147. Effect.gen(function* () {
  2148. const session = yield* setup
  2149. yield* TestLLM.push(TestLLM.textWithUsage("Earlier answer", "text-before-failed-compaction", 3_950))
  2150. yield* runPrompt(session, "Earlier question ".repeat(180))
  2151. currentModel = compactModel
  2152. requests.length = 0
  2153. yield* TestLLM.push(
  2154. [LLMEvent.providerError({ message: "Unsupported parameter: max_output_tokens" })],
  2155. TestLLM.text("Must not run", "text-after-failed-compaction"),
  2156. )
  2157. yield* admit(session, "Recent exact request ".repeat(180))
  2158. expect(yield* Effect.exit(session.resume(sessionID))).toMatchObject({ _tag: "Failure" })
  2159. expect(requests).toHaveLength(1)
  2160. expect(requests[0]?.generation).toBeUndefined()
  2161. expect(yield* session.context(sessionID)).toContainEqual(
  2162. expect.objectContaining({
  2163. type: "compaction",
  2164. status: "failed",
  2165. reason: "auto",
  2166. error: expect.objectContaining({ message: "Unsupported parameter: max_output_tokens" }),
  2167. }),
  2168. )
  2169. }),
  2170. )
  2171. it.effect("forces one compaction and retries after provider context overflow", () =>
  2172. Effect.gen(function* () {
  2173. const session = yield* setupOverflowRecovery
  2174. yield* TestLLM.push(
  2175. [
  2176. LLMEvent.stepStart({ index: 0 }),
  2177. LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }),
  2178. ],
  2179. TestLLM.text("## Objective\n- Recover overflow", "text-summary"),
  2180. TestLLM.text("Recovered", "text-final"),
  2181. )
  2182. yield* runPrompt(session, "Continue")
  2183. expect(requests).toHaveLength(3)
  2184. expect(userTexts(requests[1])[0]).toContain("## Objective")
  2185. expect(userTexts(requests[2])[0]).toContain("<summary>\n## Objective\n- Recover overflow\n</summary>")
  2186. expect(yield* session.context(sessionID)).toMatchObject([
  2187. { type: "compaction", summary: "## Objective\n- Recover overflow" },
  2188. { type: "assistant", finish: "stop" },
  2189. ])
  2190. yield* replaySessionProjection(sessionID)
  2191. expect(yield* session.context(sessionID)).toMatchObject([
  2192. { type: "compaction" },
  2193. { type: "assistant", finish: "stop" },
  2194. ])
  2195. }),
  2196. )
  2197. it.effect("recovers from provider context overflow without a configured context limit", () =>
  2198. Effect.gen(function* () {
  2199. const session = yield* setupOverflowRecovery
  2200. currentModel = model
  2201. yield* TestLLM.push(
  2202. [LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" })],
  2203. TestLLM.text("## Objective\n- Recover unknown limit", "text-summary-unknown-limit"),
  2204. TestLLM.text("Recovered", "text-final-unknown-limit"),
  2205. )
  2206. yield* runPrompt(session, "Continue")
  2207. expect(requests).toHaveLength(3)
  2208. expect(yield* session.context(sessionID)).toMatchObject([
  2209. { type: "compaction", summary: "## Objective\n- Recover unknown limit" },
  2210. { type: "assistant", finish: "stop" },
  2211. ])
  2212. }),
  2213. )
  2214. it.effect("recovers from provider context overflow despite an undersized configured context limit", () =>
  2215. Effect.gen(function* () {
  2216. const session = yield* setupOverflowRecovery
  2217. currentModel = undersizedContextModel
  2218. yield* TestLLM.push(
  2219. [LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" })],
  2220. TestLLM.text("## Objective\n- Recover undersized limit", "text-summary-undersized-limit"),
  2221. TestLLM.text("Recovered", "text-final-undersized-limit"),
  2222. )
  2223. yield* runPrompt(session, "Continue")
  2224. expect(requests).toHaveLength(3)
  2225. expect(yield* session.context(sessionID)).toMatchObject([
  2226. { type: "compaction", summary: "## Objective\n- Recover undersized limit" },
  2227. { type: "assistant", finish: "stop" },
  2228. ])
  2229. }),
  2230. )
  2231. it.effect("persists a second context overflow after one recovery", () =>
  2232. Effect.gen(function* () {
  2233. const session = yield* setupOverflowRecovery
  2234. const overflow = () => [
  2235. LLMEvent.stepStart({ index: 0 }),
  2236. LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }),
  2237. ]
  2238. yield* TestLLM.push(overflow(), TestLLM.text("## Objective\n- Recover once", "text-summary"), overflow())
  2239. yield* admit(session, "Continue")
  2240. expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("prompt too long")
  2241. expect(requests).toHaveLength(3)
  2242. expect(yield* session.context(sessionID)).toMatchObject([
  2243. { type: "compaction" },
  2244. { type: "assistant", finish: "error", error: { message: "prompt too long" } },
  2245. ])
  2246. }),
  2247. )
  2248. it.effect("recovers once from a raw context overflow failure", () =>
  2249. Effect.gen(function* () {
  2250. const session = yield* setupOverflowRecovery
  2251. yield* TestLLM.push(
  2252. Stream.fail(
  2253. new AIError({
  2254. module: "test",
  2255. method: "stream",
  2256. reason: new InvalidRequestReason({
  2257. message: "prompt too long",
  2258. classification: "context-overflow",
  2259. }),
  2260. }),
  2261. ),
  2262. )
  2263. yield* TestLLM.push(
  2264. TestLLM.text("## Objective\n- Recover raw overflow", "text-summary"),
  2265. TestLLM.text("Recovered", "text-final"),
  2266. )
  2267. yield* runPrompt(session, "Continue")
  2268. expect(requests).toHaveLength(3)
  2269. expect(yield* session.context(sessionID)).toMatchObject([
  2270. { type: "compaction", summary: "## Objective\n- Recover raw overflow" },
  2271. { type: "assistant", finish: "stop" },
  2272. ])
  2273. }),
  2274. )
  2275. it.effect("publishes the original overflow when recovery summarization fails", () =>
  2276. Effect.gen(function* () {
  2277. const session = yield* setupOverflowRecovery
  2278. yield* TestLLM.push(
  2279. [LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" })],
  2280. [LLMEvent.providerError({ message: "summary unavailable" })],
  2281. )
  2282. yield* admit(session, "Continue")
  2283. expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("prompt too long")
  2284. expect(requests).toHaveLength(2)
  2285. const context = yield* session.context(sessionID)
  2286. expect(context).toContainEqual(
  2287. expect.objectContaining({
  2288. type: "compaction",
  2289. status: "failed",
  2290. reason: "auto",
  2291. error: { type: "provider.error", message: "summary unavailable" },
  2292. }),
  2293. )
  2294. expect(context.slice(-3)).toMatchObject([
  2295. { type: "user", text: "Continue" },
  2296. { type: "compaction", status: "failed", reason: "auto" },
  2297. { type: "assistant", finish: "error", error: { message: "prompt too long" } },
  2298. ])
  2299. }),
  2300. )
  2301. it.effect("interrupts overflow recovery while the summary provider is running", () =>
  2302. Effect.gen(function* () {
  2303. const session = yield* setupOverflowRecovery
  2304. yield* TestLLM.push(
  2305. [LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" })],
  2306. TestLLM.text("## Objective\n- Interrupted", "text-summary"),
  2307. )
  2308. const first = yield* TestLLM.gate
  2309. yield* admit(session, "Continue")
  2310. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2311. yield* first.started
  2312. const summary = yield* TestLLM.gate
  2313. yield* first.release
  2314. yield* summary.started
  2315. yield* session.interrupt(sessionID)
  2316. const exit = yield* Fiber.await(run)
  2317. expect(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)).toBeTrue()
  2318. expect(yield* session.context(sessionID)).toContainEqual(
  2319. expect.objectContaining({
  2320. type: "compaction",
  2321. status: "failed",
  2322. reason: "auto",
  2323. error: { type: "compaction.interrupted", message: "Compaction was interrupted" },
  2324. }),
  2325. )
  2326. }),
  2327. )
  2328. it.effect("uses epoch values after compaction while a source is unavailable", () =>
  2329. Effect.gen(function* () {
  2330. const session = yield* setup
  2331. const bus = yield* Bus.Service
  2332. yield* runPrompt(session, "First")
  2333. systemBaseline = "Changed context"
  2334. yield* runPrompt(session, "Second")
  2335. yield* bus.publish(SessionEvent.Compaction.Started, {
  2336. sessionID,
  2337. reason: "manual",
  2338. recent: "",
  2339. })
  2340. yield* bus.publish(SessionEvent.Compaction.Ended, {
  2341. sessionID,
  2342. reason: "manual",
  2343. text: "summary",
  2344. recent: "",
  2345. })
  2346. systemUnavailable = true
  2347. yield* runPrompt(session, "Third")
  2348. // Compaction already moved current values into the new epoch before the unavailable read.
  2349. expect(requests.at(-1)?.system.map((part) => part.text)).toEqual([defaultSystem, "Changed context"])
  2350. expect(systemTexts(requests.at(-1)!)).not.toContain("Changed context")
  2351. }),
  2352. )
  2353. it.effect("projects reasoning and tool events without executing or continuing tools", () =>
  2354. Effect.gen(function* () {
  2355. const session = yield* setup
  2356. yield* admit(session, "Use tools")
  2357. yield* TestLLM.push(
  2358. TestLLM.complete(
  2359. {
  2360. reason: { normalized: "tool-calls" },
  2361. usage: {
  2362. inputTokens: 10,
  2363. nonCachedInputTokens: 8,
  2364. outputTokens: 4,
  2365. reasoningTokens: 1,
  2366. cacheReadInputTokens: 2,
  2367. },
  2368. },
  2369. LLMEvent.reasoningStart({ id: "reasoning-1" }),
  2370. LLMEvent.reasoningDelta({ id: "reasoning-1", text: "Think" }),
  2371. LLMEvent.reasoningEnd({ id: "reasoning-1" }),
  2372. LLMEvent.toolInputStart({ id: "call-error", name: "write" }),
  2373. LLMEvent.toolInputDelta({ id: "call-error", name: "write", text: '{"path":"README.md"}' }),
  2374. LLMEvent.toolInputEnd({ id: "call-error", name: "write" }),
  2375. LLMEvent.toolCall({ id: "call-error", name: "write", input: { path: "README.md" }, providerExecuted: true }),
  2376. LLMEvent.toolError({ id: "call-error", name: "write", message: "Denied" }),
  2377. LLMEvent.toolResult({ id: "call-error", name: "write", result: { type: "error", value: "Denied" } }),
  2378. LLMEvent.toolCall({
  2379. id: "call-provider",
  2380. name: "web_search",
  2381. input: { query: "hello" },
  2382. providerExecuted: true,
  2383. providerMetadata: { openai: { source: "provider" } },
  2384. }),
  2385. LLMEvent.toolResult({
  2386. id: "call-provider",
  2387. name: "web_search",
  2388. result: {
  2389. type: "content",
  2390. value: [
  2391. { type: "text", text: "Hello" },
  2392. { type: "file", uri: "data:image/png;base64,aGVsbG8=", mime: "image/png", name: "hello.png" },
  2393. ],
  2394. },
  2395. providerExecuted: true,
  2396. providerMetadata: { openai: { source: "provider" } },
  2397. }),
  2398. ),
  2399. )
  2400. yield* session.resume(sessionID)
  2401. expect(requests).toHaveLength(1)
  2402. expect(requests[0]?.tools.map((tool) => tool.name)).toEqual(["defect", "echo", "storefail"])
  2403. expect(yield* session.context(sessionID)).toMatchObject([
  2404. { type: "user", text: "Use tools" },
  2405. {
  2406. type: "assistant",
  2407. finish: "tool-calls",
  2408. cost: 0,
  2409. tokens: { input: 8, output: 3, reasoning: 1, cache: { read: 2, write: 0 } },
  2410. content: [
  2411. { type: "reasoning", text: "Think" },
  2412. {
  2413. type: "tool",
  2414. id: "call-error",
  2415. name: "write",
  2416. state: {
  2417. status: "error",
  2418. input: { path: "README.md" },
  2419. error: { type: "tool.execution", message: "Denied" },
  2420. },
  2421. },
  2422. {
  2423. type: "tool",
  2424. id: "call-provider",
  2425. name: "web_search",
  2426. executed: true,
  2427. providerState: { source: "provider" },
  2428. providerResultState: { source: "provider" },
  2429. state: {
  2430. status: "completed",
  2431. input: { query: "hello" },
  2432. content: [
  2433. { type: "text", text: "Hello" },
  2434. { type: "file", mime: "image/png", uri: "data:image/png;base64,aGVsbG8=", name: "hello.png" },
  2435. ],
  2436. },
  2437. },
  2438. ],
  2439. },
  2440. ])
  2441. }),
  2442. )
  2443. it.effect("continues with reloaded history after durably settling one local tool call", () =>
  2444. Effect.gen(function* () {
  2445. const session = yield* setup
  2446. yield* admit(session, "Echo this")
  2447. yield* TestLLM.push(TestLLM.tool("call-echo", "echo", { text: "hello" }), TestLLM.text("Done", "text-final"))
  2448. yield* session.resume(sessionID)
  2449. expect(requests).toHaveLength(2)
  2450. expect(messageRoles(requests[1])).toEqual(["user", "assistant", "tool"])
  2451. expect(authorizations).toMatchObject([{ sessionID, id: "call-echo" }])
  2452. expect(executions).toEqual(["hello"])
  2453. const context = yield* session.context(sessionID)
  2454. expect(context).toMatchObject([
  2455. { type: "user", text: "Echo this" },
  2456. {
  2457. type: "assistant",
  2458. finish: "tool-calls",
  2459. content: [
  2460. {
  2461. type: "tool",
  2462. id: "call-echo",
  2463. name: "echo",
  2464. state: {
  2465. status: "completed",
  2466. input: { text: "hello" },
  2467. content: [{ type: "text", text: "hello" }],
  2468. },
  2469. },
  2470. ],
  2471. },
  2472. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Done" }] },
  2473. ])
  2474. const assistant = requireAssistant(context)
  2475. expect(yield* recordedStepSettlementTypes(sessionID, assistant.id)).toEqual([
  2476. "session.step.started.1",
  2477. "session.tool.called.1",
  2478. "session.tool.success.2",
  2479. "session.step.ended.1",
  2480. ])
  2481. }),
  2482. )
  2483. it.effect("reloads a model switch before a tool-driven continuation step", () =>
  2484. Effect.gen(function* () {
  2485. const session = yield* setup
  2486. const bus = yield* Bus.Service
  2487. yield* admit(session, "Echo this")
  2488. yield* TestLLM.push(TestLLM.tool("call-echo", "echo", { text: "hello" }), TestLLM.stop())
  2489. const tools = yield* blockTools()
  2490. const run = yield* Effect.forkChild(session.resume(sessionID))
  2491. yield* tools.started
  2492. yield* bus.publish(SessionEvent.ModelSelected, {
  2493. sessionID,
  2494. model: { id: ID.make("replacement"), providerID: Provider.ID.make("fake") },
  2495. })
  2496. systemBaseline = "Replacement context"
  2497. yield* tools.release
  2498. yield* Fiber.join(run)
  2499. expect(requests.map((request) => request.model)).toEqual([model, replacementModel])
  2500. expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
  2501. [defaultSystem, "Initial context"],
  2502. [defaultSystem, "Initial context"],
  2503. ])
  2504. expect(systemTexts(requests[1])).toContain("Replacement context")
  2505. }),
  2506. )
  2507. it.effect("restores durable reasoning provider metadata in the next request", () =>
  2508. Effect.gen(function* () {
  2509. const session = yield* setup
  2510. yield* admit(session, "Think first")
  2511. yield* TestLLM.push(
  2512. TestLLM.stop(
  2513. LLMEvent.reasoningStart({ id: "reasoning-anthropic" }),
  2514. LLMEvent.reasoningDelta({ id: "reasoning-anthropic", text: "Signed thought" }),
  2515. LLMEvent.reasoningEnd({
  2516. id: "reasoning-anthropic",
  2517. providerMetadata: { openai: { signature: "sig_1" }, anthropic: { ignored: true } },
  2518. }),
  2519. LLMEvent.reasoningStart({
  2520. id: "reasoning-openai",
  2521. providerMetadata: {
  2522. openai: { itemId: "rs_1", reasoningEncryptedContent: null },
  2523. anthropic: { ignored: true },
  2524. },
  2525. }),
  2526. LLMEvent.reasoningDelta({ id: "reasoning-openai", text: "Encrypted thought" }),
  2527. LLMEvent.reasoningEnd({
  2528. id: "reasoning-openai",
  2529. providerMetadata: {
  2530. openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" },
  2531. anthropic: { ignored: true },
  2532. },
  2533. }),
  2534. ),
  2535. )
  2536. yield* session.resume(sessionID)
  2537. yield* replaySessionProjection(sessionID)
  2538. expect(yield* session.context(sessionID)).toMatchObject([
  2539. { type: "user", text: "Think first" },
  2540. {
  2541. type: "assistant",
  2542. content: [
  2543. {
  2544. type: "reasoning",
  2545. text: "Signed thought",
  2546. state: { signature: "sig_1" },
  2547. },
  2548. {
  2549. type: "reasoning",
  2550. text: "Encrypted thought",
  2551. state: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" },
  2552. },
  2553. ],
  2554. },
  2555. ])
  2556. yield* admit(session, "Continue")
  2557. yield* TestLLM.push([])
  2558. yield* session.resume(sessionID)
  2559. expect(requests[1]?.messages[1]?.content).toEqual([
  2560. {
  2561. type: "reasoning",
  2562. text: "Signed thought",
  2563. providerMetadata: { openai: { signature: "sig_1" } },
  2564. },
  2565. {
  2566. type: "reasoning",
  2567. text: "Encrypted thought",
  2568. providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } },
  2569. },
  2570. ])
  2571. }),
  2572. )
  2573. it.effect("restores durable text provider metadata in the next request", () =>
  2574. Effect.gen(function* () {
  2575. const session = yield* setup
  2576. yield* admit(session, "Check first")
  2577. yield* TestLLM.push(
  2578. TestLLM.stop(
  2579. LLMEvent.textStart({ id: "commentary", providerMetadata: { openai: { phase: "commentary" } } }),
  2580. LLMEvent.textDelta({ id: "commentary", text: "Checking." }),
  2581. LLMEvent.textEnd({
  2582. id: "commentary",
  2583. providerMetadata: { openai: { phase: "commentary" }, anthropic: { ignored: true } },
  2584. }),
  2585. ),
  2586. )
  2587. yield* session.resume(sessionID)
  2588. yield* replaySessionProjection(sessionID)
  2589. expect(yield* session.context(sessionID)).toMatchObject([
  2590. { type: "user", text: "Check first" },
  2591. {
  2592. type: "assistant",
  2593. content: [{ type: "text", text: "Checking.", state: { phase: "commentary" } }],
  2594. },
  2595. ])
  2596. yield* admit(session, "Continue")
  2597. yield* TestLLM.push([])
  2598. yield* session.resume(sessionID)
  2599. expect(requests[1]?.messages[1]?.content).toEqual([
  2600. {
  2601. type: "text",
  2602. text: "Checking.",
  2603. providerMetadata: { openai: { phase: "commentary" } },
  2604. },
  2605. ])
  2606. }),
  2607. )
  2608. it.effect("replays durable provider-executed tool results inline in the next request", () =>
  2609. Effect.gen(function* () {
  2610. const session = yield* setup
  2611. yield* admit(session, "Search first")
  2612. yield* TestLLM.push(
  2613. TestLLM.stop(
  2614. LLMEvent.toolCall({
  2615. id: "hosted-search",
  2616. name: "web_search",
  2617. input: { query: "Effect" },
  2618. providerExecuted: true,
  2619. providerMetadata: { openai: { itemId: "hosted-search" }, fake: { ignored: true } },
  2620. }),
  2621. LLMEvent.toolResult({
  2622. id: "hosted-search",
  2623. name: "web_search",
  2624. result: { type: "json", value: [{ title: "Effect" }] },
  2625. providerExecuted: true,
  2626. providerMetadata: { openai: { blockType: "web_search_tool_result" }, anthropic: { ignored: true } },
  2627. }),
  2628. ),
  2629. )
  2630. yield* session.resume(sessionID)
  2631. yield* replaySessionProjection(sessionID)
  2632. yield* admit(session, "Continue")
  2633. yield* TestLLM.push([])
  2634. yield* session.resume(sessionID)
  2635. expect(messageRoles(requests[1])).toEqual(["user", "assistant", "user"])
  2636. expect(requests[1]?.messages[1]?.content).toMatchObject([
  2637. {
  2638. type: "tool-call",
  2639. id: "hosted-search",
  2640. name: "web_search",
  2641. input: { query: "Effect" },
  2642. providerExecuted: true,
  2643. providerMetadata: { openai: { itemId: "hosted-search" } },
  2644. },
  2645. {
  2646. type: "tool-result",
  2647. id: "hosted-search",
  2648. name: "web_search",
  2649. // The generic replay result derives from canonical stored content.
  2650. result: { type: "text", value: '[{"title":"Effect"}]' },
  2651. providerExecuted: true,
  2652. providerMetadata: { openai: { blockType: "web_search_tool_result" } },
  2653. },
  2654. ])
  2655. }),
  2656. )
  2657. it.effect("starts recorded local tools eagerly and awaits settlement before continuing", () =>
  2658. Effect.gen(function* () {
  2659. const session = yield* setup
  2660. yield* admit(session, "Echo five times")
  2661. const tools = yield* blockTools(5)
  2662. const providerGate = yield* Deferred.make<void>()
  2663. const initial = Stream.fromIterable([
  2664. LLMEvent.stepStart({ index: 0 }),
  2665. ...Array.from({ length: 5 }, (_, index) =>
  2666. LLMEvent.toolCall({ id: `call-echo-${index}`, name: "echo", input: { text: `${index}` } }),
  2667. ),
  2668. ])
  2669. const final = Stream.fromIterable([
  2670. LLMEvent.stepFinish({ index: 0, reason: { normalized: "tool-calls" } }),
  2671. LLMEvent.finish({ reason: { normalized: "tool-calls" } }),
  2672. ])
  2673. yield* TestLLM.push(
  2674. Stream.concat(initial, Stream.fromEffect(Deferred.await(providerGate)).pipe(Stream.flatMap(() => final))),
  2675. )
  2676. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2677. yield* tools.started
  2678. expect(executions).toHaveLength(5)
  2679. expect(yield* tools.maxActive).toBe(5)
  2680. expect(yield* session.context(sessionID)).toMatchObject([
  2681. { type: "user", text: "Echo five times" },
  2682. {
  2683. type: "assistant",
  2684. content: Array.from({ length: 5 }, (_, index) => ({
  2685. type: "tool",
  2686. id: `call-echo-${index}`,
  2687. state: { status: "running", input: { text: `${index}` } },
  2688. })),
  2689. },
  2690. ])
  2691. yield* Deferred.succeed(providerGate, undefined)
  2692. yield* Effect.yieldNow
  2693. expect(requests).toHaveLength(1)
  2694. yield* tools.release
  2695. yield* Fiber.join(run)
  2696. expect(executions).toHaveLength(5)
  2697. expect(yield* tools.maxActive).toBe(5)
  2698. expect(requests).toHaveLength(2)
  2699. }),
  2700. )
  2701. it.effect("settles repeated provider-local tool call IDs against their owning assistant messages", () =>
  2702. Effect.gen(function* () {
  2703. const session = yield* setup
  2704. yield* admit(session, "Echo twice")
  2705. yield* TestLLM.push(
  2706. TestLLM.tool("tool_0", "echo", { text: "first" }),
  2707. TestLLM.tool("tool_0", "echo", { text: "second" }),
  2708. [],
  2709. )
  2710. yield* session.resume(sessionID)
  2711. const expected = [
  2712. { type: "user", text: "Echo twice" },
  2713. {
  2714. type: "assistant",
  2715. content: [
  2716. {
  2717. type: "tool",
  2718. id: "tool_0",
  2719. state: { status: "completed", content: [{ type: "text", text: "first" }] },
  2720. },
  2721. ],
  2722. },
  2723. {
  2724. type: "assistant",
  2725. content: [
  2726. {
  2727. type: "tool",
  2728. id: "tool_0",
  2729. state: { status: "completed", content: [{ type: "text", text: "second" }] },
  2730. },
  2731. ],
  2732. },
  2733. ]
  2734. expect(executions).toEqual(["first", "second"])
  2735. expect(requests).toHaveLength(3)
  2736. expect(yield* session.context(sessionID)).toMatchObject(expected)
  2737. yield* replaySessionProjection(sessionID)
  2738. expect(yield* session.context(sessionID)).toMatchObject(expected)
  2739. }),
  2740. )
  2741. it.effect("joins concurrent resume calls into one active provider run", () =>
  2742. Effect.gen(function* () {
  2743. const session = yield* setup
  2744. yield* admit(session, "Run once")
  2745. yield* TestLLM.push(TestLLM.text("Once", "text-once"))
  2746. const stream = yield* TestLLM.gate
  2747. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2748. yield* stream.started
  2749. const second = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2750. yield* Effect.yieldNow
  2751. expect(requests).toHaveLength(1)
  2752. yield* stream.release
  2753. yield* Fiber.join(first)
  2754. yield* Fiber.join(second)
  2755. expect(requests).toHaveLength(1)
  2756. expect(yield* session.context(sessionID)).toMatchObject([
  2757. { type: "user", text: "Run once" },
  2758. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Once" }] },
  2759. ])
  2760. }),
  2761. )
  2762. it.effect("steers an active step with newly recorded prompts", () =>
  2763. Effect.gen(function* () {
  2764. const session = yield* setup
  2765. yield* admit(session, "Start working")
  2766. yield* TestLLM.push(TestLLM.stop(), TestLLM.stop())
  2767. const stream = yield* TestLLM.gate
  2768. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2769. yield* stream.started
  2770. yield* session.prompt({ sessionID, text: "Change direction" })
  2771. yield* stream.release
  2772. yield* Fiber.join(first)
  2773. yield* Effect.yieldNow
  2774. expect(requests).toHaveLength(2)
  2775. expect(userTexts(requests[0])).toEqual(["Start working"])
  2776. expect(userTexts(requests[1])).toEqual(["Start working", "Change direction"])
  2777. expect((yield* session.context(sessionID)).map((message) => message.type)).toEqual([
  2778. "user",
  2779. "assistant",
  2780. "user",
  2781. "assistant",
  2782. ])
  2783. }),
  2784. )
  2785. it.effect("promotes queued input after continuation ends", () =>
  2786. Effect.gen(function* () {
  2787. const session = yield* setup
  2788. yield* admit(session, "Start working")
  2789. yield* TestLLM.push(TestLLM.tool("call-echo", "echo", { text: "hello" }), TestLLM.stop(), TestLLM.stop())
  2790. const stream = yield* TestLLM.gate
  2791. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2792. yield* stream.started
  2793. yield* session.prompt({
  2794. sessionID,
  2795. text: "Wait until continuation ends",
  2796. delivery: "queue",
  2797. })
  2798. yield* stream.release
  2799. yield* Fiber.join(first)
  2800. expect(requests).toHaveLength(3)
  2801. expect(userTexts(requests[0])).toEqual(["Start working"])
  2802. expect(userTexts(requests[1])).toEqual(["Start working"])
  2803. expect(userTexts(requests[2])).toEqual(["Start working", "Wait until continuation ends"])
  2804. }),
  2805. )
  2806. it.effect("preserves durable queued input for a later wake after interruption", () =>
  2807. Effect.gen(function* () {
  2808. const session = yield* setup
  2809. const { db } = yield* Database.Service
  2810. yield* admit(session, "Interrupt current work")
  2811. yield* TestLLM.push([], TestLLM.stop())
  2812. const stream = yield* TestLLM.gate
  2813. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2814. yield* stream.started
  2815. yield* session.prompt({
  2816. sessionID,
  2817. text: "Run after interrupt",
  2818. delivery: "queue",
  2819. })
  2820. yield* session.interrupt(sessionID)
  2821. expect(yield* Fiber.await(run)).toMatchObject({ _tag: "Failure" })
  2822. expect(requests).toHaveLength(1)
  2823. expect(yield* SessionInbox.has(db, sessionID, "queue")).toBe(true)
  2824. const resumed = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2825. yield* stream.started
  2826. yield* stream.release
  2827. yield* Fiber.join(resumed)
  2828. expect(requests).toHaveLength(2)
  2829. expect(userTexts(requests[0])).toEqual(["Interrupt current work"])
  2830. expect(userTexts(requests[1])).toEqual(["Interrupt current work", "Run after interrupt"])
  2831. }),
  2832. )
  2833. it.effect("preserves durable steering input for a later resume after interruption", () =>
  2834. Effect.gen(function* () {
  2835. const session = yield* setup
  2836. const { db } = yield* Database.Service
  2837. yield* admit(session, "Interrupt current work")
  2838. yield* TestLLM.push([], TestLLM.stop())
  2839. const stream = yield* TestLLM.gate
  2840. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2841. yield* stream.started
  2842. yield* session.prompt({
  2843. sessionID,
  2844. text: "Steer after interrupt",
  2845. })
  2846. yield* session.interrupt(sessionID)
  2847. expect(yield* Fiber.await(run)).toMatchObject({ _tag: "Failure" })
  2848. expect(requests).toHaveLength(1)
  2849. expect(yield* SessionInbox.has(db, sessionID, "steer")).toBe(true)
  2850. const resumed = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2851. yield* stream.started
  2852. yield* stream.release
  2853. yield* Fiber.join(resumed)
  2854. expect(requests).toHaveLength(2)
  2855. expect(userTexts(requests[0])).toEqual(["Interrupt current work"])
  2856. expect(userTexts(requests[1])).toEqual(["Interrupt current work", "Steer after interrupt"])
  2857. }),
  2858. )
  2859. it.effect("promotes queued inputs one at a time in FIFO order", () =>
  2860. Effect.gen(function* () {
  2861. const session = yield* setup
  2862. yield* admit(session, "Start working")
  2863. yield* TestLLM.push(TestLLM.stop(), TestLLM.stop(), TestLLM.stop())
  2864. const stream = yield* TestLLM.gate
  2865. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2866. yield* stream.started
  2867. yield* session.prompt({ sessionID, text: "Queue first", delivery: "queue" })
  2868. yield* session.prompt({ sessionID, text: "Queue second", delivery: "queue" })
  2869. yield* stream.release
  2870. yield* Fiber.join(first)
  2871. expect(requests).toHaveLength(3)
  2872. expect(userTexts(requests[0])).toEqual(["Start working"])
  2873. expect(userTexts(requests[1])).toEqual(["Start working", "Queue first"])
  2874. expect(userTexts(requests[2])).toEqual(["Start working", "Queue first", "Queue second"])
  2875. }),
  2876. )
  2877. it.effect("stops a steer-scoped drain before queued input", () =>
  2878. Effect.gen(function* () {
  2879. const session = yield* setup
  2880. const { db } = yield* Database.Service
  2881. yield* session.prompt({ sessionID, text: "Queue for later", delivery: "queue", resume: false })
  2882. yield* session.prompt({ sessionID, text: "Steer now", resume: false })
  2883. yield* TestLLM.push(TestLLM.stop())
  2884. const runner = yield* SessionRunner.Service
  2885. yield* runner.drain({ sessionID, force: false, promotable: "steer" })
  2886. expect(requests).toHaveLength(1)
  2887. expect(userTexts(requests[0])).toEqual(["Steer now"])
  2888. expect(yield* SessionInbox.has(db, sessionID, "steer")).toBe(false)
  2889. expect(yield* SessionInbox.has(db, sessionID, "queue")).toBe(true)
  2890. }),
  2891. )
  2892. it.effect("promotes queued input after steering continuation ends", () =>
  2893. Effect.gen(function* () {
  2894. const session = yield* setup
  2895. yield* admit(session, "Start steering")
  2896. yield* session.prompt({
  2897. sessionID,
  2898. text: "Queue for later",
  2899. delivery: "queue",
  2900. resume: false,
  2901. })
  2902. yield* TestLLM.push(TestLLM.stop(), TestLLM.stop())
  2903. yield* session.resume(sessionID)
  2904. expect(requests).toHaveLength(2)
  2905. expect(userTexts(requests[0])).toEqual(["Start steering"])
  2906. expect(userTexts(requests[1])).toEqual(["Start steering", "Queue for later"])
  2907. }),
  2908. )
  2909. it.effect("promotes steers before the next queued input", () =>
  2910. Effect.gen(function* () {
  2911. const session = yield* setup
  2912. yield* admit(session, "Start working")
  2913. yield* TestLLM.push(TestLLM.stop(), TestLLM.stop(), TestLLM.stop(), TestLLM.stop())
  2914. const firstStream = yield* TestLLM.gate
  2915. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2916. yield* firstStream.started
  2917. yield* session.prompt({ sessionID, text: "Queue first", delivery: "queue" })
  2918. yield* session.prompt({ sessionID, text: "Queue second", delivery: "queue" })
  2919. const secondStream = yield* TestLLM.gate
  2920. yield* firstStream.release
  2921. yield* secondStream.started
  2922. yield* session.prompt({ sessionID, text: "Steer before next queued input" })
  2923. yield* session.prompt({
  2924. sessionID,
  2925. text: "Also steer before next queued input",
  2926. })
  2927. yield* session.synthetic({ sessionID, text: "Background completion before next queued input" })
  2928. yield* secondStream.release
  2929. yield* Fiber.join(first)
  2930. expect(requests).toHaveLength(4)
  2931. expect(userTexts(requests[0])).toEqual(["Start working"])
  2932. expect(userTexts(requests[1])).toEqual(["Start working", "Queue first"])
  2933. expect(userTexts(requests[2])).toEqual([
  2934. "Start working",
  2935. "Queue first",
  2936. "Steer before next queued input",
  2937. "Also steer before next queued input",
  2938. "Background completion before next queued input",
  2939. ])
  2940. expect(userTexts(requests[3])).toEqual([
  2941. "Start working",
  2942. "Queue first",
  2943. "Steer before next queued input",
  2944. "Also steer before next queued input",
  2945. "Background completion before next queued input",
  2946. "Queue second",
  2947. ])
  2948. }),
  2949. )
  2950. it.effect("coalesces multiple active steering prompts into one continuation step", () =>
  2951. Effect.gen(function* () {
  2952. const session = yield* setup
  2953. yield* admit(session, "Start working")
  2954. yield* TestLLM.push(TestLLM.stop(), TestLLM.stop())
  2955. const stream = yield* TestLLM.gate
  2956. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2957. yield* stream.started
  2958. yield* session.prompt({ sessionID, text: "First steer" })
  2959. yield* session.prompt({ sessionID, text: "Second steer" })
  2960. yield* stream.release
  2961. yield* Fiber.join(first)
  2962. yield* Effect.yieldNow
  2963. expect(requests).toHaveLength(2)
  2964. expect(userTexts(requests[1])).toEqual(["Start working", "First steer", "Second steer"])
  2965. yield* (yield* SessionExecution.Service).wake(sessionID)
  2966. yield* Effect.yieldNow
  2967. expect(requests).toHaveLength(2)
  2968. }),
  2969. )
  2970. it.effect("runs steering input accepted while the active step fails", () =>
  2971. Effect.gen(function* () {
  2972. const session = yield* setup
  2973. yield* admit(session, "Start working")
  2974. const failure = invalidRequest()
  2975. yield* TestLLM.push(Stream.fail(failure))
  2976. const stream = yield* TestLLM.gate
  2977. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  2978. yield* stream.started
  2979. yield* session.prompt({ sessionID, text: "Recover with this" })
  2980. yield* stream.release
  2981. expect(yield* Fiber.join(first).pipe(Effect.flip)).toBe(failure)
  2982. yield* TestLLM.push([])
  2983. yield* session.wait(sessionID)
  2984. expect(requests).toHaveLength(2)
  2985. expect(userTexts(requests[1])).toEqual(["Start working", "Recover with this"])
  2986. }),
  2987. )
  2988. it.effect("durably fails local tools left running by a prior process before continuing", () =>
  2989. Effect.gen(function* () {
  2990. const session = yield* setup
  2991. const bus = yield* Bus.Service
  2992. yield* admit(session, "Recover interrupted tool")
  2993. yield* SessionInbox.promote((yield* Database.Service).db, bus, sessionID, "steer")
  2994. const assistantMessageID = SessionMessage.ID.create()
  2995. yield* bus.publish(SessionEvent.Step.Started, {
  2996. sessionID,
  2997. assistantMessageID,
  2998. agent: Agent.ID.make("build"),
  2999. model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") },
  3000. })
  3001. yield* bus.publish(SessionEvent.Tool.Input.Started, {
  3002. sessionID,
  3003. assistantMessageID,
  3004. id: "call-interrupted",
  3005. name: "echo",
  3006. })
  3007. yield* bus.publish(SessionEvent.Tool.Input.Ended, {
  3008. sessionID,
  3009. assistantMessageID,
  3010. id: "call-interrupted",
  3011. text: '{"text":"stale"}',
  3012. })
  3013. yield* bus.publish(SessionEvent.Tool.Called, {
  3014. sessionID,
  3015. assistantMessageID,
  3016. id: "call-interrupted",
  3017. input: { text: "stale" },
  3018. executed: false,
  3019. })
  3020. requests.length = 0
  3021. yield* TestLLM.push([])
  3022. yield* session.resume(sessionID)
  3023. expect(requests).toHaveLength(1)
  3024. expect(messageRoles(requests[0])).toEqual(["user", "assistant", "tool"])
  3025. expect(yield* session.context(sessionID)).toMatchObject([
  3026. { type: "user", text: "Recover interrupted tool" },
  3027. {
  3028. type: "assistant",
  3029. content: [
  3030. {
  3031. type: "tool",
  3032. id: "call-interrupted",
  3033. state: {
  3034. status: "error",
  3035. error: { type: "aborted", message: "Tool execution interrupted: echo" },
  3036. },
  3037. },
  3038. ],
  3039. },
  3040. ])
  3041. }),
  3042. )
  3043. it.effect("durably fails hosted tools left running by a prior process before continuing inline", () =>
  3044. Effect.gen(function* () {
  3045. const session = yield* setup
  3046. const bus = yield* Bus.Service
  3047. yield* admit(session, "Recover interrupted hosted tool")
  3048. yield* SessionInbox.promote((yield* Database.Service).db, bus, sessionID, "steer")
  3049. const assistantMessageID = SessionMessage.ID.create()
  3050. yield* bus.publish(SessionEvent.Step.Started, {
  3051. sessionID,
  3052. assistantMessageID,
  3053. agent: Agent.ID.make("build"),
  3054. model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") },
  3055. })
  3056. yield* bus.publish(SessionEvent.Tool.Input.Started, {
  3057. sessionID,
  3058. assistantMessageID,
  3059. id: "call-hosted-interrupted",
  3060. name: "web_search",
  3061. })
  3062. yield* bus.publish(SessionEvent.Tool.Input.Ended, {
  3063. sessionID,
  3064. assistantMessageID,
  3065. id: "call-hosted-interrupted",
  3066. text: '{"query":"stale"}',
  3067. })
  3068. yield* bus.publish(SessionEvent.Tool.Called, {
  3069. sessionID,
  3070. assistantMessageID,
  3071. id: "call-hosted-interrupted",
  3072. input: { query: "stale" },
  3073. executed: true,
  3074. state: { itemId: "call-hosted-interrupted" },
  3075. })
  3076. requests.length = 0
  3077. yield* TestLLM.push([])
  3078. yield* session.resume(sessionID)
  3079. expect(requests).toHaveLength(1)
  3080. expect(messageRoles(requests[0])).toEqual(["user", "assistant"])
  3081. expect(requests[0]?.messages[1]?.content).toMatchObject([
  3082. {
  3083. type: "tool-call",
  3084. id: "call-hosted-interrupted",
  3085. providerExecuted: true,
  3086. providerMetadata: { openai: { itemId: "call-hosted-interrupted" } },
  3087. },
  3088. { type: "tool-result", id: "call-hosted-interrupted", providerExecuted: true, result: { type: "error" } },
  3089. ])
  3090. }),
  3091. )
  3092. it.effect("durably fails pending tool input left by a prior process before continuing", () =>
  3093. Effect.gen(function* () {
  3094. const session = yield* setup
  3095. const bus = yield* Bus.Service
  3096. yield* admit(session, "Recover interrupted tool input")
  3097. yield* SessionInbox.promote((yield* Database.Service).db, bus, sessionID, "steer")
  3098. const assistantMessageID = SessionMessage.ID.create()
  3099. yield* bus.publish(SessionEvent.Step.Started, {
  3100. sessionID,
  3101. assistantMessageID,
  3102. agent: Agent.ID.make("build"),
  3103. model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") },
  3104. })
  3105. yield* bus.publish(SessionEvent.Tool.Input.Started, {
  3106. sessionID,
  3107. assistantMessageID,
  3108. id: "call-pending-interrupted",
  3109. name: "echo",
  3110. })
  3111. requests.length = 0
  3112. yield* TestLLM.push([])
  3113. yield* session.resume(sessionID)
  3114. expect(requests).toHaveLength(1)
  3115. expect(messageRoles(requests[0])).toEqual(["user", "assistant", "tool"])
  3116. expect(yield* session.context(sessionID)).toMatchObject([
  3117. { type: "user", text: "Recover interrupted tool input" },
  3118. { type: "assistant", content: [{ type: "tool", id: "call-pending-interrupted", state: { status: "error" } }] },
  3119. ])
  3120. }),
  3121. )
  3122. it.effect("promotes the first queued input when woken while idle", () =>
  3123. Effect.gen(function* () {
  3124. const session = yield* setup
  3125. yield* session.prompt({
  3126. sessionID,
  3127. text: "Wait in queue",
  3128. delivery: "queue",
  3129. resume: false,
  3130. })
  3131. const stream = yield* TestLLM.gate
  3132. yield* (yield* SessionExecution.Service).wake(sessionID)
  3133. yield* stream.started
  3134. yield* stream.release
  3135. expect(requests).toHaveLength(1)
  3136. expect(userTexts(requests[0])).toEqual(["Wait in queue"])
  3137. }),
  3138. )
  3139. it.effect("retries inbox input after prompt projection rolls back", () =>
  3140. Effect.gen(function* () {
  3141. const session = yield* setup
  3142. const bus = yield* Bus.Service
  3143. const defect = new Error("fail after prompt promotion")
  3144. let fail = true
  3145. yield* bus.project(SessionEvent.InboxDelivered, () => (fail ? Effect.die(defect) : Effect.void))
  3146. yield* admit(session, "Recover promoted input")
  3147. expect(yield* session.resume(sessionID).pipe(Effect.catchDefect(Effect.succeed))).toBe(defect)
  3148. fail = false
  3149. requests.length = 0
  3150. yield* TestLLM.push(TestLLM.stop())
  3151. const stream = yield* TestLLM.gate
  3152. yield* (yield* SessionExecution.Service).wake(sessionID)
  3153. yield* stream.started
  3154. yield* stream.release
  3155. expect(userTexts(requests[0])).toEqual(["Recover promoted input"])
  3156. }),
  3157. )
  3158. it.effect("does not strand a committed promotion when a post-commit listener defects", () =>
  3159. Effect.gen(function* () {
  3160. const session = yield* setup
  3161. const bus = yield* Bus.Service
  3162. yield* bus.listen((event) =>
  3163. event.type === SessionEvent.InboxDelivered.type
  3164. ? Effect.die("fail after prompt promotion commits")
  3165. : Effect.void,
  3166. )
  3167. yield* runPrompt(session, "Run committed promotion")
  3168. expect(requests).toHaveLength(1)
  3169. expect(userTexts(requests[0])).toEqual(["Run committed promotion"])
  3170. }),
  3171. )
  3172. it.effect("adds session correlation headers to model requests", () =>
  3173. Effect.gen(function* () {
  3174. const session = yield* setup
  3175. yield* runPrompt(session, "Run correlated request")
  3176. expect(requests[0]?.http?.headers).toEqual({
  3177. "x-session-affinity": sessionID,
  3178. "X-Session-Id": sessionID,
  3179. "User-Agent": App.useragent(App.make()),
  3180. "x-opencode-project": Project.ID.global,
  3181. "x-opencode-session": sessionID,
  3182. "x-opencode-client": "opencode",
  3183. })
  3184. }),
  3185. )
  3186. it.effect("adds the parent session header to child model requests", () =>
  3187. Effect.gen(function* () {
  3188. const session = yield* setup
  3189. const parentID = Session.ID.make("ses_runner_parent")
  3190. const { db } = yield* Database.Service
  3191. yield* db
  3192. .update(SessionTable)
  3193. .set({ parent_id: parentID })
  3194. .where(eq(SessionTable.id, sessionID))
  3195. .run()
  3196. .pipe(Effect.orDie)
  3197. yield* runPrompt(session, "Run child request")
  3198. expect(requests[0]?.http?.headers?.["x-parent-session-id"]).toBe(parentID)
  3199. }),
  3200. )
  3201. it.effect("runs different sessions concurrently", () =>
  3202. Effect.gen(function* () {
  3203. const session = yield* setup
  3204. yield* insertSession(otherSessionID)
  3205. yield* admit(session, "Run first")
  3206. yield* session.prompt({
  3207. sessionID: otherSessionID,
  3208. text: "Run second",
  3209. resume: false,
  3210. })
  3211. const stream = yield* TestLLM.gate
  3212. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3213. yield* stream.started
  3214. const second = yield* session.resume(otherSessionID).pipe(Effect.forkChild)
  3215. yield* stream.started
  3216. expect(requests).toHaveLength(2)
  3217. expect(requests.map((request) => request.promptCacheKey)).toEqual([sessionID, otherSessionID])
  3218. yield* stream.release
  3219. yield* Fiber.join(first)
  3220. yield* Fiber.join(second)
  3221. }),
  3222. )
  3223. it.effect("bounds 64-character session prompt cache keys", () =>
  3224. Effect.gen(function* () {
  3225. const session = yield* setup
  3226. const longSessionID = Session.ID.make(`ses_${"a".repeat(64)}`)
  3227. const otherLongSessionID = Session.ID.make(`ses_${"b".repeat(64)}`)
  3228. yield* insertSession(longSessionID)
  3229. yield* insertSession(otherLongSessionID)
  3230. yield* session.prompt({
  3231. sessionID: longSessionID,
  3232. text: "Run long session",
  3233. resume: false,
  3234. })
  3235. yield* session.prompt({
  3236. sessionID: otherLongSessionID,
  3237. text: "Run other long session",
  3238. resume: false,
  3239. })
  3240. yield* session.resume(longSessionID)
  3241. yield* session.resume(otherLongSessionID)
  3242. const keys = requests.map((request) => request.promptCacheKey)
  3243. expect(keys).toEqual([longSessionID.slice(4), otherLongSessionID.slice(4)])
  3244. expect(keys.every((key) => typeof key === "string" && key.length === 64)).toBe(true)
  3245. expect(keys[0]).not.toBe(keys[1])
  3246. }),
  3247. )
  3248. it.effect("fans out one failed run and allows a later retry", () =>
  3249. Effect.gen(function* () {
  3250. const session = yield* setup
  3251. yield* admit(session, "Retry after failure")
  3252. yield* TestLLM.push(Stream.fail(invalidRequest()))
  3253. const stream = yield* TestLLM.gate
  3254. const first = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3255. yield* stream.started
  3256. const second = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3257. yield* Effect.yieldNow
  3258. expect(requests).toHaveLength(1)
  3259. yield* stream.release
  3260. const [firstExit, secondExit] = yield* Effect.all([Fiber.await(first), Fiber.await(second)])
  3261. expect(secondExit).toEqual(firstExit)
  3262. yield* TestLLM.push([])
  3263. yield* session.resume(sessionID)
  3264. expect(requests).toHaveLength(2)
  3265. }),
  3266. )
  3267. it.effect("durably settles local tool failures before continuing", () =>
  3268. Effect.gen(function* () {
  3269. const session = yield* setup
  3270. yield* admit(session, "Call missing")
  3271. yield* TestLLM.push(TestLLM.tool("call-missing", "missing", {}), TestLLM.text("Recovered", "text-after-error"))
  3272. yield* session.resume(sessionID)
  3273. expect(requests).toHaveLength(2)
  3274. expect(yield* session.context(sessionID)).toMatchObject([
  3275. { type: "user", text: "Call missing" },
  3276. {
  3277. type: "assistant",
  3278. content: [
  3279. {
  3280. type: "tool",
  3281. id: "call-missing",
  3282. state: {
  3283. status: "error",
  3284. error: { type: "tool.execution", message: "Unknown tool: missing" },
  3285. },
  3286. },
  3287. ],
  3288. },
  3289. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Recovered" }] },
  3290. ])
  3291. }),
  3292. )
  3293. it.effect("returns unexpected local tool defects to the model and continues", () =>
  3294. Effect.gen(function* () {
  3295. const session = yield* setup
  3296. yield* admit(session, "Call defect")
  3297. yield* TestLLM.push(TestLLM.tool("call-defect", "defect", {}), TestLLM.text("Recovered", "text-after-defect"))
  3298. yield* session.resume(sessionID)
  3299. expect(requests).toHaveLength(2)
  3300. expect(messageRoles(requests[1])).toEqual(["user", "assistant", "tool"])
  3301. const context = yield* session.context(sessionID)
  3302. expect(context).toMatchObject([
  3303. { type: "user", text: "Call defect" },
  3304. {
  3305. type: "assistant",
  3306. content: [
  3307. {
  3308. type: "tool",
  3309. id: "call-defect",
  3310. state: {
  3311. status: "error",
  3312. error: { type: "unknown", message: "unexpected tool defect" },
  3313. },
  3314. },
  3315. ],
  3316. },
  3317. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Recovered" }] },
  3318. ])
  3319. const assistant = requireAssistant(context)
  3320. expect(yield* recordedStepSettlementTypes(sessionID, assistant.id)).toEqual([
  3321. "session.step.started.1",
  3322. "session.tool.called.1",
  3323. "session.tool.failed.2",
  3324. "session.step.ended.1",
  3325. ])
  3326. }),
  3327. )
  3328. it.effect("returns tool-wrapped policy blocks to the model and continues", () =>
  3329. Effect.gen(function* () {
  3330. const session = yield* setup
  3331. const registry = yield* Tool.Service
  3332. yield* transformTools(
  3333. registry,
  3334. {
  3335. blocked: {
  3336. name: "blocked",
  3337. description: "Fail because policy blocked execution",
  3338. input: Schema.Struct({}),
  3339. output: Schema.Struct({}),
  3340. execute: () =>
  3341. Effect.fail(new Permission.BlockedError({ rules: [], permission: "blocked", resources: ["*"] })).pipe(
  3342. Effect.mapError(() => new Tool.Error({ message: "Permission blocked" })),
  3343. ),
  3344. },
  3345. },
  3346. { codemode: false },
  3347. )
  3348. yield* admit(session, "Call blocked")
  3349. yield* TestLLM.push(TestLLM.tool("call-blocked", "blocked", {}), TestLLM.stop())
  3350. yield* session.resume(sessionID)
  3351. expect(requests).toHaveLength(2)
  3352. expect(yield* session.context(sessionID)).toMatchObject([
  3353. { type: "user", text: "Call blocked" },
  3354. {
  3355. type: "assistant",
  3356. content: [
  3357. { type: "tool", id: "call-blocked", state: { status: "error", error: { message: "Permission blocked" } } },
  3358. ],
  3359. },
  3360. { type: "assistant", finish: "stop" },
  3361. ])
  3362. }),
  3363. )
  3364. it.effect("interrupts runner continuation when permission approval is declined", () =>
  3365. Effect.gen(function* () {
  3366. const session = yield* setup
  3367. const registry = yield* Tool.Service
  3368. yield* transformTools(
  3369. registry,
  3370. {
  3371. declined: {
  3372. name: "declined",
  3373. description: "Fail because the user declined approval",
  3374. input: Schema.Struct({}),
  3375. output: Schema.Struct({}),
  3376. execute: () => Effect.die(new Permission.DeclinedError()),
  3377. },
  3378. },
  3379. { codemode: false },
  3380. )
  3381. yield* admit(session, "Call declined")
  3382. yield* TestLLM.push(TestLLM.tool("call-declined", "declined", {}))
  3383. const exit = yield* session.resume(sessionID).pipe(Effect.exit)
  3384. expect(exit._tag).toBe("Failure")
  3385. if (exit._tag === "Failure") expect(Cause.hasInterruptsOnly(exit.cause)).toBe(true)
  3386. expect(requests).toHaveLength(1)
  3387. expect(yield* session.context(sessionID)).toMatchObject([
  3388. { type: "user", text: "Call declined" },
  3389. {
  3390. type: "assistant",
  3391. content: [
  3392. {
  3393. type: "tool",
  3394. id: "call-declined",
  3395. state: { status: "error", error: { type: "aborted", message: "The user declined this tool call" } },
  3396. },
  3397. ],
  3398. },
  3399. ])
  3400. }),
  3401. )
  3402. it.effect("returns permission corrections to the model and continues", () =>
  3403. Effect.gen(function* () {
  3404. const session = yield* setup
  3405. const registry = yield* Tool.Service
  3406. yield* transformTools(
  3407. registry,
  3408. {
  3409. corrected: {
  3410. name: "corrected",
  3411. description: "Fail with user correction feedback",
  3412. input: Schema.Struct({}),
  3413. output: Schema.Struct({}),
  3414. execute: () =>
  3415. Effect.fail(new Permission.CorrectedError({ feedback: "Use another tool" })).pipe(
  3416. Effect.mapError(() => new Tool.Error({ message: "Use another tool" })),
  3417. ),
  3418. },
  3419. },
  3420. { codemode: false },
  3421. )
  3422. yield* admit(session, "Call corrected")
  3423. yield* TestLLM.push(TestLLM.tool("call-corrected", "corrected", {}), TestLLM.stop())
  3424. yield* session.resume(sessionID)
  3425. expect(requests).toHaveLength(2)
  3426. expect(yield* session.context(sessionID)).toMatchObject([
  3427. { type: "user", text: "Call corrected" },
  3428. {
  3429. type: "assistant",
  3430. content: [
  3431. { type: "tool", id: "call-corrected", state: { status: "error", error: { message: "Use another tool" } } },
  3432. ],
  3433. },
  3434. { type: "assistant", finish: "stop" },
  3435. ])
  3436. }),
  3437. )
  3438. it.effect("returns configured permission denials to the model and continues", () =>
  3439. Effect.gen(function* () {
  3440. const session = yield* setup
  3441. const registry = yield* Tool.Service
  3442. yield* transformTools(registry, { permissionfail: permissionFail }, { codemode: false })
  3443. yield* admit(session, "Reject permission")
  3444. yield* TestLLM.push(TestLLM.tool("call-permission", "permissionfail", {}), [
  3445. LLMEvent.stepStart({ index: 0 }),
  3446. LLMEvent.stepFinish({ index: 0, reason: { normalized: "stop" } }),
  3447. ])
  3448. yield* session.resume(sessionID)
  3449. expect(requests).toHaveLength(2)
  3450. expect(yield* session.context(sessionID)).toMatchObject([
  3451. { type: "user" },
  3452. {
  3453. type: "assistant",
  3454. content: [
  3455. {
  3456. type: "tool",
  3457. id: "call-permission",
  3458. state: {
  3459. status: "error",
  3460. error: {
  3461. type: "permission.rejected",
  3462. message: "Permission denied: edit",
  3463. },
  3464. },
  3465. },
  3466. ],
  3467. },
  3468. { type: "assistant", finish: "stop" },
  3469. ])
  3470. expect(yield* recordedEventTypes(sessionID)).not.toContain("session.step.failed.1")
  3471. }),
  3472. )
  3473. it.effect("interrupts runner continuation when a question is cancelled", () =>
  3474. Effect.gen(function* () {
  3475. const session = yield* setup
  3476. const registry = yield* Tool.Service
  3477. yield* transformTools(
  3478. registry,
  3479. {
  3480. question: {
  3481. name: "question",
  3482. description: "Ask the user",
  3483. input: Schema.Struct({}),
  3484. output: Schema.Struct({}),
  3485. execute: () => Effect.die(new QuestionTool.CancelledError()),
  3486. },
  3487. },
  3488. { codemode: false },
  3489. )
  3490. yield* admit(session, "Ask then stop")
  3491. yield* TestLLM.push(TestLLM.tool("call-question", "question", {}), [])
  3492. const run = yield* session.resume(sessionID).pipe(Effect.exit, Effect.forkChild)
  3493. const exit = yield* Fiber.join(run)
  3494. expect(exit._tag).toBe("Failure")
  3495. if (exit._tag === "Failure") expect(Cause.hasInterruptsOnly(exit.cause)).toBe(true)
  3496. expect(requests).toHaveLength(1)
  3497. expect(yield* session.context(sessionID)).toMatchObject([
  3498. { type: "user", text: "Ask then stop" },
  3499. {
  3500. type: "assistant",
  3501. content: [
  3502. {
  3503. type: "tool",
  3504. id: "call-question",
  3505. state: { status: "error", error: { type: "aborted", message: "The user dismissed this question" } },
  3506. },
  3507. ],
  3508. },
  3509. ])
  3510. }),
  3511. )
  3512. it.effect("awaits started local tools before surfacing provider stream failure", () =>
  3513. Effect.gen(function* () {
  3514. const session = yield* setup
  3515. yield* admit(session, "Settle before failing")
  3516. const failure = providerUnavailable()
  3517. const tools = yield* blockTools()
  3518. yield* TestLLM.push(
  3519. TestLLM.failAfter(
  3520. failure,
  3521. LLMEvent.stepStart({ index: 0 }),
  3522. LLMEvent.toolCall({ id: "call-before-failure", name: "echo", input: { text: "settle" } }),
  3523. ),
  3524. )
  3525. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3526. yield* tools.started
  3527. yield* tools.release
  3528. expect(yield* Fiber.join(run).pipe(Effect.flip)).toBe(failure)
  3529. const context = yield* session.context(sessionID)
  3530. expect(context).toMatchObject([
  3531. { type: "user", text: "Settle before failing" },
  3532. {
  3533. type: "assistant",
  3534. content: [
  3535. {
  3536. type: "tool",
  3537. id: "call-before-failure",
  3538. state: { status: "completed", content: [{ type: "text", text: "settle" }] },
  3539. },
  3540. ],
  3541. },
  3542. ])
  3543. const assistant = requireAssistant(context)
  3544. expect(yield* recordedStepSettlementTypes(sessionID, assistant.id)).toEqual([
  3545. "session.step.started.1",
  3546. "session.tool.called.1",
  3547. "session.tool.success.2",
  3548. "session.step.failed.1",
  3549. ])
  3550. }),
  3551. )
  3552. it.effect("durably fails blocked local tools when a step is interrupted", () =>
  3553. Effect.gen(function* () {
  3554. const session = yield* setup
  3555. yield* admit(session, "Interrupt blocked tool")
  3556. const tools = yield* blockTools()
  3557. yield* TestLLM.push(
  3558. TestLLM.hangAfter(
  3559. LLMEvent.stepStart({ index: 0 }),
  3560. LLMEvent.toolCall({ id: "call-before-interrupt", name: "echo", input: { text: "blocked" } }),
  3561. ),
  3562. )
  3563. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3564. yield* tools.started
  3565. yield* session.interrupt(sessionID)
  3566. expect(yield* Fiber.await(run)).toMatchObject({ _tag: "Failure" })
  3567. yield* session.interrupt(sessionID)
  3568. const context = yield* session.context(sessionID)
  3569. expect(context).toMatchObject([
  3570. { type: "user", text: "Interrupt blocked tool" },
  3571. {
  3572. type: "assistant",
  3573. content: [
  3574. {
  3575. type: "tool",
  3576. id: "call-before-interrupt",
  3577. state: { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } },
  3578. },
  3579. ],
  3580. },
  3581. ])
  3582. const assistant = requireAssistant(context)
  3583. expect(yield* recordedStepSettlementTypes(sessionID, assistant.id)).toEqual([
  3584. "session.step.started.1",
  3585. "session.tool.called.1",
  3586. "session.tool.failed.2",
  3587. "session.step.failed.1",
  3588. ])
  3589. yield* replaySessionProjection(sessionID)
  3590. expect(yield* session.context(sessionID)).toMatchObject([
  3591. { type: "user", text: "Interrupt blocked tool" },
  3592. { type: "assistant", content: [{ type: "tool", id: "call-before-interrupt", state: { status: "error" } }] },
  3593. ])
  3594. requests.length = 0
  3595. yield* TestLLM.push([])
  3596. yield* session.resume(sessionID)
  3597. expect(messageRoles(requests[0])).toEqual(["user", "assistant", "tool"])
  3598. }),
  3599. )
  3600. it.effect("interrupts a blocked step without local tool execution", () =>
  3601. Effect.gen(function* () {
  3602. const session = yield* setup
  3603. yield* admit(session, "Interrupt provider")
  3604. const stream = yield* TestLLM.gate
  3605. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3606. yield* stream.started
  3607. yield* session.interrupt(sessionID)
  3608. const exit = yield* Fiber.await(run)
  3609. expect(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)).toBeTrue()
  3610. expect(requests).toHaveLength(1)
  3611. expect(yield* session.context(sessionID)).toMatchObject([
  3612. { type: "user", text: "Interrupt provider" },
  3613. { type: "assistant", finish: "error", error: { type: "aborted", message: "Step interrupted" } },
  3614. ])
  3615. expect(yield* recordedEventTypes(sessionID)).toContain("session.step.failed.1")
  3616. yield* session.interrupt(sessionID)
  3617. }),
  3618. )
  3619. it.effect("durably fails blocked local tools when interrupted while awaiting settlement", () =>
  3620. Effect.gen(function* () {
  3621. const session = yield* setup
  3622. yield* admit(session, "Interrupt tool settlement")
  3623. const tools = yield* blockTools()
  3624. yield* TestLLM.push(TestLLM.tool("call-await-interrupt", "echo", { text: "blocked" }))
  3625. const runner = yield* SessionRunner.Service
  3626. const run = yield* runner.drain({ sessionID, force: true }).pipe(Effect.forkChild)
  3627. yield* tools.started
  3628. yield* Fiber.interrupt(run)
  3629. expect(yield* Fiber.await(run)).toMatchObject({ _tag: "Failure" })
  3630. expect(yield* session.context(sessionID)).toMatchObject([
  3631. { type: "user", text: "Interrupt tool settlement" },
  3632. {
  3633. type: "assistant",
  3634. finish: "error",
  3635. error: { type: "aborted", message: "Step interrupted" },
  3636. content: [
  3637. {
  3638. type: "tool",
  3639. id: "call-await-interrupt",
  3640. state: { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } },
  3641. },
  3642. ],
  3643. },
  3644. ])
  3645. const eventTypes = yield* recordedEventTypes(sessionID)
  3646. expect(eventTypes).toContain("session.step.failed.1")
  3647. expect(eventTypes).not.toContain("session.step.ended.1")
  3648. }),
  3649. )
  3650. it.effect("forces a text response on an agent's configured final step", () =>
  3651. Effect.gen(function* () {
  3652. const session = yield* setup
  3653. const agents = yield* Agent.Service
  3654. yield* agents.transform((editor) =>
  3655. editor.update(Agent.ID.make("build"), (agent) => {
  3656. agent.steps = 2
  3657. }),
  3658. )
  3659. yield* admit(session, "Finish at the limit")
  3660. yield* TestLLM.push(
  3661. TestLLM.tool("call-terminal", "echo", { text: "done" }),
  3662. TestLLM.tool("call-forbidden", "echo", { text: "forbidden" }),
  3663. )
  3664. yield* session.resume(sessionID)
  3665. expect(requests).toHaveLength(2)
  3666. expect(requests[0]?.toolChoice).toBeUndefined()
  3667. expect(requests[1]?.toolChoice).toMatchObject({ type: "none" })
  3668. // Protocols with native "none" keep these definitions for prompt caching.
  3669. expect(requests[1]?.tools.map((tool) => tool.name)).toContain("echo")
  3670. expect(requests[1]?.messages.at(-1)).toMatchObject({
  3671. role: "assistant",
  3672. content: [{ type: "text", text: expect.stringContaining("MAXIMUM STEPS REACHED") }],
  3673. })
  3674. expect(executions).toEqual(["done"])
  3675. expect(yield* session.context(sessionID)).toMatchObject([
  3676. { type: "user", text: "Finish at the limit" },
  3677. { type: "assistant", content: [{ type: "tool", id: "call-terminal", state: { status: "completed" } }] },
  3678. { type: "assistant", content: [{ type: "tool", id: "call-forbidden", state: { status: "error" } }] },
  3679. ])
  3680. }),
  3681. )
  3682. it.effect("resets the configured step allowance when steering input promotes", () =>
  3683. Effect.gen(function* () {
  3684. const session = yield* setup
  3685. const agents = yield* Agent.Service
  3686. yield* agents.transform((editor) =>
  3687. editor.update(Agent.ID.make("build"), (agent) => {
  3688. agent.steps = 2
  3689. }),
  3690. )
  3691. yield* admit(session, "Start work")
  3692. yield* TestLLM.push(
  3693. TestLLM.tool("call-before-steer", "echo", { text: "before" }),
  3694. TestLLM.tool("call-after-steer", "echo", { text: "after" }),
  3695. TestLLM.stop(),
  3696. )
  3697. const stream = yield* TestLLM.gate
  3698. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3699. yield* stream.started
  3700. yield* session.prompt({ sessionID, text: "Change direction" })
  3701. yield* stream.release
  3702. yield* Fiber.join(run)
  3703. expect(requests).toHaveLength(3)
  3704. expect(requests[1]?.toolChoice).toBeUndefined()
  3705. expect(requests[1]?.tools).not.toEqual([])
  3706. expect(requests[2]?.toolChoice).toMatchObject({ type: "none" })
  3707. expect(executions).toEqual(["before", "after"])
  3708. }),
  3709. )
  3710. it.effect("projects provider errors as terminal assistant step failures", () =>
  3711. Effect.gen(function* () {
  3712. const session = yield* setup
  3713. yield* TestLLM.push([
  3714. LLMEvent.stepStart({ index: 0 }),
  3715. LLMEvent.providerError({ message: "Provider unavailable" }),
  3716. ])
  3717. expect((yield* runPrompt(session, "Fail durably").pipe(Effect.flip)).message).toBe("Provider unavailable")
  3718. expect(requests).toHaveLength(1)
  3719. expect(yield* session.context(sessionID)).toMatchObject([
  3720. { type: "user", text: "Fail durably" },
  3721. { type: "assistant", finish: "error", error: { type: "provider.unknown", message: "Provider unavailable" } },
  3722. ])
  3723. }),
  3724. )
  3725. it.effect("projects provider errors emitted before assistant step start", () =>
  3726. Effect.gen(function* () {
  3727. const session = yield* setup
  3728. yield* TestLLM.push([LLMEvent.providerError({ message: "Provider unavailable" })])
  3729. expect((yield* runPrompt(session, "Fail before step").pipe(Effect.flip)).message).toBe("Provider unavailable")
  3730. expect(requests).toHaveLength(1)
  3731. expect(yield* session.context(sessionID)).toMatchObject([
  3732. { type: "user", text: "Fail before step" },
  3733. { type: "assistant", finish: "error", error: { type: "provider.unknown", message: "Provider unavailable" } },
  3734. ])
  3735. }),
  3736. )
  3737. it.effect("projects content-filter finishes as visible terminal failures", () =>
  3738. Effect.gen(function* () {
  3739. const session = yield* setup
  3740. yield* TestLLM.push(
  3741. TestLLM.complete(
  3742. {
  3743. reason: { normalized: "content-filter" },
  3744. usage: { nonCachedInputTokens: 8, outputTokens: 3, reasoningTokens: 1 },
  3745. },
  3746. LLMEvent.textStart({ id: "partial" }),
  3747. LLMEvent.textDelta({ id: "partial", text: "Partial" }),
  3748. ),
  3749. )
  3750. expect((yield* runPrompt(session, "Blocked response").pipe(Effect.flip)).message).toBe(
  3751. "Provider blocked the response",
  3752. )
  3753. expect(yield* session.context(sessionID)).toMatchObject([
  3754. { type: "user" },
  3755. {
  3756. type: "assistant",
  3757. finish: "error",
  3758. error: { type: "provider.content-filter" },
  3759. cost: 0,
  3760. tokens: { input: 8, output: 2, reasoning: 1, cache: { read: 0, write: 0 } },
  3761. content: [{ type: "text", text: "Partial" }],
  3762. },
  3763. ])
  3764. expect(yield* session.get(sessionID)).toMatchObject({
  3765. cost: 0,
  3766. tokens: { input: 8, output: 2, reasoning: 1, cache: { read: 0, write: 0 } },
  3767. })
  3768. expect(yield* recordedEventTypes(sessionID)).not.toContain("session.step.ended.1")
  3769. }),
  3770. )
  3771. it.effect("settles a local tool before one content-filter step failure", () =>
  3772. Effect.gen(function* () {
  3773. const session = yield* setup
  3774. yield* admit(session, "Tool before blocked response")
  3775. const tools = yield* blockTools()
  3776. yield* TestLLM.push(
  3777. TestLLM.complete(
  3778. { reason: { normalized: "content-filter" } },
  3779. LLMEvent.toolCall({ id: "call-before-content-filter", name: "echo", input: { text: "settled" } }),
  3780. ),
  3781. )
  3782. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3783. yield* tools.started
  3784. yield* tools.release
  3785. expect((yield* Fiber.join(run).pipe(Effect.flip)).message).toBe("Provider blocked the response")
  3786. const assistant = requireAssistant(yield* session.context(sessionID))
  3787. const bus = yield* recordedStepSettlementEvents(sessionID, assistant.id)
  3788. expect(bus.map((event) => event.type)).toEqual([
  3789. "session.step.started.1",
  3790. "session.tool.called.1",
  3791. "session.tool.success.2",
  3792. "session.step.failed.1",
  3793. ])
  3794. expect(
  3795. bus.filter((event) => event.type.startsWith("session.step.") && event.type !== "session.step.started.1"),
  3796. ).toHaveLength(1)
  3797. }),
  3798. )
  3799. it.effect("does not recover context overflow after durable assistant output", () =>
  3800. Effect.gen(function* () {
  3801. const session = yield* setup
  3802. yield* TestLLM.push([
  3803. LLMEvent.stepStart({ index: 0 }),
  3804. LLMEvent.textStart({ id: "text-partial" }),
  3805. LLMEvent.textDelta({ id: "text-partial", text: "Partial" }),
  3806. LLMEvent.textEnd({ id: "text-partial" }),
  3807. LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }),
  3808. ])
  3809. expect((yield* runPrompt(session, "Fail after output").pipe(Effect.flip)).message).toBe("prompt too long")
  3810. expect(requests).toHaveLength(1)
  3811. expect(yield* session.context(sessionID)).toMatchObject([
  3812. { type: "user", text: "Fail after output" },
  3813. {
  3814. type: "assistant",
  3815. finish: "error",
  3816. error: { message: "prompt too long" },
  3817. content: [{ type: "text", text: "Partial" }],
  3818. },
  3819. ])
  3820. }),
  3821. )
  3822. it.effect("projects raw provider stream failures as terminal assistant step failures", () =>
  3823. Effect.gen(function* () {
  3824. const session = yield* setup
  3825. const failure = invalidRequest()
  3826. yield* TestLLM.push(Stream.fail(failure))
  3827. expect(yield* runPrompt(session, "Fail raw stream durably").pipe(Effect.flip)).toBe(failure)
  3828. yield* replaySessionProjection(sessionID)
  3829. expect(yield* session.context(sessionID)).toMatchObject([
  3830. { type: "user", text: "Fail raw stream durably" },
  3831. { type: "assistant", finish: "error", error: { type: "provider.invalid-request", message: "Invalid request" } },
  3832. ])
  3833. }),
  3834. )
  3835. it.effect("bounds jittered exponential backoff for eligible pre-output failures", () =>
  3836. Effect.gen(function* () {
  3837. const session = yield* setup
  3838. yield* admit(session, "Retry transport")
  3839. yield* TestLLM.push(Stream.fail(providerUnavailable()))
  3840. yield* TestLLM.push(TestLLM.text("Recovered", "retry-success"))
  3841. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3842. yield* TestLLM.wait(1)
  3843. yield* TestClock.adjust("1599 millis")
  3844. expect(requests).toHaveLength(1)
  3845. yield* TestClock.adjust("801 millis")
  3846. yield* Fiber.join(run)
  3847. expect(requests).toHaveLength(2)
  3848. const eventTypes = yield* recordedEventTypes(sessionID)
  3849. expect(eventTypes).toContain("session.retry.scheduled.1")
  3850. expect(eventTypes.filter((type) => type === "session.step.started.1")).toHaveLength(2)
  3851. expect(yield* session.context(sessionID)).toMatchObject([
  3852. { type: "user" },
  3853. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Recovered" }] },
  3854. ])
  3855. yield* replaySessionProjection(sessionID)
  3856. expect((yield* session.context(sessionID)).filter((message) => message.type === "assistant")).toHaveLength(1)
  3857. }),
  3858. )
  3859. it.effect("immediately rebuilds once after explicit continuation rejection", () =>
  3860. Effect.gen(function* () {
  3861. const session = yield* setup
  3862. yield* TestLLM.push(Stream.fail(continuationRejected("retry-full")))
  3863. yield* TestLLM.push(TestLLM.text("Recovered", "continuation-recovery"))
  3864. yield* runPrompt(session, "Recover continuation")
  3865. expect(requests).toHaveLength(2)
  3866. expect(yield* recordedEventTypes(sessionID)).not.toContain("session.retry.scheduled.1")
  3867. expect(yield* session.context(sessionID)).toMatchObject([
  3868. { type: "user" },
  3869. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Recovered" }] },
  3870. ])
  3871. }),
  3872. )
  3873. it.effect("bounds repeated continuation rejection to one immediate recovery", () =>
  3874. Effect.gen(function* () {
  3875. const session = yield* setup
  3876. const failure = continuationRejected("rotate-and-retry-full")
  3877. yield* TestLLM.push(Stream.fail(failure), Stream.fail(failure))
  3878. expect(yield* runPrompt(session, "Reject continuation twice").pipe(Effect.flip)).toBe(failure)
  3879. expect(requests).toHaveLength(2)
  3880. expect(yield* recordedEventTypes(sessionID)).not.toContain("session.retry.scheduled.1")
  3881. }),
  3882. )
  3883. it.effect("retries an incomplete stream before output", () =>
  3884. Effect.gen(function* () {
  3885. const session = yield* setup
  3886. yield* admit(session, "Retry incomplete stream")
  3887. yield* TestLLM.push(Stream.fail(incompleteStream()))
  3888. yield* TestLLM.push(TestLLM.text("Recovered", "incomplete-stream-success"))
  3889. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3890. yield* TestLLM.wait(1)
  3891. yield* TestClock.adjust("2400 millis")
  3892. yield* Fiber.join(run)
  3893. expect(requests).toHaveLength(2)
  3894. expect(yield* session.context(sessionID)).toMatchObject([
  3895. { type: "user" },
  3896. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Recovered" }] },
  3897. ])
  3898. }),
  3899. )
  3900. it.effect("uses a larger provider retry-after delay", () =>
  3901. Effect.gen(function* () {
  3902. const session = yield* setup
  3903. yield* admit(session, "Retry rate limit")
  3904. yield* TestLLM.push(Stream.fail(rateLimited(5_000)))
  3905. yield* TestLLM.push(TestLLM.text("Recovered", "retry-after-success"))
  3906. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3907. yield* TestLLM.wait(1)
  3908. yield* TestClock.adjust("4999 millis")
  3909. expect(requests).toHaveLength(1)
  3910. yield* TestClock.adjust("1 millis")
  3911. yield* Fiber.join(run)
  3912. expect(requests).toHaveLength(2)
  3913. }),
  3914. )
  3915. it.effect("continues an incomplete stream after observable text", () =>
  3916. Effect.gen(function* () {
  3917. const session = yield* setup
  3918. const failure = incompleteStream()
  3919. yield* admit(session, "Continue partial output")
  3920. yield* TestLLM.push(
  3921. TestLLM.failAfter(
  3922. failure,
  3923. LLMEvent.stepStart({ index: 0 }),
  3924. LLMEvent.textStart({ id: "partial-rate-limit" }),
  3925. LLMEvent.textDelta({ id: "partial-rate-limit", text: "Partial" }),
  3926. ),
  3927. )
  3928. yield* TestLLM.push(TestLLM.text(" continuation", "continued-text"))
  3929. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3930. yield* TestLLM.wait(1)
  3931. yield* TestClock.adjust("2400 millis")
  3932. yield* Fiber.join(run)
  3933. expect(requests).toHaveLength(2)
  3934. expect(requests[1]?.messages.at(-2)).toMatchObject({
  3935. role: "assistant",
  3936. content: [{ type: "text", text: "Partial" }],
  3937. })
  3938. expect(requests[1]?.messages.at(-1)).toMatchObject({
  3939. role: "user",
  3940. content: [
  3941. {
  3942. type: "text",
  3943. text: INCOMPLETE_STREAM_CONTINUATION,
  3944. },
  3945. ],
  3946. })
  3947. const context = yield* session.context(sessionID)
  3948. expect(context).toMatchObject([
  3949. { type: "user", text: "Continue partial output" },
  3950. {
  3951. type: "assistant",
  3952. finish: "error",
  3953. error: { type: "provider.invalid-output" },
  3954. content: [{ type: "text", text: "Partial" }],
  3955. },
  3956. {
  3957. type: "synthetic",
  3958. text: INCOMPLETE_STREAM_CONTINUATION,
  3959. },
  3960. { type: "assistant", finish: "stop", content: [{ type: "text", text: " continuation" }] },
  3961. ])
  3962. const assistants = context.filter((message) => message.type === "assistant")
  3963. expect(new Set(assistants.map((message) => message.id)).size).toBe(2)
  3964. expect(context.find((message) => message.type === "synthetic")?.description).toBeUndefined()
  3965. expect(yield* recordedEventTypes(sessionID)).toContain("session.retry.scheduled.1")
  3966. yield* replaySessionProjection(sessionID)
  3967. expect(yield* session.context(sessionID)).toMatchObject(context)
  3968. }),
  3969. )
  3970. it.effect("lowers interrupted reasoning before continuing an incomplete stream", () =>
  3971. Effect.gen(function* () {
  3972. const session = yield* setup
  3973. yield* admit(session, "Continue interrupted reasoning")
  3974. yield* TestLLM.push(
  3975. TestLLM.failAfter(
  3976. incompleteStream(),
  3977. LLMEvent.stepStart({ index: 0 }),
  3978. LLMEvent.reasoningStart({ id: "partial-reasoning" }),
  3979. LLMEvent.reasoningDelta({ id: "partial-reasoning", text: "Partial thought" }),
  3980. ),
  3981. )
  3982. yield* TestLLM.push(TestLLM.text("Recovered", "reasoning-recovery"))
  3983. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  3984. yield* TestLLM.wait(1)
  3985. yield* TestClock.adjust("2400 millis")
  3986. yield* Fiber.join(run)
  3987. expect(requests[1]?.messages.at(-2)).toMatchObject({
  3988. role: "assistant",
  3989. content: [{ type: "text", text: "Partial thought" }],
  3990. })
  3991. expect(requests[1]?.messages.at(-1)).toMatchObject({
  3992. role: "user",
  3993. content: [
  3994. {
  3995. type: "text",
  3996. text: INCOMPLETE_STREAM_CONTINUATION,
  3997. },
  3998. ],
  3999. })
  4000. expect(yield* session.context(sessionID)).toMatchObject([
  4001. { type: "user" },
  4002. { type: "assistant", finish: "error", content: [{ type: "reasoning", text: "Partial thought" }] },
  4003. { type: "synthetic" },
  4004. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Recovered" }] },
  4005. ])
  4006. }),
  4007. )
  4008. it.effect("continues an incomplete stream after settling a local tool", () =>
  4009. Effect.gen(function* () {
  4010. const session = yield* setup
  4011. yield* admit(session, "Continue after tool")
  4012. yield* TestLLM.push(
  4013. TestLLM.failAfter(
  4014. incompleteStream(),
  4015. LLMEvent.stepStart({ index: 0 }),
  4016. LLMEvent.toolCall({ id: "call-before-close", name: "echo", input: { text: "settled" } }),
  4017. ),
  4018. )
  4019. yield* TestLLM.push(TestLLM.text("Recovered", "tool-recovery"))
  4020. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  4021. yield* TestLLM.wait(1)
  4022. while (!(yield* recordedEventTypes(sessionID)).includes("session.retry.scheduled.1")) yield* Effect.yieldNow
  4023. yield* TestClock.adjust("2400 millis")
  4024. yield* Fiber.join(run)
  4025. expect(executions).toEqual(["settled"])
  4026. expect(requests[1]?.messages.slice(-3)).toMatchObject([
  4027. {
  4028. role: "assistant",
  4029. content: [{ type: "tool-call", id: "call-before-close", name: "echo", input: { text: "settled" } }],
  4030. },
  4031. { role: "tool", content: [{ type: "tool-result", id: "call-before-close" }] },
  4032. {
  4033. role: "user",
  4034. content: [
  4035. {
  4036. type: "text",
  4037. text: INCOMPLETE_STREAM_CONTINUATION,
  4038. },
  4039. ],
  4040. },
  4041. ])
  4042. }),
  4043. )
  4044. it.effect("continues an incomplete stream after settling a local tool defect", () =>
  4045. Effect.gen(function* () {
  4046. const session = yield* setup
  4047. yield* admit(session, "Continue after tool defect")
  4048. yield* TestLLM.push(
  4049. TestLLM.failAfter(
  4050. incompleteStream(),
  4051. LLMEvent.stepStart({ index: 0 }),
  4052. LLMEvent.toolCall({ id: "call-defect-before-close", name: "defect", input: {} }),
  4053. ),
  4054. )
  4055. yield* TestLLM.push(TestLLM.text("Recovered", "tool-defect-recovery"))
  4056. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  4057. yield* TestLLM.wait(1)
  4058. while (!(yield* recordedEventTypes(sessionID)).includes("session.retry.scheduled.1")) yield* Effect.yieldNow
  4059. yield* TestClock.adjust("2400 millis")
  4060. yield* Fiber.join(run)
  4061. expect(messageRoles(requests[1])).toEqual(["user", "assistant", "tool", "user"])
  4062. expect(yield* session.context(sessionID)).toMatchObject([
  4063. { type: "user" },
  4064. {
  4065. type: "assistant",
  4066. content: [
  4067. {
  4068. type: "tool",
  4069. id: "call-defect-before-close",
  4070. state: { status: "error", error: { type: "unknown", message: "unexpected tool defect" } },
  4071. },
  4072. ],
  4073. },
  4074. { type: "synthetic", text: INCOMPLETE_STREAM_CONTINUATION },
  4075. { type: "assistant", finish: "stop", content: [{ type: "text", text: "Recovered" }] },
  4076. ])
  4077. }),
  4078. )
  4079. it.effect("stops incomplete stream continuations after five total attempts", () =>
  4080. Effect.gen(function* () {
  4081. const session = yield* setup
  4082. yield* admit(session, "Exhaust partial continuations")
  4083. const failure = incompleteStream()
  4084. yield* TestLLM.always(
  4085. TestLLM.failAfter(
  4086. failure,
  4087. LLMEvent.stepStart({ index: 0 }),
  4088. LLMEvent.textStart({ id: "partial-exhaustion" }),
  4089. LLMEvent.textDelta({ id: "partial-exhaustion", text: "Partial" }),
  4090. ),
  4091. )
  4092. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  4093. yield* TestLLM.wait(1)
  4094. for (const [index, delay] of [2_400, 4_800, 9_600, 19_200].entries()) {
  4095. yield* TestClock.adjust(delay)
  4096. yield* TestLLM.wait(index + 2)
  4097. }
  4098. expect(yield* Fiber.join(run).pipe(Effect.flip)).toBe(failure)
  4099. expect(requests).toHaveLength(5)
  4100. const context = yield* session.context(sessionID)
  4101. expect(context.filter((message) => message.type === "assistant")).toHaveLength(5)
  4102. expect(context.filter((message) => message.type === "synthetic")).toHaveLength(4)
  4103. }),
  4104. )
  4105. it.effect("stops after five total retry attempts", () =>
  4106. Effect.gen(function* () {
  4107. const session = yield* setup
  4108. yield* admit(session, "Exhaust retries")
  4109. const failure = providerUnavailable()
  4110. yield* TestLLM.always(Stream.fail(failure))
  4111. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  4112. yield* TestLLM.wait(1)
  4113. for (const [index, delay] of [2_400, 4_800, 9_600, 19_200].entries()) {
  4114. yield* TestClock.adjust(delay)
  4115. yield* TestLLM.wait(index + 2)
  4116. }
  4117. expect(yield* Fiber.join(run).pipe(Effect.flip)).toBe(failure)
  4118. expect(requests).toHaveLength(5)
  4119. const database = (yield* Database.Service).db
  4120. const retries = yield* database
  4121. .select({ data: EventTable.data })
  4122. .from(EventTable)
  4123. .where(eq(EventTable.type, "session.retry.scheduled.1"))
  4124. .orderBy(asc(EventTable.seq))
  4125. .all()
  4126. .pipe(Effect.orDie)
  4127. for (const [index, range] of [
  4128. [1_600, 2_400],
  4129. [4_800, 7_200],
  4130. [11_200, 16_800],
  4131. [24_000, 36_000],
  4132. ].entries()) {
  4133. expect(retries[index]?.data.at).toBeGreaterThanOrEqual(range[0]!)
  4134. expect(retries[index]?.data.at).toBeLessThanOrEqual(range[1]!)
  4135. }
  4136. expect((yield* recordedEventTypes(sessionID)).filter((type) => type === "session.step.started.1")).toHaveLength(5)
  4137. const assistant = requireAssistant(yield* session.context(sessionID))
  4138. expect(yield* recordedStepSettlementEvents(sessionID, assistant.id)).toMatchObject([
  4139. { type: "session.step.started.1" },
  4140. { type: "session.step.started.1" },
  4141. { type: "session.step.started.1" },
  4142. { type: "session.step.started.1" },
  4143. { type: "session.step.started.1" },
  4144. { type: "session.step.failed.1" },
  4145. ])
  4146. }),
  4147. )
  4148. it.effect("retries a model call without consuming the logical agent step", () =>
  4149. Effect.gen(function* () {
  4150. const session = yield* setup
  4151. const agents = yield* Agent.Service
  4152. yield* agents.transform((editor) =>
  4153. editor.update(Agent.ID.make("build"), (agent) => {
  4154. agent.steps = 2
  4155. }),
  4156. )
  4157. yield* admit(session, "Retry without consuming a step")
  4158. const failure = providerUnavailable()
  4159. yield* TestLLM.push(Stream.fail(failure))
  4160. yield* TestLLM.push(TestLLM.tool("call-after-retry", "echo", { text: "recovered" }), TestLLM.stop())
  4161. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  4162. yield* TestLLM.wait(1)
  4163. yield* TestClock.adjust("2400 millis")
  4164. yield* Fiber.join(run)
  4165. expect(requests).toHaveLength(3)
  4166. expect(requests[0]?.toolChoice).toBeUndefined()
  4167. expect(requests[0]?.tools.map((tool) => tool.name)).toContain("echo")
  4168. expect(requests[1]?.toolChoice).toBeUndefined()
  4169. expect(requests[1]?.tools.map((tool) => tool.name)).toContain("echo")
  4170. expect(requests[1]?.messages.at(-1)).not.toMatchObject({
  4171. role: "assistant",
  4172. content: [{ type: "text", text: expect.stringContaining("MAXIMUM STEPS REACHED") }],
  4173. })
  4174. expect(requests[2]?.toolChoice).toMatchObject({ type: "none" })
  4175. // The final step keeps tool definitions to preserve provider prompt caching.
  4176. expect(requests[2]?.tools.map((tool) => tool.name)).toContain("echo")
  4177. expect(requests[2]?.messages.at(-1)).toMatchObject({
  4178. role: "assistant",
  4179. content: [{ type: "text", text: expect.stringContaining("MAXIMUM STEPS REACHED") }],
  4180. })
  4181. expect(executions).toEqual(["recovered"])
  4182. const eventTypes = yield* recordedEventTypes(sessionID)
  4183. expect(eventTypes.filter((type) => type === "session.step.started.1")).toHaveLength(3)
  4184. expect(eventTypes.filter((type) => type === "session.retry.scheduled.1")).toHaveLength(1)
  4185. expect((yield* session.context(sessionID)).filter((message) => message.type === "assistant")).toHaveLength(2)
  4186. }),
  4187. )
  4188. it.effect("does not retry non-eligible provider failures", () =>
  4189. Effect.gen(function* () {
  4190. const session = yield* setup
  4191. const failure = invalidRequest()
  4192. yield* TestLLM.push(Stream.fail(failure))
  4193. expect(yield* runPrompt(session, "Do not retry").pipe(Effect.flip)).toBe(failure)
  4194. expect(requests).toHaveLength(1)
  4195. expect(yield* recordedEventTypes(sessionID)).not.toContain("session.retry.scheduled.1")
  4196. }),
  4197. )
  4198. it.effect("settles malformed streamed tool input before the provider failure", () =>
  4199. Effect.gen(function* () {
  4200. const session = yield* setup
  4201. const failure = new AIError({
  4202. module: "test",
  4203. method: "stream",
  4204. reason: new InvalidProviderOutputReason({ message: "Invalid JSON input for tool call echo" }),
  4205. })
  4206. yield* TestLLM.push(
  4207. TestLLM.failAfter(
  4208. failure,
  4209. LLMEvent.stepStart({ index: 0 }),
  4210. LLMEvent.toolInputStart({ id: "call-malformed", name: "echo" }),
  4211. LLMEvent.toolInputDelta({ id: "call-malformed", name: "echo", text: '{"text":"partial' }),
  4212. ),
  4213. )
  4214. expect(yield* runPrompt(session, "Call a malformed tool").pipe(Effect.flip)).toBe(failure)
  4215. const assistant = requireAssistant(yield* session.context(sessionID))
  4216. yield* TestLLM.push(TestLLM.stop())
  4217. yield* runPrompt(session, "Continue")
  4218. expect(yield* recordedStepSettlementEvents(sessionID, assistant.id)).toMatchObject([
  4219. { type: "session.step.started.1" },
  4220. {
  4221. type: "session.tool.failed.2",
  4222. data: {
  4223. id: "call-malformed",
  4224. error: { type: "provider.invalid-output", message: "Invalid JSON input for tool call echo" },
  4225. },
  4226. },
  4227. {
  4228. type: "session.step.failed.1",
  4229. data: { error: { type: "provider.invalid-output", message: "Invalid JSON input for tool call echo" } },
  4230. },
  4231. ])
  4232. }),
  4233. )
  4234. it.effect("continues after malformed local tool input without exposing raw arguments", () =>
  4235. Effect.gen(function* () {
  4236. const session = yield* setup
  4237. const marker = "raw-malformed-marker"
  4238. const raw = `{"text":"${marker}`
  4239. yield* TestLLM.push(
  4240. TestLLM.toolCalls(
  4241. LLMEvent.toolInputStart({ id: "call-malformed", name: "echo" }),
  4242. LLMEvent.toolInputDelta({ id: "call-malformed", name: "echo", text: raw }),
  4243. LLMEvent.toolInputEnd({ id: "call-malformed", name: "echo" }),
  4244. LLMEvent.toolInputError({
  4245. id: "call-malformed",
  4246. name: "echo",
  4247. raw,
  4248. }),
  4249. ),
  4250. TestLLM.stop(),
  4251. )
  4252. yield* runPrompt(session, "Recover malformed tool input")
  4253. expect(requests).toHaveLength(2)
  4254. expect(executions).toEqual([])
  4255. expect(JSON.stringify(requests[1])).not.toContain(marker)
  4256. expect(requests[1]?.messages).toEqual(
  4257. expect.arrayContaining([
  4258. expect.objectContaining({
  4259. role: "assistant",
  4260. content: expect.arrayContaining([
  4261. expect.objectContaining({ type: "tool-call", id: "call-malformed", name: "echo", input: {} }),
  4262. ]),
  4263. }),
  4264. expect.objectContaining({
  4265. role: "tool",
  4266. content: expect.arrayContaining([
  4267. expect.objectContaining({
  4268. type: "tool-result",
  4269. id: "call-malformed",
  4270. result: expect.objectContaining({
  4271. type: "error",
  4272. value: expect.objectContaining({
  4273. error: expect.objectContaining({
  4274. message: "Tool call arguments were malformed JSON and were not executed. Retry with valid JSON.",
  4275. }),
  4276. }),
  4277. }),
  4278. }),
  4279. ]),
  4280. }),
  4281. ]),
  4282. )
  4283. const context = yield* session.context(sessionID)
  4284. const failed = context.find(
  4285. (message): message is SessionMessage.Assistant =>
  4286. message.type === "assistant" && message.content.some((item) => item.type === "tool"),
  4287. )
  4288. expect(failed).toMatchObject({
  4289. content: [
  4290. {
  4291. type: "tool",
  4292. id: "call-malformed",
  4293. executed: false,
  4294. state: {
  4295. status: "error",
  4296. input: {},
  4297. error: {
  4298. type: "tool.input-json",
  4299. message: "Tool call arguments were malformed JSON and were not executed. Retry with valid JSON.",
  4300. },
  4301. },
  4302. },
  4303. ],
  4304. })
  4305. if (!failed) throw new Error("Malformed tool assistant missing")
  4306. expect(failed.error).toBeUndefined()
  4307. expect(yield* recordedStepSettlementTypes(sessionID, failed.id)).toEqual([
  4308. "session.step.started.1",
  4309. "session.tool.failed.2",
  4310. "session.step.ended.1",
  4311. ])
  4312. const database = (yield* Database.Service).db
  4313. const durable = yield* database
  4314. .select({ type: EventTable.type, data: EventTable.data })
  4315. .from(EventTable)
  4316. .where(eq(EventTable.aggregate_id, sessionID))
  4317. .all()
  4318. .pipe(Effect.orDie)
  4319. expect(durable.find((event) => event.type === "session.tool.input.ended.1")?.data).toMatchObject({
  4320. id: "call-malformed",
  4321. text: raw,
  4322. })
  4323. }),
  4324. )
  4325. it.effect("settles a valid sibling before recovering malformed tool input", () =>
  4326. Effect.gen(function* () {
  4327. const session = yield* setup
  4328. yield* admit(session, "Run parallel tools")
  4329. const tools = yield* blockTools()
  4330. yield* TestLLM.push(
  4331. TestLLM.toolCalls(
  4332. LLMEvent.toolCall({ id: "call-valid", name: "echo", input: { text: "valid" } }),
  4333. LLMEvent.toolInputError({
  4334. id: "call-malformed",
  4335. name: "echo",
  4336. raw: '{"text":"partial',
  4337. }),
  4338. ),
  4339. TestLLM.stop(),
  4340. )
  4341. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  4342. yield* tools.started
  4343. expect(requests).toHaveLength(1)
  4344. yield* tools.release
  4345. yield* Fiber.join(run)
  4346. expect(requests).toHaveLength(2)
  4347. expect(executions).toEqual(["valid"])
  4348. const request = requests[1]
  4349. if (!request) throw new Error("Malformed recovery request missing")
  4350. expect(request.messages.flatMap((message) => (message.role === "tool" ? message.content : []))).toEqual(
  4351. expect.arrayContaining([
  4352. expect.objectContaining({ id: "call-valid", type: "tool-result" }),
  4353. expect.objectContaining({ id: "call-malformed", type: "tool-result" }),
  4354. ]),
  4355. )
  4356. }),
  4357. )
  4358. it.effect("does not recover malformed input after sibling execution is interrupted", () =>
  4359. Effect.gen(function* () {
  4360. const session = yield* setup
  4361. yield* admit(session, "Interrupt malformed recovery")
  4362. const tools = yield* blockTools()
  4363. yield* TestLLM.push(
  4364. TestLLM.toolCalls(
  4365. LLMEvent.toolCall({ id: "call-valid", name: "echo", input: { text: "blocked" } }),
  4366. LLMEvent.toolInputError({
  4367. id: "call-malformed",
  4368. name: "echo",
  4369. raw: '{"text":"partial',
  4370. }),
  4371. ),
  4372. )
  4373. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  4374. yield* tools.started
  4375. while (
  4376. !(yield* session.context(sessionID)).some(
  4377. (message) =>
  4378. message.type === "assistant" &&
  4379. message.content.some((item) => item.type === "tool" && item.id === "call-malformed"),
  4380. )
  4381. )
  4382. yield* Effect.yieldNow
  4383. yield* session.interrupt(sessionID)
  4384. expect(yield* Fiber.await(run)).toMatchObject({ _tag: "Failure" })
  4385. expect(requests).toHaveLength(1)
  4386. expect(yield* session.context(sessionID)).toMatchObject([
  4387. { type: "user", text: "Interrupt malformed recovery" },
  4388. {
  4389. type: "assistant",
  4390. error: { type: "aborted", message: "Step interrupted" },
  4391. content: [
  4392. { type: "tool", id: "call-valid", state: { status: "error", error: { type: "aborted" } } },
  4393. { type: "tool", id: "call-malformed", state: { status: "error" } },
  4394. ],
  4395. },
  4396. ])
  4397. }),
  4398. )
  4399. it.effect("records malformed provider-executed input as executed", () =>
  4400. Effect.gen(function* () {
  4401. const session = yield* setup
  4402. const failure = new AIError({
  4403. module: "test",
  4404. method: "stream",
  4405. reason: new InvalidProviderOutputReason({ message: "Invalid hosted tool input" }),
  4406. })
  4407. yield* TestLLM.push(
  4408. TestLLM.failAfter(
  4409. failure,
  4410. LLMEvent.stepStart({ index: 0 }),
  4411. LLMEvent.toolInputStart({ id: "call-hosted", name: "web_search", providerExecuted: true }),
  4412. LLMEvent.toolInputDelta({ id: "call-hosted", name: "web_search", text: '{"query":"partial' }),
  4413. ),
  4414. )
  4415. expect(yield* runPrompt(session, "Fail malformed hosted input").pipe(Effect.flip)).toBe(failure)
  4416. expect(requireAssistant(yield* session.context(sessionID))).toMatchObject({
  4417. error: { type: "provider.invalid-output", message: "Invalid hosted tool input" },
  4418. content: [
  4419. {
  4420. type: "tool",
  4421. id: "call-hosted",
  4422. executed: true,
  4423. state: { status: "error", error: { type: "provider.invalid-output" } },
  4424. },
  4425. ],
  4426. })
  4427. }),
  4428. )
  4429. it.effect("records a provider failure after malformed input", () =>
  4430. Effect.gen(function* () {
  4431. const session = yield* setup
  4432. const failure = new AIError({
  4433. module: "test",
  4434. method: "stream",
  4435. reason: new InvalidProviderOutputReason({ message: "Provider failed after malformed input" }),
  4436. })
  4437. yield* TestLLM.push(
  4438. TestLLM.failAfter(
  4439. failure,
  4440. LLMEvent.stepStart({ index: 0 }),
  4441. LLMEvent.toolInputError({
  4442. id: "call-malformed",
  4443. name: "echo",
  4444. raw: '{"text":"partial',
  4445. }),
  4446. ),
  4447. )
  4448. expect(yield* runPrompt(session, "Fail after malformed input").pipe(Effect.flip)).toBe(failure)
  4449. expect(requireAssistant(yield* session.context(sessionID))).toMatchObject({
  4450. error: { type: "provider.invalid-output", message: "Provider failed after malformed input" },
  4451. content: [
  4452. {
  4453. type: "tool",
  4454. id: "call-malformed",
  4455. executed: false,
  4456. state: { status: "error", error: { type: "tool.input-json" } },
  4457. },
  4458. ],
  4459. })
  4460. expect(requests).toHaveLength(1)
  4461. }),
  4462. )
  4463. it.effect("continues after repeated malformed tool input", () =>
  4464. Effect.gen(function* () {
  4465. const session = yield* setup
  4466. const malformed = (id: string) =>
  4467. TestLLM.toolCalls(
  4468. LLMEvent.toolInputError({
  4469. id,
  4470. name: "echo",
  4471. raw: '{"text":"partial',
  4472. }),
  4473. )
  4474. yield* TestLLM.push(
  4475. malformed("call-first"),
  4476. TestLLM.tool("call-valid-between", "echo", { text: "valid" }),
  4477. malformed("call-second"),
  4478. TestLLM.stop(),
  4479. )
  4480. yield* runPrompt(session, "Keep producing malformed tools")
  4481. expect(requests).toHaveLength(4)
  4482. expect(executions).toEqual(["valid"])
  4483. expect((yield* recordedEventTypes(sessionID)).filter((type) => type === "session.step.failed.1")).toHaveLength(0)
  4484. }),
  4485. )
  4486. it.effect("does not continue malformed tool input past the agent step limit", () =>
  4487. Effect.gen(function* () {
  4488. const session = yield* setup
  4489. const agents = yield* Agent.Service
  4490. yield* agents.transform((editor) =>
  4491. editor.update(Agent.ID.make("build"), (agent) => {
  4492. agent.steps = 2
  4493. }),
  4494. )
  4495. const malformed = (id: string) =>
  4496. TestLLM.toolCalls(
  4497. LLMEvent.toolInputError({
  4498. id,
  4499. name: "echo",
  4500. raw: '{"text":"partial',
  4501. }),
  4502. )
  4503. yield* TestLLM.push(malformed("call-first"), malformed("call-at-limit"))
  4504. yield* runPrompt(session, "Stop malformed tools at the step limit")
  4505. expect(requests).toHaveLength(2)
  4506. expect(requests[0]?.toolChoice).toBeUndefined()
  4507. expect(requests[1]?.toolChoice).toMatchObject({ type: "none" })
  4508. expect((yield* recordedEventTypes(sessionID)).filter((type) => type === "session.tool.failed.2")).toHaveLength(2)
  4509. }),
  4510. )
  4511. it.effect("does not continue automatically after a provider error follows a local tool call", () =>
  4512. Effect.gen(function* () {
  4513. const session = yield* setup
  4514. yield* admit(session, "Do not continue failed provider")
  4515. const tools = yield* blockTools()
  4516. yield* TestLLM.push([
  4517. LLMEvent.stepStart({ index: 0 }),
  4518. LLMEvent.toolCall({ id: "call-before-provider-error", name: "echo", input: { text: "settled" } }),
  4519. LLMEvent.providerError({ message: "Provider unavailable" }),
  4520. ])
  4521. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  4522. yield* tools.started
  4523. yield* tools.release
  4524. expect((yield* Fiber.join(run).pipe(Effect.flip)).message).toBe("Provider unavailable")
  4525. expect(requests).toHaveLength(1)
  4526. expect(executions).toEqual(["settled"])
  4527. const context = yield* session.context(sessionID)
  4528. const assistant = requireAssistant(context)
  4529. expect(yield* recordedStepSettlementTypes(sessionID, assistant.id)).toEqual([
  4530. "session.step.started.1",
  4531. "session.tool.called.1",
  4532. "session.tool.success.2",
  4533. "session.step.failed.1",
  4534. ])
  4535. }),
  4536. )
  4537. it.effect("durably fails a hosted tool when its provider errors before returning a result", () =>
  4538. Effect.gen(function* () {
  4539. const session = yield* setup
  4540. yield* TestLLM.push([
  4541. LLMEvent.stepStart({ index: 0 }),
  4542. hostedCall("call-hosted-provider-error", "effect"),
  4543. LLMEvent.providerError({ message: "Provider unavailable" }),
  4544. ])
  4545. expect((yield* runPrompt(session, "Fail hosted tool durably").pipe(Effect.flip)).message).toBe(
  4546. "Provider unavailable",
  4547. )
  4548. expect(requests).toHaveLength(1)
  4549. const context = yield* session.context(sessionID)
  4550. expect(context).toMatchObject([
  4551. { type: "user", text: "Fail hosted tool durably" },
  4552. {
  4553. type: "assistant",
  4554. content: [{ type: "tool", id: "call-hosted-provider-error", state: { status: "error" } }],
  4555. },
  4556. ])
  4557. const assistant = requireAssistant(context)
  4558. expect(yield* recordedStepSettlementTypes(sessionID, assistant.id)).toEqual([
  4559. "session.step.started.1",
  4560. "session.tool.called.1",
  4561. "session.tool.failed.2",
  4562. "session.step.failed.1",
  4563. ])
  4564. }),
  4565. )
  4566. it.effect("preserves a tool defect before provider failure settlement", () =>
  4567. Effect.gen(function* () {
  4568. const session = yield* setup
  4569. yield* TestLLM.push([
  4570. LLMEvent.stepStart({ index: 0 }),
  4571. LLMEvent.toolCall({ id: "call-defect-provider-error", name: "defect", input: {} }),
  4572. LLMEvent.providerError({ message: "Provider unavailable" }),
  4573. ])
  4574. expect((yield* runPrompt(session, "Defect while provider fails").pipe(Effect.flip)).message).toBe(
  4575. "Provider unavailable",
  4576. )
  4577. const context = yield* session.context(sessionID)
  4578. const assistant = requireAssistant(context)
  4579. const bus = yield* recordedStepSettlementEvents(sessionID, assistant.id)
  4580. expect(bus.map((event) => event.type)).toEqual([
  4581. "session.step.started.1",
  4582. "session.tool.called.1",
  4583. "session.tool.failed.2",
  4584. "session.step.failed.1",
  4585. ])
  4586. expect(bus[2]?.data.error).toMatchObject({ type: "unknown", message: "unexpected tool defect" })
  4587. }),
  4588. )
  4589. it.effect("preserves the provider failure when tool output persistence also fails", () =>
  4590. Effect.gen(function* () {
  4591. const session = yield* setup
  4592. yield* admit(session, "Storage fails while provider fails")
  4593. yield* TestLLM.push([
  4594. LLMEvent.stepStart({ index: 0 }),
  4595. LLMEvent.toolCall({ id: "call-store-provider-error", name: "storefail", input: {} }),
  4596. LLMEvent.providerError({ message: "Provider unavailable" }),
  4597. ])
  4598. expect(yield* session.resume(sessionID).pipe(Effect.exit)).toMatchObject({
  4599. _tag: "Failure",
  4600. })
  4601. expect(requireAssistant(yield* session.context(sessionID))).toMatchObject({
  4602. error: { type: "provider.unknown", message: "Provider unavailable" },
  4603. })
  4604. }),
  4605. )
  4606. it.effect("durably fails a hosted tool left unresolved at normal provider EOF", () =>
  4607. Effect.gen(function* () {
  4608. const session = yield* setup
  4609. yield* TestLLM.push([LLMEvent.stepStart({ index: 0 }), hostedCall("call-hosted-eof", "effect")])
  4610. expect((yield* runPrompt(session, "Fail hosted tool at EOF").pipe(Effect.flip)).message).toBe(
  4611. "Provider did not return a tool result",
  4612. )
  4613. const assistant = requireAssistant(yield* session.context(sessionID))
  4614. const bus = yield* recordedStepSettlementEvents(sessionID, assistant.id)
  4615. expect(bus.map((event) => event.type)).toEqual([
  4616. "session.step.started.1",
  4617. "session.tool.called.1",
  4618. "session.tool.failed.2",
  4619. "session.step.failed.1",
  4620. ])
  4621. expect(
  4622. bus.filter((event) => event.type.startsWith("session.step.") && event.type !== "session.step.started.1"),
  4623. ).toHaveLength(1)
  4624. yield* replaySessionProjection(sessionID)
  4625. expect(yield* session.context(sessionID)).toMatchObject([
  4626. { type: "user", text: "Fail hosted tool at EOF" },
  4627. {
  4628. type: "assistant",
  4629. finish: "error",
  4630. error: { type: "tool.result-missing" },
  4631. content: [{ type: "tool", id: "call-hosted-eof", state: { status: "error" } }],
  4632. },
  4633. ])
  4634. }),
  4635. )
  4636. it.effect("fails an unresolved hosted tool before one clean step end", () =>
  4637. Effect.gen(function* () {
  4638. const session = yield* setup
  4639. yield* TestLLM.push(TestLLM.stop(hostedCall("call-hosted-clean-end", "effect")))
  4640. yield* runPrompt(session, "Settle hosted tool before ending")
  4641. const assistant = requireAssistant(yield* session.context(sessionID))
  4642. const bus = yield* recordedStepSettlementEvents(sessionID, assistant.id)
  4643. expect(bus.map((event) => event.type)).toEqual([
  4644. "session.step.started.1",
  4645. "session.tool.called.1",
  4646. "session.tool.failed.2",
  4647. "session.step.ended.1",
  4648. ])
  4649. expect(
  4650. bus.filter((event) => event.type.startsWith("session.step.") && event.type !== "session.step.started.1"),
  4651. ).toHaveLength(1)
  4652. }),
  4653. )
  4654. it.effect("settles unresolved local and hosted tools before one raw provider failure", () =>
  4655. Effect.gen(function* () {
  4656. const session = yield* setup
  4657. yield* admit(session, "Fail unresolved tools")
  4658. const failure = invalidRequest()
  4659. const providerFailed = yield* Deferred.make<void>()
  4660. const tools = yield* blockTools()
  4661. yield* TestLLM.push(
  4662. Stream.concat(
  4663. Stream.fromIterable([
  4664. LLMEvent.stepStart({ index: 0 }),
  4665. LLMEvent.toolCall({ id: "call-local-raw-failure", name: "defect", input: {} }),
  4666. hostedCall("call-hosted-raw-failure-pair", "effect"),
  4667. ]),
  4668. Stream.fromEffect(Deferred.succeed(providerFailed, undefined)).pipe(
  4669. Stream.flatMap(() => Stream.fail(failure)),
  4670. ),
  4671. ),
  4672. )
  4673. const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
  4674. yield* Deferred.await(providerFailed)
  4675. yield* tools.release
  4676. expect(yield* Fiber.join(run).pipe(Effect.flip)).toBe(failure)
  4677. const assistant = requireAssistant(yield* session.context(sessionID))
  4678. const bus = yield* recordedStepSettlementEvents(sessionID, assistant.id)
  4679. expect(bus.map((event) => ({ type: event.type, id: event.data.id }))).toEqual([
  4680. { type: "session.step.started.1", id: undefined },
  4681. { type: "session.tool.called.1", id: "call-local-raw-failure" },
  4682. { type: "session.tool.called.1", id: "call-hosted-raw-failure-pair" },
  4683. { type: "session.tool.failed.2", id: "call-local-raw-failure" },
  4684. { type: "session.tool.failed.2", id: "call-hosted-raw-failure-pair" },
  4685. { type: "session.step.failed.1", id: undefined },
  4686. ])
  4687. expect(
  4688. bus.filter((event) => event.type.startsWith("session.step.") && event.type !== "session.step.started.1"),
  4689. ).toHaveLength(1)
  4690. }),
  4691. )
  4692. it.effect("durably fails a hosted tool left unresolved by a raw provider stream failure", () =>
  4693. Effect.gen(function* () {
  4694. const session = yield* setup
  4695. const failure = providerUnavailable()
  4696. yield* TestLLM.push(
  4697. Stream.concat(
  4698. Stream.fromIterable([LLMEvent.stepStart({ index: 0 }), hostedCall("call-hosted-raw-failure", "effect")]),
  4699. Stream.fail(failure),
  4700. ),
  4701. )
  4702. expect(yield* runPrompt(session, "Fail hosted tool on raw failure").pipe(Effect.flip)).toBe(failure)
  4703. expect(requests).toHaveLength(1)
  4704. const assistant = requireAssistant(yield* session.context(sessionID))
  4705. const bus = yield* recordedStepSettlementEvents(sessionID, assistant.id)
  4706. expect(bus.map((event) => event.type)).toEqual([
  4707. "session.step.started.1",
  4708. "session.tool.called.1",
  4709. "session.tool.failed.2",
  4710. "session.step.failed.1",
  4711. ])
  4712. expect(
  4713. bus.filter((event) => event.type.startsWith("session.step.") && event.type !== "session.step.started.1"),
  4714. ).toHaveLength(1)
  4715. yield* replaySessionProjection(sessionID)
  4716. expect(yield* session.context(sessionID)).toMatchObject([
  4717. { type: "user", text: "Fail hosted tool on raw failure" },
  4718. {
  4719. type: "assistant",
  4720. finish: "error",
  4721. error: { type: "provider.transport", message: "Provider unavailable" },
  4722. content: [{ type: "tool", id: "call-hosted-raw-failure", state: { status: "error" } }],
  4723. },
  4724. ])
  4725. }),
  4726. )
  4727. it.effect("rejects a second text start before the open fragment ends", () =>
  4728. Effect.gen(function* () {
  4729. const session = yield* setup
  4730. yield* TestLLM.push([
  4731. LLMEvent.stepStart({ index: 0 }),
  4732. LLMEvent.textStart({ id: "text-1" }),
  4733. LLMEvent.textStart({ id: "text-2" }),
  4734. ])
  4735. const defect = yield* runPrompt(session, "Two blocks").pipe(Effect.catchDefect(Effect.succeed))
  4736. expect(defect).toBeInstanceOf(Error)
  4737. if (!(defect instanceof Error)) return
  4738. expect(defect.message).toBe("text start before end: text-2")
  4739. }),
  4740. )
  4741. it.effect("projects sequential text fragments as separate content parts", () =>
  4742. Effect.gen(function* () {
  4743. const session = yield* setup
  4744. yield* TestLLM.push(
  4745. TestLLM.stop(
  4746. LLMEvent.textStart({ id: "text-1" }),
  4747. LLMEvent.textDelta({ id: "text-1", text: "First" }),
  4748. LLMEvent.textEnd({ id: "text-1" }),
  4749. LLMEvent.textStart({ id: "text-2" }),
  4750. LLMEvent.textDelta({ id: "text-2", text: "Second" }),
  4751. LLMEvent.textEnd({ id: "text-2" }),
  4752. ),
  4753. )
  4754. yield* runPrompt(session, "Two blocks")
  4755. expect(yield* session.context(sessionID)).toMatchObject([
  4756. { type: "user", text: "Two blocks" },
  4757. {
  4758. type: "assistant",
  4759. content: [
  4760. { type: "text", text: "First" },
  4761. { type: "text", text: "Second" },
  4762. ],
  4763. },
  4764. ])
  4765. }),
  4766. )
  4767. for (const kind of fragmentKinds) {
  4768. it.effect(
  4769. kind === "tool input"
  4770. ? "does not broadcast provider tool input deltas"
  4771. : `batches provider ${kind} deltas without storing projection rewrites`,
  4772. () => verifyEphemeralDeltas(kind),
  4773. )
  4774. it.effect(`durably closes partial ${kind} when the provider stream fails`, () => verifyPartialFlushOnFailure(kind))
  4775. it.effect(`durably closes partial ${kind} when the provider stream is interrupted`, () =>
  4776. verifyPartialFlushOnInterruption(kind),
  4777. )
  4778. }
  4779. it.effect("rejects duplicate streamed text starts", () =>
  4780. Effect.gen(function* () {
  4781. const session = yield* setup
  4782. yield* TestLLM.push([LLMEvent.textStart({ id: "text-1" }), LLMEvent.textStart({ id: "text-1" })])
  4783. const defect = yield* session.resume(sessionID).pipe(Effect.catchDefect(Effect.succeed))
  4784. expect(defect).toBeInstanceOf(Error)
  4785. if (!(defect instanceof Error)) return
  4786. expect(defect.message).toBe("Duplicate text start: text-1")
  4787. }),
  4788. )
  4789. it.effect("transitions streamed raw tool input to parsed called input", () =>
  4790. Effect.gen(function* () {
  4791. const session = yield* setup
  4792. yield* TestLLM.push(
  4793. TestLLM.stop(
  4794. LLMEvent.toolInputStart({ id: "call-parsed", name: "web_search" }),
  4795. LLMEvent.toolInputDelta({ id: "call-parsed", name: "web_search", text: '{"query":"hello"}' }),
  4796. LLMEvent.toolInputEnd({ id: "call-parsed", name: "web_search" }),
  4797. hostedCall("call-parsed", "hello"),
  4798. ),
  4799. )
  4800. yield* runPrompt(session, "Call provider tool")
  4801. expect(yield* session.context(sessionID)).toMatchObject([
  4802. { type: "user", text: "Call provider tool" },
  4803. {
  4804. type: "assistant",
  4805. content: [{ type: "tool", id: "call-parsed", state: { status: "error", input: { query: "hello" } } }],
  4806. },
  4807. ])
  4808. }),
  4809. )
  4810. it.effect("rejects malformed streamed tool input ordering", () =>
  4811. Effect.gen(function* () {
  4812. const session = yield* setup
  4813. yield* TestLLM.push([LLMEvent.toolInputDelta({ id: "call-1", name: "read", text: "{}" })])
  4814. const defect = yield* session.resume(sessionID).pipe(Effect.catchDefect(Effect.succeed))
  4815. expect(defect).toBeInstanceOf(Error)
  4816. if (!(defect instanceof Error)) return
  4817. expect(defect.message).toBe("Tool input delta before start: call-1")
  4818. }),
  4819. )
  4820. })