session-execution.test.ts 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380
  1. import { describe, expect, test } from "bun:test"
  2. import { AIError, TransportReason } from "@opencode-ai/ai"
  3. import { Database } from "@opencode-ai/core/database/database"
  4. import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder"
  5. import { LayerNode } from "@opencode-ai/util/effect/layer-node"
  6. import { Bus } from "@opencode-ai/core/bus"
  7. import { LocationServiceMap } from "@opencode-ai/core/location-service-map"
  8. import type { LocationServices } from "@opencode-ai/core/location-services"
  9. import { Project } from "@opencode-ai/core/project"
  10. import { ProjectTable } from "@opencode-ai/core/project/sql"
  11. import { AbsolutePath } from "@opencode-ai/core/schema"
  12. import { Session } from "@opencode-ai/core/session"
  13. import { SessionExecution } from "@opencode-ai/core/session/execution"
  14. import { SessionRestart } from "@opencode-ai/core/session/execution/restart"
  15. import { UserInterruptedError } from "@opencode-ai/core/session/error"
  16. import { SessionEvent } from "@opencode-ai/core/session/event"
  17. import { SessionRunner } from "@opencode-ai/core/session/runner/index"
  18. import { SessionTable } from "@opencode-ai/core/session/sql"
  19. import { SessionStore } from "@opencode-ai/core/session/store"
  20. import { Context, Deferred, Effect, Exit, Fiber, Layer, LayerMap, Scope } from "effect"
  21. import { eq } from "drizzle-orm"
  22. import { testEffect } from "./lib/effect"
  23. const it = testEffect(AppNodeBuilder.build(LayerNode.group([Database.node, Bus.node, SessionStore.node])))
  24. describe("SessionExecution lifecycle", () => {
  25. test("classifies success and typed failure terminals", () => {
  26. expect(SessionExecution.terminal(Exit.succeed(undefined))).toEqual({ type: "succeeded" })
  27. expect(
  28. SessionExecution.terminal(
  29. Exit.fail(
  30. new AIError({
  31. module: "test",
  32. method: "stream",
  33. reason: new TransportReason({ message: "Disconnected", transport: "http", operation: "request" }),
  34. }),
  35. ),
  36. ),
  37. ).toEqual({ type: "failed", error: { type: "provider.transport", message: "Disconnected" } })
  38. })
  39. test("defaults owner-scope interruption to shutdown and preserves explicit reasons", () => {
  40. const interrupted = Effect.runSyncExit(Effect.interrupt)
  41. expect(SessionExecution.terminal(interrupted)).toEqual({ type: "interrupted", reason: "shutdown" })
  42. expect(SessionExecution.terminal(interrupted, "user")).toEqual({ type: "interrupted", reason: "user" })
  43. expect(SessionExecution.terminal(interrupted, "superseded")).toEqual({ type: "interrupted", reason: "superseded" })
  44. expect(SessionExecution.terminal(Exit.fail(new UserInterruptedError()))).toEqual({
  45. type: "interrupted",
  46. reason: "user",
  47. })
  48. })
  49. it.effect("the sweep only lists claimed top-level Sessions", () =>
  50. Effect.gen(function* () {
  51. const database = yield* Database.Service
  52. const store = yield* SessionStore.Service
  53. const parent = Session.ID.make("ses_recover_parent")
  54. const child = Session.ID.make("ses_recover_child")
  55. const idle = Session.ID.make("ses_recover_idle")
  56. yield* seedSessions(database, [parent], { time_suspended: Date.now() })
  57. yield* seedSessions(database, [idle])
  58. // An orphaned child is never resumed: the resumed parent re-runs its
  59. // tool call and spawns a fresh child instead.
  60. yield* seedSessions(database, [child], { time_suspended: Date.now(), parent_id: parent })
  61. expect(yield* store.listSuspended()).toEqual([parent])
  62. // The sweep clears orphaned child claims outright; parents keep theirs.
  63. yield* store.releaseChildClaims
  64. expect(yield* claims(database)).toEqual({ [parent]: true, [child]: false, [idle]: false })
  65. }),
  66. )
  67. it.effect("claims at execution start, releases on completion, and preserves through teardown", () =>
  68. Effect.gen(function* () {
  69. const database = yield* Database.Service
  70. const interrupted = Session.ID.make("ses_claim_interrupted")
  71. const completed = Session.ID.make("ses_claim_completed")
  72. yield* seedSessions(database, [interrupted, completed])
  73. // Each drain signals once it runs; the claim commits before the drain starts.
  74. const interruptedRunning = yield* Deferred.make<void>()
  75. const completedRunning = yield* Deferred.make<void>()
  76. const release = yield* Deferred.make<void>()
  77. const scope = yield* Scope.make()
  78. const context = yield* buildExecution(scope, ({ sessionID }) =>
  79. sessionID === completed
  80. ? Deferred.succeed(completedRunning, undefined).pipe(Effect.andThen(Deferred.await(release)))
  81. : Deferred.succeed(interruptedRunning, undefined).pipe(Effect.andThen(Effect.never)),
  82. )
  83. const execution = Context.get(context, SessionExecution.Service)
  84. yield* execution.resume(interrupted).pipe(Effect.forkScoped)
  85. const completing = yield* execution.resume(completed).pipe(Effect.forkIn(scope))
  86. yield* Deferred.await(interruptedRunning)
  87. yield* Deferred.await(completedRunning)
  88. // The write-ahead claim exists WHILE the turns run — no shutdown hook involved.
  89. expect(yield* claims(database)).toEqual({ [interrupted]: true, [completed]: true })
  90. // A drain that finishes on its own releases its claim.
  91. yield* Deferred.succeed(release, undefined)
  92. yield* Fiber.join(completing)
  93. yield* execution.awaitIdle(completed)
  94. expect((yield* claims(database))[completed]).toBe(false)
  95. // Teardown interruption (graceful twin of an unclean death) preserves the claim
  96. // for the next server start.
  97. yield* Scope.close(scope, Exit.void)
  98. expect((yield* claims(database))[interrupted]).toBe(true)
  99. }),
  100. )
  101. it.effect("a user interrupt releases the claim so the turn never resurrects", () =>
  102. Effect.gen(function* () {
  103. const database = yield* Database.Service
  104. const sessionID = Session.ID.make("ses_claim_user_cancel")
  105. yield* seedSessions(database, [sessionID])
  106. const draining = yield* Deferred.make<void>()
  107. const scope = yield* Scope.make()
  108. yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
  109. const context = yield* buildExecution(scope, () =>
  110. Deferred.succeed(draining, undefined).pipe(Effect.andThen(Effect.never)),
  111. )
  112. const execution = Context.get(context, SessionExecution.Service)
  113. yield* execution.resume(sessionID).pipe(Effect.forkScoped)
  114. yield* Deferred.await(draining)
  115. expect((yield* claims(database))[sessionID]).toBe(true)
  116. yield* execution.interrupt(sessionID)
  117. yield* execution.awaitIdle(sessionID)
  118. expect((yield* claims(database))[sessionID]).toBe(false)
  119. }),
  120. )
  121. it.effect("starts every claimed execution without waiting for earlier drains to finish", () =>
  122. Effect.gen(function* () {
  123. const database = yield* Database.Service
  124. const sessionIDs = Array.from({ length: 5 }, (_, index) => Session.ID.make(`ses_resume_concurrent_${index}`))
  125. yield* seedSessions(database, sessionIDs, { time_suspended: Date.now() })
  126. const fourStarted = yield* Deferred.make<void>()
  127. const started: Session.ID[] = []
  128. const scope = yield* Scope.make()
  129. yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
  130. const context = yield* buildExecution(scope, ({ sessionID }) =>
  131. Effect.sync(() => {
  132. started.push(sessionID)
  133. if (started.length === 4) Deferred.doneUnsafe(fourStarted, Effect.void)
  134. }).pipe(Effect.andThen(Effect.never)),
  135. )
  136. const execution = Context.get(context, SessionExecution.Service)
  137. const restart = Context.get(context, SessionRestart.Service)
  138. yield* restart.resumeSuspendedSessions.pipe(Effect.forkIn(scope))
  139. yield* Deferred.await(fourStarted)
  140. expect([...(yield* execution.active)].toSorted()).toEqual(sessionIDs.toSorted())
  141. }),
  142. )
  143. it.effect("resumes each claimed Session at most once", () =>
  144. Effect.gen(function* () {
  145. const database = yield* Database.Service
  146. const bus = yield* Bus.Service
  147. const first = Session.ID.make("ses_resume_first")
  148. const second = Session.ID.make("ses_resume_second")
  149. yield* seedSessions(database, [first, second], { time_suspended: Date.now() })
  150. const drained: string[] = []
  151. const bothDraining = yield* Deferred.make<void>()
  152. const continued: SessionEvent.Synthetic[] = []
  153. const scope = yield* Scope.make()
  154. const context = yield* buildExecution(scope, ({ sessionID }) =>
  155. Effect.sync(() => {
  156. drained.push(sessionID)
  157. if (drained.length === 2) Deferred.doneUnsafe(bothDraining, Effect.void)
  158. }),
  159. )
  160. const execution = Context.get(context, SessionExecution.Service)
  161. const restart = Context.get(context, SessionRestart.Service)
  162. yield* bus.project(SessionEvent.Synthetic, (event) => Effect.sync(() => void continued.push(event)))
  163. // The sweep forks resumed drains, so completion is observed through the executions.
  164. yield* restart.resumeSuspendedSessions
  165. yield* Deferred.await(bothDraining)
  166. yield* Effect.forEach([first, second], execution.awaitIdle, { discard: true })
  167. expect(drained.toSorted()).toEqual([first, second])
  168. expect(continued.map((event) => event.data).toSorted((a, b) => a.sessionID.localeCompare(b.sessionID))).toEqual(
  169. [first, second].map((sessionID) => ({
  170. sessionID,
  171. text: "The server restarted while you were working. Continue from where you left off without repeating completed work.",
  172. description: "Continuing after restart",
  173. })),
  174. )
  175. // Drains completed naturally, so claims are released and counters reset.
  176. expect(yield* claims(database)).toEqual({ [first]: false, [second]: false })
  177. expect(yield* attempts(database, first)).toBe(0)
  178. yield* restart.resumeSuspendedSessions
  179. expect(drained.length).toBe(2)
  180. expect(continued.length).toBe(2)
  181. yield* Scope.close(scope, Exit.void)
  182. }),
  183. )
  184. it.effect("terminalizes a turn that exhausts its resume budget instead of crash-looping", () =>
  185. Effect.gen(function* () {
  186. const database = yield* Database.Service
  187. const bus = yield* Bus.Service
  188. const sessionID = Session.ID.make("ses_resume_exhausted")
  189. // A claim from a dead process, already resumed twice without completing.
  190. yield* seedSessions(database, [sessionID], { time_suspended: Date.now(), resume_attempts: 2 })
  191. const drained: string[] = []
  192. const failures: SessionEvent.Execution.Failed[] = []
  193. const scope = yield* Scope.make()
  194. yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
  195. const context = yield* buildExecution(scope, ({ sessionID: id }) => Effect.sync(() => void drained.push(id)), {
  196. maxAttempts: 2,
  197. })
  198. const restart = Context.get(context, SessionRestart.Service)
  199. yield* bus.project(SessionEvent.Execution.Failed, (event) => Effect.sync(() => void failures.push(event)))
  200. yield* restart.resumeSuspendedSessions
  201. expect(drained).toEqual([])
  202. expect(failures.map((event) => event.data.error.type)).toEqual(["aborted"])
  203. // The terminal released the claim and reset the counter atomically.
  204. expect(yield* claims(database)).toEqual({ [sessionID]: false })
  205. expect(yield* attempts(database, sessionID)).toBe(0)
  206. }),
  207. )
  208. it.effect("counts every resume durably and never consumes the claim it recovers", () =>
  209. Effect.gen(function* () {
  210. const database = yield* Database.Service
  211. const sessionID = Session.ID.make("ses_resume_counted")
  212. yield* seedSessions(database, [sessionID], { time_suspended: Date.now() })
  213. const draining = yield* Deferred.make<void>()
  214. const scope = yield* Scope.make()
  215. yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
  216. // The drain never terminalizes (mirrors a process that will die mid-turn).
  217. const context = yield* buildExecution(scope, () =>
  218. Deferred.succeed(draining, undefined).pipe(Effect.andThen(Effect.never)),
  219. )
  220. const restart = Context.get(context, SessionRestart.Service)
  221. yield* restart.resumeSuspendedSessions.pipe(Effect.forkIn(scope))
  222. yield* Deferred.await(draining)
  223. // The attempt is durable before the drain runs, and the claim is held
  224. // throughout: a crash anywhere in the resume path leaves both intact.
  225. expect(yield* attempts(database, sessionID)).toBe(1)
  226. expect((yield* claims(database))[sessionID]).toBe(true)
  227. // Teardown (a graceful shutdown's interrupt) preserves both, so the next
  228. // boot counts attempt 2 against the same turn.
  229. yield* Scope.close(scope, Exit.void)
  230. expect((yield* claims(database))[sessionID]).toBe(true)
  231. expect(yield* attempts(database, sessionID)).toBe(1)
  232. }),
  233. )
  234. it.effect("the sweep leaves Sessions already draining in this process untouched", () =>
  235. Effect.gen(function* () {
  236. const database = yield* Database.Service
  237. const bus = yield* Bus.Service
  238. const sessionID = Session.ID.make("ses_resume_local_active")
  239. yield* seedSessions(database, [sessionID])
  240. const draining = yield* Deferred.make<void>()
  241. const continued: SessionEvent.Synthetic[] = []
  242. const scope = yield* Scope.make()
  243. yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
  244. const context = yield* buildExecution(scope, () =>
  245. Deferred.succeed(draining, undefined).pipe(Effect.andThen(Effect.never)),
  246. )
  247. const execution = Context.get(context, SessionExecution.Service)
  248. const restart = Context.get(context, SessionRestart.Service)
  249. yield* bus.project(SessionEvent.Synthetic, (event) => Effect.sync(() => void continued.push(event)))
  250. // A live local turn holds a claim; the sweep must not count, continue, or terminalize it.
  251. yield* execution.resume(sessionID).pipe(Effect.forkScoped)
  252. yield* Deferred.await(draining)
  253. yield* restart.resumeSuspendedSessions
  254. expect(continued).toEqual([])
  255. expect(yield* attempts(database, sessionID)).toBe(0)
  256. expect((yield* claims(database))[sessionID]).toBe(true)
  257. }),
  258. )
  259. })
  260. function seedSessions(
  261. database: Database.Service["Service"],
  262. sessionIDs: ReadonlyArray<Session.ID>,
  263. values: Partial<Pick<typeof SessionTable.$inferInsert, "time_suspended" | "resume_attempts" | "parent_id">> = {},
  264. ) {
  265. return Effect.gen(function* () {
  266. yield* database.db
  267. .insert(ProjectTable)
  268. .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] })
  269. .onConflictDoNothing()
  270. .run()
  271. .pipe(Effect.orDie)
  272. yield* database.db
  273. .insert(SessionTable)
  274. .values(
  275. sessionIDs.map((id) => ({
  276. id,
  277. project_id: Project.ID.global,
  278. slug: id,
  279. directory: "/project",
  280. title: id,
  281. version: "test",
  282. ...values,
  283. })),
  284. )
  285. .run()
  286. .pipe(Effect.orDie)
  287. })
  288. }
  289. function claims(database: Database.Service["Service"]) {
  290. return database.db
  291. .select({ id: SessionTable.id, claimed: SessionTable.time_suspended })
  292. .from(SessionTable)
  293. .all()
  294. .pipe(
  295. Effect.orDie,
  296. Effect.map((rows) => Object.fromEntries(rows.map((row) => [row.id, row.claimed !== null]))),
  297. )
  298. }
  299. function attempts(database: Database.Service["Service"], sessionID: Session.ID) {
  300. return database.db
  301. .select({ attempts: SessionTable.resume_attempts })
  302. .from(SessionTable)
  303. .where(eq(SessionTable.id, sessionID))
  304. .get()
  305. .pipe(
  306. Effect.orDie,
  307. Effect.map((row) => row?.attempts),
  308. )
  309. }
  310. /** Builds the local execution layer plus the restart actions against the test harness services. */
  311. function buildExecution(
  312. scope: Scope.Closeable,
  313. drain: (input: Parameters<SessionRunner.Interface["drain"]>[0]) => Effect.Effect<void, SessionRunner.RunError>,
  314. options?: SessionRestart.Options,
  315. ) {
  316. return Effect.gen(function* () {
  317. const database = yield* Database.Service
  318. const bus = yield* Bus.Service
  319. const store = yield* SessionStore.Service
  320. const runner = Layer.succeed(
  321. SessionRunner.Service,
  322. SessionRunner.Service.of({ drain: (input) => drain(input).pipe(Effect.as({ type: "complete" as const })) }),
  323. )
  324. const locations = Layer.effect(
  325. LocationServiceMap.Service,
  326. LayerMap.make(
  327. () =>
  328. // The local execution test only needs the Session runner from the Location graph.
  329. // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion
  330. runner as unknown as Layer.Layer<LocationServices>,
  331. ),
  332. )
  333. return yield* Layer.buildWithScope(
  334. SessionRestart.layer(options).pipe(
  335. Layer.provideMerge(SessionExecution.layer),
  336. Layer.provide(Layer.succeed(Database.Service, database)),
  337. Layer.provide(Layer.succeed(Bus.Service, bus)),
  338. Layer.provide(Layer.succeed(SessionStore.Service, store)),
  339. Layer.provide(locations),
  340. ),
  341. scope,
  342. )
  343. })
  344. }