service.test.ts 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360
  1. import { NodeFileSystem } from "@effect/platform-node"
  2. import { afterEach, expect, test } from "bun:test"
  3. import { Effect } from "effect"
  4. import { mkdtemp, rm, writeFile } from "node:fs/promises"
  5. import { tmpdir } from "node:os"
  6. import { join } from "node:path"
  7. import { Service, type EnsureReason } from "../src/effect/service"
  8. import { accelerate, waitForExit } from "./fixture/service-timing"
  9. const fixture = join(import.meta.dir, "fixture/service.ts")
  10. const ensure = accelerate(Service.ensure)
  11. const processes: Bun.Subprocess[] = []
  12. const directories: string[] = []
  13. afterEach(async () => {
  14. processes.forEach((process) => process.kill("SIGTERM"))
  15. await Promise.all(processes.splice(0).map((process) => process.exited))
  16. await Promise.all(directories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })))
  17. })
  18. test("a concurrent same-version start cannot invalidate a resolved endpoint", async () => {
  19. const directory = await temp()
  20. const registration = join(directory, "service.json")
  21. spawn(registration, "modern")
  22. await waitForFile(registration)
  23. const original = await Bun.file(registration).json()
  24. const starts: EnsureReason[] = []
  25. const first = run(
  26. ensure({
  27. file: registration,
  28. version: "test",
  29. command: [],
  30. onStart: (reason) => starts.push(reason),
  31. }),
  32. )
  33. await waitForFile(registration + ".first-request")
  34. const resolved = await run(ensure({ file: registration, version: "test" }))
  35. expect(resolved.url).toBe(original.url)
  36. await writeFile(registration + ".release", "")
  37. await first
  38. expect(starts).toEqual([])
  39. expect(await Bun.file(registration).json()).toEqual(original)
  40. expect(await health(resolved.url)).toEqual({ healthy: true, version: "test", pid: original.pid })
  41. })
  42. test("reuses a compatible registered service", async () => {
  43. const directory = await temp()
  44. const registration = join(directory, "service.json")
  45. const existing = spawn(registration, "compatible")
  46. await waitForFile(registration)
  47. const starts: EnsureReason[] = []
  48. const endpoint = await run(
  49. ensure({
  50. file: registration,
  51. version: (version) => version.startsWith("2."),
  52. command: [],
  53. onStart: (reason) => starts.push(reason),
  54. }),
  55. )
  56. expect(endpoint.url).toBe((await Bun.file(registration).json()).url)
  57. expect(starts).toEqual([])
  58. expect(existing.exitCode).toBe(null)
  59. })
  60. test("replaces an incompatible registered service", async () => {
  61. const directory = await temp()
  62. const registration = join(directory, "service.json")
  63. const existing = spawn(registration, "incompatible")
  64. await waitForFile(registration)
  65. const starts: EnsureReason[] = []
  66. const endpoint = await run(
  67. ensure({
  68. file: registration,
  69. version: (version) => version.startsWith("2."),
  70. command: [process.execPath, fixture, registration, "delayed-compatible", "10"],
  71. onStart: (reason) => starts.push(reason),
  72. }),
  73. )
  74. const replacement = await Bun.file(registration).json()
  75. expect(await existing.exited).toBe(0)
  76. expect(replacement.version).toBe("2.1.0-next.1")
  77. expect(endpoint.url).toBe(replacement.url)
  78. expect(starts).toEqual(["version-mismatch"])
  79. process.kill(replacement.pid, "SIGTERM")
  80. await waitForExit(replacement.pid)
  81. })
  82. test("waits for a registered service to finish starting", async () => {
  83. const directory = await temp()
  84. const registration = join(directory, "service.json")
  85. const process = spawn(registration, "starting")
  86. await waitForFile(registration)
  87. const result = run(ensure({ file: registration, version: "test", command: [] }))
  88. await waitForFile(registration + ".health-request")
  89. expect(process.exitCode).toBe(null)
  90. await writeFile(registration + ".release", "")
  91. expect((await result).url).toBe((await Bun.file(registration).json()).url)
  92. })
  93. test("reports a failed registered service without spawning", async () => {
  94. const directory = await temp()
  95. const registration = join(directory, "service.json")
  96. const process = spawn(registration, "failed-owner")
  97. await waitForFile(registration)
  98. await expect(run(ensure({ file: registration, version: "test", command: [] }))).rejects.toThrow(
  99. "Background service failed to start",
  100. )
  101. expect(process.exitCode).toBe(null)
  102. })
  103. test("evicts an unresponsive registered service before starting its replacement", async () => {
  104. const directory = await temp()
  105. const registration = join(directory, "service.json")
  106. const existing = spawn(registration, "hanging")
  107. await waitForFile(registration)
  108. const original = await Bun.file(registration).json()
  109. const endpoint = await run(
  110. ensure({
  111. file: registration,
  112. version: "test",
  113. command: [process.execPath, fixture, registration, "delayed", "10"],
  114. }),
  115. )
  116. const replacement = await Bun.file(registration).json()
  117. expect((await Bun.file(registration + ".requests").text()).trim().split("\n")).toHaveLength(3)
  118. expect(await existing.exited).toBe(0)
  119. expect(replacement.pid).not.toBe(original.pid)
  120. expect(endpoint.url).toBe(replacement.url)
  121. expect(await health(endpoint.url)).toEqual({ healthy: true, version: "test", pid: replacement.pid })
  122. process.kill(replacement.pid, "SIGTERM")
  123. await waitForExit(replacement.pid)
  124. })
  125. test("requests graceful stop of the exact service instance", async () => {
  126. const directory = await temp()
  127. const registration = join(directory, "service.json")
  128. const process = spawn(registration, "graceful")
  129. await waitForFile(registration)
  130. const info = await Bun.file(registration).json()
  131. await run(Service.stop({ file: registration }))
  132. await process.exited
  133. expect(await Bun.file(registration + ".stop").json()).toEqual({ instanceID: info.id })
  134. })
  135. test("does not spawn contenders while an incompatible service rejects replacement", async () => {
  136. const directory = await temp()
  137. const registration = join(directory, "service.json")
  138. const contender = join(directory, "contender.json")
  139. const existing = spawn(registration, "reject-stop")
  140. await waitForFile(registration)
  141. const controller = new AbortController()
  142. const starting = Effect.runPromise(
  143. ensure({
  144. file: registration,
  145. version: "test",
  146. command: [process.execPath, fixture, contender, "record-start"],
  147. }).pipe(Effect.provide(NodeFileSystem.layer)),
  148. { signal: controller.signal },
  149. )
  150. await waitForLines(registration + ".stop-attempts", 2)
  151. controller.abort()
  152. await starting.catch(() => undefined)
  153. expect(await Bun.file(contender + ".started").exists()).toBe(false)
  154. expect(existing.exitCode).toBe(null)
  155. })
  156. test("a legacy health response is still replaced", async () => {
  157. const directory = await temp()
  158. const registration = join(directory, "service.json")
  159. const existing = spawn(registration, "legacy")
  160. await waitForFile(registration)
  161. const starts: EnsureReason[] = []
  162. const result = run(ensure({ file: registration, command: [], onStart: (reason) => starts.push(reason) }))
  163. await expect(result).rejects.toThrow("Missing service command")
  164. expect(starts).toEqual(["version-mismatch"])
  165. await existing.exited
  166. })
  167. test("waits for a slow winner while bounding lock probes", async () => {
  168. const directory = await temp()
  169. const registration = join(directory, "service.json")
  170. const endpoint = await run(
  171. ensure({
  172. file: registration,
  173. version: "test",
  174. command: [process.execPath, fixture, registration, "coordinated"],
  175. }),
  176. )
  177. const info = await Bun.file(registration).json()
  178. try {
  179. expect(endpoint.url).toBe(info.url)
  180. expect(await health(endpoint.url)).toEqual({ healthy: true, version: "test", pid: info.pid })
  181. expect((await Bun.file(registration + ".starts").text()).trim().split("\n")).toHaveLength(2)
  182. } finally {
  183. process.kill(info.pid, "SIGTERM")
  184. await waitForExit(info.pid)
  185. }
  186. })
  187. test("waits for a live contender when another contender fails", async () => {
  188. const directory = await temp()
  189. const registration = join(directory, "service.json")
  190. const endpoint = await run(
  191. ensure({
  192. file: registration,
  193. version: "test",
  194. command: [process.execPath, fixture, registration, "coordinated-failed-loser", "300"],
  195. }),
  196. )
  197. const info = await Bun.file(registration).json()
  198. try {
  199. expect(endpoint.url).toBe(info.url)
  200. } finally {
  201. process.kill(info.pid, "SIGTERM")
  202. await waitForExit(info.pid)
  203. }
  204. })
  205. test("reports a contender that fails to start", async () => {
  206. const directory = await temp()
  207. const registration = join(directory, "service.json")
  208. await expect(
  209. run(
  210. ensure({
  211. file: registration,
  212. version: "test",
  213. command: [process.execPath, fixture, registration, "failed"],
  214. }),
  215. ),
  216. ).rejects.toThrow("Server process exited with code 1")
  217. })
  218. test("reports a bounded contender stderr tail", async () => {
  219. const directory = await temp()
  220. const registration = join(directory, "service.json")
  221. const error = await run(
  222. Service.ensure({
  223. file: registration,
  224. version: "test",
  225. command: [process.execPath, fixture, registration, "stderr-failed"],
  226. }),
  227. ).catch((error: unknown) => error)
  228. expect(error).toBeInstanceOf(Error)
  229. if (!(error instanceof Error)) throw error
  230. expect(error.message).toContain("actionable startup failure")
  231. expect(error.message.length).toBeLessThan(9_000)
  232. }, 10_000)
  233. test("reports a contender terminated by a signal", async () => {
  234. const directory = await temp()
  235. const registration = join(directory, "service.json")
  236. await expect(
  237. run(
  238. ensure({
  239. file: registration,
  240. version: "test",
  241. command: [process.execPath, fixture, registration, "signal"],
  242. }),
  243. ),
  244. ).rejects.toThrow(/Server process (terminated by|exited with code)/)
  245. })
  246. test("reports a slow contender that eventually fails", async () => {
  247. const directory = await temp()
  248. const registration = join(directory, "service.json")
  249. await expect(
  250. run(
  251. ensure({
  252. file: registration,
  253. version: "test",
  254. command: [process.execPath, fixture, registration, "delayed-failed", "500"],
  255. }),
  256. ),
  257. ).rejects.toThrow("Server process exited with code 1")
  258. })
  259. test("replaces an incompatible owner that appears during startup", async () => {
  260. const directory = await temp()
  261. const registration = join(directory, "service.json")
  262. const starting = run(
  263. ensure({
  264. file: registration,
  265. version: "test",
  266. command: [process.execPath, fixture, registration, "delayed", "500"],
  267. }),
  268. )
  269. await waitForFile(registration + ".starts")
  270. const old = spawn(registration, "old")
  271. await waitForFile(registration)
  272. const endpoint = await starting
  273. const info = await Bun.file(registration).json()
  274. try {
  275. expect(endpoint.url).toBe(info.url)
  276. expect(info.version).toBe("test")
  277. await old.exited
  278. } finally {
  279. process.kill(info.pid, "SIGTERM")
  280. await waitForExit(info.pid)
  281. }
  282. })
  283. function run<A, E>(effect: Effect.Effect<A, E>) {
  284. return Effect.runPromise(effect.pipe(Effect.provide(NodeFileSystem.layer)))
  285. }
  286. function spawn(registration: string, mode: string, ...args: string[]) {
  287. const subprocess = Bun.spawn([process.execPath, fixture, registration, mode, ...args], {
  288. stdout: "ignore",
  289. stderr: "inherit",
  290. })
  291. processes.push(subprocess)
  292. return subprocess
  293. }
  294. async function temp() {
  295. const directory = await mkdtemp(join(tmpdir(), "opencode-client-service-"))
  296. directories.push(directory)
  297. return directory
  298. }
  299. async function waitForFile(file: string) {
  300. for (let attempt = 0; attempt < 600; attempt++) {
  301. if (await Bun.file(file).exists()) return
  302. await Bun.sleep(5)
  303. }
  304. throw new Error(`Timed out waiting for ${file}`)
  305. }
  306. async function waitForLines(file: string, count: number) {
  307. for (let attempt = 0; attempt < 600; attempt++) {
  308. const text = await Bun.file(file)
  309. .text()
  310. .catch(() => "")
  311. if (text.trim().split("\n").length >= count) return
  312. await Bun.sleep(5)
  313. }
  314. throw new Error(`Timed out waiting for ${count} lines in ${file}`)
  315. }
  316. async function health(url: string) {
  317. return fetch(new URL("/api/health", url), { signal: AbortSignal.timeout(1_000) }).then((response) => response.json())
  318. }