zen: migrate to new inference

packages/console/app/src/routes/zen/util/handler.ts

@@ -196,6 +196,7 @@ export async function handler(
           providerInfo.modifyHeaders(headers, providerInfo.apiKey, stickyId)
           Object.entries(providerInfo.headerModifier ?? {}).forEach(([k, v]) => {
             if (v === "$ip") return headers.set(k, ip)
+            if (v === "$caller") return headers.set(k, `caller:${ip}`)
             if (v === "$session") return headers.set(k, sessionId)
             if (v === "$model") return headers.set(k, model)
             if (v === "$request") return headers.set(k, requestId)
@@ -354,7 +355,7 @@ export async function handler(
               responseLength += value.length
               buffer += decoder.decode(value, { stream: true })
 
-              const parts = buffer.split(providerInfo.streamSeparator)
+              const parts = buffer.split(/\r\n\r\n|\n\n|\r\r/)
               buffer = parts.pop() ?? ""
 
               for (let part of parts) {

packages/console/app/src/routes/zen/util/provider/anthropic.ts

@@ -139,7 +139,6 @@ export const anthropicHelper: ProviderHelper = ({ reqModel, providerModel }) =>
         return encoder.encode(messages.join(""))
       }
     },
-    streamSeparator: "\n\n",
     createUsageParser: () => {
       let usage: Usage
 

packages/console/app/src/routes/zen/util/provider/google.ts

@@ -37,7 +37,6 @@ export const googleHelper: ProviderHelper = ({ providerModel }) => ({
     return body
   },
   createBinaryStreamDecoder: () => undefined,
-  streamSeparator: "\r\n\r\n",
   createUsageParser: () => {
     let usage: Usage
 

packages/console/app/src/routes/zen/util/provider/openai-compatible.ts

@@ -37,7 +37,6 @@ export const oaCompatHelper: ProviderHelper = ({ adjustCacheUsage }) => ({
     }
   },
   createBinaryStreamDecoder: () => undefined,
-  streamSeparator: "\n\n",
   createUsageParser: () => {
     let usage: Usage
 

packages/console/app/src/routes/zen/util/provider/openai.ts

@@ -20,7 +20,6 @@ export const openaiHelper: ProviderHelper = ({ workspaceID }) => ({
   },
   modifyBody: (body: Record<string, any>) => body,
   createBinaryStreamDecoder: () => undefined,
-  streamSeparator: "\n\n",
   createUsageParser: () => {
     let usage: Usage
 

packages/console/app/src/routes/zen/util/provider/provider.ts

@@ -44,7 +44,6 @@ export type ProviderHelper = (input: {
   modifyHeaders: (headers: Headers, apiKey: string, stickyId: string) => void
   modifyBody: (body: Record<string, any>) => Record<string, any>
   createBinaryStreamDecoder: () => ((chunk: Uint8Array) => Uint8Array | undefined) | undefined
-  streamSeparator: string
   createUsageParser: () => {
     parse: (chunk: string) => void
     retrieve: () => any

packages/schema/sst-env.d.ts

@@ -0,0 +1,10 @@
+/* This file is auto-generated by SST. Do not edit. */
+/* tslint:disable */
+/* eslint-disable */
+/* deno-fmt-ignore-file */
+/* biome-ignore-all lint: auto-generated */
+
+/// <reference path="../../sst-env.d.ts" />
+
+import "sst"
+export {}