Просмотр исходного кода

fix(stats): correct r2 daily totals

Adam 4 дней назад
Родитель
Сommit
7e0353cca9
2 измененных файлов с 58 добавлено и 13 удалено
  1. 38 0
      packages/stats/core/src/domain/inference.test.ts
  2. 20 13
      packages/stats/core/src/domain/inference.ts

+ 38 - 0
packages/stats/core/src/domain/inference.test.ts

@@ -103,6 +103,44 @@ describe("inference stat normalization", () => {
     expect(queries[1]).toContain("'geo_model' ELSE 'geo'")
     expect(queries[1]).toContain("'geo_model' ELSE 'geo'")
     expect(queries[1]).toContain("0 AS sessions")
     expect(queries[1]).toContain("0 AS sessions")
   })
   })
+
+  test("aligns periods to UTC calendar boundaries", () => {
+    const queries = buildStatsQueries(
+      new Date("2026-06-17T15:56:00.000Z"),
+      new Date("2026-06-19T15:56:00.000Z"),
+      {
+        namespace: "inference",
+        table: "generation",
+        dataset: "zen",
+      },
+    )
+
+    expect(queries).toHaveLength(8)
+    expect(queries[0]).toContain("'2026-W25' AS period_key")
+    expect(queries[0]).toContain("started_at >= '2026-06-15T00:00:00.000Z'")
+    expect(queries[2]).toContain("'2026-06-17' AS period_key")
+    expect(queries[2]).toContain("started_at >= '2026-06-17T00:00:00.000Z'")
+    expect(queries[2]).toContain("started_at < '2026-06-18T00:00:00.000Z'")
+    expect(queries[6]).toContain("'2026-06-19' AS period_key")
+    expect(queries[6]).toContain("started_at < '2026-06-19T15:56:00.000Z'")
+  })
+
+  test("uses an exclusive live and legacy source handoff", () => {
+    const [query] = buildStatsQueries(
+      new Date("2026-08-11T00:00:00.000Z"),
+      new Date("2026-08-12T00:00:00.000Z"),
+      {
+        namespace: "inference",
+        table: "generation",
+        dataset: "zen",
+      },
+    )
+
+    expect(query).toContain(
+      "(source = 'inference-legacy' AND started_at < '2026-08-11T10:57:48.186Z')",
+    )
+    expect(query).toContain("(source = 'inference' AND started_at >= '2026-08-11T10:57:48.186Z')")
+  })
 })
 })
 
 
 function aggregate(model: string, provider: string) {
 function aggregate(model: string, provider: string) {

+ 20 - 13
packages/stats/core/src/domain/inference.ts

@@ -10,7 +10,14 @@ import {
   statProvider,
   statProvider,
 } from "./model-normalization"
 } from "./model-normalization"
 import type { ProviderStatAggregate } from "./provider"
 import type { ProviderStatAggregate } from "./provider"
-import { normalizeCountry, normalizeTier, type StatBaseAggregate } from "./stat"
+import {
+  normalizeCountry,
+  normalizeTier,
+  periodKeyFor,
+  startOfIsoWeek,
+  startOfUtcDay,
+  type StatBaseAggregate,
+} from "./stat"
 
 
 export type StatDimension = "model" | "provider" | "geo" | "geo_model"
 export type StatDimension = "model" | "provider" | "geo" | "geo_model"
 export type StatsQuerySource = { namespace: string; table: string; dataset: string }
 export type StatsQuerySource = { namespace: string; table: string; dataset: string }
@@ -18,6 +25,10 @@ type StatsQueryFamily = "usage" | "geo"
 
 
 const DAY_MS = 86_400_000
 const DAY_MS = 86_400_000
 const WEEK_MS = 7 * DAY_MS
 const WEEK_MS = 7 * DAY_MS
+// The typed production stream began before the legacy backfill's original end
+// boundary. Use one exclusive handoff so the overlapping rows are never counted
+// from both sources.
+const LIVE_SOURCE_START = "2026-08-11T10:57:48.186Z"
 
 
 // R2 SQL limits result sets to 10,000 rows and does not support OFFSET. Two
 // R2 SQL limits result sets to 10,000 rows and does not support OFFSET. Two
 // queries per day/week keep each result bounded and avoid combining the costly
 // queries per day/week keep each result bounded and avoid combining the costly
@@ -123,6 +134,10 @@ WITH normalized AS (
   FROM ${sourceTable}
   FROM ${sourceTable}
   WHERE event_type = 'generation.completed'
   WHERE event_type = 'generation.completed'
     AND source IN ('inference', 'inference-legacy')
     AND source IN ('inference', 'inference-legacy')
+    AND (
+      (source = 'inference-legacy' AND started_at < ${sqlString(LIVE_SOURCE_START)})
+      OR (source = 'inference' AND started_at >= ${sqlString(LIVE_SOURCE_START)})
+    )
     AND product = 'go'
     AND product = 'go'
     AND model_requested IS NOT NULL
     AND model_requested IS NOT NULL
     AND model_requested <> ''
     AND model_requested <> ''
@@ -264,27 +279,19 @@ function sqlString(value: string) {
 
 
 function statPeriods(grain: "day" | "week", periodStart: Date, periodEnd: Date) {
 function statPeriods(grain: "day" | "week", periodStart: Date, periodEnd: Date) {
   const interval = grain === "day" ? DAY_MS : WEEK_MS
   const interval = grain === "day" ? DAY_MS : WEEK_MS
-  const count = Math.max(0, Math.ceil((periodEnd.getTime() - periodStart.getTime()) / interval))
+  const first = grain === "day" ? startOfUtcDay(periodStart) : startOfIsoWeek(periodStart)
+  const count = Math.max(0, Math.ceil((periodEnd.getTime() - first.getTime()) / interval))
   return Array.from({ length: count }, (_, index) => {
   return Array.from({ length: count }, (_, index) => {
-    const start = new Date(periodStart.getTime() + index * interval)
+    const start = new Date(first.getTime() + index * interval)
     return {
     return {
       grain,
       grain,
-      key: grain === "day" ? start.toISOString().slice(0, 10) : isoWeekKey(start),
+      key: periodKeyFor(grain, start),
       start,
       start,
       end: new Date(Math.min(start.getTime() + interval, periodEnd.getTime())),
       end: new Date(Math.min(start.getTime() + interval, periodEnd.getTime())),
     }
     }
   })
   })
 }
 }
 
 
-function isoWeekKey(date: Date) {
-  const thursday = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()))
-  const day = thursday.getUTCDay() || 7
-  thursday.setUTCDate(thursday.getUTCDate() + 4 - day)
-  const year = thursday.getUTCFullYear()
-  const week = Math.ceil((thursday.getTime() - Date.UTC(year, 0, 1) + DAY_MS) / WEEK_MS)
-  return `${year}-W${String(week).padStart(2, "0")}`
-}
-
 function statModelSql(model: string, providerModel: string) {
 function statModelSql(model: string, providerModel: string) {
   return `COALESCE(NULLIF(regexp_replace(CASE
   return `COALESCE(NULLIF(regexp_replace(CASE
       WHEN lower(${model}) = 'big-pickle' THEN NULLIF(${providerModel}, '')
       WHEN lower(${model}) = 'big-pickle' THEN NULLIF(${providerModel}, '')