chore: reactivate alerts

.github/workflows/deploy.yml

@@ -37,7 +37,6 @@ jobs:
           PLANETSCALE_SERVICE_TOKEN: ${{ secrets.PLANETSCALE_SERVICE_TOKEN }}
           STRIPE_SECRET_KEY: ${{ github.ref_name == 'production' && secrets.STRIPE_SECRET_KEY_PROD || secrets.STRIPE_SECRET_KEY_DEV }}
           HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY }}
-          INCIDENT_API_KEY: ${{ secrets.INCIDENT_API_KEY }}
           SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
           SENTRY_ORG: ${{ vars.SENTRY_ORG }}
           SENTRY_PROJECT: ${{ vars.WEB_SENTRY_PROJECT }}

infra/monitoring.ts

@@ -45,7 +45,7 @@ const modelHttpErrorsQuery = (product: "go" | "zen") => {
       { op: "COUNT", name: "TOTAL", filterCombination: "AND", filters },
       { op: "SUM", name: "FAILED", column: "is_failed_http_status", filterCombination: "AND", filters },
     ],
-    formulas: [{ name: "ERROR", expression: "IF(GTE($TOTAL, 2500), DIV($FAILED, $TOTAL), 0)" }],
+    formulas: [{ name: "ERROR", expression: "IF(GTE($TOTAL, 500), DIV($FAILED, $TOTAL), 0)" }],
     timeRange: 900,
   }).json
 }
@@ -60,14 +60,14 @@ new honeycomb.Trigger("IncreasedModelHttpErrorsGo", {
   frequency: 300,
   thresholds: [{ op: ">=", value: 0.8, exceededLimit: 1 }],
   recipients: [
-    // {
-    //   id: webhookRecipient.id,
-    //   notificationDetails: [
-    //     {
-    //       variables: [{ name: "type", value: "model_http_errors" }],
-    //     },
-    //   ],
-    // },
+    {
+      id: webhookRecipient.id,
+      notificationDetails: [
+        {
+          variables: [{ name: "type", value: "model_http_errors" }],
+        },
+      ],
+    },
   ],
 })
 
@@ -79,13 +79,13 @@ new honeycomb.Trigger("IncreasedModelHttpErrorsZen", {
   frequency: 300,
   thresholds: [{ op: ">=", value: 0.8, exceededLimit: 1 }],
   recipients: [
-    // {
-    //   id: webhookRecipient.id,
-    //   notificationDetails: [
-    //     {
-    //       variables: [{ name: "type", value: "model_http_errors" }],
-    //     },
-    //   ],
-    // },
+    {
+      id: webhookRecipient.id,
+      notificationDetails: [
+        {
+          variables: [{ name: "type", value: "model_http_errors" }],
+        },
+      ],
+    },
   ],
 })

sst.config.ts

@@ -14,7 +14,6 @@ export default $config({
         random: "4.19.2",
         planetscale: "0.4.1",
         honeycomb: "0.49.0",
-        incident: "5.35.0",
       },
     }
   },