Batch metrics sending and add CI

Author: Ankcorn

.github/workflows/ci.yml

@@ -0,0 +1,21 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+          cache: npm
+      - run: npm ci
+      - run: npm run lint
+      - run: npm run typecheck
+      - run: npm test

src/utils.ts

@@ -0,0 +1,10 @@
+/**
+ * Split an array into chunks of a specified size.
+ */
+export function chunk<T>(array: T[], size: number): T[][] {
+	const chunks: T[][] = [];
+	for (let i = 0; i < array.length; i += size) {
+		chunks.push(array.slice(i, i + size));
+	}
+	return chunks;
+}

src/workflow.ts

@@ -3,9 +3,11 @@ import {
 	type WorkflowEvent,
 	type WorkflowStep,
 } from "cloudflare:workers";
+import pAll from "p-all";
 import { createCloudflareApi } from "./api/cloudflare";
 import { createDatadogApi } from "./api/datadog";
 import { formatHealthMetrics, formatMetricsForContainer } from "./metrics";
+import { chunk } from "./utils";
 
 /**
  * Calculate the metrics time window.
@@ -40,6 +42,11 @@ export class MetricsExporterWorkflow extends WorkflowEntrypoint<Env> {
 			this.env.CLOUDFLARE_API_TOKEN,
 		);
 
+		const datadog = createDatadogApi(
+			this.env.DATADOG_API_KEY,
+			this.env.DATADOG_SITE,
+		);
+
 		const { start, end } = getMetricsTimeWindow();
 		console.log("Workflow started", {
 			start: start.toISOString(),
@@ -57,10 +64,6 @@ export class MetricsExporterWorkflow extends WorkflowEntrypoint<Env> {
 					this.env.CLOUDFLARE_ACCOUNT_ID,
 					result,
 				);
-				const datadog = createDatadogApi(
-					this.env.DATADOG_API_KEY,
-					this.env.DATADOG_SITE,
-				);
 				await datadog.sendMetrics(healthMetrics);
 
 				return result.map((c) => ({ id: c.id, name: c.name }));
@@ -86,18 +89,25 @@ export class MetricsExporterWorkflow extends WorkflowEntrypoint<Env> {
 						metricsGroups,
 					);
 
-					if (metrics.length > 0) {
-						const datadog = createDatadogApi(
-							this.env.DATADOG_API_KEY,
-							this.env.DATADOG_SITE,
-						);
-						await datadog.sendMetrics(metrics);
-					}
+					const batches = chunk(metrics, 25000);
+
+					await pAll(
+						batches.map(
+							(batch, i) => () =>
+								step.do(
+									`send batch ${i + 1}/${batches.length} (${batch.length} metrics)`,
+									STEP_CONFIG,
+									async () => {
+										await datadog.sendMetrics(batch);
+									},
+								),
+						),
+						{ concurrency: 6 },
+					);
 
 					return metrics.length;
 				},
 			);
-
 			totalMetrics += count;
 		}