Automate tests in CI

import { setTimeout as sleep } from "node:timers/promises";

function required(name) {
  const value = process.env[name]?.trim();
  if (!value) throw new Error(`Set ${name}.`);
  return value;
}

const origin = required("EGMA_URL");
const key = required("EGMA_API_KEY");
const suiteId = required("EGMA_SUITE_ID");
const agentId = required("EGMA_AGENT_ID");
const connectionId = required("EGMA_CONNECTION_ID");
const runLabel = required("EGMA_CI_RUN_KEY");
const deadline = Date.now() + 15 * 60 * 1000;
const headers = {
  Authorization: `Bearer ${key}`,
  "Content-Type": "application/json",
};

async function request(path, { method = "GET", body } = {}) {
  // Only reads are retried; repeating creation can start another run.
  for (let attempt = 0; attempt < 3; attempt++) {
    const remaining = deadline - Date.now();
    if (remaining <= 0) throw new Error("Timed out waiting for Egma.");
    let response;
    try {
      response = await fetch(new URL(path, origin), {
        method,
        headers,
        body: body === undefined ? undefined : JSON.stringify(body),
        signal: AbortSignal.timeout(Math.min(30_000, remaining)),
      });
    } catch (error) {
      if (method !== "GET" || attempt === 2) throw error;
      await sleep(1000 * (attempt + 1));
      continue;
    }
    if (response.ok) return response.json();
    if (method !== "GET" || ![429, 500, 502, 503, 504].includes(response.status) || attempt === 2) {
      throw new Error(`${method} ${path}: HTTP ${response.status}`);
    }
    const retryAfter = Number(response.headers.get("retry-after"));
    const wait = Number.isFinite(retryAfter) && retryAfter > 0
      ? retryAfter * 1000
      : 1000 * (attempt + 1);
    if (Date.now() + wait >= deadline) throw new Error("Timed out waiting for Egma.");
    await sleep(wait);
  }
}

async function allPages(path, field) {
  const items = [];
  let pageToken;
  do {
    const url = new URL(path, origin);
    url.searchParams.set("pageSize", "200");
    if (pageToken) url.searchParams.set("pageToken", pageToken);
    const page = await request(url.pathname + url.search);
    items.push(...page[field]);
    pageToken = page.nextPageToken;
  } while (pageToken);
  return items;
}

function checkSimulation(simulation) {
  if (simulation.status !== "completed" || simulation.gradingState !== "complete") {
    throw new Error(`${simulation.id}: execution or grading did not complete.`);
  }
  if (simulation.agentPovComplete !== true || simulation.agentPovIncomplete !== false) {
    throw new Error(`${simulation.id}: agent evidence is missing or unknown.`);
  }
  const plan = simulation.gradingPlan?.items;
  if (!plan?.length) throw new Error(`${simulation.id}: no selected graders.`);
  for (const item of plan) {
    const grade = simulation.grades.find(
      (value) => value.projectGraderId === item.projectGraderId,
    );
    const matchesPlan = grade &&
      grade.graderDefinitionId === item.graderDefinitionId &&
      grade.graderDefinitionVersion === item.graderDefinitionVersion &&
      grade.passThreshold === item.passThreshold;
    if (!matchesPlan || grade.result !== "passed" ||
        typeof grade.score !== "number" || grade.score < item.passThreshold) {
      throw new Error(`${simulation.id}: ${item.graderName} did not pass.`);
    }
  }
}

let run;
let creationRequested = false;
try {
  const tests = await allPages(
    `/v1/tests?suiteId=${encodeURIComponent(suiteId)}`, "tests",
  );
  if (!tests.length) throw new Error("The suite has no tests.");
  creationRequested = true;
  run = await request("/v1/runs", {
    method: "POST",
    body: {
      suiteId, agentId, connectionId,
      name: `CI ${runLabel}`,
      expectedTestVersions: tests.map((test) => ({
        testId: test.id, versionId: test.versionId,
      })),
    },
  });
  console.log(`Run ${run.id}: ${run.resultsUrl}`);
  for (;;) {
    run = await request(`/v1/runs/${encodeURIComponent(run.id)}`);
    if (run.status === "canceled") throw new Error("The run was canceled.");
    if (run.status === "completed") {
      const simulations = await allPages(
        `/v1/runs/${encodeURIComponent(run.id)}/simulations`, "simulations",
      );
      if (simulations.length !== run.expectedSimulationCount || !simulations.length) {
        throw new Error("The run is missing simulations.");
      }
      if (simulations.some((one) => one.status !== "completed")) {
        throw new Error("At least one simulation failed or was canceled.");
      }
      const waiting = simulations.some(
        (one) => ["pending", "running"].includes(one.gradingState),
      );
      if (!waiting) {
        for (const one of simulations) {
          checkSimulation(await request(`/v1/simulations/${encodeURIComponent(one.id)}`));
        }
        console.log(`CI policy passed for ${simulations.length} simulations.`);
        break;
      }
    }
    await sleep(5000);
  }
} catch (error) {
  console.error(error.message);
  process.exitCode = 1;
  if (run && !["completed", "canceled"].includes(run.status)) {
    try {
      const response = await fetch(
        new URL(`/v1/runs/${encodeURIComponent(run.id)}/cancel`, origin),
        { method: "POST", headers, signal: AbortSignal.timeout(10_000) },
      );
      if (!response.ok) throw new Error(`HTTP ${response.status}`);
      console.error(`Canceled run ${run.id}.`);
    } catch {
      console.error(`Could not confirm cancellation. Check run ${run.id}.`);
    }
  }
  if (!run && creationRequested) {
    console.error("If creation lost its response, check Runs before submitting again. Repeating creation starts another run.");
  }
}