gatekeep/scripts/merge-findings.js

#!/usr/bin/env node
/**
 * Merge parallel model outputs into agreed findings + disputed + fail-closed PR risk tier.
 * Zero npm dependencies (Node 20 stdlib only).
 *
 * Finding-level severity (Critical/High/Medium/Low) is distinct from PR risk tier.
 * Every finding carries: severity, exact location, why_it_matters.
 *
 * Agreement: fingerprint must appear in >= agreementMin distinct models (default 2).
 * Security: agreed security findings block; scan hard-fail always blocks.
 */
import { readFileSync, writeFileSync, readdirSync } from "node:fs";
import { join } from "node:path";

const args = parseArgs(process.argv.slice(2));
const dir = args.dir;
const floor = args.floor || "low";
const agreementMin = Number(args["agreement-min"] || 2);
const scanHardFail = args["scan-hard-fail"] === "true";
const language = (args.language || process.env.GATEKEEP_LANGUAGE || "en").trim() || "en";
const outPath = args.out;

const tierRank = { low: 0, medium: 1, high: 2 };
const sevRank = { Critical: 0, High: 1, Medium: 2, Low: 3 };
const maxTier = (a, b) => (tierRank[a] >= tierRank[b] ? a : b);
const normTier = (t) => {
  const s = String(t || "").toLowerCase();
  if (s === "low" || s === "medium" || s === "high") return s;
  if (s === "easy") return "low";
  if (s === "hard") return "high";
  return "high";
};

const files = readdirSync(dir).filter((f) => f.startsWith("model-") && f.endsWith(".json"));
const models = [];
for (const f of files) {
  try {
    const j = JSON.parse(readFileSync(join(dir, f), "utf8"));
    if (j.error || j.parse_error) {
      models.push({
        model: j._model || f,
        error: j.error || j.parse_error,
        findings: [],
        risk_tier: "high",
      });
      continue;
    }
    models.push({
      model: j._model || f,
      risk_tier: normTier(j.risk_tier),
      summary: j.summary || "",
      rationale: j.rationale || "",
      findings: Array.isArray(j.findings) ? j.findings : [],
      human_focus: Array.isArray(j.human_focus) ? j.human_focus : [],
    });
  } catch (e) {
    models.push({ model: f, error: String(e), findings: [], risk_tier: "high" });
  }
}

function fingerprint(f) {
  const path = String(f.path || "").replace(/\\/g, "/").toLowerCase();
  const lineBucket = f.line ? String(Math.floor(Number(f.line) / 5) * 5) : "file";
  const kind = String(f.kind || "other").toLowerCase();
  const title = String(f.title || "")
    .toLowerCase()
    .replace(/[^a-z0-9]+/g, " ")
    .trim()
    .split(/\s+/)
    .slice(0, 8)
    .join(" ");
  return `${kind}|${path}|${lineBucket}|${title}`;
}

function normSeverity(raw, kind, tag) {
  const s = String(raw || "").trim();
  const map = {
    critical: "Critical",
    crit: "Critical",
    high: "High",
    medium: "Medium",
    med: "Medium",
    low: "Low",
    important: "High",
    nit: "Low",
  };
  if (map[s.toLowerCase()]) return map[s.toLowerCase()];
  // Infer from kind/tag when model omitted severity
  if (String(kind).toLowerCase() === "security") return "Critical";
  if (tag === "Important") return "High";
  if (tag === "Nit") return "Low";
  if (tag === "Pre-existing") return "Medium";
  return "Medium";
}

function formatLocation(path, line, lineEnd) {
  const p = String(path || "").trim() || "(unknown file)";
  if (line == null || line === "") return p;
  if (lineEnd != null && Number(lineEnd) > Number(line)) return `${p}:${line}-${lineEnd}`;
  return `${p}:${line}`;
}

function normalizeFinding(f, model) {
  const kind = ["security", "correctness", "behavior", "other"].includes(String(f.kind).toLowerCase())
    ? String(f.kind).toLowerCase()
    : "other";
  let tag = ["Important", "Nit", "Pre-existing"].includes(f.tag) ? f.tag : "Important";
  if (f.pre_existing === true) tag = "Pre-existing";
  const severity = normSeverity(f.severity, kind, tag);
  const path = String(f.path || "");
  const line = typeof f.line === "number" ? f.line : undefined;
  const line_end = typeof f.line_end === "number" ? f.line_end : undefined;
  const why =
    String(f.why_it_matters || f.why || f.impact || "").trim() ||
    // Fall back: last sentence-ish of body, else body itself
    (() => {
      const body = String(f.body || "").trim();
      if (!body) return "Impact not spelled out — treat as unresolved until clarified.";
      const parts = body.split(/\n+/).map((x) => x.trim()).filter(Boolean);
      return parts[parts.length - 1] || body;
    })();
  const title = String(f.title || "Finding");
  const body = String(f.body || "");
  return {
    path,
    line,
    line_end,
    location: formatLocation(path, line, line_end),
    severity,
    tag,
    kind,
    title,
    body,
    why_it_matters: why,
    blocking: Boolean(f.blocking) || kind === "security" || severity === "Critical",
    models: [model],
    fingerprint: fingerprint({ ...f, kind, title }),
  };
}

function preferSeverity(a, b) {
  return (sevRank[a] ?? 9) <= (sevRank[b] ?? 9) ? a : b;
}

/** @type {Map<string, any>} */
const bucket = new Map();
for (const m of models) {
  for (const raw of m.findings) {
    const n = normalizeFinding(raw, m.model);
    const prev = bucket.get(n.fingerprint);
    if (!prev) {
      bucket.set(n.fingerprint, n);
    } else {
      if (!prev.models.includes(m.model)) prev.models.push(m.model);
      prev.severity = preferSeverity(prev.severity, n.severity);
      if (n.tag === "Important") prev.tag = "Important";
      if (n.tag === "Pre-existing" && prev.tag !== "Important") prev.tag = "Pre-existing";
      if ((n.why_it_matters || "").length > (prev.why_it_matters || "").length) {
        prev.why_it_matters = n.why_it_matters;
      }
      if ((n.body || "").length > (prev.body || "").length) prev.body = n.body;
      if (n.blocking) prev.blocking = true;
      if (!prev.line && n.line) prev.line = n.line;
      if (!prev.line_end && n.line_end) prev.line_end = n.line_end;
      prev.location = formatLocation(prev.path, prev.line, prev.line_end);
    }
  }
}

// Prefer successful model tiers; only if every call failed, fail-closed high.
const okModels = models.filter((m) => !m.error);
const modelErrors = models.filter((m) => m.error).length;
const degraded = modelErrors > 0 || okModels.length < agreementMin;

// When some models 429/fail, lower the bar to surface usable findings from survivors
// (still require at least 1 successful model). Full fail-closed high only if zero ok.
let effectiveAgreement = agreementMin;
if (okModels.length > 0 && okModels.length < agreementMin) {
  effectiveAgreement = Math.max(1, okModels.length);
}

const agreed = [];
const disputed = [];
for (const f of bucket.values()) {
  if (f.models.length >= effectiveAgreement) agreed.push(f);
  else disputed.push(f);
}

const bySeverity = (a, b) => (sevRank[a.severity] ?? 9) - (sevRank[b.severity] ?? 9);
agreed.sort(bySeverity);
disputed.sort(bySeverity);

const tierSources = okModels.length ? okModels : models;
let risk_tier = floor;
for (const m of tierSources) risk_tier = maxTier(risk_tier, m.risk_tier);
if (!okModels.length) risk_tier = maxTier(risk_tier, "high");
for (const f of agreed) {
  if (f.kind === "security" || f.severity === "Critical") risk_tier = maxTier(risk_tier, "high");
  else if (f.severity === "High" || f.tag === "Important") risk_tier = maxTier(risk_tier, "medium");
}
if (scanHardFail) risk_tier = maxTier(risk_tier, "high");

const openSecurityBlocking =
  scanHardFail || agreed.some((f) => (f.kind === "security" || f.severity === "Critical") && f.blocking);

const result = {
  risk_tier,
  policy_floor: floor,
  agreement_min: agreementMin,
  effective_agreement_min: effectiveAgreement,
  language,
  degraded,
  model_errors: modelErrors,
  models_ok: okModels.length,
  models: models.map((m) => ({
    id: m.model,
    risk_tier: m.risk_tier,
    finding_count: m.findings.length,
    error: m.error || null,
  })),
  agreed_findings: agreed,
  disputed_findings: disputed,
  open_security_blocking: openSecurityBlocking,
  scan_hard_fail: scanHardFail,
  human_focus: [...new Set(models.flatMap((m) => m.human_focus || []))].slice(0, 12),
  summaries: models.map((m) => ({ model: m.model, summary: m.summary, rationale: m.rationale })),
};

writeFileSync(outPath, JSON.stringify(result, null, 2));
process.stdout.write(
  JSON.stringify({
    risk_tier: result.risk_tier,
    security_blocking: openSecurityBlocking,
    agreed_count: agreed.length,
    disputed_count: disputed.length,
  }) + "\n",
);

function parseArgs(argv) {
  const out = {};
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a.startsWith("--")) {
      const key = a.slice(2);
      const val = argv[i + 1] && !argv[i + 1].startsWith("--") ? argv[++i] : "true";
      out[key] = val;
    }
  }
  return out;
}