layue13
05f8fad38a
Gatekeep evals / eval-floor (push) Successful in 8s
Add .github/workflows/gatekeep-pr.yml with uses: ./ and OPENROUTER_API_KEY from secrets. Forge-agnostic API helper for Gitea.
115 lines
4.0 KiB
Bash
115 lines
4.0 KiB
Bash
#!/usr/bin/env bash
|
|
# Verify layer: discover or run caller-configured benchmarks vs thresholds.
|
|
# Architecture-level performance debt is out of scope — metrics only.
|
|
set -euo pipefail
|
|
|
|
ROOT="${GATEKEEP_ROOT:-.}"
|
|
ACTION="${GATEKEEP_ACTION:-.}"
|
|
OUT="$ROOT/.gatekeep/verify"
|
|
mkdir -p "$OUT"
|
|
|
|
# shellcheck source=lib-config.sh
|
|
source "$ACTION/scripts/lib-config.sh"
|
|
BENCH_CFG="$(gatekeep_config_file benchmarks.json)"
|
|
|
|
ENABLED=$(python3 -c 'import json,sys; print(str(json.load(open(sys.argv[1])).get("enabled", True)).lower())' "$BENCH_CFG")
|
|
if [[ "$ENABLED" != "true" ]]; then
|
|
echo "Benchmarks disabled" | tee "$OUT/result.txt"
|
|
exit 0
|
|
fi
|
|
|
|
python3 - "$BENCH_CFG" "$OUT" "$ROOT" <<'PY'
|
|
import json, os, pathlib, subprocess, time, re, sys
|
|
|
|
cfg = json.load(open(sys.argv[1]))
|
|
out = pathlib.Path(sys.argv[2])
|
|
root = pathlib.Path(sys.argv[3])
|
|
thr = cfg.get("thresholds") or {}
|
|
cmds = cfg.get("commands") or {}
|
|
discover = {}
|
|
dpath = root / ".gatekeep/out/discover.json"
|
|
if dpath.exists():
|
|
discover = json.loads(dpath.read_text())
|
|
|
|
def pick_command(kind_prefixes):
|
|
explicit = cmds.get("benchmark") or cmds.get(kind_prefixes[0].split(":")[0]) or ""
|
|
if explicit:
|
|
return explicit, "config"
|
|
for e in discover.get("entries") or []:
|
|
k = e.get("kind") or ""
|
|
if any(k.startswith(p) or p in k for p in kind_prefixes):
|
|
return e.get("command"), e.get("source")
|
|
return None, None
|
|
|
|
results = {"metrics": {}, "thresholds": thr, "ran": [], "skipped": [], "pass": True}
|
|
|
|
# Optional explicit startup / memory / fps harnesses — callers wire real collectors.
|
|
# Default: run discovered bench/test if present; parse common patterns when printed.
|
|
|
|
bench_cmd, src = pick_command(["npm:bench", "npm:benchmark", "make:bench", "make:benchmark", "cargo:bench"])
|
|
test_cmd, tsrc = pick_command(["npm:test", "make:test", "gradle:test", "cargo:test", "go:test", "pytest", "mvn:test"])
|
|
|
|
def run(label, cmd):
|
|
if not cmd:
|
|
results["skipped"].append(label)
|
|
return ""
|
|
results["ran"].append({"label": label, "command": cmd})
|
|
t0 = time.time()
|
|
p = subprocess.run(cmd, shell=True, cwd=root, capture_output=True, text=True)
|
|
dt = int((time.time() - t0) * 1000)
|
|
(out / f"{label}.stdout.txt").write_text(p.stdout[-50000:])
|
|
(out / f"{label}.stderr.txt").write_text(p.stderr[-20000:])
|
|
results["metrics"][f"{label}_exit"] = p.returncode
|
|
results["metrics"][f"{label}_wall_ms"] = dt
|
|
if p.returncode != 0:
|
|
results["pass"] = False
|
|
return p.stdout + "\n" + p.stderr
|
|
|
|
log = ""
|
|
if bench_cmd:
|
|
log += run("benchmark", bench_cmd)
|
|
elif test_cmd:
|
|
# Fall back: treat test suite wall time as coarse startup/regression signal only when no bench
|
|
log += run("verify_tests", test_cmd)
|
|
else:
|
|
results["skipped"].append("no discoverable benchmark/test command")
|
|
|
|
# Parse metrics if present in output
|
|
def grab(name, patterns):
|
|
for pat in patterns:
|
|
m = re.search(pat, log, re.I | re.M)
|
|
if m:
|
|
try:
|
|
results["metrics"][name] = float(m.group(1))
|
|
return
|
|
except Exception:
|
|
pass
|
|
|
|
grab("fps", [r"fps[=:\s]+([0-9.]+)", r"frames?/s[=:\s]+([0-9.]+)"])
|
|
grab("startup_ms", [r"startup[^0-9]*([0-9.]+)\s*ms", r"started in ([0-9.]+)\s*ms"])
|
|
grab("memory_mb", [r"memory[^0-9]*([0-9.]+)\s*mb", r"rss[=:\s]+([0-9.]+)\s*mb"])
|
|
|
|
# Threshold checks (null skips)
|
|
def check(metric, op, key):
|
|
lim = thr.get(key)
|
|
if lim is None:
|
|
return
|
|
if metric not in results["metrics"]:
|
|
results["skipped"].append(f"missing metric for {key}")
|
|
return
|
|
val = results["metrics"][metric]
|
|
ok = val <= lim if op == "max" else val >= lim
|
|
results["metrics"][f"{key}_ok"] = ok
|
|
if not ok:
|
|
results["pass"] = False
|
|
|
|
check("startup_ms", "max", "startup_ms_max")
|
|
check("memory_mb", "max", "memory_mb_max")
|
|
check("fps", "min", "fps_min")
|
|
|
|
(out / "benchmarks.json").write_text(json.dumps(results, indent=2))
|
|
print(json.dumps({"pass": results["pass"], "metrics": results["metrics"]}, indent=2))
|
|
if not results["pass"]:
|
|
sys.exit(1)
|
|
PY
|