layue13
05f8fad38a
Gatekeep evals / eval-floor (push) Successful in 8s
Add .github/workflows/gatekeep-pr.yml with uses: ./ and OPENROUTER_API_KEY from secrets. Forge-agnostic API helper for Gitea.
84 lines
2.5 KiB
Bash
84 lines
2.5 KiB
Bash
#!/usr/bin/env bash
|
|
# Offline eval of deterministic policy floor (fail-closed). No OpenRouter calls.
|
|
set -euo pipefail
|
|
|
|
CASES_DIR="${1:?evals/cases dir}"
|
|
POLICY="${2:-}"
|
|
ACTION_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
export GATEKEEP_ACTION="$ACTION_ROOT"
|
|
export GATEKEEP_ROOT="${GATEKEEP_ROOT:-$ACTION_ROOT}"
|
|
export CONFIG_DIR="${CONFIG_DIR:-}"
|
|
# shellcheck source=lib-config.sh
|
|
source "$ACTION_ROOT/scripts/lib-config.sh"
|
|
POLICY="${POLICY:-$(gatekeep_config_file policy.json)}"
|
|
|
|
failed=0
|
|
total=0
|
|
|
|
for case in "$CASES_DIR"/*.json; do
|
|
[[ -f "$case" ]] || continue
|
|
[[ "$(basename "$case")" == _template.json ]] && continue
|
|
total=$((total + 1))
|
|
python3 - "$case" "$POLICY" <<'PY' || failed=$((failed + 1))
|
|
import json, fnmatch, sys
|
|
|
|
case = json.load(open(sys.argv[1]))
|
|
policy = json.load(open(sys.argv[2]))
|
|
fx = case["fixture"]
|
|
files = fx.get("files") or []
|
|
labels = [x.lower() for x in (fx.get("labels") or [])]
|
|
add = int(fx.get("additions") or 10)
|
|
delete = int(fx.get("deletions") or 2)
|
|
ci = fx.get("ci_status") or "success"
|
|
|
|
rank = {"low": 0, "medium": 1, "high": 2}
|
|
floor = "low"
|
|
reasons = []
|
|
|
|
def bump(t, why):
|
|
global floor
|
|
if rank[t] > rank[floor]:
|
|
floor = t
|
|
reasons.append(why)
|
|
|
|
def match(path, glob):
|
|
g = glob.replace("**/", "*").replace("**", "*")
|
|
return fnmatch.fnmatch(path, glob) or fnmatch.fnmatch(path, g)
|
|
|
|
for lab in labels:
|
|
if lab in [x.lower() for x in policy.get("force_high_labels", [])]:
|
|
bump("high", f"label {lab}")
|
|
if lab in [x.lower() for x in policy.get("force_medium_labels", [])]:
|
|
bump("medium", f"label {lab}")
|
|
|
|
for f in files:
|
|
for g in policy.get("high_path_globs", []):
|
|
if match(f, g):
|
|
bump("high", f"{f}~{g}")
|
|
for g in policy.get("medium_path_globs", []):
|
|
if match(f, g):
|
|
bump("medium", f"{f}~{g}")
|
|
|
|
if len(files) > int(policy.get("low_max_files", 8)):
|
|
bump("medium", "file count")
|
|
if add + delete > int(policy.get("low_max_diff_lines", 200)):
|
|
bump("medium", "diff lines")
|
|
if ci in ("failure", "error"):
|
|
bump("high", f"ci {ci}")
|
|
|
|
min_t = case["expect_min_tier"]
|
|
exact = case.get("expect_tier")
|
|
ok = rank[floor] >= rank[min_t] and (exact is None or floor == exact)
|
|
status = "PASS" if ok else "FAIL"
|
|
print(f"{status} {case['id']}: floor={floor} min={min_t}" + (f" expect={exact}" if exact else ""))
|
|
if not ok:
|
|
print(" " + "; ".join(reasons) or "(none)")
|
|
print(" " + case.get("description", ""))
|
|
sys.exit(1)
|
|
PY
|
|
done
|
|
|
|
echo ""
|
|
echo "$((total - failed))/$total passed"
|
|
exit "$failed"
|