k-skill/tools/k-skill-qa-bot/bin/classify-skill.py
Jeffrey (Dongkyu) Kim 7f73e55011 feat(qa-bot): add k-skill-qa-bot under tools/
External macOS daemon that clones NomaDamas/k-skill main every 3 days, runs
each skill through codex exec, has an LLM judge grade pass/fail/skip via
codex exec --output-schema, and files dedup'd GitHub issues for true failures.

Layout:
- install.sh copies tools/k-skill-qa-bot/ to ~/.local/share/k-skill-qa-bot/
  and registers a LaunchAgent at ~/Library/LaunchAgents/.
- update-clone.sh has a hard guard: refuses any K_SKILL_CLONE outside
  K_QA_HOME/k-skill-clone unless ALLOW_EXTERNAL_CLONE_TARGET=1.
- Force-skip 10 destructive/login-required skills (ktx-booking, srt-booking,
  catchtable-sniper, kakaotalk-mac, hipass-receipt, toss-securities, etc.)
  so the bot never triggers reservation abuse.
- Deprecated skills (strike-through + 지원 중단 in README) auto-detected
  and skipped, never failed.
- First-run safety: CREATE_ISSUES=false by default.
- mkdir-based concurrency lock with atomic stale reclaim.
- Issue dedup: sha1(skill_name + symptom_class)[:12] body marker.
- Deterministic gates override LLM judge to FAIL on exit_code != 0, missing
  VERDICT line, or near-timeout duration.
2026-05-17 18:24:11 +09:00

215 lines
7 KiB
Python
Executable file

#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import os
import re
import sys
from pathlib import Path
_HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(_HERE / "lib"))
import qa_utils # type: ignore # noqa: E402
LOCATION_REQUIRED = {
"blue-ribbon-nearby", "cheap-gas-nearby", "kakao-bar-nearby",
"public-restroom-nearby", "parking-lot-search", "fine-dust-location",
"daangn-cars-search", "daangn-jobs-search", "daangn-realty-search",
"daangn-used-goods-search", "donation-place-search",
"korean-transit-route", "delivery-tracking",
}
LOGIN_REQUIRED = {
"catchtable-sniper", "kakaotalk-mac", "hipass-receipt", "toss-securities",
"iros-registry-automation", "ktx-booking", "srt-booking",
"foresttrip-vacancy",
}
DESTRUCTIVE = {
"ktx-booking", "srt-booking", "express-bus-booking",
"intercity-bus-booking", "catchtable-sniper", "foresttrip-vacancy",
}
API_KEY_ENV_BY_SKILL = {
"k-dart": "API_K_DART",
"korean-patent-search": "KIPRIS_API_KEY",
"korean-transit-route": "ODSAY_API_KEY",
"korean-stock-search": "KRX_API_KEY",
"kosis-stats": "KOSIS_API_KEY",
}
PROXY_DEPENDENT = {
"blue-ribbon-nearby", "cheap-gas-nearby", "daangn-cars-search",
"daangn-jobs-search", "daangn-realty-search", "daangn-used-goods-search",
"daishin-report-search", "donation-place-search", "fine-dust-location",
"gangnamunni-clinic-search", "gongsijiga-search", "han-river-water-level",
"household-waste-info", "k-schoollunch-menu", "kbl-results", "kbo-results",
"kleague-results", "korea-weather", "korean-marathon-schedule",
"korean-stock-search", "korean-transit-route", "kosis-stats",
"lh-notice-search", "library-book-search", "mfds-drug-safety",
"mfds-food-safety", "naver-news-search", "naver-shopping-search",
"nts-business-registration", "seoul-density", "seoul-subway-arrival",
"toss-securities",
}
_API_VAR_RE = re.compile(r"\b(API_[A-Z][A-Z0-9_]+)\b")
def _is_read_only(flags: dict) -> bool:
return not (
flags["login"]
or flags["destructive"]
or flags["api_key"]
or flags["location"]
)
def _read_skill_md(md_path):
if not md_path:
return ""
p = Path(md_path)
if not p.is_file():
return ""
try:
return p.read_text(encoding="utf-8-sig", errors="replace")
except OSError:
return ""
def classify(entry: dict, overrides: dict, deprecated: set) -> dict:
name = entry.get("name") or ""
flags = {
"location": name in LOCATION_REQUIRED,
"login": name in LOGIN_REQUIRED,
"destructive": name in DESTRUCTIVE,
"api_key": name in API_KEY_ENV_BY_SKILL,
"proxy_dependent": name in PROXY_DEPENDENT,
"read_only": False,
}
env_required = []
if name in API_KEY_ENV_BY_SKILL:
env_required.append(API_KEY_ENV_BY_SKILL[name])
md_text = _read_skill_md(entry.get("skill_md_path"))
if "k-skill-proxy" in md_text:
flags["proxy_dependent"] = True
for m in _API_VAR_RE.finditer(md_text):
env_required.append(m.group(1))
flags["api_key"] = True
env_required = sorted(set(env_required))
flags["read_only"] = _is_read_only(flags)
skip_reason = None
override_applied = False
if name in deprecated:
skip_reason = "deprecated in README"
override_applied = True
else:
ov = overrides.get(name) if isinstance(overrides, dict) else None
if isinstance(ov, dict):
if isinstance(ov.get("category_override"), dict):
for k, v in ov["category_override"].items():
if k in flags:
flags[k] = bool(v)
flags["read_only"] = _is_read_only(flags)
override_applied = True
extra_env = ov.get("env_required")
if isinstance(extra_env, list) and extra_env:
env_required = sorted(set(env_required) | {str(e) for e in extra_env})
flags["api_key"] = True
if ov.get("force_skip"):
skip_reason = str(ov.get("reason") or "force_skip override")
override_applied = True
if skip_reason is None and (flags["login"] or flags["destructive"]):
skip_reason = "requires user login or executes destructive actions"
if skip_reason is None and flags["api_key"]:
missing = [v for v in env_required if not os.environ.get(v)]
if missing:
skip_reason = f"missing required env: {', '.join(missing)}"
description = ""
fm = entry.get("frontmatter")
if isinstance(fm, dict):
d = fm.get("description")
if isinstance(d, str):
description = d
when_to_use = entry.get("when_to_use") or []
default_inputs = {}
ov = overrides.get(name) if isinstance(overrides, dict) else None
if isinstance(ov, dict) and isinstance(ov.get("default_inputs"), dict):
default_inputs = ov["default_inputs"]
prompt = qa_utils.synthesize_test_prompt(
name=name,
when_to_use=when_to_use,
description=description,
category_flags=flags,
default_inputs=default_inputs,
)
return {
"name": name,
"category_flags": flags,
"env_required": env_required,
"default_test_prompt": prompt,
"skip_reason": skip_reason,
"override_applied": override_applied,
}
def _default_overrides_path() -> Path:
home = os.environ.get("K_QA_HOME")
if home:
p = Path(home) / "config" / "skill-overrides.yml"
if p.is_file():
return p
return _HERE.parent / "config" / "skill-overrides.yml"
def _default_readme_path() -> Path:
clone = os.environ.get("K_SKILL_CLONE")
if clone:
p = Path(clone) / "README.md"
if p.is_file():
return p
return Path("README.md")
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description="Classify a k-skill manifest entry.")
ap.add_argument("--overrides", type=Path, default=None)
ap.add_argument("--readme", type=Path, default=None)
args = ap.parse_args(argv)
overrides_path = args.overrides or _default_overrides_path()
readme_path = args.readme or _default_readme_path()
raw = sys.stdin.read()
try:
entry = json.loads(raw)
except json.JSONDecodeError as exc:
print(f"classify-skill.py: invalid JSON on stdin: {exc}", file=sys.stderr)
return 2
if not isinstance(entry, dict):
print("classify-skill.py: stdin JSON must be an object", file=sys.stderr)
return 2
try:
overrides = qa_utils.load_overrides(overrides_path)
except RuntimeError as exc:
print(f"classify-skill.py: {exc}", file=sys.stderr)
return 2
deprecated = qa_utils.parse_readme_deprecations(readme_path)
result = classify(entry, overrides, deprecated)
json.dump(result, sys.stdout, ensure_ascii=False)
sys.stdout.write("\n")
return 0
if __name__ == "__main__":
sys.exit(main())