#!/usr/bin/env python3
# 夜间监督看门狗：检查并行主题的进度/产出/异常关键字。只读，不改任何工作树。
import json, os, re, subprocess, sys, time
from datetime import datetime, timezone, timedelta

ROOT = '/vol1/ai/projects/wyszz-new'
TASKS = '/vol1/ai/projects/wyszz-new/frontend-rebuild-hm/tasks-out'
NOW = datetime.now(timezone(timedelta(hours=8)))
ERR_RE = re.compile(r'(?i)(\berror\b|错误|失败|崩溃|OOM|\b429\b|rate.?limit|额度不足|超时|timeout|截断|truncat|模型不可用|断线|killed|HTTP\s*5\d{2})')
OK_RE = re.compile(r'(?i)(全绿|通过|PASS|✓|✅)')
STALE_MIN = 25

WATCH = [
    ('S-24F', 'hm034',        f'{TASKS}/S-24F'),
    ('S-28',  'hm040-city',   f'{TASKS}/S-28'),
    ('S-29',  'hm041-city2',  f'{TASKS}/S-29'),
    ('S-25',  'hm035-island', f'{TASKS}/S-25'),
]

def age_min(p):
    if not os.path.exists(p):
        return None
    return (time.time() - os.path.getmtime(p)) / 60.0

def tail(p, n=40):
    try:
        with open(p, encoding='utf-8', errors='replace') as f:
            return ''.join(f.readlines()[-n:])
    except Exception:
        return ''

def latest(tdir):
    """最近改动的文件（工作树 + tasks-out），用于判断是否还在动。"""
    best, bp = 0.0, ''
    for base in (tdir,):
        for dp, dn, fn in os.walk(base):
            if 'node_modules' in dp or '/.git' in dp or '/dist' in dp:
                continue
            for f in fn:
                p = os.path.join(dp, f)
                try:
                    m = os.path.getmtime(p)
                except OSError:
                    continue
                if m > best:
                    best, bp = m, p
    return best, bp

def check(tag, tree, tdir):
    r = {'tag': tag, 'tree': tree, 'taskdir': tdir, 'issues': [], 'ok': True}
    prog = os.path.join(tdir, '进度.md')
    report = os.path.join(tdir, '总报告.md')
    payload = os.path.join(tdir, 'payload.tar.gz')
    r['progress_exists'] = os.path.exists(prog)
    r['progress_age_min'] = round(age_min(prog), 1) if os.path.exists(prog) else None
    r['report_exists'] = os.path.exists(report)
    r['payload_exists'] = os.path.exists(payload)
    best, bp = latest(tdir if os.path.isdir(tdir) else os.path.join(ROOT, tree))
    r['taskdir_last_write_min'] = round((time.time() - best) / 60.0, 1) if best else None
    r['last_file'] = bp.replace(ROOT + '/', '')
    # 工作树最近写入
    tdir_tree = os.path.join(ROOT, 'frontend-rebuild-' + tree)
    wbest, wbp = latest(tdir_tree) if os.path.isdir(tdir_tree) else (0.0, '')
    r['tree_last_write_min'] = round((time.time() - wbest) / 60.0, 1) if wbest else None
    # 异常关键字（只看最近 200 行进度 + 报告尾部）
    hits = []
    for p in (prog, report):
        if os.path.exists(p):
            for line in tail(p, 200).splitlines():
                if ERR_RE.search(line) and not OK_RE.search(line):
                    hits.append(line.strip()[:160])
    r['error_lines'] = hits[-5:]
    if hits[-3:]:
        r['issues'].append('进度/报告出现异常关键字：' + hits[-1][:100])
    # 判定
    finished = r['payload_exists'] or r['report_exists']
    if not finished:
        if r['progress_age_min'] is not None and r['progress_age_min'] > STALE_MIN:
            r['issues'].append(f'进度 {r["progress_age_min"]} 分钟没更新（>{STALE_MIN}）')
        if r['tree_last_write_min'] is not None and r['tree_last_write_min'] > STALE_MIN and r['taskdir_last_write_min'] and r['taskdir_last_write_min'] > STALE_MIN:
            r['issues'].append(f'工作树与任务目录都停了（树 {r["tree_last_write_min"]} 分钟 / 目录 {r["taskdir_last_write_min"]} 分钟）')
    r['finished'] = finished
    r['ok'] = not r['issues'] or finished
    return r

def main():
    send = '--send' in sys.argv
    as_json = '--json' in sys.argv
    rows = [check(*w) for w in WATCH]
    bad = [r for r in rows if r['issues']]
    if as_json:
        out = {'time': NOW.strftime('%F %T'), 'bad': [r['tag'] for r in bad], 'rows': rows}
        print(json.dumps(out, ensure_ascii=False, indent=1))
    else:
        print('# 夜间监督 ' + NOW.strftime('%F %H:%M') + '\n')
        print('| 任务 | 树 | 产出 | 进度龄(min) | 树写入龄(min) | 判定 |')
        print('|---|---|---|---|---|---|')
        for r in rows:
            verdict = '完成' if r['finished'] else ('异常' if r['issues'] else '进行中')
            print(f'| {r["tag"]} | {r["tree"]} | {"payload/报告✓" if r["finished"] else "-"} | {r["progress_age_min"]} | {r["tree_last_write_min"]} | {verdict} |')
        for r in bad:
            print(f'\n**{r["tag"]} 异常**')
            for i in r['issues']:
                print(f'- {i}')
            for e in r['error_lines']:
                print(f'  > {e}')
    if send:
        key = None
        envp = '/vol1/ai/agents/hermes/.env'
        if os.path.exists(envp):
            for line in open(envp, encoding='utf-8', errors='replace'):
                line = line.strip()
                if line.startswith('API_SERVER_KEY='):
                    key = line.split('=', 1)[1].strip().strip('"\'')
        if not key:
            print('!! 找不到 API_SERVER_KEY，跳过发送', file=sys.stderr)
        else:
            body = json.dumps({'model': 'hermes', 'stream': False, 'messages': [{'role': 'user', 'content': '【夜间看门狗】' + json.dumps([{'tag': r['tag'], 'finished': r['finished'], 'issues': r['issues']} for r in rows], ensure_ascii=False)}]})
            import urllib.request
            req = urllib.request.Request('http://127.0.0.1:8642/v1/chat/completions', data=body.encode(),
                                         headers={'Authorization': f'Bearer {key}', 'Content-Type': 'application/json',
                                                  'X-Hermes-Session-Id': 's-wyszz-overnight-watch'})
            try:
                with urllib.request.urlopen(req, timeout=120) as resp:
                    resp.read()
                print('已发送给 Hermes')
            except Exception as e:
                print(f'!! 发送失败：{e}', file=sys.stderr)
    sys.exit(1 if bad and not any(r['finished'] for r in bad) else 0)

if __name__ == '__main__':
    main()
