"""π0.5 の推論の記録を Gemini Decisions API で先回り判定する(2版)。 1版からの変更: 最後の1コマを高い解像度の画像として動画と一緒に送り、「最後もつかんだままか」を聞く。 使い方: python3 judge_v2.py manifest.csv … 人の判定がある19本で、直したことが効くか確かめる python3 judge_v2.py manifest_unlabeled.csv … 人の判定がまだない記録に先回り判定を付ける 出力: v2_{manifestの名前}.csv(1本1行)と v2_{manifestの名前}.jsonl(生の答え) 判定の表(pi05_policy_verdicts_v1.csv)には書かない。 """ import base64, csv, getpass, json, os, pathlib, sys, time, urllib.error, urllib.request KEY_FILE = pathlib.Path.home() / ".config" / "p2m" / "gemini_api_key" key = os.environ.get("GEMINI_API_KEY", "") if not (key and key.isascii()) and KEY_FILE.exists(): key = KEY_FILE.read_text().strip() if not (key and key.isascii()): key = getpass.getpass("Gemini の API キーを貼って Enter(画面には出ません。次回からは聞きません): ").strip() KEY_FILE.parent.mkdir(parents=True, exist_ok=True) KEY_FILE.write_text(key); KEY_FILE.chmod(0o600) THRESHOLD = 0.5 # これ未満は人が見る QUESTIONS = { "grasped": {"type": "noul", "instructions": "In the video, does the gripper grasp the red block and lift it off the mat at any point?"}, "holding_at_end": {"type": "noul", "instructions": "Look at the final frame image. Is the red block still held between the gripper fingers?"}, "in_box_at_end": {"type": "noul", "instructions": "Look at the final frame image. Is the red block resting inside the white box (not held by the gripper, not beside the box)?"}, "outcome": {"type": "choice", "instructions": "The robot's task is: put the red block in the white box. Judge only by what is visible at the end. What best describes the result?", "criteria": { "success": "At the end the red block is inside the white box and the gripper has released it.", "failed_grasp": "The robot never grasps and lifts the red block.", "grasped_not_placed": "The robot lifts the red block, but at the end it is not inside the white box (still held, dropped, or beside the box).", "no_motion": "The robot arm barely moves, or the video is too blurry to see what happens.", }}, } def b64(p): return base64.b64encode(open(p, "rb").read()).decode() def ask(stem): body = {"model": "gemini-decision-1", "input": [ {"type": "text", "text": "Top-down camera. First: the whole attempt as a video. Second: the final frame at high resolution."}, {"type": "video", "mime_type": "video/mp4", "resolution": "low", "data": b64(f"videos/{stem}.mp4")}, {"type": "image", "mime_type": "image/jpeg", "resolution": "high", "data": b64(f"last_frames/{stem}.jpg")}, ], "questions": QUESTIONS} req = urllib.request.Request("https://generativelanguage.googleapis.com/v1beta/decisions", data=json.dumps(body).encode(), method="POST", headers={"x-goog-api-key": key, "Content-Type": "application/json"}) t = time.time() with urllib.request.urlopen(req, timeout=180) as r: res = json.load(r) return res, round(1000 * (time.time() - t)) manifest = sys.argv[1] if len(sys.argv) > 1 else "manifest.csv" name = pathlib.Path(manifest).stem rows = list(csv.DictReader(open(manifest))) has_human = "human_outcome" in rows[0] raw = open(f"v2_{name}.jsonl", "w") out_rows = [] for i, row in enumerate(rows, 1): s = row["run_stem"] if not (os.path.exists(f"videos/{s}.mp4") and os.path.exists(f"last_frames/{s}.jpg")): print(f"[{i}] {s}: 動画か最後のコマがない(飛ばす)"); continue try: res, ms = ask(s) except urllib.error.HTTPError as e: msg = e.read().decode()[:500] print(f"[{i}] {s}: HTTP {e.code} {msg}") if i == 1: break continue a = res["answers"] ai = a["outcome"]["choice"]; conf = a["outcome"]["confidence"] rec = {"run_stem": s, "ai_outcome": ai, "ai_success_or_failure": "success" if ai == "success" else "failure", "ai_conf": round(conf, 3), "route": "auto" if conf >= THRESHOLD else "human", "p_grasped": round(a["grasped"]["noul"], 3), "p_holding_at_end": round(a["holding_at_end"]["noul"], 3), "p_in_box_at_end": round(a["in_box_at_end"]["noul"], 3), "ms": ms, "tokens": res["usage"]["total_input_tokens"]} if has_human: rec["human"] = row["human_outcome"]; rec["match"] = rec["ai_success_or_failure"] == row["human_outcome"] else: rec["previous_label"] = f"{row['latest_labeler']}:{row['latest_outcome']}" raw.write(json.dumps({**rec, "raw": res}, ensure_ascii=False) + "\n"); raw.flush() out_rows.append(rec) tail = (" " + ("○" if rec["match"] else "×") + f" 人:{rec['human']}") if has_human else f" 前の印:{rec['previous_label']}" print(f"[{i}] {s} AI:{ai:18} 確信{conf:.2f} 持ったまま{rec['p_holding_at_end']:.2f} 箱の中{rec['p_in_box_at_end']:.2f} → {'自動' if rec['route']=='auto' else '人へ'}{tail}") if out_rows: with open(f"v2_{name}.csv", "w", newline="") as f: w = csv.DictWriter(f, fieldnames=list(out_rows[0].keys())); w.writeheader(); w.writerows(out_rows) n = len(out_rows); toks = sum(r["tokens"] for r in out_rows) auto = [r for r in out_rows if r["route"] == "auto"] print(f"\n本数 {n} / 自動 {len(auto)} / 人へ {n-len(auto)} / 入力トークン {toks}(約 ${toks/1e6*0.10:.4f})") if has_human: print(f"合った割合: {sum(r['match'] for r in out_rows)}/{n} 自動のうち合った: {sum(r['match'] for r in auto)}/{len(auto)}")