"""π0.5 の推論の動画を Gemini Decisions API に判定させ、人の判定と照らし合わせる(お試し)。 使い方: cd ~/r2r2r-scripts/decisions_test && python3 judge_videos.py 入力: manifest.csv(人の判定)と videos/{run_stem}.mp4(640px・2fps に縮めた写し) 出力: results.jsonl(1本ずつの生の答え)と summary.txt(合った割合) """ import base64, csv, getpass, json, os, time, urllib.error, urllib.request import getpass, os, pathlib KEY_FILE = pathlib.Path.home() / ".config" / "p2m" / "gemini_api_key" key = os.environ.get("GEMINI_API_KEY", "") if not (key and key.isascii()) and KEY_FILE.exists(): key = KEY_FILE.read_text().strip() if not (key and key.isascii()): key = getpass.getpass("Gemini の API キーを貼って Enter(画面には出ません。次回からは聞きません): ").strip() KEY_FILE.parent.mkdir(parents=True, exist_ok=True) KEY_FILE.write_text(key); KEY_FILE.chmod(0o600) QUESTIONS = { "grasped": {"type": "noul", "instructions": "This is a top-down video of a robot arm. At any point in the video, does the gripper grasp the red block and lift it off the mat?"}, "in_box_at_end": {"type": "noul", "instructions": "At the end of the video, is the red block inside the white box?"}, "outcome": {"type": "choice", "instructions": "The robot's task is: put the red block in the white box. What best describes the result of this attempt?", "criteria": { "success": "The red block ends up inside the white box.", "failed_grasp": "The robot never manages to grasp and lift the red block.", "grasped_not_placed": "The robot lifts the red block but it does not end up inside the white box.", }}, } def ask(path): body = {"model": "gemini-decision-1", "input": [{"type": "video", "mime_type": "video/mp4", "resolution": "low", "data": base64.b64encode(open(path, "rb").read()).decode()}], "questions": QUESTIONS} req = urllib.request.Request("https://generativelanguage.googleapis.com/v1beta/decisions", data=json.dumps(body).encode(), method="POST", headers={"x-goog-api-key": key, "Content-Type": "application/json"}) t = time.time() with urllib.request.urlopen(req, timeout=180) as r: res = json.load(r) return res, round(1000 * (time.time() - t)) rows = list(csv.DictReader(open("manifest.csv"))) out = open("results.jsonl", "a") lines = [] for i, row in enumerate(rows, 1): s = row["run_stem"] path = f"videos/{s}.mp4" if not os.path.exists(path): print(f"[{i}] {s}: 動画なし(飛ばす)"); continue try: res, ms = ask(path) except urllib.error.HTTPError as e: msg = e.read().decode()[:500] print(f"[{i}] {s}: HTTP {e.code} {msg}") out.write(json.dumps({"run_stem": s, "error": e.code, "msg": msg}, ensure_ascii=False) + "\n"); out.flush() if i == 1: break # 1本目で失敗したら形が違う。止める continue a = res["answers"] ai = a["outcome"]["choice"] ai_ok = "success" if ai == "success" else "failure" rec = {"run_stem": s, "human": row["human_outcome"], "human_pick": row["pick"], "human_place": row["place"], "ai_outcome": ai, "ai_conf": round(a["outcome"]["confidence"], 3), "p_grasped": round(a["grasped"]["noul"], 3), "p_in_box": round(a["in_box_at_end"]["noul"], 3), "match": ai_ok == row["human_outcome"], "ms": ms, "tokens": res["usage"]["total_input_tokens"], "raw": res} out.write(json.dumps(rec, ensure_ascii=False) + "\n"); out.flush() line = f"[{i}] {s} 人:{row['human_outcome']:7} AI:{ai:18} 確信{rec['ai_conf']:.2f} 箱の中{rec['p_in_box']:.2f} {'○' if rec['match'] else '×'} {ms}ms {rec['tokens']}tok" print(line); lines.append((rec, line)) if lines: n = len(lines); m = sum(r["match"] for r, _ in lines) toks = sum(r["tokens"] for r, _ in lines) summary = [f"合った割合: {m}/{n}", f"入力トークン合計: {toks}(約 ${toks/1e6*0.10:.4f})", ""] + [l for _, l in lines] open("summary.txt", "w").write("\n".join(summary) + "\n") print("\n".join(summary[:2]))