実況モード実装: getDisplayMediaで画面共有し数秒ごとのクリップをGemini/Gemmaへ送信、反応をチャットに表示。Gemmaは音声入力非対応と判明したためGemini解放時のみ利用可能に制限

gemini.go

@@ -36,11 +36,16 @@ var personaTones = map[string]string{
 	"neutral":   "落ち着いていて素直な性格。",
 }
 
-func buildSystemInstruction(videoTitle, videoDescription, persona string) *genai.Content {
+func personaTone(persona string) string {
 	tone, ok := personaTones[persona]
 	if !ok {
-		tone = personaTones["neutral"]
+		return personaTones["neutral"]
 	}
+	return tone
+}
+
+func buildSystemInstruction(videoTitle, videoDescription, persona string) *genai.Content {
+	tone := personaTone(persona)
 	prompt := fmt.Sprintf(
 		"あなたはユーザーと一緒に動画を見ている友達です。タメ口で短く、実況や相槌のように反応してください。\n"+
 			"性格: %s\n"+
@@ -165,3 +170,23 @@ func chatReply(ctx context.Context, client *genai.Client, model, ytAPIKey, video
 
 	return nil, fmt.Errorf("tool call iteration limit reached")
 }
+
+func watchReaction(ctx context.Context, client *genai.Client, model string, clipBytes []byte, mimeType, videoTitle, videoDescription, persona string) (string, error) {
+	prompt := fmt.Sprintf(
+		"あなたはユーザーと一緒に動画を見ている友達です。今渡す数秒ぶんの映像・音声クリップを見て、"+
+			"実況・相槌のような短い一言だけ返してください(1文程度、説明はしない、喋ってないなら無音を報告しなくていい)。\n"+
+			"性格: %s\n動画タイトル: %s\n概要: %s",
+		personaTone(persona), videoTitle, videoDescription,
+	)
+	contents := []*genai.Content{
+		genai.NewContentFromParts([]*genai.Part{
+			genai.NewPartFromText(prompt),
+			genai.NewPartFromBytes(clipBytes, mimeType),
+		}, genai.RoleUser),
+	}
+	resp, err := client.Models.GenerateContent(ctx, model, contents, nil)
+	if err != nil {
+		return "", err
+	}
+	return resp.Text(), nil
+}

main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"io"
 	"log"
 	"net/http"
 	"os"
@@ -38,6 +39,7 @@ func main() {
 	http.HandleFunc("/api/search", handleSearch)
 	http.HandleFunc("/api/chat", handleChat)
 	http.HandleFunc("/api/unlock", handleUnlock)
+	http.HandleFunc("/api/watch", handleWatch)
 	http.Handle("/", http.FileServer(http.Dir("web")))
 
 	log.Printf("listening on :%s", port)
@@ -134,6 +136,48 @@ func handleChat(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
+type watchResponse struct {
+	Reaction string `json:"reaction"`
+}
+
+func handleWatch(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if err := r.ParseMultipartForm(32 << 20); err != nil {
+		http.Error(w, "bad multipart form", http.StatusBadRequest)
+		return
+	}
+	file, header, err := r.FormFile("clip")
+	if err != nil {
+		http.Error(w, "missing clip", http.StatusBadRequest)
+		return
+	}
+	defer file.Close()
+
+	clipBytes, err := io.ReadAll(file)
+	if err != nil {
+		http.Error(w, "failed to read clip", http.StatusBadRequest)
+		return
+	}
+
+	mimeType := header.Header.Get("Content-Type")
+	if mimeType == "" {
+		mimeType = "video/webm"
+	}
+
+	model := selectModel(r.FormValue("unlockCode"), os.Getenv("UNLOCK_CODE"))
+	reaction, err := watchReaction(r.Context(), geminiClient, model, clipBytes, mimeType, r.FormValue("videoTitle"), r.FormValue("videoDescription"), r.FormValue("persona"))
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadGateway)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(watchResponse{Reaction: reaction})
+}
+
 type unlockRequest struct {
 	Code string `json:"code"`
 }

web/index.html

@@ -78,6 +78,9 @@
     display: flex; align-items: center; justify-content: center;
   }
   .iconBtn:hover { filter: brightness(1.08); }
+  .iconBtn.recording { background: #f28b82; color: #410002; animation: recPulse 1.5s infinite; }
+  .iconBtn.locked { background: var(--surface-3); color: var(--on-surface-dim); }
+  @keyframes recPulse { 0%, 100% { box-shadow: 0 0 0 0 rgba(242,139,130,.5); } 50% { box-shadow: 0 0 0 6px rgba(242,139,130,0); } }
 
   .ytRow { display: flex; gap: 10px; padding: 8px; border-radius: 10px; cursor: pointer; align-items: flex-start; white-space: normal; }
   .ytRow:hover { background: var(--surface-3); }
@@ -171,6 +174,7 @@
   <div class="videoPane">
     <div class="topBar">
       <div class="brand"><span class="material-symbols-outlined">smart_toy</span>AIbow</div>
+      <button id="watchToggleBtn" class="iconBtn" title="実況モード(画面共有)"><span class="material-symbols-outlined">screen_share</span></button>
     </div>
     <div class="pillInput">
       <input id="urlInput" type="text" placeholder="YouTubeのURL・動画ID・検索キーワード">
@@ -387,6 +391,7 @@ modelSelect.addEventListener('change', () => {
   }
   lockIcon.textContent = (modelSelect.value === 'gemini' && unlockCode) ? 'lock_open' : (modelSelect.value === 'gemini' ? 'lock' : 'lock_open');
   lockIcon.classList.toggle('unlocked', modelSelect.value === 'gemini' && !!unlockCode);
+  updateWatchButton();
 });
 
 document.getElementById('unlockInput').addEventListener('keydown', (e) => {
@@ -408,6 +413,7 @@ document.getElementById('unlockBtn').addEventListener('click', async () => {
       document.getElementById('unlockRow').classList.remove('show');
       lockIcon.textContent = 'lock_open';
       lockIcon.classList.add('unlocked');
+      updateWatchButton();
     } else {
       alert('コードが違います');
       modelSelect.value = 'gemma';
@@ -520,6 +526,119 @@ document.getElementById('chatForm').addEventListener('submit', (e) => {
   input.value = '';
   sendChatMessage(message);
 });
+
+const WATCH_SEGMENT_MS = 8000;
+let watchStream = null;
+let watchRecorder = null;
+let watchActive = false;
+let watchTimer = null;
+
+function updateWatchButton() {
+  const btn = document.getElementById('watchToggleBtn');
+  const icon = btn.querySelector('.material-symbols-outlined');
+  const available = modelSelect.value === 'gemini' && !!unlockCode;
+  btn.classList.toggle('recording', watchActive);
+  btn.classList.toggle('locked', !available && !watchActive);
+  icon.textContent = watchActive ? 'stop_screen_share' : 'screen_share';
+  btn.title = watchActive
+    ? '実況モードを終了'
+    : (available ? '実況モード(画面共有)' : '実況モードはGemini解放後に使えます(Gemmaは音声非対応)');
+}
+
+function pickRecorderMimeType() {
+  const candidates = ['video/webm;codecs=vp9,opus', 'video/webm;codecs=vp8,opus', 'video/webm'];
+  for (const type of candidates) {
+    if (window.MediaRecorder && MediaRecorder.isTypeSupported(type)) return type;
+  }
+  return '';
+}
+
+function recordWatchSegment() {
+  if (!watchActive || !watchStream) return;
+  const chunks = [];
+  const mimeType = pickRecorderMimeType();
+  let recorder;
+  try {
+    recorder = mimeType ? new MediaRecorder(watchStream, { mimeType }) : new MediaRecorder(watchStream);
+  } catch (e) {
+    console.error('MediaRecorder init failed', e);
+    stopWatchMode();
+    return;
+  }
+  watchRecorder = recorder;
+  recorder.ondataavailable = (e) => { if (e.data && e.data.size > 0) chunks.push(e.data); };
+  recorder.onstop = () => {
+    if (chunks.length > 0) {
+      sendWatchClip(new Blob(chunks, { type: mimeType || 'video/webm' }));
+    }
+    if (watchActive) recordWatchSegment();
+  };
+  recorder.start();
+  watchTimer = setTimeout(() => {
+    if (recorder.state !== 'inactive') recorder.stop();
+  }, WATCH_SEGMENT_MS);
+}
+
+async function sendWatchClip(blob) {
+  const form = new FormData();
+  form.append('clip', blob, 'clip.webm');
+  form.append('videoTitle', videoTitle);
+  form.append('videoDescription', videoDescription);
+  form.append('persona', document.getElementById('personaSelect').value);
+  form.append('unlockCode', modelSelect.value === 'gemini' ? unlockCode : '');
+  try {
+    const res = await fetch('/api/watch', { method: 'POST', body: form });
+    if (!res.ok) return;
+    const data = await res.json();
+    const reaction = (data.reaction || '').trim();
+    if (reaction) {
+      appendBubble('model', reaction);
+      chatHistory.push({ role: 'model', text: reaction });
+    }
+  } catch (e) {
+    console.error('watch clip error', e);
+  }
+}
+
+async function startWatchMode() {
+  if (!(modelSelect.value === 'gemini' && unlockCode)) {
+    alert('実況モードは音声理解が必要なのでGeminiを解放してから使ってね(Gemmaは音声非対応)');
+    return;
+  }
+  try {
+    watchStream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: true });
+  } catch (e) {
+    console.error('getDisplayMedia failed', e);
+    return;
+  }
+  watchActive = true;
+  updateWatchButton();
+  watchStream.getVideoTracks()[0].addEventListener('ended', stopWatchMode);
+  recordWatchSegment();
+}
+
+function stopWatchMode() {
+  watchActive = false;
+  clearTimeout(watchTimer);
+  if (watchRecorder && watchRecorder.state !== 'inactive') {
+    watchRecorder.stop();
+  }
+  if (watchStream) {
+    watchStream.getTracks().forEach((t) => t.stop());
+    watchStream = null;
+  }
+  updateWatchButton();
+}
+
+document.getElementById('watchToggleBtn').addEventListener('click', () => {
+  if (watchActive) {
+    stopWatchMode();
+  } else {
+    startWatchMode();
+  }
+});
+
+updateWatchButton();
 </script>
 </body>
 </html>