diff --git a/gemini.go b/gemini.go index 4e3ac8c..13db1ef 100644 --- a/gemini.go +++ b/gemini.go @@ -36,11 +36,16 @@ var personaTones = map[string]string{ "neutral": "落ち着いていて素直な性格。", } -func buildSystemInstruction(videoTitle, videoDescription, persona string) *genai.Content { +func personaTone(persona string) string { tone, ok := personaTones[persona] if !ok { - tone = personaTones["neutral"] + return personaTones["neutral"] } + return tone +} + +func buildSystemInstruction(videoTitle, videoDescription, persona string) *genai.Content { + tone := personaTone(persona) prompt := fmt.Sprintf( "あなたはユーザーと一緒に動画を見ている友達です。タメ口で短く、実況や相槌のように反応してください。\n"+ "性格: %s\n"+ @@ -165,3 +170,23 @@ func chatReply(ctx context.Context, client *genai.Client, model, ytAPIKey, video return nil, fmt.Errorf("tool call iteration limit reached") } + +func watchReaction(ctx context.Context, client *genai.Client, model string, clipBytes []byte, mimeType, videoTitle, videoDescription, persona string) (string, error) { + prompt := fmt.Sprintf( + "あなたはユーザーと一緒に動画を見ている友達です。今渡す数秒ぶんの映像・音声クリップを見て、"+ + "実況・相槌のような短い一言だけ返してください(1文程度、説明はしない、喋ってないなら無音を報告しなくていい)。\n"+ + "性格: %s\n動画タイトル: %s\n概要: %s", + personaTone(persona), videoTitle, videoDescription, + ) + contents := []*genai.Content{ + genai.NewContentFromParts([]*genai.Part{ + genai.NewPartFromText(prompt), + genai.NewPartFromBytes(clipBytes, mimeType), + }, genai.RoleUser), + } + resp, err := client.Models.GenerateContent(ctx, model, contents, nil) + if err != nil { + return "", err + } + return resp.Text(), nil +} diff --git a/main.go b/main.go index 04aa774..867e481 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "io" "log" "net/http" "os" @@ -38,6 +39,7 @@ func main() { http.HandleFunc("/api/search", handleSearch) http.HandleFunc("/api/chat", handleChat) http.HandleFunc("/api/unlock", handleUnlock) + http.HandleFunc("/api/watch", handleWatch) http.Handle("/", http.FileServer(http.Dir("web"))) log.Printf("listening on :%s", port) @@ -134,6 +136,48 @@ func handleChat(w http.ResponseWriter, r *http.Request) { }) } +type watchResponse struct { + Reaction string `json:"reaction"` +} + +func handleWatch(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + if err := r.ParseMultipartForm(32 << 20); err != nil { + http.Error(w, "bad multipart form", http.StatusBadRequest) + return + } + file, header, err := r.FormFile("clip") + if err != nil { + http.Error(w, "missing clip", http.StatusBadRequest) + return + } + defer file.Close() + + clipBytes, err := io.ReadAll(file) + if err != nil { + http.Error(w, "failed to read clip", http.StatusBadRequest) + return + } + + mimeType := header.Header.Get("Content-Type") + if mimeType == "" { + mimeType = "video/webm" + } + + model := selectModel(r.FormValue("unlockCode"), os.Getenv("UNLOCK_CODE")) + reaction, err := watchReaction(r.Context(), geminiClient, model, clipBytes, mimeType, r.FormValue("videoTitle"), r.FormValue("videoDescription"), r.FormValue("persona")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(watchResponse{Reaction: reaction}) +} + type unlockRequest struct { Code string `json:"code"` } diff --git a/web/index.html b/web/index.html index 464de71..53c3df6 100644 --- a/web/index.html +++ b/web/index.html @@ -78,6 +78,9 @@ display: flex; align-items: center; justify-content: center; } .iconBtn:hover { filter: brightness(1.08); } + .iconBtn.recording { background: #f28b82; color: #410002; animation: recPulse 1.5s infinite; } + .iconBtn.locked { background: var(--surface-3); color: var(--on-surface-dim); } + @keyframes recPulse { 0%, 100% { box-shadow: 0 0 0 0 rgba(242,139,130,.5); } 50% { box-shadow: 0 0 0 6px rgba(242,139,130,0); } } .ytRow { display: flex; gap: 10px; padding: 8px; border-radius: 10px; cursor: pointer; align-items: flex-start; white-space: normal; } .ytRow:hover { background: var(--surface-3); } @@ -171,6 +174,7 @@
smart_toyAIbow
+
@@ -387,6 +391,7 @@ modelSelect.addEventListener('change', () => { } lockIcon.textContent = (modelSelect.value === 'gemini' && unlockCode) ? 'lock_open' : (modelSelect.value === 'gemini' ? 'lock' : 'lock_open'); lockIcon.classList.toggle('unlocked', modelSelect.value === 'gemini' && !!unlockCode); + updateWatchButton(); }); document.getElementById('unlockInput').addEventListener('keydown', (e) => { @@ -408,6 +413,7 @@ document.getElementById('unlockBtn').addEventListener('click', async () => { document.getElementById('unlockRow').classList.remove('show'); lockIcon.textContent = 'lock_open'; lockIcon.classList.add('unlocked'); + updateWatchButton(); } else { alert('コードが違います'); modelSelect.value = 'gemma'; @@ -520,6 +526,119 @@ document.getElementById('chatForm').addEventListener('submit', (e) => { input.value = ''; sendChatMessage(message); }); + +const WATCH_SEGMENT_MS = 8000; +let watchStream = null; +let watchRecorder = null; +let watchActive = false; +let watchTimer = null; + +function updateWatchButton() { + const btn = document.getElementById('watchToggleBtn'); + const icon = btn.querySelector('.material-symbols-outlined'); + const available = modelSelect.value === 'gemini' && !!unlockCode; + btn.classList.toggle('recording', watchActive); + btn.classList.toggle('locked', !available && !watchActive); + icon.textContent = watchActive ? 'stop_screen_share' : 'screen_share'; + btn.title = watchActive + ? '実況モードを終了' + : (available ? '実況モード(画面共有)' : '実況モードはGemini解放後に使えます(Gemmaは音声非対応)'); +} + +function pickRecorderMimeType() { + const candidates = ['video/webm;codecs=vp9,opus', 'video/webm;codecs=vp8,opus', 'video/webm']; + for (const type of candidates) { + if (window.MediaRecorder && MediaRecorder.isTypeSupported(type)) return type; + } + return ''; +} + +function recordWatchSegment() { + if (!watchActive || !watchStream) return; + const chunks = []; + const mimeType = pickRecorderMimeType(); + let recorder; + try { + recorder = mimeType ? new MediaRecorder(watchStream, { mimeType }) : new MediaRecorder(watchStream); + } catch (e) { + console.error('MediaRecorder init failed', e); + stopWatchMode(); + return; + } + watchRecorder = recorder; + recorder.ondataavailable = (e) => { if (e.data && e.data.size > 0) chunks.push(e.data); }; + recorder.onstop = () => { + if (chunks.length > 0) { + sendWatchClip(new Blob(chunks, { type: mimeType || 'video/webm' })); + } + if (watchActive) recordWatchSegment(); + }; + recorder.start(); + watchTimer = setTimeout(() => { + if (recorder.state !== 'inactive') recorder.stop(); + }, WATCH_SEGMENT_MS); +} + +async function sendWatchClip(blob) { + const form = new FormData(); + form.append('clip', blob, 'clip.webm'); + form.append('videoTitle', videoTitle); + form.append('videoDescription', videoDescription); + form.append('persona', document.getElementById('personaSelect').value); + form.append('unlockCode', modelSelect.value === 'gemini' ? unlockCode : ''); + try { + const res = await fetch('/api/watch', { method: 'POST', body: form }); + if (!res.ok) return; + const data = await res.json(); + const reaction = (data.reaction || '').trim(); + if (reaction) { + appendBubble('model', reaction); + chatHistory.push({ role: 'model', text: reaction }); + } + } catch (e) { + console.error('watch clip error', e); + } +} + +async function startWatchMode() { + if (!(modelSelect.value === 'gemini' && unlockCode)) { + alert('実況モードは音声理解が必要なのでGeminiを解放してから使ってね(Gemmaは音声非対応)'); + return; + } + try { + watchStream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: true }); + } catch (e) { + console.error('getDisplayMedia failed', e); + return; + } + watchActive = true; + updateWatchButton(); + watchStream.getVideoTracks()[0].addEventListener('ended', stopWatchMode); + recordWatchSegment(); +} + +function stopWatchMode() { + watchActive = false; + clearTimeout(watchTimer); + if (watchRecorder && watchRecorder.state !== 'inactive') { + watchRecorder.stop(); + } + if (watchStream) { + watchStream.getTracks().forEach((t) => t.stop()); + watchStream = null; + } + updateWatchButton(); +} + +document.getElementById('watchToggleBtn').addEventListener('click', () => { + if (watchActive) { + stopWatchMode(); + } else { + startWatchMode(); + } +}); + +updateWatchButton();