diff --git a/gemini.go b/gemini.go index 13db1ef..d3e0e08 100644 --- a/gemini.go +++ b/gemini.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "fmt" "google.golang.org/genai" @@ -171,13 +172,26 @@ func chatReply(ctx context.Context, client *genai.Client, model, ytAPIKey, video return nil, fmt.Errorf("tool call iteration limit reached") } -func watchReaction(ctx context.Context, client *genai.Client, model string, clipBytes []byte, mimeType, videoTitle, videoDescription, persona string) (string, error) { +func watchReaction(ctx context.Context, client *genai.Client, model string, clipBytes []byte, mimeType, videoTitle, videoDescription, persona string, recentReactions []chatTurn) (string, error) { prompt := fmt.Sprintf( - "あなたはユーザーと一緒に動画を見ている友達です。今渡す数秒ぶんの映像・音声クリップを見て、"+ - "実況・相槌のような短い一言だけ返してください(1文程度、説明はしない、喋ってないなら無音を報告しなくていい)。\n"+ - "性格: %s\n動画タイトル: %s\n概要: %s", + "あなたはユーザーと一緒に動画を見ている友達です。今渡す数秒ぶんの映像・音声クリップを見て反応してください。\n"+ + "性格: %s\n動画タイトル: %s\n概要: %s\n\n"+ + "厳守ルール:\n"+ + "- 「楽しみだね」「いいね」「気になる」のような当たり障りのない薄い相槌・感想は絶対に言わない。\n"+ + "- 具体的に何が映った・話されたかに基づく一言のみ許可(1文、短く)。\n"+ + "- 特に語ることがない、映像に変化がない、さっきと同じ内容の繰り返しになる場合は、何も返さず空文字だけを返す。無理にコメントしなくていい。\n"+ + "- 毎回喋る必要はない。むしろ喋らない方が普通。\n"+ + "- 直近で自分が言ったこと(下記)と同じ/似た内容を繰り返さない。", personaTone(persona), videoTitle, videoDescription, ) + if len(recentReactions) > 0 { + prompt += "\n\n直近の自分の発言:\n" + for _, h := range recentReactions { + if h.Role == genai.RoleModel { + prompt += "- " + h.Text + "\n" + } + } + } contents := []*genai.Content{ genai.NewContentFromParts([]*genai.Part{ genai.NewPartFromText(prompt), @@ -190,3 +204,51 @@ func watchReaction(ctx context.Context, client *genai.Client, model string, clip } return resp.Text(), nil } + +type voiceOutcome struct { + Transcript string `json:"transcript"` + Reply string `json:"reply"` +} + +func voiceChat(ctx context.Context, client *genai.Client, model string, audioBytes []byte, mimeType, videoTitle, videoDescription, persona string, history []chatTurn) (*voiceOutcome, error) { + prompt := fmt.Sprintf( + "あなたはユーザーと一緒に動画を見ている友達です。ユーザーが今、音声で話しかけてきました。音声を聞いて、\n"+ + "1. transcript: 話した内容の文字起こし\n"+ + "2. reply: タメ口で短い、友達としての返答\n"+ + "の2つをJSONで返してください。\n"+ + "性格: %s\n今見ている動画:\nタイトル: %s\n概要: %s", + personaTone(persona), videoTitle, videoDescription, + ) + + contents := make([]*genai.Content, 0, len(history)+1) + for _, h := range history { + contents = append(contents, genai.NewContentFromText(h.Text, genai.Role(h.Role))) + } + contents = append(contents, genai.NewContentFromParts([]*genai.Part{ + genai.NewPartFromText(prompt), + genai.NewPartFromBytes(audioBytes, mimeType), + }, genai.RoleUser)) + + config := &genai.GenerateContentConfig{ + ResponseMIMEType: "application/json", + ResponseSchema: &genai.Schema{ + Type: genai.TypeObject, + Properties: map[string]*genai.Schema{ + "transcript": {Type: genai.TypeString}, + "reply": {Type: genai.TypeString}, + }, + Required: []string{"transcript", "reply"}, + }, + } + + resp, err := client.Models.GenerateContent(ctx, model, contents, config) + if err != nil { + return nil, err + } + + var out voiceOutcome + if err := json.Unmarshal([]byte(resp.Text()), &out); err != nil { + return nil, fmt.Errorf("failed to parse voice response: %w", err) + } + return &out, nil +} diff --git a/main.go b/main.go index 867e481..dce01f7 100644 --- a/main.go +++ b/main.go @@ -40,6 +40,7 @@ func main() { http.HandleFunc("/api/chat", handleChat) http.HandleFunc("/api/unlock", handleUnlock) http.HandleFunc("/api/watch", handleWatch) + http.HandleFunc("/api/voice-chat", handleVoiceChat) http.Handle("/", http.FileServer(http.Dir("web"))) log.Printf("listening on :%s", port) @@ -167,8 +168,13 @@ func handleWatch(w http.ResponseWriter, r *http.Request) { mimeType = "video/webm" } + var recentReactions []chatTurn + if raw := r.FormValue("history"); raw != "" { + json.Unmarshal([]byte(raw), &recentReactions) + } + model := selectModel(r.FormValue("unlockCode"), os.Getenv("UNLOCK_CODE")) - reaction, err := watchReaction(r.Context(), geminiClient, model, clipBytes, mimeType, r.FormValue("videoTitle"), r.FormValue("videoDescription"), r.FormValue("persona")) + reaction, err := watchReaction(r.Context(), geminiClient, model, clipBytes, mimeType, r.FormValue("videoTitle"), r.FormValue("videoDescription"), r.FormValue("persona"), recentReactions) if err != nil { http.Error(w, err.Error(), http.StatusBadGateway) return @@ -178,6 +184,55 @@ func handleWatch(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(watchResponse{Reaction: reaction}) } +type voiceChatResponse struct { + Transcript string `json:"transcript"` + Reply string `json:"reply"` + Model string `json:"model"` +} + +func handleVoiceChat(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + if err := r.ParseMultipartForm(16 << 20); err != nil { + http.Error(w, "bad multipart form", http.StatusBadRequest) + return + } + file, header, err := r.FormFile("audio") + if err != nil { + http.Error(w, "missing audio", http.StatusBadRequest) + return + } + defer file.Close() + + audioBytes, err := io.ReadAll(file) + if err != nil { + http.Error(w, "failed to read audio", http.StatusBadRequest) + return + } + + mimeType := header.Header.Get("Content-Type") + if mimeType == "" { + mimeType = "audio/webm" + } + + var history []chatTurn + if raw := r.FormValue("history"); raw != "" { + json.Unmarshal([]byte(raw), &history) + } + + model := selectModel(r.FormValue("unlockCode"), os.Getenv("UNLOCK_CODE")) + outcome, err := voiceChat(r.Context(), geminiClient, model, audioBytes, mimeType, r.FormValue("videoTitle"), r.FormValue("videoDescription"), r.FormValue("persona"), history) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(voiceChatResponse{Transcript: outcome.Transcript, Reply: outcome.Reply, Model: model}) +} + type unlockRequest struct { Code string `json:"code"` } diff --git a/web/index.html b/web/index.html index 66633ca..5c308c2 100644 --- a/web/index.html +++ b/web/index.html @@ -219,6 +219,7 @@
+
@@ -255,6 +256,22 @@ let videoDescription = ''; let chatHistory = []; let unlockCode = ''; +const MAX_HISTORY_TURNS = 24; +function pushHistory(role, text) { + chatHistory.push({ role, text }); + if (chatHistory.length > MAX_HISTORY_TURNS) { + chatHistory = chatHistory.slice(chatHistory.length - MAX_HISTORY_TURNS); + } +} + +const MAX_CHAT_BUBBLES = 40; +function trimChatLog() { + const log = document.getElementById('chatLog'); + while (log.children.length > MAX_CHAT_BUBBLES) { + log.removeChild(log.firstChild); + } +} + async function loadVideoMeta(id) { videoTitle = ''; videoDescription = ''; @@ -461,6 +478,7 @@ function finalizeBubble(div, text, videos) { } function scrollChatToBottom() { + trimChatLog(); const log = document.getElementById('chatLog'); log.scrollTop = log.scrollHeight; } @@ -503,8 +521,8 @@ async function sendChatMessage(message, opts) { setStatusText(statusDiv, evt.text); } else if (evt.type === 'final') { finalizeBubble(statusDiv, evt.reply, evt.videos); - chatHistory.push({ role: 'user', text: message }); - chatHistory.push({ role: 'model', text: evt.reply }); + pushHistory('user', message); + pushHistory('model', evt.reply); if (evt.autoPlayVideoId) { loadVideo(evt.autoPlayVideoId, { notifyAI: false }); } @@ -586,6 +604,7 @@ async function sendWatchClip(blob) { form.append('videoDescription', videoDescription); form.append('persona', document.getElementById('personaSelect').value); form.append('unlockCode', modelSelect.value === 'gemini' ? unlockCode : ''); + form.append('history', JSON.stringify(chatHistory.slice(-6))); try { const res = await fetch('/api/watch', { method: 'POST', body: form }); if (!res.ok) return; @@ -593,7 +612,7 @@ async function sendWatchClip(blob) { const reaction = (data.reaction || '').trim(); if (reaction) { appendBubble('model', reaction); - chatHistory.push({ role: 'model', text: reaction }); + pushHistory('model', reaction); } } catch (e) { console.error('watch clip error', e); @@ -645,6 +664,103 @@ document.getElementById('watchToggleBtn').addEventListener('click', () => { }); updateWatchButton(); + +let micStream = null; +let micRecorder = null; +let micChunks = []; +let micActive = false; + +function voiceAvailable() { + return modelSelect.value === 'gemini' && !!unlockCode; +} + +function micMimeType() { + const candidates = ['audio/webm;codecs=opus', 'audio/webm']; + for (const type of candidates) { + if (window.MediaRecorder && MediaRecorder.isTypeSupported(type)) return type; + } + return ''; +} + +async function startVoiceRecording() { + if (micActive) return; + if (!voiceAvailable()) { + alert('音声発言は音声理解が必要なのでGeminiを解放してから使ってね(Gemmaは音声非対応)'); + return; + } + try { + micStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + } catch (e) { + console.error('getUserMedia failed', e); + return; + } + micChunks = []; + const mimeType = micMimeType(); + try { + micRecorder = mimeType ? new MediaRecorder(micStream, { mimeType }) : new MediaRecorder(micStream); + } catch (e) { + console.error('MediaRecorder init failed', e); + micStream.getTracks().forEach((t) => t.stop()); + micStream = null; + return; + } + micRecorder.ondataavailable = (e) => { if (e.data && e.data.size > 0) micChunks.push(e.data); }; + micRecorder.onstop = () => { + micStream.getTracks().forEach((t) => t.stop()); + micStream = null; + if (micChunks.length > 0) { + sendVoiceMessage(new Blob(micChunks, { type: micRecorder.mimeType || 'audio/webm' })); + } + }; + micRecorder.start(); + micActive = true; + document.getElementById('micBtn').classList.add('recording'); +} + +function stopVoiceRecording() { + if (!micActive) return; + micActive = false; + document.getElementById('micBtn').classList.remove('recording'); + if (micRecorder && micRecorder.state !== 'inactive') { + micRecorder.stop(); + } +} + +async function sendVoiceMessage(blob) { + const statusDiv = appendStatusBubble(); + setStatusText(statusDiv, '🎤 聞き取り中…'); + const form = new FormData(); + form.append('audio', blob, 'voice.webm'); + form.append('videoTitle', videoTitle); + form.append('videoDescription', videoDescription); + form.append('persona', document.getElementById('personaSelect').value); + form.append('unlockCode', unlockCode); + form.append('history', JSON.stringify(chatHistory.slice(-12))); + try { + const res = await fetch('/api/voice-chat', { method: 'POST', body: form }); + if (!res.ok) throw new Error('voice chat failed: ' + res.status); + const data = await res.json(); + statusDiv.remove(); + const transcript = (data.transcript || '').trim(); + if (transcript) { + appendBubble('user', transcript); + pushHistory('user', transcript); + } + const reply = (data.reply || '').trim(); + if (reply) { + appendBubble('model', reply); + pushHistory('model', reply); + } + } catch (err) { + finalizeBubble(statusDiv, '(エラー: ' + err.message + ')'); + } +} + +const micBtn = document.getElementById('micBtn'); +micBtn.addEventListener('pointerdown', (e) => { e.preventDefault(); startVoiceRecording(); }); +micBtn.addEventListener('pointerup', stopVoiceRecording); +micBtn.addEventListener('pointerleave', stopVoiceRecording); +micBtn.addEventListener('pointercancel', stopVoiceRecording);