検索UIをYouTube風PIPレイアウトに再設計、AI自律再生(play_video)、意図判別強化、動画切替時の自動反応、NDJSONストリーミングで状態表示

This commit is contained in:
tmk3ki 2026-07-22 10:26:25 +09:00
parent e5d55f59ef
commit aff02e99a8
3 changed files with 305 additions and 123 deletions

160
gemini.go
View file

@ -17,6 +17,9 @@ const (
modelGemini = "gemini-flash-latest"
searchFuncName = "search_youtube"
playFuncName = "play_video"
maxToolIterations = 4
)
func selectModel(unlockCode, unlockSecret string) string {
@ -41,26 +44,44 @@ func buildSystemInstruction(videoTitle, videoDescription, persona string) *genai
prompt := fmt.Sprintf(
"あなたはユーザーと一緒に動画を見ている友達です。タメ口で短く、実況や相槌のように反応してください。\n"+
"性格: %s\n"+
"今見ている動画:\nタイトル: %s\n概要: %s\n"+
"ユーザーが「次の動画」「こういうの見たい」「〇〇のやつ見せて」のように次に見る動画を探してほしそうな時は"+
"%s関数でYouTubeを検索し、見つかったものから軽くおすすめして。",
tone, videoTitle, videoDescription, searchFuncName,
"今見ている動画:\nタイトル: %s\n概要: %s\n\n"+
"動画検索・再生ツールの使い分けルール:\n"+
"- ユーザーが今の動画の内容について話したり感想を言っているだけの時は、%sも%sも呼ばない。ただ会話する。\n"+
"- ユーザーが「次の動画」「こういうの見たい」「〇〇のやつ見せて」のように次に見る動画を探してほしそうな時だけ%sを呼ぶ。\n"+
"- %sの結果に対してユーザーがまだ何も選んでいないなら、%sは呼ばずに候補を紹介して選んでもらう。\n"+
"- ユーザーが「任せる」「なんでもいいから」「それにして」「1個目で」のように選択をAIに委ねている、または検索結果から明確にどれかを選んだ時だけ%sを呼んで良い。\n"+
"- 判断に迷う時は再生せず、必ず候補を提示するだけにする(誤って動画を切り替えない方を優先)。",
tone, videoTitle, videoDescription,
searchFuncName, playFuncName, searchFuncName, searchFuncName, playFuncName, playFuncName,
)
return genai.NewContentFromText(prompt, genai.RoleUser)
}
func searchTool() *genai.Tool {
return &genai.Tool{
FunctionDeclarations: []*genai.FunctionDeclaration{
{
Name: searchFuncName,
Description: "ユーザーが次に見たい動画・関連動画・代替の動画を探すためにYouTubeを検索する。",
Parameters: &genai.Schema{
Type: genai.TypeObject,
Properties: map[string]*genai.Schema{
"query": {Type: genai.TypeString, Description: "YouTube検索キーワード"},
func chatTools() []*genai.Tool {
return []*genai.Tool{
{
FunctionDeclarations: []*genai.FunctionDeclaration{
{
Name: searchFuncName,
Description: "ユーザーが次に見たい動画・関連動画・代替の動画を探すためにYouTubeを検索し、候補を提示する。まだ動画は切り替えない。",
Parameters: &genai.Schema{
Type: genai.TypeObject,
Properties: map[string]*genai.Schema{
"query": {Type: genai.TypeString, Description: "YouTube検索キーワード"},
},
Required: []string{"query"},
},
},
{
Name: playFuncName,
Description: "検索結果や会話から選ばれた動画に実際に切り替えて再生する。ユーザーが選択をAIに委ねた時、または明確にどれかを選んだ時のみ使う。",
Parameters: &genai.Schema{
Type: genai.TypeObject,
Properties: map[string]*genai.Schema{
"videoId": {Type: genai.TypeString, Description: "再生するYouTube動画ID"},
},
Required: []string{"videoId"},
},
Required: []string{"query"},
},
},
},
@ -68,40 +89,13 @@ func searchTool() *genai.Tool {
}
type chatOutcome struct {
Reply string
Videos []searchResult
Model string
Reply string
Videos []searchResult
AutoPlayVideoID string
Model string
}
func chatReply(ctx context.Context, client *genai.Client, model, ytAPIKey, videoTitle, videoDescription, persona string, history []chatTurn, message string) (*chatOutcome, error) {
contents := make([]*genai.Content, 0, len(history)+1)
for _, h := range history {
contents = append(contents, genai.NewContentFromText(h.Text, genai.Role(h.Role)))
}
contents = append(contents, genai.NewContentFromText(message, genai.RoleUser))
config := &genai.GenerateContentConfig{
SystemInstruction: buildSystemInstruction(videoTitle, videoDescription, persona),
Tools: []*genai.Tool{searchTool()},
}
resp, err := client.Models.GenerateContent(ctx, model, contents, config)
if err != nil {
return nil, err
}
calls := resp.FunctionCalls()
if len(calls) == 0 {
return &chatOutcome{Reply: resp.Text(), Model: model}, nil
}
call := calls[0]
query, _ := call.Args["query"].(string)
results, err := searchVideos(query, ytAPIKey)
if err != nil {
results = nil
}
func searchResultsToFuncResponse(results []searchResult) map[string]any {
items := make([]map[string]any, len(results))
for i, r := range results {
items[i] = map[string]any{
@ -110,16 +104,64 @@ func chatReply(ctx context.Context, client *genai.Client, model, ytAPIKey, video
"channelTitle": r.ChannelTitle,
}
}
if len(resp.Candidates) > 0 {
contents = append(contents, resp.Candidates[0].Content)
}
contents = append(contents, genai.NewContentFromFunctionResponse(call.Name, map[string]any{"results": items}, genai.RoleUser))
resp2, err := client.Models.GenerateContent(ctx, model, contents, config)
if err != nil {
return nil, err
}
return &chatOutcome{Reply: resp2.Text(), Videos: results, Model: model}, nil
return map[string]any{"results": items}
}
func chatReply(ctx context.Context, client *genai.Client, model, ytAPIKey, videoTitle, videoDescription, persona string, history []chatTurn, message string, onStatus func(string)) (*chatOutcome, error) {
contents := make([]*genai.Content, 0, len(history)+1)
for _, h := range history {
contents = append(contents, genai.NewContentFromText(h.Text, genai.Role(h.Role)))
}
contents = append(contents, genai.NewContentFromText(message, genai.RoleUser))
config := &genai.GenerateContentConfig{
SystemInstruction: buildSystemInstruction(videoTitle, videoDescription, persona),
Tools: chatTools(),
}
outcome := &chatOutcome{Model: model}
for i := 0; i < maxToolIterations; i++ {
resp, err := client.Models.GenerateContent(ctx, model, contents, config)
if err != nil {
return nil, err
}
calls := resp.FunctionCalls()
if len(calls) == 0 {
outcome.Reply = resp.Text()
return outcome, nil
}
if len(resp.Candidates) > 0 {
contents = append(contents, resp.Candidates[0].Content)
}
for _, call := range calls {
switch call.Name {
case searchFuncName:
if onStatus != nil {
onStatus("🔍 探してるよ…")
}
query, _ := call.Args["query"].(string)
results, err := searchVideos(query, ytAPIKey)
if err != nil {
results = nil
}
outcome.Videos = results
contents = append(contents, genai.NewContentFromFunctionResponse(call.Name, searchResultsToFuncResponse(results), genai.RoleUser))
case playFuncName:
if onStatus != nil {
onStatus("▶ 切り替えてるよ…")
}
videoID, _ := call.Args["videoId"].(string)
outcome.AutoPlayVideoID = videoID
contents = append(contents, genai.NewContentFromFunctionResponse(call.Name, map[string]any{"ok": true, "videoId": videoID}, genai.RoleUser))
default:
contents = append(contents, genai.NewContentFromFunctionResponse(call.Name, map[string]any{"ok": false}, genai.RoleUser))
}
}
}
return nil, fmt.Errorf("tool call iteration limit reached")
}

37
main.go
View file

@ -85,10 +85,13 @@ type chatRequest struct {
Persona string `json:"persona"`
}
type chatResponse struct {
Reply string `json:"reply"`
Model string `json:"model"`
Videos []searchResult `json:"videos,omitempty"`
type chatStreamEvent struct {
Type string `json:"type"` // "status" | "final" | "error"
Text string `json:"text,omitempty"`
Reply string `json:"reply,omitempty"`
Model string `json:"model,omitempty"`
Videos []searchResult `json:"videos,omitempty"`
AutoPlayVideoID string `json:"autoPlayVideoId,omitempty"`
}
func handleChat(w http.ResponseWriter, r *http.Request) {
@ -102,15 +105,33 @@ func handleChat(w http.ResponseWriter, r *http.Request) {
return
}
w.Header().Set("Content-Type", "application/x-ndjson")
flusher, _ := w.(http.Flusher)
encoder := json.NewEncoder(w)
writeEvent := func(ev chatStreamEvent) {
encoder.Encode(ev)
if flusher != nil {
flusher.Flush()
}
}
model := selectModel(req.UnlockCode, os.Getenv("UNLOCK_CODE"))
outcome, err := chatReply(r.Context(), geminiClient, model, os.Getenv("YOUTUBE_API_KEY"), req.VideoTitle, req.VideoDescription, req.Persona, req.History, req.Message)
outcome, err := chatReply(r.Context(), geminiClient, model, os.Getenv("YOUTUBE_API_KEY"), req.VideoTitle, req.VideoDescription, req.Persona, req.History, req.Message, func(status string) {
writeEvent(chatStreamEvent{Type: "status", Text: status})
})
if err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
writeEvent(chatStreamEvent{Type: "error", Text: err.Error()})
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(chatResponse{Reply: outcome.Reply, Model: outcome.Model, Videos: outcome.Videos})
writeEvent(chatStreamEvent{
Type: "final",
Reply: outcome.Reply,
Model: outcome.Model,
Videos: outcome.Videos,
AutoPlayVideoID: outcome.AutoPlayVideoID,
})
}
type unlockRequest struct {

View file

@ -79,31 +79,45 @@
}
.iconBtn:hover { filter: brightness(1.08); }
.searchPanel {
display: none; position: absolute; top: 62px; left: 12px; z-index: 20;
width: min(420px, calc(100% - 24px)); max-height: 420px; overflow-y: auto;
background: var(--surface-2); border: 1px solid var(--outline); border-radius: var(--radius-md);
box-shadow: var(--elevation-3); padding: 6px;
}
.searchPanel.show { display: block; }
.ytRow { display: flex; gap: 10px; padding: 8px; border-radius: 10px; cursor: pointer; align-items: flex-start; }
.ytRow { display: flex; gap: 10px; padding: 8px; border-radius: 10px; cursor: pointer; align-items: flex-start; white-space: normal; }
.ytRow:hover { background: var(--surface-3); }
.ytRow img { width: 120px; aspect-ratio: 16/9; object-fit: cover; border-radius: 8px; background: #000; flex-shrink: 0; }
.ytRow .meta { min-width: 0; }
.ytRow .title { font-size: 13px; line-height: 1.4; display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; }
.ytRow .channel { font-size: 11.5px; color: var(--on-surface-dim); margin-top: 4px; }
.ytRow .title { font-size: 13px; line-height: 1.4; display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; white-space: normal; }
.ytRow .channel { font-size: 11.5px; color: var(--on-surface-dim); margin-top: 4px; white-space: normal; }
.inlineSearch { margin-top: 6px; background: var(--surface-2); border-radius: var(--radius-md); padding: 4px; }
.inlineSearch { margin-top: 6px; background: var(--surface-2); border-radius: var(--radius-md); padding: 4px; white-space: normal; }
.inlineSearch .ytRow img { width: 90px; }
.inlineSearch .ytRow .title { font-size: 12px; }
.playerStage { flex: 1; display: flex; align-items: center; justify-content: center; min-height: 0; }
.playerStage { flex: 1; display: flex; align-items: center; justify-content: center; min-height: 0; position: relative; }
.playerWrap {
position: relative; width: 100%; max-width: 1100px; aspect-ratio: 16/9;
background: #000; border-radius: var(--radius-md); overflow: hidden; box-shadow: var(--elevation-2);
transition: width .25s ease, height .25s ease, right .25s ease, bottom .25s ease, box-shadow .25s ease;
}
.playerWrap.pip {
position: fixed; right: 20px; bottom: 20px; width: 260px; height: 146px; max-width: none;
z-index: 40; box-shadow: var(--elevation-3); border: 1px solid var(--outline); cursor: pointer;
}
#player { position: absolute; inset: 0; width: 100%; height: 100%; }
.searchStage {
display: none; flex-direction: column; width: 100%; height: 100%;
background: var(--surface-2); border-radius: var(--radius-md); overflow: hidden; box-shadow: var(--elevation-1);
}
.searchStage.show { display: flex; }
.searchStage .searchHeader {
display: flex; align-items: center; gap: 8px; padding: 10px 14px; border-bottom: 1px solid var(--outline); flex-shrink: 0;
}
.searchStage .searchHeader button {
background: none; border: none; color: var(--on-surface); cursor: pointer;
display: flex; align-items: center; justify-content: center; border-radius: 50%; width: 32px; height: 32px;
}
.searchStage .searchHeader button:hover { background: var(--surface-3); }
.searchStage .searchHeader .label { font-size: 13px; color: var(--on-surface-dim); }
.searchStage .searchList { flex: 1; overflow-y: auto; padding: 8px; }
.chatPane {
display: flex; flex-direction: column; min-width: 0;
background: var(--surface); border-left: 1px solid var(--outline);
@ -128,12 +142,23 @@
.bubble { padding: 10px 14px; border-radius: var(--radius-md); max-width: 92%; white-space: pre-wrap; font-size: 14px; line-height: 1.5; }
.bubble.user { align-self: flex-end; background: var(--primary); color: var(--on-primary); border-bottom-right-radius: 4px; }
.bubble.model { align-self: flex-start; background: var(--surface-3); border-bottom-left-radius: 4px; max-width: 100%; }
.bubble.status { display: flex; align-items: center; gap: 8px; }
.bubble .statusText { font-size: 13px; color: var(--on-surface-dim); }
.bubble .dots { display: inline-flex; gap: 3px; }
.bubble .dots .dot {
width: 6px; height: 6px; border-radius: 50%; background: var(--on-surface-dim);
animation: dotBlink 1.2s infinite ease-in-out;
}
.bubble .dots .dot:nth-child(2) { animation-delay: .2s; }
.bubble .dots .dot:nth-child(3) { animation-delay: .4s; }
@keyframes dotBlink { 0%, 80%, 100% { opacity: .2; } 40% { opacity: 1; } }
#chatForm { padding: 12px 16px 16px; border-top: 1px solid var(--outline); }
@media (max-width: 900px) {
.app { grid-template-columns: 1fr; grid-template-rows: 1fr 340px; }
.chatPane { border-left: none; border-top: 1px solid var(--outline); }
.playerWrap.pip { width: 180px; height: 101px; right: 12px; bottom: 12px; }
}
</style>
</head>
@ -147,9 +172,15 @@
<input id="urlInput" type="text" placeholder="YouTubeのURL・動画ID・検索キーワード">
<button id="loadBtn" class="iconBtn" title="読み込み/検索"><span class="material-symbols-outlined">search</span></button>
</div>
<div id="searchPanel" class="searchPanel"></div>
<div class="playerStage">
<div class="playerWrap"><div id="player"></div></div>
<div id="searchStage" class="searchStage">
<div class="searchHeader">
<button id="searchCancelBtn" title="動画に戻る"><span class="material-symbols-outlined">arrow_back</span></button>
<span class="label">検索結果 - 動画をクリックして再生</span>
</div>
<div id="searchList" class="searchList"></div>
</div>
<div id="playerWrap" class="playerWrap"><div id="player"></div></div>
</div>
</div>
@ -230,30 +261,72 @@ async function loadVideoMeta(id) {
}
}
function hideSearchPanel() {
const box = document.getElementById('searchPanel');
box.classList.remove('show');
box.innerHTML = '';
function setPip(active) {
document.getElementById('playerWrap').classList.toggle('pip', active);
}
function loadVideo(id) {
function showSearchStage(items) {
const list = document.getElementById('searchList');
list.innerHTML = '';
for (const item of (items || [])) {
list.appendChild(buildYtRow(item));
}
document.getElementById('searchStage').classList.add('show');
setPip(true);
}
function exitSearchMode() {
document.getElementById('searchStage').classList.remove('show');
setPip(false);
}
document.getElementById('playerWrap').addEventListener('click', () => {
if (document.getElementById('playerWrap').classList.contains('pip')) {
exitSearchMode();
}
});
document.getElementById('searchCancelBtn').addEventListener('click', exitSearchMode);
function loadVideo(id, opts) {
opts = opts || {};
const notifyAI = opts.notifyAI !== false;
if (player && player.loadVideoById) {
player.loadVideoById(id);
}
loadVideoMeta(id);
hideSearchPanel();
exitSearchMode();
document.getElementById('urlInput').value = id;
loadVideoMeta(id).then(() => {
if (notifyAI) {
sendChatMessage('(視聴中の動画が切り替わりました。新しい動画について一言リアクションして)', { showUserBubble: false });
}
});
}
const entityDecoder = document.createElement('textarea');
function decodeEntities(text) {
entityDecoder.innerHTML = text || '';
return entityDecoder.value;
}
function buildYtRow(item) {
const row = document.createElement('div');
row.className = 'ytRow';
row.innerHTML = `
<img src="${item.thumbnail}" alt="">
<div class="meta">
<div class="title">${item.title}</div>
<div class="channel">${item.channelTitle}</div>
</div>`;
const title = document.createElement('div');
title.className = 'title';
title.textContent = decodeEntities(item.title);
const channel = document.createElement('div');
channel.className = 'channel';
channel.textContent = decodeEntities(item.channelTitle);
const meta = document.createElement('div');
meta.className = 'meta';
meta.appendChild(title);
meta.appendChild(channel);
const img = document.createElement('img');
img.src = item.thumbnail;
img.alt = '';
row.appendChild(img);
row.appendChild(meta);
row.addEventListener('click', () => loadVideo(item.videoId));
return row;
}
@ -263,12 +336,7 @@ async function runSearch(query) {
const res = await fetch('/api/search?q=' + encodeURIComponent(query));
if (!res.ok) throw new Error('search failed');
const items = await res.json();
const box = document.getElementById('searchPanel');
box.innerHTML = '';
for (const item of (items || [])) {
box.appendChild(buildYtRow(item));
}
box.classList.add('show');
showSearchStage(items);
} catch (e) {
console.error('search error', e);
}
@ -289,13 +357,6 @@ document.getElementById('urlInput').addEventListener('keydown', (e) => {
if (e.key === 'Enter') document.getElementById('loadBtn').click();
});
document.addEventListener('click', (e) => {
const panel = document.getElementById('searchPanel');
if (!panel.contains(e.target) && e.target.id !== 'urlInput' && e.target.id !== 'loadBtn') {
hideSearchPanel();
}
});
const modelSelect = document.getElementById('modelSelect');
const lockIcon = document.getElementById('lockIcon');
modelSelect.addEventListener('change', () => {
@ -333,10 +394,31 @@ document.getElementById('unlockBtn').addEventListener('click', async () => {
}
});
function appendBubble(role, text, videos) {
function appendBubble(role, text) {
const div = document.createElement('div');
div.className = 'bubble ' + role;
div.textContent = text;
document.getElementById('chatLog').appendChild(div);
scrollChatToBottom();
return div;
}
function appendStatusBubble() {
const div = document.createElement('div');
div.className = 'bubble model status';
div.innerHTML = '<span class="statusText">考え中</span><span class="dots"><span class="dot"></span><span class="dot"></span><span class="dot"></span></span>';
document.getElementById('chatLog').appendChild(div);
scrollChatToBottom();
return div;
}
function setStatusText(div, text) {
div.querySelector('.statusText').textContent = text;
}
function finalizeBubble(div, text, videos) {
div.className = 'bubble model';
div.textContent = text;
if (videos && videos.length) {
const box = document.createElement('div');
box.className = 'inlineSearch';
@ -345,20 +427,22 @@ function appendBubble(role, text, videos) {
}
div.appendChild(box);
}
scrollChatToBottom();
}
function scrollChatToBottom() {
const log = document.getElementById('chatLog');
log.appendChild(div);
log.scrollTop = log.scrollHeight;
}
document.getElementById('chatForm').addEventListener('submit', async (e) => {
e.preventDefault();
const input = document.getElementById('chatInput');
const message = input.value.trim();
if (!message) return;
input.value = '';
appendBubble('user', message);
async function sendChatMessage(message, opts) {
opts = opts || {};
if (opts.showUserBubble !== false) {
appendBubble('user', message);
}
const statusDiv = appendStatusBubble();
const wantsGemini = modelSelect.value === 'gemini';
try {
const res = await fetch('/api/chat', {
method: 'POST',
@ -371,14 +455,49 @@ document.getElementById('chatForm').addEventListener('submit', async (e) => {
persona: document.getElementById('personaSelect').value,
}),
});
if (!res.ok) throw new Error('chat failed: ' + res.status);
const data = await res.json();
appendBubble('model', data.reply, data.videos);
chatHistory.push({ role: 'user', text: message });
chatHistory.push({ role: 'model', text: data.reply });
if (!res.ok || !res.body) throw new Error('chat failed: ' + res.status);
const reader = res.body.getReader();
const decoder = new TextDecoder();
let buf = '';
while (true) {
const { done, value } = await reader.read();
if (done) break;
buf += decoder.decode(value, { stream: true });
const lines = buf.split('\n');
buf = lines.pop();
for (const line of lines) {
if (!line.trim()) continue;
const evt = JSON.parse(line);
if (evt.type === 'status') {
setStatusText(statusDiv, evt.text);
} else if (evt.type === 'final') {
finalizeBubble(statusDiv, evt.reply, evt.videos);
chatHistory.push({ role: 'user', text: message });
chatHistory.push({ role: 'model', text: evt.reply });
if (evt.videos && evt.videos.length) {
setPip(true);
}
if (evt.autoPlayVideoId) {
loadVideo(evt.autoPlayVideoId, { notifyAI: false });
}
} else if (evt.type === 'error') {
finalizeBubble(statusDiv, '(エラー: ' + evt.text + ')');
}
}
}
} catch (err) {
appendBubble('model', '(エラー: ' + err.message + ')');
finalizeBubble(statusDiv, '(エラー: ' + err.message + ')');
}
}
document.getElementById('chatForm').addEventListener('submit', (e) => {
e.preventDefault();
const input = document.getElementById('chatInput');
const message = input.value.trim();
if (!message) return;
input.value = '';
sendChatMessage(message);
});
</script>
</body>