From 45456650d4b873d4d770db579e897b16377ca4c6 Mon Sep 17 00:00:00 2001 From: xRangerDE Date: Sun, 10 May 2026 20:40:16 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20VOD=20chat-replay=20download=20?= =?UTF-8?q?=E2=80=94=20keep=20the=20chat=20alongside=20the=20video?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twitch retains chat replay on the same VOD-lifetime clock — when the VOD vanishes after 7-60 days, the chat goes with it. Anyone archiving the video usually wants the chat too. Added an opt-in setting that saves a paginated GQL pull of the chat as a JSON file next to the .mp4 download. Server: - new fetchVodChatReplay(videoId, onProgress, cancelCheck) — uses the existing fetchPublicTwitchGql helper (so the retry-on-transient logic from cycle 4 applies here too) with the standard video.comments(contentOffsetSeconds, cursor) query, paginated via edge cursors. Each message is normalised to a small flat shape: id, offset (seconds-into-VOD), createdAt, user (display name), login, color, text (assembled from fragments). Hard-capped at 500 pages (~50k messages) so a single runaway stream can't fill memory; hitting the cap sets truncated:true in the result. Honours a cancelCheck() callback so removing the queue item also cancels the in-flight chat fetch. - new chatReplayPathFor() helper produces sibling .chat.json path. - processOneQueueItem fires the chat fetch after a successful, non- live, non-merge VOD download whose URL parses to a VOD id. Progress shows up in the queue item via existing download-progress IPC: "Fetching chat replay..." then "Chat messages fetched: N". Output file is added to item.outputFiles so the existing Open file / Show in folder UI lists the chat right next to the video. A failed chat fetch is logged but does NOT mark the queue item as failed — the video itself is fine, the chat is a bonus. - Atomic write via writeFileAtomicSync so a crash mid-fetch can't leave a half-written .chat.json next to the video. Renderer: - new download_chat_replay: boolean in Config (default false because long streams can take a few minutes of chat-page pulls and we don't want to surprise users on upgrade). Settings -> Download card gets the toggle with hint tooltip explaining the trade-off. - AppConfig type, settings autosave fingerprint, syncSettingsForm, applyLanguageToStaticUI all updated. - DE + EN labels and the two new backend status strings (statusFetchingChatReplay, statusChatMessagesFetched). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/index.html | 4 + src/main.ts | 171 +++++++++++++++++++++++++++++++++++++- src/renderer-globals.d.ts | 1 + src/renderer-locale-de.ts | 2 + src/renderer-locale-en.ts | 2 + src/renderer-settings.ts | 4 + src/renderer-texts.ts | 3 + 7 files changed, 185 insertions(+), 2 deletions(-) diff --git a/src/index.html b/src/index.html index 1904f1c..0352cae 100644 --- a/src/index.html +++ b/src/index.html @@ -527,6 +527,10 @@ Twitch-Ads beim Download ueberspringen +
diff --git a/src/main.ts b/src/main.ts index 59c1453..6c78634 100644 --- a/src/main.ts +++ b/src/main.ts @@ -119,6 +119,8 @@ const BACKEND_MESSAGES = { statusCheckingTools: 'Prufe Download-Tools...', statusDownloadStarted: 'Download gestartet', statusBytesDownloaded: '{bytes} heruntergeladen', + statusFetchingChatReplay: 'Chat-Replay wird heruntergeladen...', + statusChatMessagesFetched: 'Chat-Nachrichten geladen: {count}', preflightNoInternet: 'Keine Internetverbindung erkannt.', preflightStreamlinkMissing: 'Streamlink fehlt oder ist nicht startbar.', preflightFfmpegMissing: 'FFmpeg fehlt oder ist nicht startbar.', @@ -156,6 +158,8 @@ const BACKEND_MESSAGES = { statusCheckingTools: 'Checking download tools...', statusDownloadStarted: 'Download started', statusBytesDownloaded: '{bytes} downloaded', + statusFetchingChatReplay: 'Fetching chat replay...', + statusChatMessagesFetched: 'Chat messages fetched: {count}', preflightNoInternet: 'No internet connection detected.', preflightStreamlinkMissing: 'Streamlink is missing or not runnable.', preflightFfmpegMissing: 'FFmpeg is missing or not runnable.', @@ -210,6 +214,7 @@ interface Config { streamlink_disable_ads: boolean; auto_record_streamers: string[]; auto_record_poll_seconds: number; + download_chat_replay: boolean; } interface RuntimeMetrics { @@ -328,7 +333,8 @@ const defaultConfig: Config = { notify_on_each_completion: false, streamlink_disable_ads: true, auto_record_streamers: [], - auto_record_poll_seconds: 90 + auto_record_poll_seconds: 90, + download_chat_replay: false }; const AUTO_RECORD_POLL_MIN_SECONDS = 30; @@ -425,7 +431,8 @@ function normalizeConfigTemplates(input: Config): Config { // an explicit `false` from the loaded config. streamlink_disable_ads: input.streamlink_disable_ads !== false, auto_record_streamers: normalizeAutoRecordList(input.auto_record_streamers), - auto_record_poll_seconds: normalizeAutoRecordPollSeconds(input.auto_record_poll_seconds) + auto_record_poll_seconds: normalizeAutoRecordPollSeconds(input.auto_record_poll_seconds), + download_chat_replay: input.download_chat_replay === true }; } @@ -2966,6 +2973,104 @@ async function runAutoRecordPoll(): Promise { } } +// ========================================== +// CHAT REPLAY DOWNLOAD +// ========================================== +// Twitch retains chat replay alongside the VOD itself — same 7-60 day TTL. +// Anyone archiving the video usually wants the chat too. fetchVodChatReplay +// pulls the entire chat for a VOD via the public GQL endpoint, paginated +// via edge cursors (Twitch returns ~100 comments per page). +interface ChatReplayMessage { + id: string; + offset: number; // contentOffsetSeconds — when in the VOD + createdAt: string; // ISO timestamp + user: string; // display name + login: string; // login (lowercase) + color: string; // user chat color + text: string; // assembled message text +} + +interface ChatReplayResult { + messages: ChatReplayMessage[]; + truncated: boolean; + pages: number; +} + +async function fetchVodChatReplay( + videoId: string, + onProgress?: (count: number) => void, + cancelCheck?: () => boolean +): Promise { + const messages: ChatReplayMessage[] = []; + let cursor: string | null = null; + let pages = 0; + let truncated = false; + // Hard cap to keep one runaway stream from filling memory. 200 pages = + // ~20k messages which covers typical 6-hour streams. Above that we + // stop and mark truncated. + const MAX_PAGES = 500; + + type CommentNode = { + id: string; + contentOffsetSeconds: number; + createdAt: string; + message?: { fragments?: Array<{ text?: string }>; userColor?: string }; + commenter?: { displayName?: string; login?: string }; + }; + type CommentEdge = { node: CommentNode; cursor: string }; + type CommentsPage = { + video: { comments: { edges: CommentEdge[]; pageInfo: { hasNextPage: boolean } } } | null; + }; + + const query = 'query($videoID:ID!,$cursor:Cursor){video(id:$videoID){comments(contentOffsetSeconds:0,cursor:$cursor){edges{node{id contentOffsetSeconds createdAt message{fragments{text} userColor} commenter{displayName login}} cursor} pageInfo{hasNextPage}}}}'; + + while (pages < MAX_PAGES) { + if (cancelCheck && cancelCheck()) { + truncated = true; + break; + } + const data: CommentsPage | null = await fetchPublicTwitchGql(query, { + videoID: videoId, + cursor + }); + if (!data || !data.video || !data.video.comments) break; + + const edges: CommentEdge[] = Array.isArray(data.video.comments.edges) ? data.video.comments.edges : []; + for (const edge of edges) { + const node = edge.node; + const fragments = node.message?.fragments || []; + const text = fragments.map((f: { text?: string }) => (typeof f.text === 'string' ? f.text : '')).join(''); + messages.push({ + id: node.id, + offset: Number(node.contentOffsetSeconds) || 0, + createdAt: node.createdAt || '', + user: node.commenter?.displayName || '', + login: node.commenter?.login || '', + color: node.message?.userColor || '', + text + }); + } + + pages += 1; + if (onProgress) onProgress(messages.length); + + const last: CommentEdge | undefined = edges[edges.length - 1]; + if (!data.video.comments.pageInfo.hasNextPage || !last) break; + cursor = last.cursor; + } + + if (pages >= MAX_PAGES) truncated = true; + return { messages, truncated, pages }; +} + +function chatReplayPathFor(vodFilePath: string): string { + // Strip the final extension and append .chat.json so the chat file + // lives next to the video and is easy to find. + const ext = path.extname(vodFilePath); + const base = ext ? vodFilePath.slice(0, -ext.length) : vodFilePath; + return `${base}.chat.json`; +} + async function downloadLiveStream( item: QueueItem, onProgress: (progress: DownloadProgress) => void @@ -3630,6 +3735,68 @@ async function processOneQueueItem(item: QueueItem): Promise { const id = parseVodId(item.url); if (id) recordDownloadedVodId(id); } + + // Optional chat-replay download. Only for non-live, non-merge + // VODs that have a parseable VOD id and produced at least one + // output file. Saved as {video_basename}.chat.json next to the + // video. Truncation is logged but not fatal. + if (config.download_chat_replay && !item.isLive && !item.mergeGroup) { + const vodIdForChat = parseVodId(item.url); + const firstOutput = item.outputFiles?.[0]; + if (vodIdForChat && firstOutput) { + try { + mainWindow?.webContents.send('download-progress', { + id: item.id, + progress: 100, + speed: '', + eta: '', + status: tBackend('statusFetchingChatReplay'), + currentPart: 0, + totalParts: 0 + } as DownloadProgress); + + const replay = await fetchVodChatReplay(vodIdForChat, (count) => { + mainWindow?.webContents.send('download-progress', { + id: item.id, + progress: 100, + speed: '', + eta: '', + status: tBackend('statusChatMessagesFetched', { count: String(count) }), + currentPart: 0, + totalParts: 0 + } as DownloadProgress); + }, () => cancelledItemIds.has(item.id)); + + const chatPath = chatReplayPathFor(firstOutput); + const payload = { + videoId: vodIdForChat, + videoUrl: item.url, + streamer: item.streamer, + title: item.title, + fetchedAt: new Date().toISOString(), + messageCount: replay.messages.length, + truncated: replay.truncated, + pages: replay.pages, + messages: replay.messages + }; + writeFileAtomicSync(chatPath, JSON.stringify(payload, null, 2)); + appendDebugLog('chat-replay-saved', { + itemId: item.id, + videoId: vodIdForChat, + messages: replay.messages.length, + pages: replay.pages, + truncated: replay.truncated, + path: chatPath + }); + if (Array.isArray(item.outputFiles)) { + item.outputFiles = [...item.outputFiles, chatPath]; + } + } catch (e) { + // Non-fatal: video download still succeeded. + appendDebugLog('chat-replay-failed', { itemId: item.id, error: String(e) }); + } + } + } } if (finalResult.success) { diff --git a/src/renderer-globals.d.ts b/src/renderer-globals.d.ts index 0c1bfdc..78174d2 100644 --- a/src/renderer-globals.d.ts +++ b/src/renderer-globals.d.ts @@ -23,6 +23,7 @@ interface AppConfig { streamlink_disable_ads?: boolean; auto_record_streamers?: string[]; auto_record_poll_seconds?: number; + download_chat_replay?: boolean; [key: string]: unknown; } diff --git a/src/renderer-locale-de.ts b/src/renderer-locale-de.ts index f8697b7..487886b 100644 --- a/src/renderer-locale-de.ts +++ b/src/renderer-locale-de.ts @@ -73,6 +73,8 @@ const UI_TEXT_DE = { notifyEachCompletionHint: 'Standardmaessig aus — bei langen Queues wuerde das System-Notifications-Panel sonst zugespammt. Die Queue-End-Zusammenfassung erscheint trotzdem.', streamlinkDisableAdsLabel: 'Twitch-Ads beim Download ueberspringen', streamlinkDisableAdsHint: 'Gibt --twitch-disable-ads an streamlink weiter, damit Mid-Roll-Ads nicht ins VOD eingebettet werden. Empfohlen aktiv lassen.', + downloadChatReplayLabel: 'Chat-Replay parallel zum VOD speichern (.chat.json)', + downloadChatReplayHint: 'Nach erfolgreichem VOD-Download wird der oeffentliche Chat-Replay via Twitch GQL geholt und als JSON neben dem Video gespeichert. Twitch behaelt Chat-Replays nur solange wie das VOD selbst.', streamlinkQualityLabel: 'Stream-Qualitaet', streamlinkQualityHint: 'Streamlink versucht erst diese Qualitaet; falls das VOD sie nicht anbietet, faellt es auf "best" zurueck.', streamlinkQualityBest: 'Best (Standard)', diff --git a/src/renderer-locale-en.ts b/src/renderer-locale-en.ts index 2d46e16..fa290fa 100644 --- a/src/renderer-locale-en.ts +++ b/src/renderer-locale-en.ts @@ -73,6 +73,8 @@ const UI_TEXT_EN = { notifyEachCompletionHint: 'Off by default — long queues would otherwise spam the OS notifications panel. The end-of-queue summary notification fires either way.', streamlinkDisableAdsLabel: 'Skip Twitch ads while downloading', streamlinkDisableAdsHint: 'Passes --twitch-disable-ads to streamlink so mid-roll ads do not get embedded into the VOD output. Recommended on.', + downloadChatReplayLabel: 'Save chat replay alongside each VOD (.chat.json)', + downloadChatReplayHint: 'After a VOD download completes, fetches the public chat replay via Twitch GQL and saves it as JSON next to the video. Twitch keeps chat replay only as long as the VOD itself.', streamlinkQualityLabel: 'Stream quality', streamlinkQualityHint: 'Streamlink will try this quality first; if the VOD does not offer it, falls back to "best".', streamlinkQualityBest: 'Best (default)', diff --git a/src/renderer-settings.ts b/src/renderer-settings.ts index e64b4d3..4e60e5b 100644 --- a/src/renderer-settings.ts +++ b/src/renderer-settings.ts @@ -389,6 +389,7 @@ function collectDownloadSettingsPayload(): Partial { auto_resume_queue_on_startup: byId('autoResumeQueueToggle').checked, notify_on_each_completion: byId('notifyEachCompletionToggle').checked, streamlink_disable_ads: byId('streamlinkDisableAdsToggle').checked, + download_chat_replay: byId('downloadChatReplayToggle').checked, streamlink_quality: byId('streamlinkQuality').value, metadata_cache_minutes: parseInt(byId('metadataCacheMinutes').value, 10) || 10 }; @@ -435,6 +436,7 @@ function getSettingsFingerprint(payload: Partial): string { effective.auto_resume_queue_on_startup === true, effective.notify_on_each_completion === true, effective.streamlink_disable_ads !== false, + effective.download_chat_replay === true, effective.streamlink_quality ?? 'best', effective.metadata_cache_minutes ?? 10, effective.filename_template_vod ?? '{title}.mp4', @@ -456,6 +458,7 @@ function syncSettingsFormFromConfig(): void { byId('autoResumeQueueToggle').checked = (config.auto_resume_queue_on_startup as boolean) === true; byId('notifyEachCompletionToggle').checked = (config.notify_on_each_completion as boolean) === true; byId('streamlinkDisableAdsToggle').checked = (config.streamlink_disable_ads as boolean) !== false; + byId('downloadChatReplayToggle').checked = (config.download_chat_replay as boolean) === true; byId('streamlinkQuality').value = (config.streamlink_quality as string) || 'best'; byId('metadataCacheMinutes').value = String((config.metadata_cache_minutes as number) || 10); byId('vodFilenameTemplate').value = (config.filename_template_vod as string) || '{title}.mp4'; @@ -571,6 +574,7 @@ function initSettingsAutoSave(): void { 'autoResumeQueueToggle', 'notifyEachCompletionToggle', 'streamlinkDisableAdsToggle', + 'downloadChatReplayToggle', 'streamlinkQuality' ] as const; diff --git a/src/renderer-texts.ts b/src/renderer-texts.ts index 02bbb3b..584555e 100644 --- a/src/renderer-texts.ts +++ b/src/renderer-texts.ts @@ -123,6 +123,9 @@ function applyLanguageToStaticUI(): void { setText('streamlinkDisableAdsLabel', UI_TEXT.static.streamlinkDisableAdsLabel); setTitle('streamlinkDisableAdsLabel', UI_TEXT.static.streamlinkDisableAdsHint); setTitle('streamlinkDisableAdsToggle', UI_TEXT.static.streamlinkDisableAdsHint); + setText('downloadChatReplayLabel', UI_TEXT.static.downloadChatReplayLabel); + setTitle('downloadChatReplayLabel', UI_TEXT.static.downloadChatReplayHint); + setTitle('downloadChatReplayToggle', UI_TEXT.static.downloadChatReplayHint); setText('streamlinkQualityLabel', UI_TEXT.static.streamlinkQualityLabel); setTitle('streamlinkQualityLabel', UI_TEXT.static.streamlinkQualityHint); setTitle('streamlinkQuality', UI_TEXT.static.streamlinkQualityHint);