From 09ce80357f9683d7386ab52f339fd2c51d30d522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jim=20Liu=20=E5=AE=9D=E7=8E=89?= Date: Tue, 24 Mar 2026 20:59:04 -0500 Subject: [PATCH] feat(baoyu-youtube-transcript): add yt-dlp fallback and modularize codebase Retry with alternate InnerTube client identities when YouTube returns anti-bot responses, then fall back to yt-dlp when available. Split monolithic main.ts into typed modules (youtube, transcript, storage, shared, types) and add unit tests. --- skills/baoyu-youtube-transcript/SKILL.md | 11 +- .../scripts/main.test.ts | 125 +++ .../baoyu-youtube-transcript/scripts/main.ts | 792 ++---------------- .../scripts/shared.ts | 83 ++ .../scripts/storage.ts | 60 ++ .../scripts/transcript.ts | 349 ++++++++ .../baoyu-youtube-transcript/scripts/types.ts | 123 +++ .../scripts/youtube.ts | 477 +++++++++++ 8 files changed, 1320 insertions(+), 700 deletions(-) create mode 100644 skills/baoyu-youtube-transcript/scripts/main.test.ts create mode 100644 skills/baoyu-youtube-transcript/scripts/shared.ts create mode 100644 skills/baoyu-youtube-transcript/scripts/storage.ts create mode 100644 skills/baoyu-youtube-transcript/scripts/transcript.ts create mode 100644 skills/baoyu-youtube-transcript/scripts/types.ts create mode 100644 skills/baoyu-youtube-transcript/scripts/youtube.ts diff --git a/skills/baoyu-youtube-transcript/SKILL.md b/skills/baoyu-youtube-transcript/SKILL.md index c37b4cb..35a0564 100644 --- a/skills/baoyu-youtube-transcript/SKILL.md +++ b/skills/baoyu-youtube-transcript/SKILL.md @@ -13,7 +13,7 @@ metadata: # YouTube Transcript -Downloads transcripts (subtitles/captions) from YouTube videos. Works with both manually created and auto-generated transcripts. No API key or browser required — uses YouTube's InnerTube API directly. +Downloads transcripts (subtitles/captions) from YouTube videos. Works with both manually created and auto-generated transcripts. No API key or browser required — uses YouTube's InnerTube API directly and automatically falls back to `yt-dlp` when YouTube blocks the direct API path. Fetches video metadata and cover image on first run, caches raw data for fast re-formatting. @@ -75,6 +75,12 @@ ${BUN_X} {baseDir}/scripts/main.ts --refresh | `-o, --output ` | Save to specific file path | auto-generated | | `--output-dir ` | Base output directory | `youtube-transcript` | +## Optional Environment Variables + +| Variable | Description | +|----------|-------------| +| `YOUTUBE_TRANSCRIPT_COOKIES_FROM_BROWSER` | Passed to `yt-dlp --cookies-from-browser` during fallback, e.g. `chrome`, `safari`, `firefox`, or `chrome:Profile 1` | + ## Input Formats Accepts any of these as video input: @@ -121,6 +127,8 @@ On first fetch, the script saves: Subsequent runs for the same video use cached data (no network calls). Use `--refresh` to force re-fetch. If a different language is requested, the cache is automatically refreshed. +When YouTube returns anti-bot / blocked responses on the direct InnerTube path, the script retries with alternate client identities and then falls back to `yt-dlp` if available. If fallback is needed but `yt-dlp` is unavailable, the agent should decide how to make `yt-dlp` available and continue rather than pushing the installation decision to the user. + SRT output (`--format srt`) is generated from `transcript-raw.json`. Text/markdown output uses `transcript-sentences.json` for natural sentence boundaries. ## Workflow @@ -175,3 +183,4 @@ When `--speakers` is used, `--chapters` is implied — the processed output alwa | Video unavailable | Video deleted, private, or region-locked | | IP blocked | Too many requests, try again later | | Age restricted | Video requires login for age verification | +| bot detected | The script retries alternate clients and then `yt-dlp`; if fallback tooling is missing, the agent should resolve that itself, otherwise if it still fails try `YOUTUBE_TRANSCRIPT_COOKIES_FROM_BROWSER=safari` (or your browser) | diff --git a/skills/baoyu-youtube-transcript/scripts/main.test.ts b/skills/baoyu-youtube-transcript/scripts/main.test.ts new file mode 100644 index 0000000..e3b3df7 --- /dev/null +++ b/skills/baoyu-youtube-transcript/scripts/main.test.ts @@ -0,0 +1,125 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { findTranscript, parseTranscriptJson3, parseWebVtt } from "./transcript.ts"; +import { buildTranscriptListFromYtDlp, resolveVideoSource, selectYtDlpTrack } from "./youtube.ts"; + +test("selectYtDlpTrack prefers json3 over xml and vtt", () => { + const track = selectYtDlpTrack([ + { ext: "vtt", url: "https://example.com/subs.vtt" }, + { ext: "srv3", url: "https://example.com/subs.srv3" }, + { ext: "json3", url: "https://example.com/subs.json3" }, + ]); + + assert.equal(track?.ext, "json3"); +}); + +test("buildTranscriptListFromYtDlp keeps manual and generated tracks separate", () => { + const transcripts = buildTranscriptListFromYtDlp({ + subtitles: { + en: [ + { ext: "json3", url: "https://example.com/en.json3", name: "English" }, + ], + }, + automatic_captions: { + "zh-Hans": [ + { ext: "json3", url: "https://example.com/zh.json3", name: "Chinese (Simplified)" }, + ], + }, + }); + + assert.equal(transcripts.length, 2); + assert.equal(transcripts[0].isGenerated, false); + assert.equal(transcripts[1].isGenerated, true); + assert.equal(transcripts[0].translationLanguages[0]?.languageCode, "zh-Hans"); + + const translated = findTranscript(transcripts, ["zh-Hans"], false, false); + assert.equal(translated.languageCode, "zh-Hans"); + assert.equal(translated.isGenerated, true); +}); + +test("parseTranscriptJson3 reads youtube timedtext json3 payloads", () => { + const snippets = parseTranscriptJson3(JSON.stringify({ + events: [ + { + tStartMs: 80, + dDurationMs: 3120, + segs: [{ utf8: "hello\nworld" }], + }, + { + tStartMs: 4000, + dDurationMs: 1800, + segs: [{ utf8: "again" }], + }, + ], + })); + + assert.deepEqual(snippets, [ + { text: "hello world", start: 0.08, duration: 3.12 }, + { text: "again", start: 4, duration: 1.8 }, + ]); +}); + +test("parseWebVtt strips tags and cue settings", () => { + const snippets = parseWebVtt(`WEBVTT + +00:00:00.080 --> 00:00:03.200 align:start position:0% +Hello world + +00:00:04.000 --> 00:00:05.800 +Again +`); + + assert.equal(snippets.length, 2); + assert.equal(snippets[0].text, "Hello world"); + assert.equal(snippets[0].start, 0.08); + assert.equal(snippets[0].duration, 3.12); + assert.equal(snippets[1].text, "Again"); + assert.equal(snippets[1].start, 4); + assert.equal(Number(snippets[1].duration.toFixed(1)), 1.8); +}); + +test("resolveVideoSource prefers primary InnerTube result before fallback", async () => { + let fallbackCalled = false; + const source = await resolveVideoSource( + "video12345ab", + async () => ({ kind: "innertube", data: { videoDetails: { title: "Primary" } }, transcripts: [] }), + () => { + fallbackCalled = true; + return { + subtitles: { + en: [{ ext: "json3", url: "https://example.com/en.json3", name: "English" }], + }, + }; + }, + () => {} + ); + + assert.equal(source.kind, "innertube"); + assert.equal(fallbackCalled, false); +}); + +test("resolveVideoSource falls back to yt-dlp only after fallback-eligible errors", async () => { + let fallbackCalled = false; + const source = await resolveVideoSource( + "video12345ab", + async () => { + const error = new Error("Request blocked for video12345ab: bot detected"); + (error as Error & { code?: string }).code = "BOT_DETECTED"; + throw error; + }, + () => { + fallbackCalled = true; + return { + automatic_captions: { + en: [{ ext: "json3", url: "https://example.com/en.json3", name: "English (auto-generated)" }], + }, + }; + }, + () => {} + ); + + assert.equal(source.kind, "yt-dlp"); + assert.equal(fallbackCalled, true); + assert.equal(source.transcripts[0].languageCode, "en"); +}); diff --git a/skills/baoyu-youtube-transcript/scripts/main.ts b/skills/baoyu-youtube-transcript/scripts/main.ts index 3e618e7..22aea19 100644 --- a/skills/baoyu-youtube-transcript/scripts/main.ts +++ b/skills/baoyu-youtube-transcript/scripts/main.ts @@ -1,659 +1,55 @@ #!/usr/bin/env bun -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; -import { dirname, join, resolve } from "path"; - -type Format = "text" | "srt"; - -interface Options { - videoIds: string[]; - languages: string[]; - format: Format; - translate: string; - list: boolean; - excludeGenerated: boolean; - excludeManual: boolean; - output: string; - outputDir: string; - timestamps: boolean; - chapters: boolean; - speakers: boolean; - refresh: boolean; -} - -interface Snippet { - text: string; - start: number; - duration: number; -} - -interface Sentence { - text: string; - start: string; - end: string; -} - -interface TranscriptInfo { - language: string; - languageCode: string; - isGenerated: boolean; - isTranslatable: boolean; - baseUrl: string; - translationLanguages: { language: string; languageCode: string }[]; -} - -interface Chapter { - title: string; - start: number; - end: number; -} - -interface VideoMeta { - videoId: string; - title: string; - channel: string; - channelId: string; - description: string; - duration: number; - publishDate: string; - url: string; - coverImage: string; - thumbnailUrl: string; - language: { code: string; name: string; isGenerated: boolean }; - chapters: Chapter[]; -} - -interface VideoResult { - videoId: string; - title?: string; - filePath?: string; - content?: string; - error?: string; -} - -const WATCH_URL = "https://www.youtube.com/watch?v="; -const INNERTUBE_URL = "https://www.youtube.com/youtubei/v1/player"; -const INNERTUBE_CTX = { client: { clientName: "ANDROID", clientVersion: "20.10.38" } }; - -function extractVideoId(input: string): string { - input = input.replace(/\\/g, "").trim(); - const patterns = [ - /(?:youtube\.com\/watch\?.*v=|youtu\.be\/|youtube\.com\/embed\/|youtube\.com\/v\/|youtube\.com\/shorts\/)([a-zA-Z0-9_-]{11})/, - /^([a-zA-Z0-9_-]{11})$/, - ]; - for (const p of patterns) { - const m = input.match(p); - if (m) return m[1]; - } - return input; -} - -function slugify(s: string): string { - return s - .toLowerCase() - .replace(/[^\w\s-]/g, "") - .replace(/\s+/g, "-") - .replace(/-+/g, "-") - .replace(/^-|-$/g, "") || "untitled"; -} - -function htmlUnescape(s: string): string { - return s - .replace(/&/g, "&") - .replace(/</g, "<") - .replace(/>/g, ">") - .replace(/"/g, '"') - .replace(/'/g, "'") - .replace(/'/g, "'") - .replace(///g, "/") - .replace(/'/g, "'") - .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n))) - .replace(/&#x([0-9a-fA-F]+);/g, (_, n) => String.fromCharCode(parseInt(n, 16))); -} - -function stripTags(s: string): string { - return s.replace(/<[^>]*>/g, ""); -} - -function parseTranscriptXml(xml: string): Snippet[] { - const snippets: Snippet[] = []; - const re = /]*>([\s\S]*?)<\/text>/g; - let m: RegExpExecArray | null; - while ((m = re.exec(xml)) !== null) { - const raw = m[3]; - if (!raw) continue; - snippets.push({ - text: htmlUnescape(stripTags(raw)), - start: parseFloat(m[1]), - duration: parseFloat(m[2] || "0"), - }); - } - return snippets; -} - -// --- YouTube API --- - -async function fetchHtml(videoId: string): Promise { - const r = await fetch(WATCH_URL + videoId, { - headers: { "Accept-Language": "en-US", "User-Agent": "Mozilla/5.0" }, - }); - if (!r.ok) throw new Error(`HTTP ${r.status} fetching video page`); - let html = await r.text(); - if (html.includes('action="https://consent.youtube.com/s"')) { - const cv = html.match(/name="v" value="(.*?)"/); - if (!cv) throw new Error("Failed to create consent cookie"); - const r2 = await fetch(WATCH_URL + videoId, { - headers: { - "Accept-Language": "en-US", - "User-Agent": "Mozilla/5.0", - Cookie: `CONSENT=YES+${cv[1]}`, - }, - }); - if (!r2.ok) throw new Error(`HTTP ${r2.status} fetching video page (consent)`); - html = await r2.text(); - } - return html; -} - -function extractApiKey(html: string, videoId: string): string { - const m = html.match(/"INNERTUBE_API_KEY":\s*"([a-zA-Z0-9_-]+)"/); - if (!m) { - if (html.includes('class="g-recaptcha"')) throw new Error(`IP blocked for ${videoId} (reCAPTCHA)`); - throw new Error(`Cannot extract API key for ${videoId}`); - } - return m[1]; -} - -async function fetchInnertubeData(videoId: string, apiKey: string): Promise { - const r = await fetch(`${INNERTUBE_URL}?key=${apiKey}`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ context: INNERTUBE_CTX, videoId }), - }); - if (r.status === 429) throw new Error(`IP blocked for ${videoId} (429)`); - if (!r.ok) throw new Error(`HTTP ${r.status} from InnerTube API`); - return r.json(); -} - -function assertPlayability(data: any, videoId: string) { - const ps = data?.playabilityStatus; - if (!ps) return; - const status = ps.status; - if (status === "OK" || !status) return; - const reason = ps.reason || ""; - if (status === "LOGIN_REQUIRED") { - if (reason.includes("bot")) throw new Error(`Request blocked for ${videoId}: bot detected`); - if (reason.includes("inappropriate")) throw new Error(`Age restricted: ${videoId}`); - } - if (status === "ERROR" && reason.includes("unavailable")) { - if (videoId.startsWith("http")) throw new Error(`Invalid video ID: pass the ID, not the URL`); - throw new Error(`Video unavailable: ${videoId}`); - } - const subreasons = ps.errorScreen?.playerErrorMessageRenderer?.subreason?.runs?.map((r: any) => r.text).join("") || ""; - throw new Error(`Video unplayable (${videoId}): ${reason} ${subreasons}`.trim()); -} - -function extractCaptionsJson(data: any, videoId: string): any { - assertPlayability(data, videoId); - const cj = data?.captions?.playerCaptionsTracklistRenderer; - if (!cj || !cj.captionTracks) throw new Error(`Transcripts disabled for ${videoId}`); - return cj; -} - -function buildTranscriptList(captionsJson: any): TranscriptInfo[] { - const tlLangs = (captionsJson.translationLanguages || []).map((tl: any) => ({ - language: tl.languageName?.runs?.[0]?.text || tl.languageName?.simpleText || "", - languageCode: tl.languageCode, - })); - return (captionsJson.captionTracks || []).map((t: any) => ({ - language: t.name?.runs?.[0]?.text || t.name?.simpleText || "", - languageCode: t.languageCode, - isGenerated: t.kind === "asr", - isTranslatable: !!t.isTranslatable, - baseUrl: (t.baseUrl || "").replace(/&fmt=srv3/g, ""), - translationLanguages: t.isTranslatable ? tlLangs : [], - })); -} - -function findTranscript( - transcripts: TranscriptInfo[], - languages: string[], - excludeGenerated: boolean, - excludeManual: boolean -): TranscriptInfo { - let filtered = transcripts; - if (excludeGenerated) filtered = filtered.filter((t) => !t.isGenerated); - if (excludeManual) filtered = filtered.filter((t) => t.isGenerated); - for (const lang of languages) { - const found = filtered.find((t) => t.languageCode === lang); - if (found) return found; - } - const available = filtered.map((t) => `${t.languageCode} ("${t.language}")`).join(", "); - throw new Error(`No transcript found for languages [${languages.join(", ")}]. Available: ${available || "none"}`); -} - -async function fetchTranscriptSnippets(info: TranscriptInfo, translateTo?: string): Promise<{ snippets: Snippet[]; language: string; languageCode: string }> { - let url = info.baseUrl; - let lang = info.language; - let langCode = info.languageCode; - if (translateTo) { - if (!info.isTranslatable) throw new Error(`Transcript ${info.languageCode} is not translatable`); - const tl = info.translationLanguages.find((t) => t.languageCode === translateTo); - if (!tl) throw new Error(`Translation language ${translateTo} not available`); - url += `&tlang=${translateTo}`; - lang = tl.language; - langCode = translateTo; - } - const r = await fetch(url, { headers: { "Accept-Language": "en-US" } }); - if (!r.ok) throw new Error(`HTTP ${r.status} fetching transcript`); - return { snippets: parseTranscriptXml(await r.text()), language: lang, languageCode: langCode }; -} - -// --- Metadata & chapters --- - -function parseChapters(description: string, duration: number = 0): Chapter[] { - const raw: { title: string; start: number }[] = []; - for (const line of description.split("\n")) { - const m = line.trim().match(/^(?:(\d{1,2}):)?(\d{1,2}):(\d{2})\s+(.+)$/); - if (m) { - const h = m[1] ? parseInt(m[1]) : 0; - raw.push({ title: m[4].trim(), start: h * 3600 + parseInt(m[2]) * 60 + parseInt(m[3]) }); - } - } - if (raw.length < 2) return []; - return raw.map((ch, i) => ({ - title: ch.title, - start: ch.start, - end: i < raw.length - 1 ? raw[i + 1].start : Math.max(duration, ch.start), - })); -} - -function getThumbnailUrls(videoId: string, data: any): string[] { - const urls = [ - `https://i.ytimg.com/vi/${videoId}/maxresdefault.jpg`, - `https://i.ytimg.com/vi/${videoId}/hqdefault.jpg`, - ]; - const thumbnails = data?.videoDetails?.thumbnail?.thumbnails || - data?.microformat?.playerMicroformatRenderer?.thumbnail?.thumbnails || []; - if (thumbnails.length) { - const sorted = [...thumbnails].sort((a: any, b: any) => (b.width || 0) - (a.width || 0)); - for (const t of sorted) if (t.url && !urls.includes(t.url)) urls.push(t.url); - } - return urls; -} - -function buildVideoMeta(data: any, videoId: string, langInfo: { code: string; name: string; isGenerated: boolean }, chapters: Chapter[]): VideoMeta { - const vd = data?.videoDetails || {}; - const mf = data?.microformat?.playerMicroformatRenderer || {}; - return { - videoId, - title: vd.title || mf.title?.simpleText || "", - channel: vd.author || mf.ownerChannelName || "", - channelId: vd.channelId || mf.externalChannelId || "", - description: vd.shortDescription || mf.description?.simpleText || "", - duration: parseInt(vd.lengthSeconds || "0"), - publishDate: mf.publishDate || mf.uploadDate || "", - url: `https://www.youtube.com/watch?v=${videoId}`, - coverImage: "", - thumbnailUrl: getThumbnailUrls(videoId, data)[0], - language: langInfo, - chapters, - }; -} - -async function downloadCoverImage(urls: string[], outputPath: string): Promise { - for (const u of urls) { - try { - const r = await fetch(u); - if (r.ok) { - writeFileSync(outputPath, Buffer.from(await r.arrayBuffer())); - return true; - } - } catch {} - } - return false; -} - -function parseSrt(srt: string): Snippet[] { - const blocks = srt.trim().split(/\n\n+/); - const snippets: Snippet[] = []; - for (const block of blocks) { - const lines = block.split("\n"); - if (lines.length < 3) continue; - const m = lines[1].match(/(\d{2}):(\d{2}):(\d{2}),(\d{3})\s*-->\s*(\d{2}):(\d{2}):(\d{2}),(\d{3})/); - if (!m) continue; - const start = parseInt(m[1]) * 3600 + parseInt(m[2]) * 60 + parseInt(m[3]) + parseInt(m[4]) / 1000; - const end = parseInt(m[5]) * 3600 + parseInt(m[6]) * 60 + parseInt(m[7]) + parseInt(m[8]) / 1000; - snippets.push({ text: lines.slice(2).join(" "), start, duration: end - start }); - } - return snippets; -} - -// --- Timestamp formatting --- - -function ts(t: number): string { - const h = Math.floor(t / 3600); - const m = Math.floor((t % 3600) / 60); - const s = Math.floor(t % 60); - return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`; -} - -function tsMs(t: number, sep: string): string { - const h = Math.floor(t / 3600); - const m = Math.floor((t % 3600) / 60); - const s = Math.floor(t % 60); - const ms = Math.round((t - Math.floor(t)) * 1000); - return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}${sep}${String(ms).padStart(3, "0")}`; -} - -// --- Paragraph grouping --- - -interface Paragraph { - text: string; - start: number; - end: number; -} - -function groupIntoParagraphs(snippets: Snippet[]): Paragraph[] { - if (!snippets.length) return []; - const paras: Paragraph[] = []; - let buf: Snippet[] = []; - for (let i = 0; i < snippets.length; i++) { - buf.push(snippets[i]); - const last = i === snippets.length - 1; - const gap = !last && snippets[i + 1].start - (snippets[i].start + snippets[i].duration) > 1.5; - if (last || gap || buf.length >= 8) { - const lastS = buf[buf.length - 1]; - paras.push({ text: buf.map(s => s.text).join(" "), start: buf[0].start, end: lastS.start + lastS.duration }); - buf = []; - } - } - return paras; -} - -// --- Sentence segmentation --- - -const SENTENCE_END_RE = /[.?!…。？！⁈⁇‼‽．]/; - -function isCJK(ch: string): boolean { - const code = ch.charCodeAt(0); - return (code >= 0x4E00 && code <= 0x9FFF) || - (code >= 0x3040 && code <= 0x309F) || - (code >= 0x30A0 && code <= 0x30FF) || - (code >= 0xAC00 && code <= 0xD7AF) || - (code >= 0x3400 && code <= 0x4DBF) || - (code >= 0xF900 && code <= 0xFAFF); -} - -function splitSnippetAtPunctuation(s: Snippet): { text: string; start: number; end: number }[] { - const { text, start, duration } = s; - const end = start + duration; - if (!text.length) return []; - - const splitPoints: number[] = []; - for (let i = 0; i < text.length; i++) { - if (SENTENCE_END_RE.test(text[i])) { - while (i + 1 < text.length && SENTENCE_END_RE.test(text[i + 1])) i++; - if (i < text.length - 1) splitPoints.push(i); - } - } - - if (!splitPoints.length) return [{ text, start, end }]; - - const parts: { text: string; start: number; end: number }[] = []; - let prev = 0; - for (const pos of splitPoints) { - const partText = text.slice(prev, pos + 1).trim(); - if (partText) { - parts.push({ - text: partText, - start: start + (prev / text.length) * duration, - end: start + ((pos + 1) / text.length) * duration, - }); - } - prev = pos + 1; - } - - const remaining = text.slice(prev).trim(); - if (remaining) { - parts.push({ text: remaining, start: start + (prev / text.length) * duration, end }); - } - - return parts; -} - -function mergeTexts(texts: string[]): string { - if (!texts.length) return ""; - let result = texts[0]; - for (let i = 1; i < texts.length; i++) { - const next = texts[i]; - if (!next) continue; - const lastChar = result[result.length - 1]; - const firstChar = next[0]; - if (isCJK(lastChar) || isCJK(firstChar)) { - result += next; - } else { - result = result.trimEnd() + " " + next.trimStart(); - } - } - return result.replace(/ {2,}/g, " "); -} - -function segmentIntoSentences(snippets: Snippet[]): Sentence[] { - const parts: { text: string; start: number; end: number }[] = []; - for (const s of snippets) parts.push(...splitSnippetAtPunctuation(s)); - - const sentences: Sentence[] = []; - let buf: { text: string; start: number; end: number }[] = []; - - for (const part of parts) { - buf.push(part); - if (SENTENCE_END_RE.test(part.text[part.text.length - 1])) { - sentences.push({ - text: mergeTexts(buf.map(b => b.text)), - start: ts(buf[0].start), - end: ts(buf[buf.length - 1].end), - }); - buf = []; - } - } - - if (buf.length) { - sentences.push({ - text: mergeTexts(buf.map(b => b.text)), - start: ts(buf[0].start), - end: ts(buf[buf.length - 1].end), - }); - } - - return sentences; -} - -function parseTs(t: string): number { - const [h, m, s] = t.split(":").map(Number); - return h * 3600 + m * 60 + s; -} - -function groupSentenceParas(sentences: Sentence[]): Paragraph[] { - if (!sentences.length) return []; - const paras: Paragraph[] = []; - let buf: Sentence[] = []; - for (let i = 0; i < sentences.length; i++) { - buf.push(sentences[i]); - const last = i === sentences.length - 1; - const gap = !last && parseTs(sentences[i + 1].start) - parseTs(sentences[i].end) > 2; - if (last || gap || buf.length >= 5) { - paras.push({ - text: mergeTexts(buf.map(s => s.text)), - start: parseTs(buf[0].start), - end: parseTs(buf[buf.length - 1].end), - }); - buf = []; - } - } - return paras; -} - -// --- Format functions --- - -function formatSrt(snippets: Snippet[]): string { - return snippets - .map((s, i) => { - const end = i < snippets.length - 1 && snippets[i + 1].start < s.start + s.duration - ? snippets[i + 1].start - : s.start + s.duration; - return `${i + 1}\n${tsMs(s.start, ",")} --> ${tsMs(end, ",")}\n${s.text}`; - }) - .join("\n\n") + "\n"; -} - -function yamlEscape(s: string): string { - if (/[:"'{}\[\]#&*!|>%@`\n]/.test(s) || s.trim() !== s) return `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; - return s; -} - -function extractSummary(description: string): string { - if (!description) return ""; - const firstPara = description.split(/\n\s*\n/)[0].trim(); - const lines = firstPara.split("\n").filter(l => !/^\s*(https?:\/\/|#|@|\d+:\d+)/.test(l) && l.trim()); - return lines.join(" ").slice(0, 300).trim(); -} - -function formatMarkdown(sentences: Sentence[], meta: VideoMeta, opts: { timestamps: boolean; chapters: boolean; speakers: boolean }, snippets?: Snippet[]): string { - const summary = extractSummary(meta.description); - let md = "---\n"; - md += `title: ${yamlEscape(meta.title)}\n`; - md += `channel: ${yamlEscape(meta.channel)}\n`; - if (meta.publishDate) md += `date: ${meta.publishDate}\n`; - md += `url: ${yamlEscape(meta.url)}\n`; - if (meta.coverImage) md += `cover: ${meta.coverImage}\n`; - if (summary) md += `description: ${yamlEscape(summary)}\n`; - if (meta.language) md += `language: ${meta.language.code}\n`; - md += "---\n\n"; - - if (opts.speakers) { - md += `# ${meta.title}\n\n`; - if (summary) md += `${summary}\n\n`; - if (meta.description) md += "# Description\n\n" + meta.description.trim() + "\n\n"; - if (meta.chapters.length) { - md += "# Chapters\n\n"; - for (const ch of meta.chapters) md += `* [${ts(ch.start)}] ${ch.title}\n`; - md += "\n"; - } - md += "# Transcript\n\n"; - md += snippets ? formatSrt(snippets) : ""; - return md; - } - - md += `# ${meta.title}\n\n`; - if (summary) md += `${summary}\n\n`; - - const chapters = opts.chapters ? meta.chapters : []; - - if (chapters.length) { - md += "## Table of Contents\n\n"; - for (const ch of chapters) md += opts.timestamps ? `* [${ts(ch.start)}] ${ch.title}\n` : `* ${ch.title}\n`; - md += "\n"; - if (meta.coverImage) md += `\n![cover](${meta.coverImage})\n`; - md += "\n"; - for (let i = 0; i < chapters.length; i++) { - const nextStart = i < chapters.length - 1 ? chapters[i + 1].start : Infinity; - const chSentences = sentences.filter(s => parseTs(s.start) >= chapters[i].start && parseTs(s.start) < nextStart); - const paras = groupSentenceParas(chSentences); - md += opts.timestamps - ? `## [${ts(chapters[i].start)}] ${chapters[i].title}\n\n` - : `## ${chapters[i].title}\n\n`; - for (const p of paras) md += opts.timestamps ? `${p.text} [${ts(p.start)} → ${ts(p.end)}]\n\n` : `${p.text}\n\n`; - md += "\n"; - } - } else { - const paras = groupSentenceParas(sentences); - for (const p of paras) md += opts.timestamps ? `${p.text} [${ts(p.start)} → ${ts(p.end)}]\n\n` : `${p.text}\n\n`; - } - - return md.trimEnd() + "\n"; -} - -function formatListOutput(videoId: string, title: string, transcripts: TranscriptInfo[]): string { - const manual = transcripts.filter((t) => !t.isGenerated); - const generated = transcripts.filter((t) => t.isGenerated); - const tlLangs = transcripts.find((t) => t.translationLanguages.length > 0)?.translationLanguages || []; - const fmtList = (list: TranscriptInfo[]) => - list.length ? list.map((t) => ` - ${t.languageCode} ("${t.language}")${t.isTranslatable ? " [TRANSLATABLE]" : ""}`).join("\n") : "None"; - const fmtTl = tlLangs.length - ? tlLangs.map((t) => ` - ${t.languageCode} ("${t.language}")`).join("\n") - : "None"; - return `Transcripts for ${videoId}${title ? ` (${title})` : ""}:\n\n(MANUALLY CREATED)\n${fmtList(manual)}\n\n(GENERATED)\n${fmtList(generated)}\n\n(TRANSLATION LANGUAGES)\n${fmtTl}`; -} - -// --- File helpers --- - -function ensureDir(p: string) { - const dir = dirname(p); - if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); -} - -function resolveBaseDir(outputDir: string): string { - return resolve(outputDir || "youtube-transcript"); -} - -function loadIndex(baseDir: string): Record { - try { return JSON.parse(readFileSync(join(baseDir, ".index.json"), "utf-8")); } catch { return {}; } -} - -function saveIndex(baseDir: string, index: Record) { - const p = join(baseDir, ".index.json"); - ensureDir(p); - writeFileSync(p, JSON.stringify(index, null, 2)); -} - -function lookupVideoDir(videoId: string, baseDir: string): string | null { - const rel = loadIndex(baseDir)[videoId]; - if (rel) { - const dir = resolve(baseDir, rel); - if (existsSync(dir)) return dir; - } - return null; -} - -function registerVideoDir(videoId: string, channelSlug: string, titleSlug: string, baseDir: string): string { - const rel = join(channelSlug, titleSlug); - const index = loadIndex(baseDir); - index[videoId] = rel; - saveIndex(baseDir, index); - return resolve(baseDir, rel); -} - -function hasCachedData(videoDir: string): boolean { - return existsSync(join(videoDir, "meta.json")) && existsSync(join(videoDir, "transcript-raw.json")); -} - -function loadMeta(videoDir: string): VideoMeta { - return JSON.parse(readFileSync(join(videoDir, "meta.json"), "utf-8")); -} - -function loadSnippets(videoDir: string): Snippet[] { - return JSON.parse(readFileSync(join(videoDir, "transcript-raw.json"), "utf-8")); -} - -function loadSentences(videoDir: string): Sentence[] { - return JSON.parse(readFileSync(join(videoDir, "transcript-sentences.json"), "utf-8")); -} - -// --- Main processing --- - -async function fetchAndCache(videoId: string, baseDir: string, opts: Options): Promise<{ meta: VideoMeta; snippets: Snippet[]; sentences: Sentence[]; videoDir: string }> { - const html = await fetchHtml(videoId); - const apiKey = extractApiKey(html, videoId); - const data = await fetchInnertubeData(videoId, apiKey); - const captionsJson = extractCaptionsJson(data, videoId); - const transcripts = buildTranscriptList(captionsJson); - const info = findTranscript(transcripts, opts.languages, opts.excludeGenerated, opts.excludeManual); - const result = await fetchTranscriptSnippets(info, opts.translate || undefined); - const description = data?.videoDetails?.shortDescription || ""; - const duration = parseInt(data?.videoDetails?.lengthSeconds || "0"); +import { writeFileSync } from "fs"; +import { join, resolve } from "path"; + +import { extractVideoId, slugify } from "./shared.ts"; +import { + ensureDir, + hasCachedData, + loadMeta, + loadSentences, + loadSnippets, + lookupVideoDir, + registerVideoDir, + resolveBaseDir, +} from "./storage.ts"; +import { findTranscript, formatListOutput, formatMarkdown, formatSrt, segmentIntoSentences } from "./transcript.ts"; +import type { Options, Sentence, Snippet, VideoMeta, VideoResult } from "./types.ts"; +import { + buildVideoMeta, + buildVideoMetaFromYtDlp, + downloadCoverImage, + fetchTranscriptSnippets, + fetchVideoSource, + getThumbnailUrls, + getYtDlpThumbnailUrls, + parseChapters, +} from "./youtube.ts"; + +async function fetchAndCache( + videoId: string, + baseDir: string, + opts: Options +): Promise<{ meta: VideoMeta; snippets: Snippet[]; sentences: Sentence[]; videoDir: string }> { + const source = await fetchVideoSource(videoId); + const requestedLanguages = source.kind === "yt-dlp" && opts.translate ? [opts.translate] : opts.languages; + const transcript = findTranscript(source.transcripts, requestedLanguages, opts.excludeGenerated, opts.excludeManual); + const result = await fetchTranscriptSnippets(transcript, source.kind === "yt-dlp" ? undefined : opts.translate || undefined); + const description = source.kind === "yt-dlp" + ? source.info.description || "" + : source.data?.videoDetails?.shortDescription || ""; + const duration = source.kind === "yt-dlp" + ? Number(source.info.duration || 0) + : parseInt(source.data?.videoDetails?.lengthSeconds || "0"); const chapters = parseChapters(description, duration); - const langInfo = { code: result.languageCode, name: result.language, isGenerated: info.isGenerated }; - const meta = buildVideoMeta(data, videoId, langInfo, chapters); + const language = { + code: result.languageCode, + name: result.language, + isGenerated: transcript.isGenerated, + }; + const meta = source.kind === "yt-dlp" + ? buildVideoMetaFromYtDlp(source.info, videoId, language, chapters) + : buildVideoMeta(source.data, videoId, language, chapters); const videoDir = registerVideoDir(videoId, slugify(meta.channel), slugify(meta.title), baseDir); ensureDir(join(videoDir, "meta.json")); @@ -663,9 +59,12 @@ async function fetchAndCache(videoId: string, baseDir: string, opts: Options): P const sentences = segmentIntoSentences(result.snippets); writeFileSync(join(videoDir, "transcript-sentences.json"), JSON.stringify(sentences, null, 2)); - const imgPath = join(videoDir, "imgs", "cover.jpg"); - ensureDir(imgPath); - const downloaded = await downloadCoverImage(getThumbnailUrls(videoId, data), imgPath); + const imagePath = join(videoDir, "imgs", "cover.jpg"); + ensureDir(imagePath); + const downloaded = await downloadCoverImage( + source.kind === "yt-dlp" ? getYtDlpThumbnailUrls(videoId, source.info) : getThumbnailUrls(videoId, source.data), + imagePath + ); meta.coverImage = downloaded ? "imgs/cover.jpg" : ""; writeFileSync(join(videoDir, "meta.json"), JSON.stringify(meta, null, 2)); @@ -676,15 +75,10 @@ async function fetchAndCache(videoId: string, baseDir: string, opts: Options): P async function processVideo(videoId: string, opts: Options): Promise { const baseDir = resolveBaseDir(opts.outputDir); - // --list: always fetch fresh if (opts.list) { - const html = await fetchHtml(videoId); - const apiKey = extractApiKey(html, videoId); - const data = await fetchInnertubeData(videoId, apiKey); - const title = data?.videoDetails?.title || ""; - const captionsJson = extractCaptionsJson(data, videoId); - const transcripts = buildTranscriptList(captionsJson); - return { videoId, title, content: formatListOutput(videoId, title, transcripts) }; + const source = await fetchVideoSource(videoId); + const title = source.kind === "yt-dlp" ? source.info.title || "" : source.data?.videoDetails?.title || ""; + return { videoId, title, content: formatListOutput(videoId, title, source.transcripts) }; } let videoDir = lookupVideoDir(videoId, baseDir); @@ -697,16 +91,17 @@ async function processVideo(videoId: string, opts: Options): Promise 0 && meta.chapters.some((ch: any) => ch.end === undefined)) { + const wantedLanguages = opts.translate ? [opts.translate] : opts.languages; + if (!wantedLanguages.includes(meta.language.code)) needsFetch = true; + if (!needsFetch && meta.chapters.length > 0 && meta.chapters.some((chapter: any) => chapter.end === undefined)) { for (let i = 0; i < meta.chapters.length; i++) { meta.chapters[i].end = i < meta.chapters.length - 1 ? meta.chapters[i + 1].start : Math.max(meta.duration, meta.chapters[i].start); } - try { writeFileSync(join(videoDir, "meta.json"), JSON.stringify(meta, null, 2)); } catch {} + try { + writeFileSync(join(videoDir, "meta.json"), JSON.stringify(meta, null, 2)); + } catch {} } } @@ -722,21 +117,19 @@ async function processVideo(videoId: string, opts: Options): Promise [options] @@ -789,13 +180,13 @@ function parseArgs(argv: string[]): Options | null { printHelp(); process.exit(0); } else if (arg === "--languages") { - const v = argv[++i]; - if (v) opts.languages = v.split(",").map((s) => s.trim()); + const value = argv[++i]; + if (value) opts.languages = value.split(",").map((entry) => entry.trim()); } else if (arg === "--format") { - const v = argv[++i]?.toLowerCase(); - if (v === "text" || v === "srt") opts.format = v; + const value = argv[++i]?.toLowerCase(); + if (value === "text" || value === "srt") opts.format = value; else { - console.error(`Invalid format: ${v}. Use: text, srt`); + console.error(`Invalid format: ${value}. Use: text, srt`); return null; } } else if (arg === "--translate") { @@ -830,6 +221,7 @@ function parseArgs(argv: string[]): Options | null { printHelp(); return null; } + return opts; } @@ -844,14 +236,16 @@ async function main() { for (const videoId of opts.videoIds) { try { - const r = await processVideo(videoId, opts); - if (r.error) console.error(`Error (${r.videoId}): ${r.error}`); - else if (r.filePath) console.log(r.filePath); - else if (r.content) console.log(r.content); - } catch (e) { - console.error(`Error (${videoId}): ${(e as Error).message}`); + const result = await processVideo(videoId, opts); + if (result.error) console.error(`Error (${result.videoId}): ${result.error}`); + else if (result.filePath) console.log(result.filePath); + else if (result.content) console.log(result.content); + } catch (error) { + console.error(`Error (${videoId}): ${(error as Error).message}`); } } } -main(); +if (import.meta.main) { + main(); +} diff --git a/skills/baoyu-youtube-transcript/scripts/shared.ts b/skills/baoyu-youtube-transcript/scripts/shared.ts new file mode 100644 index 0000000..9efda86 --- /dev/null +++ b/skills/baoyu-youtube-transcript/scripts/shared.ts @@ -0,0 +1,83 @@ +import type { TranscriptError } from "./types.ts"; + +export function extractVideoId(input: string): string { + input = input.replace(/\\/g, "").trim(); + const patterns = [ + /(?:youtube\.com\/watch\?.*v=|youtu\.be\/|youtube\.com\/embed\/|youtube\.com\/v\/|youtube\.com\/shorts\/)([a-zA-Z0-9_-]{11})/, + /^([a-zA-Z0-9_-]{11})$/, + ]; + for (const pattern of patterns) { + const match = input.match(pattern); + if (match) return match[1]; + } + return input; +} + +export function slugify(value: string): string { + return value + .toLowerCase() + .replace(/[^\w\s-]/g, "") + .replace(/\s+/g, "-") + .replace(/-+/g, "-") + .replace(/^-|-$/g, "") || "untitled"; +} + +export function htmlUnescape(value: string): string { + return value + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(///g, "/") + .replace(/'/g, "'") + .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n))) + .replace(/&#x([0-9a-fA-F]+);/g, (_, n) => String.fromCharCode(parseInt(n, 16))); +} + +export function stripTags(value: string): string { + return value.replace(/<[^>]*>/g, ""); +} + +export function makeError(message: string, code?: string): Error { + const error = new Error(message) as TranscriptError; + if (code) error.code = code; + return error; +} + +export function normalizeError(error: unknown): TranscriptError { + if (error instanceof Error) { + const known = error as TranscriptError; + if (known.code) return known; + const message = known.message || String(error); + const lower = message.toLowerCase(); + if (lower.includes("bot detected")) known.code = "BOT_DETECTED"; + else if (lower.includes("age restricted")) known.code = "AGE_RESTRICTED"; + else if (lower.includes("video unavailable")) known.code = "VIDEO_UNAVAILABLE"; + else if (lower.includes("transcripts disabled")) known.code = "TRANSCRIPTS_DISABLED"; + else if (lower.includes("no transcript found")) known.code = "NO_TRANSCRIPT"; + else if (lower.includes("invalid video id")) known.code = "INVALID_VIDEO_ID"; + else if (lower.includes("ip blocked") || lower.includes("recaptcha") || lower.includes("http 429")) known.code = "IP_BLOCKED"; + else if (lower.includes("cannot extract api key")) known.code = "PAGE_FETCH_FAILED"; + else if (lower.includes("innertube api") || lower.includes("http 403")) known.code = "INNERTUBE_REJECTED"; + else if (lower.includes("yt-dlp fallback failed")) known.code = "YT_DLP_FAILED"; + return known; + } + return makeError(String(error), "UNKNOWN") as TranscriptError; +} + +export function shouldTryAlternateClient(error: unknown): boolean { + const code = normalizeError(error).code; + return code === "BOT_DETECTED" || code === "IP_BLOCKED" || code === "INNERTUBE_REJECTED" || code === "AGE_RESTRICTED" || code === "VIDEO_UNAVAILABLE"; +} + +export function shouldTryYtDlpFallback(error: unknown): boolean { + const code = normalizeError(error).code; + return code === "BOT_DETECTED" || code === "IP_BLOCKED" || code === "INNERTUBE_REJECTED" || code === "PAGE_FETCH_FAILED" || code === "AGE_RESTRICTED" || code === "VIDEO_UNAVAILABLE"; +} + +export function normalizePublishDate(uploadDate?: string): string { + if (!uploadDate || !/^\d{8}$/.test(uploadDate)) return uploadDate || ""; + return `${uploadDate.slice(0, 4)}-${uploadDate.slice(4, 6)}-${uploadDate.slice(6, 8)}`; +} diff --git a/skills/baoyu-youtube-transcript/scripts/storage.ts b/skills/baoyu-youtube-transcript/scripts/storage.ts new file mode 100644 index 0000000..cea0e10 --- /dev/null +++ b/skills/baoyu-youtube-transcript/scripts/storage.ts @@ -0,0 +1,60 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; +import { dirname, join, resolve } from "path"; + +import type { Sentence, Snippet, VideoMeta } from "./types.ts"; + +export function ensureDir(path: string) { + const dir = dirname(path); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); +} + +export function resolveBaseDir(outputDir: string): string { + return resolve(outputDir || "youtube-transcript"); +} + +function loadIndex(baseDir: string): Record { + try { + return JSON.parse(readFileSync(join(baseDir, ".index.json"), "utf-8")); + } catch { + return {}; + } +} + +function saveIndex(baseDir: string, index: Record) { + const path = join(baseDir, ".index.json"); + ensureDir(path); + writeFileSync(path, JSON.stringify(index, null, 2)); +} + +export function lookupVideoDir(videoId: string, baseDir: string): string | null { + const rel = loadIndex(baseDir)[videoId]; + if (rel) { + const dir = resolve(baseDir, rel); + if (existsSync(dir)) return dir; + } + return null; +} + +export function registerVideoDir(videoId: string, channelSlug: string, titleSlug: string, baseDir: string): string { + const rel = join(channelSlug, titleSlug); + const index = loadIndex(baseDir); + index[videoId] = rel; + saveIndex(baseDir, index); + return resolve(baseDir, rel); +} + +export function hasCachedData(videoDir: string): boolean { + return existsSync(join(videoDir, "meta.json")) && existsSync(join(videoDir, "transcript-raw.json")); +} + +export function loadMeta(videoDir: string): VideoMeta { + return JSON.parse(readFileSync(join(videoDir, "meta.json"), "utf-8")); +} + +export function loadSnippets(videoDir: string): Snippet[] { + return JSON.parse(readFileSync(join(videoDir, "transcript-raw.json"), "utf-8")); +} + +export function loadSentences(videoDir: string): Sentence[] { + return JSON.parse(readFileSync(join(videoDir, "transcript-sentences.json"), "utf-8")); +} diff --git a/skills/baoyu-youtube-transcript/scripts/transcript.ts b/skills/baoyu-youtube-transcript/scripts/transcript.ts new file mode 100644 index 0000000..661b935 --- /dev/null +++ b/skills/baoyu-youtube-transcript/scripts/transcript.ts @@ -0,0 +1,349 @@ +import { htmlUnescape, makeError, stripTags } from "./shared.ts"; +import type { Sentence, Snippet, TranscriptInfo, VideoMeta } from "./types.ts"; + +interface Paragraph { + text: string; + start: number; + end: number; +} + +const SENTENCE_END_RE = /[.?!…。？！⁈⁇‼‽．]/; + +export function parseTranscriptXml(xml: string): Snippet[] { + const snippets: Snippet[] = []; + const pattern = /]*>([\s\S]*?)<\/text>/g; + let match: RegExpExecArray | null; + while ((match = pattern.exec(xml)) !== null) { + const raw = match[3]; + if (!raw) continue; + snippets.push({ + text: htmlUnescape(stripTags(raw)), + start: parseFloat(match[1]), + duration: parseFloat(match[2] || "0"), + }); + } + return snippets; +} + +export function parseTranscriptJson3(text: string): Snippet[] { + const data = JSON.parse(text); + const events = Array.isArray(data?.events) ? data.events : []; + const snippets: Snippet[] = []; + for (const event of events) { + const segs = Array.isArray(event?.segs) ? event.segs : []; + const textParts = segs + .map((seg: any) => htmlUnescape(String(seg?.utf8 || "").replace(/\n+/g, " ").trim())) + .filter(Boolean); + const merged = mergeTexts(textParts).trim(); + if (!merged) continue; + snippets.push({ + text: merged, + start: Number(event?.tStartMs || 0) / 1000, + duration: Number(event?.dDurationMs || 0) / 1000, + }); + } + return snippets; +} + +function parseSrt(srt: string): Snippet[] { + const blocks = srt.trim().split(/\n\n+/); + const snippets: Snippet[] = []; + for (const block of blocks) { + const lines = block.split("\n"); + if (lines.length < 3) continue; + const match = lines[1].match(/(\d{2}):(\d{2}):(\d{2}),(\d{3})\s*-->\s*(\d{2}):(\d{2}):(\d{2}),(\d{3})/); + if (!match) continue; + const start = parseInt(match[1]) * 3600 + parseInt(match[2]) * 60 + parseInt(match[3]) + parseInt(match[4]) / 1000; + const end = parseInt(match[5]) * 3600 + parseInt(match[6]) * 60 + parseInt(match[7]) + parseInt(match[8]) / 1000; + snippets.push({ text: lines.slice(2).join(" "), start, duration: end - start }); + } + return snippets; +} + +export function parseWebVtt(vtt: string): Snippet[] { + const blocks = vtt + .replace(/^WEBVTT\s*/m, "") + .trim() + .split(/\n\n+/); + const snippets: Snippet[] = []; + for (const block of blocks) { + const lines = block.split("\n").map((line) => line.trim()).filter(Boolean); + const tsLine = lines.find((line) => line.includes("-->")); + if (!tsLine) continue; + const match = tsLine.match( + /(?:(\d{2}):)?(\d{2}):(\d{2})\.(\d{3})\s*-->\s*(?:(\d{2}):)?(\d{2}):(\d{2})\.(\d{3})/ + ); + if (!match) continue; + const start = + (match[1] ? parseInt(match[1]) : 0) * 3600 + + parseInt(match[2]) * 60 + + parseInt(match[3]) + + parseInt(match[4]) / 1000; + const end = + (match[5] ? parseInt(match[5]) : 0) * 3600 + + parseInt(match[6]) * 60 + + parseInt(match[7]) + + parseInt(match[8]) / 1000; + const text = htmlUnescape(stripTags(lines.slice(lines.indexOf(tsLine) + 1).join(" ").replace(/\s+/g, " ").trim())); + if (!text) continue; + snippets.push({ text, start, duration: end - start }); + } + return snippets; +} + +export function parseTranscriptPayload(payload: string, url: string): Snippet[] { + const normalized = payload.trimStart(); + if (url.includes("fmt=json3") || normalized.startsWith("{")) return parseTranscriptJson3(payload); + if (normalized.startsWith("WEBVTT")) return parseWebVtt(payload); + if (/^\d+\s*\n\d{2}:\d{2}:\d{2},\d{3}\s*-->/.test(normalized)) return parseSrt(payload); + return parseTranscriptXml(payload); +} + +function isCJK(ch: string): boolean { + const code = ch.charCodeAt(0); + return (code >= 0x4E00 && code <= 0x9FFF) || + (code >= 0x3040 && code <= 0x309F) || + (code >= 0x30A0 && code <= 0x30FF) || + (code >= 0xAC00 && code <= 0xD7AF) || + (code >= 0x3400 && code <= 0x4DBF) || + (code >= 0xF900 && code <= 0xFAFF); +} + +function splitSnippetAtPunctuation(snippet: Snippet): { text: string; start: number; end: number }[] { + const { text, start, duration } = snippet; + const end = start + duration; + if (!text.length) return []; + + const splitPoints: number[] = []; + for (let i = 0; i < text.length; i++) { + if (SENTENCE_END_RE.test(text[i])) { + while (i + 1 < text.length && SENTENCE_END_RE.test(text[i + 1])) i++; + if (i < text.length - 1) splitPoints.push(i); + } + } + + if (!splitPoints.length) return [{ text, start, end }]; + + const parts: { text: string; start: number; end: number }[] = []; + let prev = 0; + for (const pos of splitPoints) { + const partText = text.slice(prev, pos + 1).trim(); + if (partText) { + parts.push({ + text: partText, + start: start + (prev / text.length) * duration, + end: start + ((pos + 1) / text.length) * duration, + }); + } + prev = pos + 1; + } + + const remaining = text.slice(prev).trim(); + if (remaining) parts.push({ text: remaining, start: start + (prev / text.length) * duration, end }); + + return parts; +} + +function mergeTexts(texts: string[]): string { + if (!texts.length) return ""; + let result = texts[0]; + for (let i = 1; i < texts.length; i++) { + const next = texts[i]; + if (!next) continue; + const lastChar = result[result.length - 1]; + const firstChar = next[0]; + if (isCJK(lastChar) || isCJK(firstChar)) { + result += next; + } else { + result = result.trimEnd() + " " + next.trimStart(); + } + } + return result.replace(/ {2,}/g, " "); +} + +export function ts(time: number): string { + const h = Math.floor(time / 3600); + const m = Math.floor((time % 3600) / 60); + const s = Math.floor(time % 60); + return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`; +} + +function tsMs(time: number, sep: string): string { + const h = Math.floor(time / 3600); + const m = Math.floor((time % 3600) / 60); + const s = Math.floor(time % 60); + const ms = Math.round((time - Math.floor(time)) * 1000); + return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}${sep}${String(ms).padStart(3, "0")}`; +} + +function parseTs(time: string): number { + const [h, m, s] = time.split(":").map(Number); + return h * 3600 + m * 60 + s; +} + +export function segmentIntoSentences(snippets: Snippet[]): Sentence[] { + const parts: { text: string; start: number; end: number }[] = []; + for (const snippet of snippets) parts.push(...splitSnippetAtPunctuation(snippet)); + + const sentences: Sentence[] = []; + let buffer: { text: string; start: number; end: number }[] = []; + + for (const part of parts) { + buffer.push(part); + if (SENTENCE_END_RE.test(part.text[part.text.length - 1])) { + sentences.push({ + text: mergeTexts(buffer.map((entry) => entry.text)), + start: ts(buffer[0].start), + end: ts(buffer[buffer.length - 1].end), + }); + buffer = []; + } + } + + if (buffer.length) { + sentences.push({ + text: mergeTexts(buffer.map((entry) => entry.text)), + start: ts(buffer[0].start), + end: ts(buffer[buffer.length - 1].end), + }); + } + + return sentences; +} + +function groupSentenceParas(sentences: Sentence[]): Paragraph[] { + if (!sentences.length) return []; + const paragraphs: Paragraph[] = []; + let buffer: Sentence[] = []; + for (let i = 0; i < sentences.length; i++) { + buffer.push(sentences[i]); + const last = i === sentences.length - 1; + const gap = !last && parseTs(sentences[i + 1].start) - parseTs(sentences[i].end) > 2; + if (last || gap || buffer.length >= 5) { + paragraphs.push({ + text: mergeTexts(buffer.map((sentence) => sentence.text)), + start: parseTs(buffer[0].start), + end: parseTs(buffer[buffer.length - 1].end), + }); + buffer = []; + } + } + return paragraphs; +} + +export function formatSrt(snippets: Snippet[]): string { + return snippets + .map((snippet, index) => { + const end = index < snippets.length - 1 && snippets[index + 1].start < snippet.start + snippet.duration + ? snippets[index + 1].start + : snippet.start + snippet.duration; + return `${index + 1}\n${tsMs(snippet.start, ",")} --> ${tsMs(end, ",")}\n${snippet.text}`; + }) + .join("\n\n") + "\n"; +} + +function yamlEscape(value: string): string { + if (/[:"'{}\[\]#&*!|>%@`\n]/.test(value) || value.trim() !== value) { + return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; + } + return value; +} + +function extractSummary(description: string): string { + if (!description) return ""; + const firstPara = description.split(/\n\s*\n/)[0].trim(); + const lines = firstPara.split("\n").filter((line) => !/^\s*(https?:\/\/|#|@|\d+:\d+)/.test(line) && line.trim()); + return lines.join(" ").slice(0, 300).trim(); +} + +export function formatMarkdown( + sentences: Sentence[], + meta: VideoMeta, + opts: { timestamps: boolean; chapters: boolean; speakers: boolean }, + snippets?: Snippet[] +): string { + const summary = extractSummary(meta.description); + let md = "---\n"; + md += `title: ${yamlEscape(meta.title)}\n`; + md += `channel: ${yamlEscape(meta.channel)}\n`; + if (meta.publishDate) md += `date: ${meta.publishDate}\n`; + md += `url: ${yamlEscape(meta.url)}\n`; + if (meta.coverImage) md += `cover: ${meta.coverImage}\n`; + if (summary) md += `description: ${yamlEscape(summary)}\n`; + if (meta.language) md += `language: ${meta.language.code}\n`; + md += "---\n\n"; + + if (opts.speakers) { + md += `# ${meta.title}\n\n`; + if (summary) md += `${summary}\n\n`; + if (meta.description) md += `# Description\n\n${meta.description.trim()}\n\n`; + if (meta.chapters.length) { + md += "# Chapters\n\n"; + for (const chapter of meta.chapters) md += `* [${ts(chapter.start)}] ${chapter.title}\n`; + md += "\n"; + } + md += "# Transcript\n\n"; + md += snippets ? formatSrt(snippets) : ""; + return md; + } + + md += `# ${meta.title}\n\n`; + if (summary) md += `${summary}\n\n`; + + const chapters = opts.chapters ? meta.chapters : []; + if (chapters.length) { + md += "## Table of Contents\n\n"; + for (const chapter of chapters) md += opts.timestamps ? `* [${ts(chapter.start)}] ${chapter.title}\n` : `* ${chapter.title}\n`; + md += "\n"; + if (meta.coverImage) md += `\n![cover](${meta.coverImage})\n`; + md += "\n"; + for (let i = 0; i < chapters.length; i++) { + const nextStart = i < chapters.length - 1 ? chapters[i + 1].start : Infinity; + const chapterSentences = sentences.filter((sentence) => parseTs(sentence.start) >= chapters[i].start && parseTs(sentence.start) < nextStart); + const paragraphs = groupSentenceParas(chapterSentences); + md += opts.timestamps ? `## [${ts(chapters[i].start)}] ${chapters[i].title}\n\n` : `## ${chapters[i].title}\n\n`; + for (const paragraph of paragraphs) { + md += opts.timestamps ? `${paragraph.text} [${ts(paragraph.start)} → ${ts(paragraph.end)}]\n\n` : `${paragraph.text}\n\n`; + } + md += "\n"; + } + } else { + const paragraphs = groupSentenceParas(sentences); + for (const paragraph of paragraphs) { + md += opts.timestamps ? `${paragraph.text} [${ts(paragraph.start)} → ${ts(paragraph.end)}]\n\n` : `${paragraph.text}\n\n`; + } + } + + return md.trimEnd() + "\n"; +} + +export function formatListOutput(videoId: string, title: string, transcripts: TranscriptInfo[]): string { + const manual = transcripts.filter((transcript) => !transcript.isGenerated); + const generated = transcripts.filter((transcript) => transcript.isGenerated); + const translationLanguages = transcripts.find((transcript) => transcript.translationLanguages.length > 0)?.translationLanguages || []; + const formatList = (list: TranscriptInfo[]) => + list.length + ? list.map((transcript) => ` - ${transcript.languageCode} ("${transcript.language}")${transcript.isTranslatable ? " [TRANSLATABLE]" : ""}`).join("\n") + : "None"; + const formatTranslations = translationLanguages.length + ? translationLanguages.map((language) => ` - ${language.languageCode} ("${language.language}")`).join("\n") + : "None"; + return `Transcripts for ${videoId}${title ? ` (${title})` : ""}:\n\n(MANUALLY CREATED)\n${formatList(manual)}\n\n(GENERATED)\n${formatList(generated)}\n\n(TRANSLATION LANGUAGES)\n${formatTranslations}`; +} + +export function findTranscript( + transcripts: TranscriptInfo[], + languages: string[], + excludeGenerated: boolean, + excludeManual: boolean +): TranscriptInfo { + let filtered = transcripts; + if (excludeGenerated) filtered = filtered.filter((transcript) => !transcript.isGenerated); + if (excludeManual) filtered = filtered.filter((transcript) => transcript.isGenerated); + for (const language of languages) { + const found = filtered.find((transcript) => transcript.languageCode === language); + if (found) return found; + } + const available = filtered.map((transcript) => `${transcript.languageCode} ("${transcript.language}")`).join(", "); + throw makeError(`No transcript found for languages [${languages.join(", ")}]. Available: ${available || "none"}`, "NO_TRANSCRIPT"); +} diff --git a/skills/baoyu-youtube-transcript/scripts/types.ts b/skills/baoyu-youtube-transcript/scripts/types.ts new file mode 100644 index 0000000..83efdbf --- /dev/null +++ b/skills/baoyu-youtube-transcript/scripts/types.ts @@ -0,0 +1,123 @@ +export type Format = "text" | "srt"; + +export interface Options { + videoIds: string[]; + languages: string[]; + format: Format; + translate: string; + list: boolean; + excludeGenerated: boolean; + excludeManual: boolean; + output: string; + outputDir: string; + timestamps: boolean; + chapters: boolean; + speakers: boolean; + refresh: boolean; +} + +export interface Snippet { + text: string; + start: number; + duration: number; +} + +export interface Sentence { + text: string; + start: string; + end: string; +} + +export interface TranscriptLanguage { + language: string; + languageCode: string; +} + +export interface TranscriptInfo { + language: string; + languageCode: string; + isGenerated: boolean; + isTranslatable: boolean; + baseUrl: string; + translationLanguages: TranscriptLanguage[]; +} + +export interface Chapter { + title: string; + start: number; + end: number; +} + +export interface LanguageMeta { + code: string; + name: string; + isGenerated: boolean; +} + +export interface VideoMeta { + videoId: string; + title: string; + channel: string; + channelId: string; + description: string; + duration: number; + publishDate: string; + url: string; + coverImage: string; + thumbnailUrl: string; + language: LanguageMeta; + chapters: Chapter[]; +} + +export interface VideoResult { + videoId: string; + title?: string; + filePath?: string; + content?: string; + error?: string; +} + +export interface InnerTubeSession { + apiKey: string; + webClientVersion: string; + visitorData: string; +} + +export interface InnerTubeClient { + id: string; + clientName: string; + clientVersion?: string; + clientHeaderName?: string; + userAgent: string; + extraContext?: Record; +} + +export interface TranscriptError extends Error { + code?: string; +} + +export interface YtDlpTrack { + ext?: string; + url?: string; + name?: string; +} + +export interface YtDlpInfo { + title?: string; + channel?: string; + channel_id?: string; + uploader?: string; + uploader_id?: string; + description?: string; + duration?: number; + upload_date?: string; + webpage_url?: string; + thumbnail?: string; + thumbnails?: { url?: string; width?: number; height?: number }[]; + subtitles?: Record; + automatic_captions?: Record; +} + +export type VideoSource = + | { kind: "innertube"; data: any; transcripts: TranscriptInfo[] } + | { kind: "yt-dlp"; info: YtDlpInfo; transcripts: TranscriptInfo[] }; diff --git a/skills/baoyu-youtube-transcript/scripts/youtube.ts b/skills/baoyu-youtube-transcript/scripts/youtube.ts new file mode 100644 index 0000000..5a2f682 --- /dev/null +++ b/skills/baoyu-youtube-transcript/scripts/youtube.ts @@ -0,0 +1,477 @@ +import { spawnSync } from "child_process"; +import { writeFileSync } from "fs"; + +import { makeError, normalizeError, normalizePublishDate, shouldTryAlternateClient, shouldTryYtDlpFallback } from "./shared.ts"; +import { parseTranscriptPayload } from "./transcript.ts"; +import type { + Chapter, + InnerTubeClient, + InnerTubeSession, + LanguageMeta, + Snippet, + TranscriptInfo, + VideoMeta, + VideoSource, + YtDlpInfo, + YtDlpTrack, +} from "./types.ts"; + +const WATCH_URL = "https://www.youtube.com/watch?v="; +const INNERTUBE_URL = "https://www.youtube.com/youtubei/v1/player"; +const WATCH_PAGE_USER_AGENT = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36"; +const DEFAULT_WEB_CLIENT_VERSION = "2.20260320.08.00"; +const YT_DLP_MAX_BUFFER = 32 * 1024 * 1024; + +let cachedYtDlpCommand: { command: string; args: string[]; label: string } | null | undefined; + +const INNER_TUBE_CLIENTS: InnerTubeClient[] = [ + { + id: "android", + clientName: "ANDROID", + clientHeaderName: "3", + clientVersion: "20.10.38", + userAgent: + "com.google.android.youtube/20.10.38 (Linux; U; Android 14; en_US; Pixel 8 Pro; Build/AP1A.240405.002)", + extraContext: { + clientFormFactor: "SMALL_FORM_FACTOR", + androidSdkVersion: 34, + osName: "Android", + osVersion: "14", + platform: "MOBILE", + }, + }, + { + id: "web", + clientName: "WEB", + clientHeaderName: "1", + userAgent: WATCH_PAGE_USER_AGENT, + }, + { + id: "ios", + clientName: "IOS", + clientHeaderName: "5", + clientVersion: "20.10.4", + userAgent: + "com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3 like Mac OS X; en_US)", + extraContext: { + deviceMake: "Apple", + deviceModel: "iPhone16,2", + osName: "iPhone", + osVersion: "18.3.0.22D5054f", + platform: "MOBILE", + }, + }, +]; + +async function fetchHtml(videoId: string): Promise { + const watchUrl = `${WATCH_URL}${videoId}&hl=en&persist_hl=1&has_verified=1&bpctr=9999999999`; + const baseHeaders = { + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Cache-Control": "no-cache", + Pragma: "no-cache", + "User-Agent": WATCH_PAGE_USER_AGENT, + }; + const response = await fetch(watchUrl, { headers: baseHeaders }); + if (!response.ok) throw new Error(`HTTP ${response.status} fetching video page`); + let html = await response.text(); + if (html.includes('action="https://consent.youtube.com/s"')) { + const consentValue = html.match(/name="v" value="(.*?)"/); + if (!consentValue) throw new Error("Failed to create consent cookie"); + const consentResponse = await fetch(watchUrl, { + headers: { + ...baseHeaders, + Cookie: `CONSENT=YES+${consentValue[1]}`, + }, + }); + if (!consentResponse.ok) throw new Error(`HTTP ${consentResponse.status} fetching video page (consent)`); + html = await consentResponse.text(); + } + return html; +} + +function extractSession(html: string, videoId: string): InnerTubeSession { + const apiKey = html.match(/"INNERTUBE_API_KEY":\s*"([a-zA-Z0-9_-]+)"/)?.[1]; + if (!apiKey) { + if (html.includes('class="g-recaptcha"')) throw new Error(`IP blocked for ${videoId} (reCAPTCHA)`); + throw new Error(`Cannot extract API key for ${videoId}`); + } + const webClientVersion = + html.match(/"INNERTUBE_CLIENT_VERSION":\s*"([^"]+)"/)?.[1] || + html.match(/"clientVersion":"([^"]+)"/)?.[1] || + DEFAULT_WEB_CLIENT_VERSION; + const visitorData = + html.match(/"VISITOR_DATA":"([^"]+)"/)?.[1] || + html.match(/"visitorData":"([^"]+)"/)?.[1] || + ""; + return { apiKey, webClientVersion, visitorData }; +} + +function buildInnerTubeContext(client: InnerTubeClient, session: InnerTubeSession, videoId: string) { + return { + context: { + client: { + hl: "en", + gl: "US", + utcOffsetMinutes: 0, + visitorData: session.visitorData, + clientName: client.clientName, + clientVersion: client.clientVersion || session.webClientVersion, + ...client.extraContext, + }, + request: { useSsl: true }, + }, + videoId, + }; +} + +async function fetchInnertubeData(videoId: string, session: InnerTubeSession, client: InnerTubeClient): Promise { + const clientVersion = client.clientVersion || session.webClientVersion; + const headers: Record = { + Accept: "application/json", + "Accept-Language": "en-US,en;q=0.9", + "Content-Type": "application/json", + Origin: "https://www.youtube.com", + Referer: `${WATCH_URL}${videoId}`, + "User-Agent": client.userAgent, + "X-YouTube-Client-Name": client.clientHeaderName || "1", + "X-YouTube-Client-Version": clientVersion, + }; + if (session.visitorData) headers["X-Goog-Visitor-Id"] = session.visitorData; + const response = await fetch(`${INNERTUBE_URL}?key=${session.apiKey}&prettyPrint=false`, { + method: "POST", + headers, + body: JSON.stringify(buildInnerTubeContext(client, session, videoId)), + }); + if (response.status === 429) throw new Error(`IP blocked for ${videoId} (429)`); + if (!response.ok) throw new Error(`HTTP ${response.status} from InnerTube API`); + return response.json(); +} + +function assertPlayability(data: any, videoId: string) { + const playabilityStatus = data?.playabilityStatus; + if (!playabilityStatus) return; + const status = playabilityStatus.status; + if (status === "OK" || !status) return; + const reason = playabilityStatus.reason || ""; + const reasonLower = reason.toLowerCase(); + if (status === "LOGIN_REQUIRED") { + if (reasonLower.includes("bot")) throw makeError(`Request blocked for ${videoId}: bot detected`, "BOT_DETECTED"); + if (reasonLower.includes("inappropriate")) throw makeError(`Age restricted: ${videoId}`, "AGE_RESTRICTED"); + } + if (status === "ERROR" && reasonLower.includes("unavailable")) { + if (videoId.startsWith("http")) throw makeError("Invalid video ID: pass the ID, not the URL", "INVALID_VIDEO_ID"); + throw makeError(`Video unavailable: ${videoId}`, "VIDEO_UNAVAILABLE"); + } + const subreasons = playabilityStatus.errorScreen?.playerErrorMessageRenderer?.subreason?.runs?.map((run: any) => run.text).join("") || ""; + throw new Error(`Video unplayable (${videoId}): ${reason} ${subreasons}`.trim()); +} + +function extractCaptionsJson(data: any, videoId: string): any { + assertPlayability(data, videoId); + const captionsJson = data?.captions?.playerCaptionsTracklistRenderer; + if (!captionsJson || !captionsJson.captionTracks) throw makeError(`Transcripts disabled for ${videoId}`, "TRANSCRIPTS_DISABLED"); + return captionsJson; +} + +function buildTranscriptList(captionsJson: any): TranscriptInfo[] { + const translationLanguages = (captionsJson.translationLanguages || []).map((language: any) => ({ + language: language.languageName?.runs?.[0]?.text || language.languageName?.simpleText || "", + languageCode: language.languageCode, + })); + return (captionsJson.captionTracks || []).map((track: any) => ({ + language: track.name?.runs?.[0]?.text || track.name?.simpleText || "", + languageCode: track.languageCode, + isGenerated: track.kind === "asr", + isTranslatable: !!track.isTranslatable, + baseUrl: track.baseUrl || "", + translationLanguages: track.isTranslatable ? translationLanguages : [], + })); +} + +export async function fetchTranscriptSnippets( + info: TranscriptInfo, + translateTo?: string +): Promise<{ snippets: Snippet[]; language: string; languageCode: string }> { + let url = info.baseUrl; + let language = info.language; + let languageCode = info.languageCode; + if (translateTo) { + if (!info.isTranslatable) throw new Error(`Transcript ${info.languageCode} is not translatable`); + const translatedLanguage = info.translationLanguages.find((entry) => entry.languageCode === translateTo); + if (!translatedLanguage) throw new Error(`Translation language ${translateTo} not available`); + url += `&tlang=${translateTo}`; + language = translatedLanguage.language; + languageCode = translateTo; + } + const response = await fetch(url, { + headers: { + "Accept-Language": "en-US,en;q=0.9", + "User-Agent": WATCH_PAGE_USER_AGENT, + }, + }); + if (!response.ok) throw new Error(`HTTP ${response.status} fetching transcript`); + return { + snippets: parseTranscriptPayload(await response.text(), url), + language, + languageCode, + }; +} + +export function detectYtDlpCommand(): { command: string; args: string[]; label: string } | null { + if (cachedYtDlpCommand !== undefined) return cachedYtDlpCommand; + const candidates = [ + { command: "yt-dlp", args: [], label: "yt-dlp" }, + { command: "uvx", args: ["--from", "yt-dlp", "yt-dlp"], label: "uvx --from yt-dlp yt-dlp" }, + { command: "python3", args: ["-m", "yt_dlp"], label: "python3 -m yt_dlp" }, + ]; + for (const candidate of candidates) { + const probe = spawnSync(candidate.command, [...candidate.args, "--version"], { + encoding: "utf8", + maxBuffer: 1024 * 1024, + }); + if (probe.status !== 0) continue; + + const helpProbe = spawnSync(candidate.command, [...candidate.args, "--help"], { + encoding: "utf8", + maxBuffer: 2 * 1024 * 1024, + }); + const helpText = `${helpProbe.stdout || ""}\n${helpProbe.stderr || ""}`; + const supportsRequiredFlags = + helpProbe.status === 0 && + helpText.includes("--js-runtimes") && + helpText.includes("--remote-components"); + + if (supportsRequiredFlags) { + cachedYtDlpCommand = candidate; + return candidate; + } + } + cachedYtDlpCommand = null; + return cachedYtDlpCommand; +} + +export function selectYtDlpTrack(entries: YtDlpTrack[]): YtDlpTrack | null { + const preferredExts = ["json3", "srv3", "srv2", "srv1", "ttml", "vtt"]; + for (const ext of preferredExts) { + const match = entries.find((entry) => entry.url && entry.ext === ext); + if (match) return match; + } + return entries.find((entry) => !!entry.url) || null; +} + +export function buildTranscriptListFromYtDlp(info: YtDlpInfo): TranscriptInfo[] { + const translationLanguages = Object.entries(info.automatic_captions || {}).map(([languageCode, entries]) => ({ + language: entries.find((entry) => entry.name)?.name || languageCode, + languageCode, + })); + const manual = Object.entries(info.subtitles || {}).flatMap(([languageCode, entries]) => { + const selected = selectYtDlpTrack(entries); + if (!selected?.url) return []; + return [{ + language: selected.name || languageCode, + languageCode, + isGenerated: false, + isTranslatable: translationLanguages.length > 0, + baseUrl: selected.url, + translationLanguages, + }]; + }); + const generated = Object.entries(info.automatic_captions || {}).flatMap(([languageCode, entries]) => { + const selected = selectYtDlpTrack(entries); + if (!selected?.url) return []; + return [{ + language: selected.name || languageCode, + languageCode, + isGenerated: true, + isTranslatable: translationLanguages.length > 0, + baseUrl: selected.url, + translationLanguages, + }]; + }); + return [...manual, ...generated]; +} + +function fetchYtDlpInfo(videoId: string): YtDlpInfo { + const command = detectYtDlpCommand(); + if (!command) { + throw makeError( + `Request blocked for ${videoId}: bot detected. yt-dlp fallback unavailable (install yt-dlp or uv).`, + "YT_DLP_UNAVAILABLE" + ); + } + + const args = [ + ...command.args, + "-J", + "--skip-download", + "--js-runtimes", + "bun", + "--remote-components", + "ejs:github", + ]; + const cookiesFromBrowser = process.env.YOUTUBE_TRANSCRIPT_COOKIES_FROM_BROWSER?.trim(); + if (cookiesFromBrowser) args.push("--cookies-from-browser", cookiesFromBrowser); + args.push(`${WATCH_URL}${videoId}`); + + const result = spawnSync(command.command, args, { + encoding: "utf8", + maxBuffer: YT_DLP_MAX_BUFFER, + }); + if (result.status !== 0) { + const stderr = (result.stderr || "").trim(); + const stdout = (result.stdout || "").trim(); + const detail = stderr || stdout || `exit ${result.status ?? "unknown"}`; + throw makeError(`yt-dlp fallback failed for ${videoId} (${command.label}): ${detail}`, "YT_DLP_FAILED"); + } + return JSON.parse(result.stdout); +} + +async function fetchInnertubeSource(videoId: string): Promise { + const html = await fetchHtml(videoId); + const session = extractSession(html, videoId); + const attempts: string[] = []; + let lastError: Error | null = null; + + for (const client of INNER_TUBE_CLIENTS) { + try { + const data = await fetchInnertubeData(videoId, session, client); + const captionsJson = extractCaptionsJson(data, videoId); + return { kind: "innertube", data, transcripts: buildTranscriptList(captionsJson) }; + } catch (error) { + const normalized = normalizeError(error); + attempts.push(`${client.id}: ${normalized.message}`); + lastError = normalized; + if (!shouldTryAlternateClient(normalized)) break; + } + } + + if (!lastError) throw makeError(`Unable to fetch transcript metadata for ${videoId}`, "UNKNOWN"); + if (attempts.length > 1) { + throw makeError(`${lastError.message}. Tried clients: ${attempts.join("; ")}`, normalizeError(lastError).code); + } + throw lastError; +} + +export async function resolveVideoSource( + videoId: string, + fetchPrimary: (videoId: string) => Promise, + fetchFallback: (videoId: string) => YtDlpInfo, + logWarning: (message: string) => void = (message) => console.error(message) +): Promise { + try { + return await fetchPrimary(videoId); + } catch (error) { + const normalized = normalizeError(error); + if (!shouldTryYtDlpFallback(normalized)) throw normalized; + logWarning(`Warning (${videoId}): ${normalized.message}. Retrying with yt-dlp fallback.`); + const info = fetchFallback(videoId); + const transcripts = buildTranscriptListFromYtDlp(info); + if (!transcripts.length) throw makeError(`Transcripts disabled for ${videoId}`, "TRANSCRIPTS_DISABLED"); + return { kind: "yt-dlp", info, transcripts }; + } +} + +export async function fetchVideoSource(videoId: string): Promise { + return resolveVideoSource(videoId, fetchInnertubeSource, fetchYtDlpInfo); +} + +export function parseChapters(description: string, duration: number = 0): Chapter[] { + const raw: { title: string; start: number }[] = []; + for (const line of description.split("\n")) { + const match = line.trim().match(/^(?:(\d{1,2}):)?(\d{1,2}):(\d{2})\s+(.+)$/); + if (match) { + const hours = match[1] ? parseInt(match[1]) : 0; + raw.push({ title: match[4].trim(), start: hours * 3600 + parseInt(match[2]) * 60 + parseInt(match[3]) }); + } + } + if (raw.length < 2) return []; + return raw.map((chapter, index) => ({ + title: chapter.title, + start: chapter.start, + end: index < raw.length - 1 ? raw[index + 1].start : Math.max(duration, chapter.start), + })); +} + +export function getThumbnailUrls(videoId: string, data: any): string[] { + const urls = [ + `https://i.ytimg.com/vi/${videoId}/maxresdefault.jpg`, + `https://i.ytimg.com/vi/${videoId}/hqdefault.jpg`, + ]; + const thumbnails = data?.videoDetails?.thumbnail?.thumbnails || + data?.microformat?.playerMicroformatRenderer?.thumbnail?.thumbnails || + []; + if (thumbnails.length) { + const sorted = [...thumbnails].sort((a: any, b: any) => (b.width || 0) - (a.width || 0)); + for (const thumbnail of sorted) { + if (thumbnail.url && !urls.includes(thumbnail.url)) urls.push(thumbnail.url); + } + } + return urls; +} + +export function getYtDlpThumbnailUrls(videoId: string, info: YtDlpInfo): string[] { + const urls = getThumbnailUrls(videoId, null); + const thumbnails = Array.isArray(info.thumbnails) ? info.thumbnails : []; + const sorted = [...thumbnails].sort((a, b) => (b?.width || 0) - (a?.width || 0)); + for (const thumbnail of sorted) { + if (thumbnail?.url && !urls.includes(thumbnail.url)) urls.push(thumbnail.url); + } + if (info.thumbnail && !urls.includes(info.thumbnail)) urls.push(info.thumbnail); + return urls; +} + +export function buildVideoMeta(data: any, videoId: string, language: LanguageMeta, chapters: Chapter[]): VideoMeta { + const videoDetails = data?.videoDetails || {}; + const microformat = data?.microformat?.playerMicroformatRenderer || {}; + return { + videoId, + title: videoDetails.title || microformat.title?.simpleText || "", + channel: videoDetails.author || microformat.ownerChannelName || "", + channelId: videoDetails.channelId || microformat.externalChannelId || "", + description: videoDetails.shortDescription || microformat.description?.simpleText || "", + duration: parseInt(videoDetails.lengthSeconds || "0"), + publishDate: microformat.publishDate || microformat.uploadDate || "", + url: `${WATCH_URL}${videoId}`, + coverImage: "", + thumbnailUrl: getThumbnailUrls(videoId, data)[0], + language, + chapters, + }; +} + +export function buildVideoMetaFromYtDlp( + info: YtDlpInfo, + videoId: string, + language: LanguageMeta, + chapters: Chapter[] +): VideoMeta { + return { + videoId, + title: info.title || "", + channel: info.channel || info.uploader || "", + channelId: info.channel_id || info.uploader_id || "", + description: info.description || "", + duration: Number(info.duration || 0), + publishDate: normalizePublishDate(info.upload_date), + url: info.webpage_url || `${WATCH_URL}${videoId}`, + coverImage: "", + thumbnailUrl: getYtDlpThumbnailUrls(videoId, info)[0] || "", + language, + chapters, + }; +} + +export async function downloadCoverImage(urls: string[], outputPath: string): Promise { + for (const url of urls) { + try { + const response = await fetch(url); + if (response.ok) { + writeFileSync(outputPath, Buffer.from(await response.arrayBuffer())); + return true; + } + } catch {} + } + return false; +}