diff --git a/skills/baoyu-danger-x-to-markdown/SKILL.md b/skills/baoyu-danger-x-to-markdown/SKILL.md index 392b743..157ee16 100644 --- a/skills/baoyu-danger-x-to-markdown/SKILL.md +++ b/skills/baoyu-danger-x-to-markdown/SKILL.md @@ -170,7 +170,7 @@ coverImage: "https://pbs.twimg.com/media/example.jpg" Content... ``` -**File structure**: `x-to-markdown/{username}/{tweet-id}.md` +**File structure**: `x-to-markdown/{username}/{tweet-id}/{content-slug}.md` When `--download-media` is enabled: - Images are saved to `imgs/` next to the markdown file diff --git a/skills/baoyu-danger-x-to-markdown/scripts/main.ts b/skills/baoyu-danger-x-to-markdown/scripts/main.ts index 5667093..74b7d34 100644 --- a/skills/baoyu-danger-x-to-markdown/scripts/main.ts +++ b/skills/baoyu-danger-x-to-markdown/scripts/main.ts @@ -7,6 +7,7 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; import { fetchXArticle } from "./graphql.js"; import { formatArticleMarkdown } from "./markdown.js"; import { localizeMarkdownMedia, type LocalizeMarkdownMediaResult } from "./media-localizer.js"; +import { resolveReferencedTweetsFromArticle } from "./referenced-tweets.js"; import { hasRequiredXCookies, loadXCookies, refreshXCookies } from "./cookies.js"; import { resolveXToMarkdownConsentPath } from "./paths.js"; import { tweetToMarkdown } from "./tweet-to-markdown.js"; @@ -203,6 +204,134 @@ function extractContentSlug(markdown: string): string { return "untitled"; } +function resolveSlugAndId(normalizedUrl: string, kind: "tweet" | "article"): { slug: string; idPart: string } { + const articleId = kind === "article" ? parseArticleId(normalizedUrl) : null; + const tweetId = kind === "tweet" ? parseTweetId(normalizedUrl) : null; + const username = kind === "tweet" ? parseTweetUsername(normalizedUrl) : null; + + const idPart = articleId ?? tweetId ?? String(Date.now()); + const userSlug = username ? sanitizeSlug(username) : null; + const slug = userSlug ?? idPart; + return { slug, idPart }; +} + +function extractFrontmatterUrls(markdown: string): string[] { + const match = markdown.match(/^---\n([\s\S]*?)\n---/); + if (!match?.[1]) return []; + + const lines = match[1].split("\n"); + const urls: string[] = []; + for (const line of lines) { + const m = line.match(/^(url|requestedUrl):\s*["']([^"']+)["']\s*$/); + if (m?.[2]) { + urls.push(m[2]); + } + } + return urls; +} + +function frontmatterMatchesTarget( + markdown: string, + normalizedUrl: string, + kind: "tweet" | "article" +): boolean { + const urls = extractFrontmatterUrls(markdown); + if (urls.length === 0) return false; + + const targetId = kind === "article" ? parseArticleId(normalizedUrl) : parseTweetId(normalizedUrl); + if (!targetId) return false; + + for (const url of urls) { + const candidateId = kind === "article" ? parseArticleId(url) : parseTweetId(url); + if (candidateId && candidateId === targetId) { + return true; + } + } + + return false; +} + +function listMarkdownFiles(dirPath: string): string[] { + try { + return fs + .readdirSync(dirPath) + .filter((name) => name.toLowerCase().endsWith(".md")) + .map((name) => path.join(dirPath, name)) + .sort(); + } catch { + return []; + } +} + +function resolveExistingMarkdownPath( + normalizedUrl: string, + kind: "tweet" | "article", + argsOutput: string | null +): string | null { + const { slug, idPart } = resolveSlugAndId(normalizedUrl, kind); + const candidateDirs = new Set(); + const candidateFiles = new Set(); + + if (argsOutput) { + const resolved = path.resolve(argsOutput); + const looksDir = argsOutput.endsWith("/") || argsOutput.endsWith("\\"); + try { + if (fs.existsSync(resolved)) { + const stat = fs.statSync(resolved); + if (stat.isFile()) { + candidateFiles.add(resolved); + } else if (stat.isDirectory()) { + candidateDirs.add(path.join(resolved, slug, idPart)); + candidateDirs.add(resolved); + } + } else if (looksDir) { + candidateDirs.add(path.join(resolved, slug, idPart)); + } + } catch { + // ignore and continue + } + } else { + candidateDirs.add(path.resolve(process.cwd(), "x-to-markdown", slug, idPart)); + } + + for (const filePath of candidateFiles) { + if (!filePath.toLowerCase().endsWith(".md")) continue; + try { + const markdown = fs.readFileSync(filePath, "utf8"); + if (frontmatterMatchesTarget(markdown, normalizedUrl, kind)) { + return filePath; + } + } catch { + // ignore and continue + } + } + + for (const dirPath of candidateDirs) { + if (!fs.existsSync(dirPath)) continue; + let stat: fs.Stats; + try { + stat = fs.statSync(dirPath); + } catch { + continue; + } + if (!stat.isDirectory()) continue; + + const markdownFiles = listMarkdownFiles(dirPath); + for (const filePath of markdownFiles) { + try { + const markdown = fs.readFileSync(filePath, "utf8"); + if (frontmatterMatchesTarget(markdown, normalizedUrl, kind)) { + return filePath; + } + } catch { + // ignore and continue + } + } + } + + return null; +} + async function resolveOutputPath( normalizedUrl: string, kind: "tweet" | "article", @@ -218,14 +347,14 @@ async function resolveOutputPath( const idPart = articleId ?? tweetId ?? String(Date.now()); const slug = userSlug ?? idPart; - const defaultFileName = `${idPart}.md`; + const defaultFileName = `${contentSlug}.md`; if (argsOutput) { const wantsDir = argsOutput.endsWith("/") || argsOutput.endsWith("\\"); const resolved = path.resolve(argsOutput); try { if (wantsDir || (fs.existsSync(resolved) && fs.statSync(resolved).isDirectory())) { - const outputDir = path.join(resolved, slug, contentSlug); + const outputDir = path.join(resolved, slug, idPart); await mkdir(outputDir, { recursive: true }); return { outputDir, markdownPath: path.join(outputDir, defaultFileName), slug }; } @@ -238,7 +367,7 @@ async function resolveOutputPath( return { outputDir, markdownPath: resolved, slug }; } - const outputDir = path.resolve(process.cwd(), "x-to-markdown", slug, contentSlug); + const outputDir = path.resolve(process.cwd(), "x-to-markdown", slug, idPart); await mkdir(outputDir, { recursive: true }); return { outputDir, markdownPath: path.join(outputDir, defaultFileName), slug }; } @@ -349,7 +478,8 @@ async function convertArticleToMarkdown( log(`[x-to-markdown] Fetching article ${articleId}...`); const article = await fetchXArticle(articleId, cookieMap, false); - const { markdown: body, coverUrl } = formatArticleMarkdown(article); + const referencedTweets = await resolveReferencedTweetsFromArticle(article, cookieMap, { log }); + const { markdown: body, coverUrl } = formatArticleMarkdown(article, { referencedTweets }); const title = typeof (article as any)?.title === "string" ? String((article as any).title).trim() : ""; const meta = formatMetaMarkdown({ @@ -389,6 +519,58 @@ async function main(): Promise { const kind = articleId ? ("article" as const) : ("tweet" as const); + if (args.downloadMedia) { + const existingMarkdownPath = resolveExistingMarkdownPath(normalizedUrl, kind, args.output); + if (existingMarkdownPath) { + log(`[x-to-markdown] Reusing existing markdown: ${existingMarkdownPath}`); + const existingMarkdown = await readFile(existingMarkdownPath, "utf8"); + const mediaResult = await localizeMarkdownMedia(existingMarkdown, { + markdownPath: existingMarkdownPath, + log, + }); + const didLocalize = + mediaResult.downloadedImages > 0 || + mediaResult.downloadedVideos > 0 || + mediaResult.markdown !== existingMarkdown; + + if (didLocalize) { + await writeFile(existingMarkdownPath, mediaResult.markdown, "utf8"); + log( + `[x-to-markdown] Media localized: images=${mediaResult.downloadedImages}, videos=${mediaResult.downloadedVideos}` + ); + log(`[x-to-markdown] Saved: ${existingMarkdownPath}`); + + const { slug } = resolveSlugAndId(normalizedUrl, kind); + if (args.json) { + console.log( + JSON.stringify( + { + url: articleId ? `https://x.com/i/article/${articleId}` : normalizedUrl, + requestedUrl: normalizedUrl, + type: kind, + slug, + outputDir: path.dirname(existingMarkdownPath), + markdownPath: existingMarkdownPath, + downloadMedia: true, + downloadedImages: mediaResult.downloadedImages, + downloadedVideos: mediaResult.downloadedVideos, + imageDir: mediaResult.imageDir, + videoDir: mediaResult.videoDir, + }, + null, + 2 + ) + ); + } else { + console.log(existingMarkdownPath); + } + return; + } + + log("[x-to-markdown] Existing markdown already localized; rebuilding content to refresh placement."); + } + } + let markdown = kind === "article" && articleId ? await convertArticleToMarkdown(normalizedUrl, articleId, log) diff --git a/skills/baoyu-danger-x-to-markdown/scripts/markdown.ts b/skills/baoyu-danger-x-to-markdown/scripts/markdown.ts index 644f875..946e708 100644 --- a/skills/baoyu-danger-x-to-markdown/scripts/markdown.ts +++ b/skills/baoyu-danger-x-to-markdown/scripts/markdown.ts @@ -2,9 +2,22 @@ import type { ArticleBlock, ArticleContentState, ArticleEntity, + ArticleEntityMapEntry, ArticleMediaInfo, } from "./types.js"; +export type ReferencedTweetInfo = { + id: string; + url: string; + authorName?: string; + authorUsername?: string; + text?: string; +}; + +export type FormatArticleOptions = { + referencedTweets?: Map; +}; + function coerceArticleEntity(value: unknown): ArticleEntity | null { if (!value || typeof value !== "object") return null; const candidate = value as ArticleEntity; @@ -29,6 +42,73 @@ function normalizeCaption(caption?: string): string { return trimmed.replace(/\s+/g, " "); } +function summarizeTweetText(text?: string): string { + const trimmed = text?.trim(); + if (!trimmed) return ""; + const normalized = trimmed + .split(/\r?\n+/) + .map((line) => line.trim()) + .filter(Boolean) + .join(" "); + if (normalized.length <= 280) return normalized; + return `${normalized.slice(0, 277)}...`; +} + +function buildTweetUrl(tweetId?: string, username?: string): string | null { + if (!tweetId) return null; + if (username) { + return `https://x.com/${username}/status/${tweetId}`; + } + return `https://x.com/i/web/status/${tweetId}`; +} + +type EntityLookup = { + byIndex: Map; + byLogicalKey: Map; +}; + +function buildEntityLookup( + entityMap: ArticleContentState["entityMap"] | undefined +): EntityLookup { + const lookup: EntityLookup = { + byIndex: new Map(), + byLogicalKey: new Map(), + }; + + if (!entityMap) return lookup; + + for (const [idx, entry] of Object.entries(entityMap)) { + const idxNum = Number(idx); + if (Number.isFinite(idxNum)) { + lookup.byIndex.set(idxNum, entry); + } + + const logicalKey = parseInt(entry?.key ?? "", 10); + if (Number.isFinite(logicalKey) && !lookup.byLogicalKey.has(logicalKey)) { + lookup.byLogicalKey.set(logicalKey, entry); + } + } + + return lookup; +} + +function resolveEntityEntry( + entityKey: number | undefined, + entityMap: ArticleContentState["entityMap"] | undefined, + lookup: EntityLookup +): ArticleEntityMapEntry | undefined { + if (entityKey === undefined) return undefined; + + const byLogicalKey = lookup.byLogicalKey.get(entityKey); + if (byLogicalKey) return byLogicalKey; + + const byIndex = lookup.byIndex.get(entityKey); + if (byIndex) return byIndex; + + if (!entityMap) return undefined; + return entityMap[String(entityKey)]; +} + function resolveMediaUrl(info?: ArticleMediaInfo): string | undefined { if (!info) return undefined; if (info.original_img_url) return info.original_img_url; @@ -79,11 +159,12 @@ function collectMediaUrls( function resolveEntityMediaLines( entityKey: number | undefined, entityMap: ArticleContentState["entityMap"] | undefined, + entityLookup: EntityLookup, mediaById: Map, usedUrls: Set ): string[] { - if (entityKey === undefined || !entityMap) return []; - const entry = entityMap[String(entityKey)]; + if (entityKey === undefined) return []; + const entry = resolveEntityEntry(entityKey, entityMap, entityLookup); const value = entry?.value; if (!value) return []; const type = value.type; @@ -117,6 +198,45 @@ function resolveEntityMediaLines( return lines; } +function resolveEntityTweetLines( + entityKey: number | undefined, + entityMap: ArticleContentState["entityMap"] | undefined, + entityLookup: EntityLookup, + referencedTweets?: Map +): string[] { + if (entityKey === undefined) return []; + const entry = resolveEntityEntry(entityKey, entityMap, entityLookup); + const value = entry?.value; + if (!value || value.type !== "TWEET") return []; + + const tweetId = typeof value.data?.tweetId === "string" ? value.data.tweetId : ""; + if (!tweetId) return []; + + const referenced = referencedTweets?.get(tweetId); + const url = + referenced?.url ?? + buildTweetUrl(tweetId, referenced?.authorUsername) ?? + `https://x.com/i/web/status/${tweetId}`; + + const authorText = + referenced?.authorName && referenced?.authorUsername + ? `${referenced.authorName} (@${referenced.authorUsername})` + : referenced?.authorUsername + ? `@${referenced.authorUsername}` + : referenced?.authorName; + + const lines: string[] = []; + lines.push(`> 引用推文${authorText ? `:${authorText}` : ""}`); + + const summary = summarizeTweetText(referenced?.text); + if (summary) { + lines.push(`> ${summary}`); + } + + lines.push(`> ${url}`); + return lines; +} + function buildMediaLinkMap( entityMap: ArticleContentState["entityMap"] | undefined ): Map { @@ -151,6 +271,7 @@ function buildMediaLinkMap( if (linkIdx === -1) linkIdx = 0; const link = pool.splice(linkIdx, 1)[0]!; map.set(media.idx, link.url); + map.set(media.key, link.url); } return map; @@ -160,6 +281,7 @@ function renderInlineLinks( text: string, entityRanges: Array<{ key?: number; offset?: number; length?: number }>, entityMap: ArticleContentState["entityMap"] | undefined, + entityLookup: EntityLookup, mediaLinkMap: Map ): string { if (!entityMap || entityRanges.length === 0) return text; @@ -181,7 +303,7 @@ function renderInlineLinks( const length = range.length!; const key = range.key!; - const entry = entityMap[String(key)]; + const entry = resolveEntityEntry(key, entityMap, entityLookup); const value = entry?.value; if (!value) continue; @@ -204,27 +326,14 @@ function renderInlineLinks( return result; } -function buildAtomicMediaQueue( - article: ArticleEntity, - usedUrls: Set -): string[] { - const queue: string[] = []; - for (const entity of article.media_entities ?? []) { - const url = resolveMediaUrl(entity?.media_info); - if (url && !usedUrls.has(url)) { - queue.push(url); - } - } - return queue; -} - function renderContentBlocks( blocks: ArticleBlock[], entityMap: ArticleContentState["entityMap"] | undefined, + entityLookup: EntityLookup, mediaById: Map, usedUrls: Set, - atomicMediaQueue: string[], - mediaLinkMap: Map + mediaLinkMap: Map, + referencedTweets?: Map ): string[] { const lines: string[] = []; let previousKind: "list" | "quote" | "heading" | "text" | "code" | "media" | null = null; @@ -253,18 +362,54 @@ function renderContentBlocks( const mediaLines: string[] = []; for (const range of ranges) { if (typeof range?.key !== "number") continue; - mediaLines.push(...resolveEntityMediaLines(range.key, entityMap, mediaById, usedUrls)); + mediaLines.push( + ...resolveEntityMediaLines(range.key, entityMap, entityLookup, mediaById, usedUrls) + ); } return mediaLines; }; + const collectTweetLines = (block: ArticleBlock): string[] => { + const ranges = Array.isArray(block.entityRanges) ? block.entityRanges : []; + const tweetLines: string[] = []; + for (const range of ranges) { + if (typeof range?.key !== "number") continue; + tweetLines.push( + ...resolveEntityTweetLines(range.key, entityMap, entityLookup, referencedTweets) + ); + } + return tweetLines; + }; + + const collectLinkLines = (block: ArticleBlock): string[] => { + const ranges = Array.isArray(block.entityRanges) ? block.entityRanges : []; + const linkLines: string[] = []; + for (const range of ranges) { + if (typeof range?.key !== "number") continue; + const entry = resolveEntityEntry(range.key, entityMap, entityLookup); + const value = entry?.value; + if (value?.type !== "LINK") continue; + const url = typeof value.data?.url === "string" ? value.data.url : ""; + if (url) { + linkLines.push(url); + } + } + return [...new Set(linkLines)]; + }; + + const pushTrailingMedia = (mediaLines: string[]) => { + if (mediaLines.length > 0) { + pushBlock(mediaLines, "media"); + } + }; + for (const block of blocks) { const type = typeof block?.type === "string" ? block.type : "unstyled"; const rawText = typeof block?.text === "string" ? block.text : ""; const ranges = Array.isArray(block.entityRanges) ? block.entityRanges : []; const text = type !== "atomic" && type !== "code-block" - ? renderInlineLinks(rawText, ranges, entityMap, mediaLinkMap) + ? renderInlineLinks(rawText, ranges, entityMap, entityLookup, mediaLinkMap) : rawText; if (type === "code-block") { @@ -290,16 +435,22 @@ function renderContentBlocks( } listKind = null; orderedIndex = 0; + + const tweetLines = collectTweetLines(block); + if (tweetLines.length > 0) { + pushBlock(tweetLines, "quote"); + } + const mediaLines = collectMediaLines(block); if (mediaLines.length > 0) { pushBlock(mediaLines, "media"); - } else if (atomicMediaQueue.length > 0) { - const url = atomicMediaQueue.shift()!; - if (!usedUrls.has(url)) { - usedUrls.add(url); - pushBlock([`![](${url})`], "media"); - } } + + const linkLines = collectLinkLines(block); + if (linkLines.length > 0) { + pushBlock(linkLines, "text"); + } + continue; } @@ -313,6 +464,7 @@ function renderContentBlocks( listKind = "unordered"; orderedIndex = 0; pushBlock([`- ${text}`], "list"); + pushTrailingMedia(collectMediaLines(block)); continue; } @@ -323,6 +475,7 @@ function renderContentBlocks( listKind = "ordered"; orderedIndex += 1; pushBlock([`${orderedIndex}. ${text}`], "list"); + pushTrailingMedia(collectMediaLines(block)); continue; } @@ -332,29 +485,41 @@ function renderContentBlocks( switch (type) { case "header-one": pushBlock([`# ${text}`], "heading"); + pushTrailingMedia(collectMediaLines(block)); break; case "header-two": pushBlock([`## ${text}`], "heading"); + pushTrailingMedia(collectMediaLines(block)); break; case "header-three": pushBlock([`### ${text}`], "heading"); + pushTrailingMedia(collectMediaLines(block)); break; case "header-four": pushBlock([`#### ${text}`], "heading"); + pushTrailingMedia(collectMediaLines(block)); break; case "header-five": pushBlock([`##### ${text}`], "heading"); + pushTrailingMedia(collectMediaLines(block)); break; case "header-six": pushBlock([`###### ${text}`], "heading"); + pushTrailingMedia(collectMediaLines(block)); break; case "blockquote": { const quoteLines = text.length > 0 ? text.split("\n") : [""]; pushBlock(quoteLines.map((line) => `> ${line}`), "quote"); + pushTrailingMedia(collectMediaLines(block)); break; } default: + if (/^XIMGPH_\d+$/.test(text.trim())) { + pushTrailingMedia(collectMediaLines(block)); + break; + } pushBlock([text], "text"); + pushTrailingMedia(collectMediaLines(block)); break; } } @@ -371,7 +536,28 @@ export type FormatArticleResult = { coverUrl: string | null; }; -export function formatArticleMarkdown(article: unknown): FormatArticleResult { +export function extractReferencedTweetIds(article: unknown): string[] { + const candidate = coerceArticleEntity(article); + const entityMap = candidate?.content_state?.entityMap; + if (!entityMap) return []; + + const ids: string[] = []; + const seen = new Set(); + for (const entry of Object.values(entityMap)) { + const value = entry?.value; + if (value?.type !== "TWEET") continue; + const tweetId = typeof value.data?.tweetId === "string" ? value.data.tweetId : ""; + if (!tweetId || seen.has(tweetId)) continue; + seen.add(tweetId); + ids.push(tweetId); + } + return ids; +} + +export function formatArticleMarkdown( + article: unknown, + options: FormatArticleOptions = {} +): FormatArticleResult { const candidate = coerceArticleEntity(article); if (!candidate) { return { markdown: `\`\`\`json\n${JSON.stringify(article, null, 2)}\n\`\`\``, coverUrl: null }; @@ -392,10 +578,18 @@ export function formatArticleMarkdown(article: unknown): FormatArticleResult { const blocks = candidate.content_state?.blocks; const entityMap = candidate.content_state?.entityMap; + const entityLookup = buildEntityLookup(entityMap); if (Array.isArray(blocks) && blocks.length > 0) { - const atomicMediaQueue = buildAtomicMediaQueue(candidate, usedUrls); const mediaLinkMap = buildMediaLinkMap(entityMap); - const rendered = renderContentBlocks(blocks, entityMap, mediaById, usedUrls, atomicMediaQueue, mediaLinkMap); + const rendered = renderContentBlocks( + blocks, + entityMap, + entityLookup, + mediaById, + usedUrls, + mediaLinkMap, + options.referencedTweets + ); if (rendered.length > 0) { if (lines.length > 0) lines.push(""); lines.push(...rendered); diff --git a/skills/baoyu-danger-x-to-markdown/scripts/media-localizer.ts b/skills/baoyu-danger-x-to-markdown/scripts/media-localizer.ts index a7fec04..ae7da26 100644 --- a/skills/baoyu-danger-x-to-markdown/scripts/media-localizer.ts +++ b/skills/baoyu-danger-x-to-markdown/scripts/media-localizer.ts @@ -187,11 +187,35 @@ function buildFileName(kind: MediaKind, index: number, sourceUrl: string, extens const FRONTMATTER_COVER_RE = /^(coverImage:\s*")(https?:\/\/[^"]+)(")/m; +function toHighResUrl(rawUrl: string): string { + try { + const parsed = new URL(rawUrl); + if (parsed.hostname !== "pbs.twimg.com") return rawUrl; + const ext = path.posix.extname(parsed.pathname).replace(/^\./, "").toLowerCase(); + if (!ext || !IMAGE_EXTENSIONS.has(ext)) return rawUrl; + parsed.pathname = parsed.pathname.replace(new RegExp(`\\.${ext}$`), ""); + parsed.searchParams.set("format", ext === "jpeg" ? "jpg" : ext); + parsed.searchParams.set("name", "4096x4096"); + return parsed.toString(); + } catch { + return rawUrl; + } +} + function collectMarkdownLinkCandidates(markdown: string): MarkdownLinkCandidate[] { - MARKDOWN_LINK_RE.lastIndex = 0; const candidates: MarkdownLinkCandidate[] = []; const seen = new Set(); + const fmMatch = markdown.match(/^---\n([\s\S]*?)\n---/); + if (fmMatch) { + const coverMatch = fmMatch[1]?.match(FRONTMATTER_COVER_RE); + if (coverMatch?.[2] && !seen.has(coverMatch[2])) { + seen.add(coverMatch[2]); + candidates.push({ url: coverMatch[2], hint: "image" }); + } + } + + MARKDOWN_LINK_RE.lastIndex = 0; let match: RegExpExecArray | null; while ((match = MARKDOWN_LINK_RE.exec(markdown))) { const label = match[1] ?? ""; @@ -204,15 +228,6 @@ function collectMarkdownLinkCandidates(markdown: string): MarkdownLinkCandidate[ }); } - const fmMatch = markdown.match(/^---\n([\s\S]*?)\n---/); - if (fmMatch) { - const coverMatch = fmMatch[1]?.match(FRONTMATTER_COVER_RE); - if (coverMatch?.[2] && !seen.has(coverMatch[2])) { - seen.add(coverMatch[2]); - candidates.push({ url: coverMatch[2], hint: "image" }); - } - } - return candidates; } @@ -259,7 +274,8 @@ export async function localizeMarkdownMedia( for (const candidate of candidates) { try { - const response = await fetch(candidate.url, { + const downloadUrl = toHighResUrl(candidate.url); + const response = await fetch(downloadUrl, { method: "GET", redirect: "follow", headers: { diff --git a/skills/baoyu-danger-x-to-markdown/scripts/referenced-tweets.ts b/skills/baoyu-danger-x-to-markdown/scripts/referenced-tweets.ts new file mode 100644 index 0000000..c7518f0 --- /dev/null +++ b/skills/baoyu-danger-x-to-markdown/scripts/referenced-tweets.ts @@ -0,0 +1,81 @@ +import { fetchXTweet } from "./graphql.js"; +import { + extractReferencedTweetIds, + type ReferencedTweetInfo, +} from "./markdown.js"; + +type ResolveReferencedTweetsOptions = { + log?: (message: string) => void; +}; + +function extractReferencedTweetInfo(tweet: any, fallbackTweetId: string): ReferencedTweetInfo { + const userCore = tweet?.core?.user_results?.result?.core; + const userLegacy = tweet?.core?.user_results?.result?.legacy; + + const authorName = + typeof userCore?.name === "string" + ? userCore.name + : typeof userLegacy?.name === "string" + ? userLegacy.name + : undefined; + + const authorUsername = + typeof userCore?.screen_name === "string" + ? userCore.screen_name + : typeof userLegacy?.screen_name === "string" + ? userLegacy.screen_name + : undefined; + + const text = + tweet?.note_tweet?.note_tweet_results?.result?.text ?? + tweet?.legacy?.full_text ?? + tweet?.legacy?.text ?? + undefined; + + const tweetId = + typeof tweet?.rest_id === "string" && tweet.rest_id.length > 0 + ? tweet.rest_id + : fallbackTweetId; + + const url = authorUsername + ? `https://x.com/${authorUsername}/status/${tweetId}` + : `https://x.com/i/web/status/${tweetId}`; + + return { + id: tweetId, + url, + authorName, + authorUsername, + text: typeof text === "string" ? text : undefined, + }; +} + +export async function resolveReferencedTweetsFromArticle( + article: unknown, + cookieMap: Record, + options: ResolveReferencedTweetsOptions = {} +): Promise> { + const log = options.log ?? (() => {}); + const ids = extractReferencedTweetIds(article); + const referencedTweets = new Map(); + + for (const id of ids) { + try { + const tweet = await fetchXTweet(id, cookieMap, false); + const info = extractReferencedTweetInfo(tweet, id); + referencedTweets.set(id, info); + } catch (error) { + log( + `[x-to-markdown] Failed to fetch referenced tweet ${id}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + referencedTweets.set(id, { + id, + url: `https://x.com/i/web/status/${id}`, + }); + } + } + + return referencedTweets; +} diff --git a/skills/baoyu-danger-x-to-markdown/scripts/tweet-to-markdown.ts b/skills/baoyu-danger-x-to-markdown/scripts/tweet-to-markdown.ts index f76636a..57d0853 100644 --- a/skills/baoyu-danger-x-to-markdown/scripts/tweet-to-markdown.ts +++ b/skills/baoyu-danger-x-to-markdown/scripts/tweet-to-markdown.ts @@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url"; import { hasRequiredXCookies, loadXCookies } from "./cookies.js"; import { fetchTweetThread } from "./thread.js"; import { formatArticleMarkdown } from "./markdown.js"; +import { resolveReferencedTweetsFromArticle } from "./referenced-tweets.js"; import { formatThreadTweetsMarkdown } from "./thread-markdown.js"; import { resolveArticleEntityFromTweet } from "./tweet-article.js"; @@ -129,7 +130,8 @@ export async function tweetToMarkdown( const parts: string[] = []; if (articleEntity) { - const articleResult = formatArticleMarkdown(articleEntity); + const referencedTweets = await resolveReferencedTweetsFromArticle(articleEntity, cookieMap, { log }); + const articleResult = formatArticleMarkdown(articleEntity, { referencedTweets }); coverImage = articleResult.coverUrl; const articleMarkdown = articleResult.markdown.trimEnd(); if (articleMarkdown) { diff --git a/skills/baoyu-danger-x-to-markdown/scripts/types.ts b/skills/baoyu-danger-x-to-markdown/scripts/types.ts index fc2365d..dbdec2a 100644 --- a/skills/baoyu-danger-x-to-markdown/scripts/types.ts +++ b/skills/baoyu-danger-x-to-markdown/scripts/types.ts @@ -40,6 +40,7 @@ export type ArticleEntityMapEntry = { caption?: string; mediaItems?: ArticleEntityMapMediaItem[]; url?: string; + tweetId?: string; }; }; };