feat(baoyu-danger-x-to-markdown): add referenced tweet rendering, media reuse, and hi-res downloads

- Render embedded tweets in articles as blockquotes with author info and text summary
- Reuse existing markdown when --download-media targets already-converted URLs
- Upgrade Twitter image downloads to 4096x4096 resolution
- Improve entity resolution with logical key lookup
- Support trailing media for all block types (headings, lists, blockquotes)
- Change output path to {username}/{tweet-id}/{content-slug}.md for stable paths

skills/baoyu-danger-x-to-markdown/SKILL.md

@@ -170,7 +170,7 @@ coverImage: "https://pbs.twimg.com/media/example.jpg"
 Content...
 ```
 
-**File structure**: `x-to-markdown/{username}/{tweet-id}.md`
+**File structure**: `x-to-markdown/{username}/{tweet-id}/{content-slug}.md`
 
 When `--download-media` is enabled:
 - Images are saved to `imgs/` next to the markdown file

skills/baoyu-danger-x-to-markdown/scripts/main.ts

@@ -7,6 +7,7 @@ import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { fetchXArticle } from "./graphql.js";
 import { formatArticleMarkdown } from "./markdown.js";
 import { localizeMarkdownMedia, type LocalizeMarkdownMediaResult } from "./media-localizer.js";
+import { resolveReferencedTweetsFromArticle } from "./referenced-tweets.js";
 import { hasRequiredXCookies, loadXCookies, refreshXCookies } from "./cookies.js";
 import { resolveXToMarkdownConsentPath } from "./paths.js";
 import { tweetToMarkdown } from "./tweet-to-markdown.js";
@@ -203,6 +204,134 @@ function extractContentSlug(markdown: string): string {
   return "untitled";
 }
 
+function resolveSlugAndId(normalizedUrl: string, kind: "tweet" | "article"): { slug: string; idPart: string } {
+  const articleId = kind === "article" ? parseArticleId(normalizedUrl) : null;
+  const tweetId = kind === "tweet" ? parseTweetId(normalizedUrl) : null;
+  const username = kind === "tweet" ? parseTweetUsername(normalizedUrl) : null;
+
+  const idPart = articleId ?? tweetId ?? String(Date.now());
+  const userSlug = username ? sanitizeSlug(username) : null;
+  const slug = userSlug ?? idPart;
+  return { slug, idPart };
+}
+
+function extractFrontmatterUrls(markdown: string): string[] {
+  const match = markdown.match(/^---\n([\s\S]*?)\n---/);
+  if (!match?.[1]) return [];
+
+  const lines = match[1].split("\n");
+  const urls: string[] = [];
+  for (const line of lines) {
+    const m = line.match(/^(url|requestedUrl):\s*["']([^"']+)["']\s*$/);
+    if (m?.[2]) {
+      urls.push(m[2]);
+    }
+  }
+  return urls;
+}
+
+function frontmatterMatchesTarget(
+  markdown: string,
+  normalizedUrl: string,
+  kind: "tweet" | "article"
+): boolean {
+  const urls = extractFrontmatterUrls(markdown);
+  if (urls.length === 0) return false;
+
+  const targetId = kind === "article" ? parseArticleId(normalizedUrl) : parseTweetId(normalizedUrl);
+  if (!targetId) return false;
+
+  for (const url of urls) {
+    const candidateId = kind === "article" ? parseArticleId(url) : parseTweetId(url);
+    if (candidateId && candidateId === targetId) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+function listMarkdownFiles(dirPath: string): string[] {
+  try {
+    return fs
+      .readdirSync(dirPath)
+      .filter((name) => name.toLowerCase().endsWith(".md"))
+      .map((name) => path.join(dirPath, name))
+      .sort();
+  } catch {
+    return [];
+  }
+}
+
+function resolveExistingMarkdownPath(
+  normalizedUrl: string,
+  kind: "tweet" | "article",
+  argsOutput: string | null
+): string | null {
+  const { slug, idPart } = resolveSlugAndId(normalizedUrl, kind);
+  const candidateDirs = new Set<string>();
+  const candidateFiles = new Set<string>();
+
+  if (argsOutput) {
+    const resolved = path.resolve(argsOutput);
+    const looksDir = argsOutput.endsWith("/") || argsOutput.endsWith("\\");
+    try {
+      if (fs.existsSync(resolved)) {
+        const stat = fs.statSync(resolved);
+        if (stat.isFile()) {
+          candidateFiles.add(resolved);
+        } else if (stat.isDirectory()) {
+          candidateDirs.add(path.join(resolved, slug, idPart));
+          candidateDirs.add(resolved);
+        }
+      } else if (looksDir) {
+        candidateDirs.add(path.join(resolved, slug, idPart));
+      }
+    } catch {
+      // ignore and continue
+    }
+  } else {
+    candidateDirs.add(path.resolve(process.cwd(), "x-to-markdown", slug, idPart));
+  }
+
+  for (const filePath of candidateFiles) {
+    if (!filePath.toLowerCase().endsWith(".md")) continue;
+    try {
+      const markdown = fs.readFileSync(filePath, "utf8");
+      if (frontmatterMatchesTarget(markdown, normalizedUrl, kind)) {
+        return filePath;
+      }
+    } catch {
+      // ignore and continue
+    }
+  }
+
+  for (const dirPath of candidateDirs) {
+    if (!fs.existsSync(dirPath)) continue;
+    let stat: fs.Stats;
+    try {
+      stat = fs.statSync(dirPath);
+    } catch {
+      continue;
+    }
+    if (!stat.isDirectory()) continue;
+
+    const markdownFiles = listMarkdownFiles(dirPath);
+    for (const filePath of markdownFiles) {
+      try {
+        const markdown = fs.readFileSync(filePath, "utf8");
+        if (frontmatterMatchesTarget(markdown, normalizedUrl, kind)) {
+          return filePath;
+        }
+      } catch {
+        // ignore and continue
+      }
+    }
+  }
+
+  return null;
+}
+
 async function resolveOutputPath(
   normalizedUrl: string,
   kind: "tweet" | "article",
@@ -218,14 +347,14 @@ async function resolveOutputPath(
   const idPart = articleId ?? tweetId ?? String(Date.now());
   const slug = userSlug ?? idPart;
 
-  const defaultFileName = `${idPart}.md`;
+  const defaultFileName = `${contentSlug}.md`;
 
   if (argsOutput) {
     const wantsDir = argsOutput.endsWith("/") || argsOutput.endsWith("\\");
     const resolved = path.resolve(argsOutput);
     try {
       if (wantsDir || (fs.existsSync(resolved) && fs.statSync(resolved).isDirectory())) {
-        const outputDir = path.join(resolved, slug, contentSlug);
+        const outputDir = path.join(resolved, slug, idPart);
         await mkdir(outputDir, { recursive: true });
         return { outputDir, markdownPath: path.join(outputDir, defaultFileName), slug };
       }
@@ -238,7 +367,7 @@ async function resolveOutputPath(
     return { outputDir, markdownPath: resolved, slug };
   }
 
-  const outputDir = path.resolve(process.cwd(), "x-to-markdown", slug, contentSlug);
+  const outputDir = path.resolve(process.cwd(), "x-to-markdown", slug, idPart);
   await mkdir(outputDir, { recursive: true });
   return { outputDir, markdownPath: path.join(outputDir, defaultFileName), slug };
 }
@@ -349,7 +478,8 @@ async function convertArticleToMarkdown(
 
   log(`[x-to-markdown] Fetching article ${articleId}...`);
   const article = await fetchXArticle(articleId, cookieMap, false);
-  const { markdown: body, coverUrl } = formatArticleMarkdown(article);
+  const referencedTweets = await resolveReferencedTweetsFromArticle(article, cookieMap, { log });
+  const { markdown: body, coverUrl } = formatArticleMarkdown(article, { referencedTweets });
 
   const title = typeof (article as any)?.title === "string" ? String((article as any).title).trim() : "";
   const meta = formatMetaMarkdown({
@@ -389,6 +519,58 @@ async function main(): Promise<void> {
 
   const kind = articleId ? ("article" as const) : ("tweet" as const);
 
+  if (args.downloadMedia) {
+    const existingMarkdownPath = resolveExistingMarkdownPath(normalizedUrl, kind, args.output);
+    if (existingMarkdownPath) {
+      log(`[x-to-markdown] Reusing existing markdown: ${existingMarkdownPath}`);
+      const existingMarkdown = await readFile(existingMarkdownPath, "utf8");
+      const mediaResult = await localizeMarkdownMedia(existingMarkdown, {
+        markdownPath: existingMarkdownPath,
+        log,
+      });
+      const didLocalize =
+        mediaResult.downloadedImages > 0 ||
+        mediaResult.downloadedVideos > 0 ||
+        mediaResult.markdown !== existingMarkdown;
+
+      if (didLocalize) {
+        await writeFile(existingMarkdownPath, mediaResult.markdown, "utf8");
+        log(
+          `[x-to-markdown] Media localized: images=${mediaResult.downloadedImages}, videos=${mediaResult.downloadedVideos}`
+        );
+        log(`[x-to-markdown] Saved: ${existingMarkdownPath}`);
+
+        const { slug } = resolveSlugAndId(normalizedUrl, kind);
+        if (args.json) {
+          console.log(
+            JSON.stringify(
+              {
+                url: articleId ? `https://x.com/i/article/${articleId}` : normalizedUrl,
+                requestedUrl: normalizedUrl,
+                type: kind,
+                slug,
+                outputDir: path.dirname(existingMarkdownPath),
+                markdownPath: existingMarkdownPath,
+                downloadMedia: true,
+                downloadedImages: mediaResult.downloadedImages,
+                downloadedVideos: mediaResult.downloadedVideos,
+                imageDir: mediaResult.imageDir,
+                videoDir: mediaResult.videoDir,
+              },
+              null,
+              2
+            )
+          );
+        } else {
+          console.log(existingMarkdownPath);
+        }
+        return;
+      }
+
+      log("[x-to-markdown] Existing markdown already localized; rebuilding content to refresh placement.");
+    }
+  }
+
   let markdown =
     kind === "article" && articleId
       ? await convertArticleToMarkdown(normalizedUrl, articleId, log)

skills/baoyu-danger-x-to-markdown/scripts/markdown.ts

@@ -2,9 +2,22 @@ import type {
   ArticleBlock,
   ArticleContentState,
   ArticleEntity,
+  ArticleEntityMapEntry,
   ArticleMediaInfo,
 } from "./types.js";
 
+export type ReferencedTweetInfo = {
+  id: string;
+  url: string;
+  authorName?: string;
+  authorUsername?: string;
+  text?: string;
+};
+
+export type FormatArticleOptions = {
+  referencedTweets?: Map<string, ReferencedTweetInfo>;
+};
+
 function coerceArticleEntity(value: unknown): ArticleEntity | null {
   if (!value || typeof value !== "object") return null;
   const candidate = value as ArticleEntity;
@@ -29,6 +42,73 @@ function normalizeCaption(caption?: string): string {
   return trimmed.replace(/\s+/g, " ");
 }
 
+function summarizeTweetText(text?: string): string {
+  const trimmed = text?.trim();
+  if (!trimmed) return "";
+  const normalized = trimmed
+    .split(/\r?\n+/)
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .join(" ");
+  if (normalized.length <= 280) return normalized;
+  return `${normalized.slice(0, 277)}...`;
+}
+
+function buildTweetUrl(tweetId?: string, username?: string): string | null {
+  if (!tweetId) return null;
+  if (username) {
+    return `https://x.com/${username}/status/${tweetId}`;
+  }
+  return `https://x.com/i/web/status/${tweetId}`;
+}
+
+type EntityLookup = {
+  byIndex: Map<number, ArticleEntityMapEntry>;
+  byLogicalKey: Map<number, ArticleEntityMapEntry>;
+};
+
+function buildEntityLookup(
+  entityMap: ArticleContentState["entityMap"] | undefined
+): EntityLookup {
+  const lookup: EntityLookup = {
+    byIndex: new Map<number, ArticleEntityMapEntry>(),
+    byLogicalKey: new Map<number, ArticleEntityMapEntry>(),
+  };
+
+  if (!entityMap) return lookup;
+
+  for (const [idx, entry] of Object.entries(entityMap)) {
+    const idxNum = Number(idx);
+    if (Number.isFinite(idxNum)) {
+      lookup.byIndex.set(idxNum, entry);
+    }
+
+    const logicalKey = parseInt(entry?.key ?? "", 10);
+    if (Number.isFinite(logicalKey) && !lookup.byLogicalKey.has(logicalKey)) {
+      lookup.byLogicalKey.set(logicalKey, entry);
+    }
+  }
+
+  return lookup;
+}
+
+function resolveEntityEntry(
+  entityKey: number | undefined,
+  entityMap: ArticleContentState["entityMap"] | undefined,
+  lookup: EntityLookup
+): ArticleEntityMapEntry | undefined {
+  if (entityKey === undefined) return undefined;
+
+  const byLogicalKey = lookup.byLogicalKey.get(entityKey);
+  if (byLogicalKey) return byLogicalKey;
+
+  const byIndex = lookup.byIndex.get(entityKey);
+  if (byIndex) return byIndex;
+
+  if (!entityMap) return undefined;
+  return entityMap[String(entityKey)];
+}
+
 function resolveMediaUrl(info?: ArticleMediaInfo): string | undefined {
   if (!info) return undefined;
   if (info.original_img_url) return info.original_img_url;
@@ -79,11 +159,12 @@ function collectMediaUrls(
 function resolveEntityMediaLines(
   entityKey: number | undefined,
   entityMap: ArticleContentState["entityMap"] | undefined,
+  entityLookup: EntityLookup,
   mediaById: Map<string, string>,
   usedUrls: Set<string>
 ): string[] {
-  if (entityKey === undefined || !entityMap) return [];
-  const entry = entityMap[String(entityKey)];
+  if (entityKey === undefined) return [];
+  const entry = resolveEntityEntry(entityKey, entityMap, entityLookup);
   const value = entry?.value;
   if (!value) return [];
   const type = value.type;
@@ -117,6 +198,45 @@ function resolveEntityMediaLines(
   return lines;
 }
 
+function resolveEntityTweetLines(
+  entityKey: number | undefined,
+  entityMap: ArticleContentState["entityMap"] | undefined,
+  entityLookup: EntityLookup,
+  referencedTweets?: Map<string, ReferencedTweetInfo>
+): string[] {
+  if (entityKey === undefined) return [];
+  const entry = resolveEntityEntry(entityKey, entityMap, entityLookup);
+  const value = entry?.value;
+  if (!value || value.type !== "TWEET") return [];
+
+  const tweetId = typeof value.data?.tweetId === "string" ? value.data.tweetId : "";
+  if (!tweetId) return [];
+
+  const referenced = referencedTweets?.get(tweetId);
+  const url =
+    referenced?.url ??
+    buildTweetUrl(tweetId, referenced?.authorUsername) ??
+    `https://x.com/i/web/status/${tweetId}`;
+
+  const authorText =
+    referenced?.authorName && referenced?.authorUsername
+      ? `${referenced.authorName} (@${referenced.authorUsername})`
+      : referenced?.authorUsername
+        ? `@${referenced.authorUsername}`
+        : referenced?.authorName;
+
+  const lines: string[] = [];
+  lines.push(`> 引用推文${authorText ? `：${authorText}` : ""}`);
+
+  const summary = summarizeTweetText(referenced?.text);
+  if (summary) {
+    lines.push(`> ${summary}`);
+  }
+
+  lines.push(`> ${url}`);
+  return lines;
+}
+
 function buildMediaLinkMap(
   entityMap: ArticleContentState["entityMap"] | undefined
 ): Map<number, string> {
@@ -151,6 +271,7 @@ function buildMediaLinkMap(
     if (linkIdx === -1) linkIdx = 0;
     const link = pool.splice(linkIdx, 1)[0]!;
     map.set(media.idx, link.url);
+    map.set(media.key, link.url);
   }
 
   return map;
@@ -160,6 +281,7 @@ function renderInlineLinks(
   text: string,
   entityRanges: Array<{ key?: number; offset?: number; length?: number }>,
   entityMap: ArticleContentState["entityMap"] | undefined,
+  entityLookup: EntityLookup,
   mediaLinkMap: Map<number, string>
 ): string {
   if (!entityMap || entityRanges.length === 0) return text;
@@ -181,7 +303,7 @@ function renderInlineLinks(
     const length = range.length!;
     const key = range.key!;
 
-    const entry = entityMap[String(key)];
+    const entry = resolveEntityEntry(key, entityMap, entityLookup);
     const value = entry?.value;
     if (!value) continue;
 
@@ -204,27 +326,14 @@ function renderInlineLinks(
   return result;
 }
 
-function buildAtomicMediaQueue(
-  article: ArticleEntity,
-  usedUrls: Set<string>
-): string[] {
-  const queue: string[] = [];
-  for (const entity of article.media_entities ?? []) {
-    const url = resolveMediaUrl(entity?.media_info);
-    if (url && !usedUrls.has(url)) {
-      queue.push(url);
-    }
-  }
-  return queue;
-}
-
 function renderContentBlocks(
   blocks: ArticleBlock[],
   entityMap: ArticleContentState["entityMap"] | undefined,
+  entityLookup: EntityLookup,
   mediaById: Map<string, string>,
   usedUrls: Set<string>,
-  atomicMediaQueue: string[],
-  mediaLinkMap: Map<number, string>
+  mediaLinkMap: Map<number, string>,
+  referencedTweets?: Map<string, ReferencedTweetInfo>
 ): string[] {
   const lines: string[] = [];
   let previousKind: "list" | "quote" | "heading" | "text" | "code" | "media" | null = null;
@@ -253,18 +362,54 @@ function renderContentBlocks(
     const mediaLines: string[] = [];
     for (const range of ranges) {
       if (typeof range?.key !== "number") continue;
-      mediaLines.push(...resolveEntityMediaLines(range.key, entityMap, mediaById, usedUrls));
+      mediaLines.push(
+        ...resolveEntityMediaLines(range.key, entityMap, entityLookup, mediaById, usedUrls)
+      );
     }
     return mediaLines;
   };
 
+  const collectTweetLines = (block: ArticleBlock): string[] => {
+    const ranges = Array.isArray(block.entityRanges) ? block.entityRanges : [];
+    const tweetLines: string[] = [];
+    for (const range of ranges) {
+      if (typeof range?.key !== "number") continue;
+      tweetLines.push(
+        ...resolveEntityTweetLines(range.key, entityMap, entityLookup, referencedTweets)
+      );
+    }
+    return tweetLines;
+  };
+
+  const collectLinkLines = (block: ArticleBlock): string[] => {
+    const ranges = Array.isArray(block.entityRanges) ? block.entityRanges : [];
+    const linkLines: string[] = [];
+    for (const range of ranges) {
+      if (typeof range?.key !== "number") continue;
+      const entry = resolveEntityEntry(range.key, entityMap, entityLookup);
+      const value = entry?.value;
+      if (value?.type !== "LINK") continue;
+      const url = typeof value.data?.url === "string" ? value.data.url : "";
+      if (url) {
+        linkLines.push(url);
+      }
+    }
+    return [...new Set(linkLines)];
+  };
+
+  const pushTrailingMedia = (mediaLines: string[]) => {
+    if (mediaLines.length > 0) {
+      pushBlock(mediaLines, "media");
+    }
+  };
+
   for (const block of blocks) {
     const type = typeof block?.type === "string" ? block.type : "unstyled";
     const rawText = typeof block?.text === "string" ? block.text : "";
     const ranges = Array.isArray(block.entityRanges) ? block.entityRanges : [];
     const text =
       type !== "atomic" && type !== "code-block"
-        ? renderInlineLinks(rawText, ranges, entityMap, mediaLinkMap)
+        ? renderInlineLinks(rawText, ranges, entityMap, entityLookup, mediaLinkMap)
         : rawText;
 
     if (type === "code-block") {
@@ -290,16 +435,22 @@ function renderContentBlocks(
       }
       listKind = null;
       orderedIndex = 0;
+
+      const tweetLines = collectTweetLines(block);
+      if (tweetLines.length > 0) {
+        pushBlock(tweetLines, "quote");
+      }
+
       const mediaLines = collectMediaLines(block);
       if (mediaLines.length > 0) {
         pushBlock(mediaLines, "media");
-      } else if (atomicMediaQueue.length > 0) {
-        const url = atomicMediaQueue.shift()!;
-        if (!usedUrls.has(url)) {
-          usedUrls.add(url);
-          pushBlock([`![](${url})`], "media");
-        }
       }
+
+      const linkLines = collectLinkLines(block);
+      if (linkLines.length > 0) {
+        pushBlock(linkLines, "text");
+      }
+
       continue;
     }
 
@@ -313,6 +464,7 @@ function renderContentBlocks(
       listKind = "unordered";
       orderedIndex = 0;
       pushBlock([`- ${text}`], "list");
+      pushTrailingMedia(collectMediaLines(block));
       continue;
     }
 
@@ -323,6 +475,7 @@ function renderContentBlocks(
       listKind = "ordered";
       orderedIndex += 1;
       pushBlock([`${orderedIndex}. ${text}`], "list");
+      pushTrailingMedia(collectMediaLines(block));
       continue;
     }
 
@@ -332,29 +485,41 @@ function renderContentBlocks(
     switch (type) {
       case "header-one":
         pushBlock([`# ${text}`], "heading");
+        pushTrailingMedia(collectMediaLines(block));
         break;
       case "header-two":
         pushBlock([`## ${text}`], "heading");
+        pushTrailingMedia(collectMediaLines(block));
         break;
       case "header-three":
         pushBlock([`### ${text}`], "heading");
+        pushTrailingMedia(collectMediaLines(block));
         break;
       case "header-four":
         pushBlock([`#### ${text}`], "heading");
+        pushTrailingMedia(collectMediaLines(block));
         break;
       case "header-five":
         pushBlock([`##### ${text}`], "heading");
+        pushTrailingMedia(collectMediaLines(block));
         break;
       case "header-six":
         pushBlock([`###### ${text}`], "heading");
+        pushTrailingMedia(collectMediaLines(block));
         break;
       case "blockquote": {
         const quoteLines = text.length > 0 ? text.split("\n") : [""];
         pushBlock(quoteLines.map((line) => `> ${line}`), "quote");
+        pushTrailingMedia(collectMediaLines(block));
         break;
       }
       default:
+        if (/^XIMGPH_\d+$/.test(text.trim())) {
+          pushTrailingMedia(collectMediaLines(block));
+          break;
+        }
         pushBlock([text], "text");
+        pushTrailingMedia(collectMediaLines(block));
         break;
     }
   }
@@ -371,7 +536,28 @@ export type FormatArticleResult = {
   coverUrl: string | null;
 };
 
-export function formatArticleMarkdown(article: unknown): FormatArticleResult {
+export function extractReferencedTweetIds(article: unknown): string[] {
+  const candidate = coerceArticleEntity(article);
+  const entityMap = candidate?.content_state?.entityMap;
+  if (!entityMap) return [];
+
+  const ids: string[] = [];
+  const seen = new Set<string>();
+  for (const entry of Object.values(entityMap)) {
+    const value = entry?.value;
+    if (value?.type !== "TWEET") continue;
+    const tweetId = typeof value.data?.tweetId === "string" ? value.data.tweetId : "";
+    if (!tweetId || seen.has(tweetId)) continue;
+    seen.add(tweetId);
+    ids.push(tweetId);
+  }
+  return ids;
+}
+
+export function formatArticleMarkdown(
+  article: unknown,
+  options: FormatArticleOptions = {}
+): FormatArticleResult {
   const candidate = coerceArticleEntity(article);
   if (!candidate) {
     return { markdown: `\`\`\`json\n${JSON.stringify(article, null, 2)}\n\`\`\``, coverUrl: null };
@@ -392,10 +578,18 @@ export function formatArticleMarkdown(article: unknown): FormatArticleResult {
 
   const blocks = candidate.content_state?.blocks;
   const entityMap = candidate.content_state?.entityMap;
+  const entityLookup = buildEntityLookup(entityMap);
   if (Array.isArray(blocks) && blocks.length > 0) {
-    const atomicMediaQueue = buildAtomicMediaQueue(candidate, usedUrls);
     const mediaLinkMap = buildMediaLinkMap(entityMap);
-    const rendered = renderContentBlocks(blocks, entityMap, mediaById, usedUrls, atomicMediaQueue, mediaLinkMap);
+    const rendered = renderContentBlocks(
+      blocks,
+      entityMap,
+      entityLookup,
+      mediaById,
+      usedUrls,
+      mediaLinkMap,
+      options.referencedTweets
+    );
     if (rendered.length > 0) {
       if (lines.length > 0) lines.push("");
       lines.push(...rendered);

skills/baoyu-danger-x-to-markdown/scripts/media-localizer.ts

@@ -187,11 +187,35 @@ function buildFileName(kind: MediaKind, index: number, sourceUrl: string, extens
 
 const FRONTMATTER_COVER_RE = /^(coverImage:\s*")(https?:\/\/[^"]+)(")/m;
 
+function toHighResUrl(rawUrl: string): string {
+  try {
+    const parsed = new URL(rawUrl);
+    if (parsed.hostname !== "pbs.twimg.com") return rawUrl;
+    const ext = path.posix.extname(parsed.pathname).replace(/^\./, "").toLowerCase();
+    if (!ext || !IMAGE_EXTENSIONS.has(ext)) return rawUrl;
+    parsed.pathname = parsed.pathname.replace(new RegExp(`\\.${ext}$`), "");
+    parsed.searchParams.set("format", ext === "jpeg" ? "jpg" : ext);
+    parsed.searchParams.set("name", "4096x4096");
+    return parsed.toString();
+  } catch {
+    return rawUrl;
+  }
+}
+
 function collectMarkdownLinkCandidates(markdown: string): MarkdownLinkCandidate[] {
-  MARKDOWN_LINK_RE.lastIndex = 0;
   const candidates: MarkdownLinkCandidate[] = [];
   const seen = new Set<string>();
 
+  const fmMatch = markdown.match(/^---\n([\s\S]*?)\n---/);
+  if (fmMatch) {
+    const coverMatch = fmMatch[1]?.match(FRONTMATTER_COVER_RE);
+    if (coverMatch?.[2] && !seen.has(coverMatch[2])) {
+      seen.add(coverMatch[2]);
+      candidates.push({ url: coverMatch[2], hint: "image" });
+    }
+  }
+
+  MARKDOWN_LINK_RE.lastIndex = 0;
   let match: RegExpExecArray | null;
   while ((match = MARKDOWN_LINK_RE.exec(markdown))) {
     const label = match[1] ?? "";
@@ -204,15 +228,6 @@ function collectMarkdownLinkCandidates(markdown: string): MarkdownLinkCandidate[
     });
   }
 
-  const fmMatch = markdown.match(/^---\n([\s\S]*?)\n---/);
-  if (fmMatch) {
-    const coverMatch = fmMatch[1]?.match(FRONTMATTER_COVER_RE);
-    if (coverMatch?.[2] && !seen.has(coverMatch[2])) {
-      seen.add(coverMatch[2]);
-      candidates.push({ url: coverMatch[2], hint: "image" });
-    }
-  }
-
   return candidates;
 }
 
@@ -259,7 +274,8 @@ export async function localizeMarkdownMedia(
 
   for (const candidate of candidates) {
     try {
-      const response = await fetch(candidate.url, {
+      const downloadUrl = toHighResUrl(candidate.url);
+      const response = await fetch(downloadUrl, {
         method: "GET",
         redirect: "follow",
         headers: {

skills/baoyu-danger-x-to-markdown/scripts/referenced-tweets.ts

@@ -0,0 +1,81 @@
+import { fetchXTweet } from "./graphql.js";
+import {
+  extractReferencedTweetIds,
+  type ReferencedTweetInfo,
+} from "./markdown.js";
+
+type ResolveReferencedTweetsOptions = {
+  log?: (message: string) => void;
+};
+
+function extractReferencedTweetInfo(tweet: any, fallbackTweetId: string): ReferencedTweetInfo {
+  const userCore = tweet?.core?.user_results?.result?.core;
+  const userLegacy = tweet?.core?.user_results?.result?.legacy;
+
+  const authorName =
+    typeof userCore?.name === "string"
+      ? userCore.name
+      : typeof userLegacy?.name === "string"
+        ? userLegacy.name
+        : undefined;
+
+  const authorUsername =
+    typeof userCore?.screen_name === "string"
+      ? userCore.screen_name
+      : typeof userLegacy?.screen_name === "string"
+        ? userLegacy.screen_name
+        : undefined;
+
+  const text =
+    tweet?.note_tweet?.note_tweet_results?.result?.text ??
+    tweet?.legacy?.full_text ??
+    tweet?.legacy?.text ??
+    undefined;
+
+  const tweetId =
+    typeof tweet?.rest_id === "string" && tweet.rest_id.length > 0
+      ? tweet.rest_id
+      : fallbackTweetId;
+
+  const url = authorUsername
+    ? `https://x.com/${authorUsername}/status/${tweetId}`
+    : `https://x.com/i/web/status/${tweetId}`;
+
+  return {
+    id: tweetId,
+    url,
+    authorName,
+    authorUsername,
+    text: typeof text === "string" ? text : undefined,
+  };
+}
+
+export async function resolveReferencedTweetsFromArticle(
+  article: unknown,
+  cookieMap: Record<string, string>,
+  options: ResolveReferencedTweetsOptions = {}
+): Promise<Map<string, ReferencedTweetInfo>> {
+  const log = options.log ?? (() => {});
+  const ids = extractReferencedTweetIds(article);
+  const referencedTweets = new Map<string, ReferencedTweetInfo>();
+
+  for (const id of ids) {
+    try {
+      const tweet = await fetchXTweet(id, cookieMap, false);
+      const info = extractReferencedTweetInfo(tweet, id);
+      referencedTweets.set(id, info);
+    } catch (error) {
+      log(
+        `[x-to-markdown] Failed to fetch referenced tweet ${id}: ${
+          error instanceof Error ? error.message : String(error)
+        }`
+      );
+      referencedTweets.set(id, {
+        id,
+        url: `https://x.com/i/web/status/${id}`,
+      });
+    }
+  }
+
+  return referencedTweets;
+}

skills/baoyu-danger-x-to-markdown/scripts/tweet-to-markdown.ts

@@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url";
 import { hasRequiredXCookies, loadXCookies } from "./cookies.js";
 import { fetchTweetThread } from "./thread.js";
 import { formatArticleMarkdown } from "./markdown.js";
+import { resolveReferencedTweetsFromArticle } from "./referenced-tweets.js";
 import { formatThreadTweetsMarkdown } from "./thread-markdown.js";
 import { resolveArticleEntityFromTweet } from "./tweet-article.js";
 
@@ -129,7 +130,8 @@ export async function tweetToMarkdown(
   const parts: string[] = [];
 
   if (articleEntity) {
-    const articleResult = formatArticleMarkdown(articleEntity);
+    const referencedTweets = await resolveReferencedTweetsFromArticle(articleEntity, cookieMap, { log });
+    const articleResult = formatArticleMarkdown(articleEntity, { referencedTweets });
     coverImage = articleResult.coverUrl;
     const articleMarkdown = articleResult.markdown.trimEnd();
     if (articleMarkdown) {

skills/baoyu-danger-x-to-markdown/scripts/types.ts

@@ -40,6 +40,7 @@ export type ArticleEntityMapEntry = {
       caption?: string;
       mediaItems?: ArticleEntityMapMediaItem[];
       url?: string;
+      tweetId?: string;
     };
   };
 };