mirror of
https://github.com/JimLiu/baoyu-skills.git
synced 2026-07-12 13:59:47 +08:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 13707fa2cf | |||
| b56e503b16 | |||
| 7d03685ade | |||
| b305c386bc | |||
| 240dd7d314 | |||
| e737c4a611 | |||
| 2eec4f3639 |
@@ -6,7 +6,7 @@
|
||||
},
|
||||
"metadata": {
|
||||
"description": "Skills shared by Baoyu for improving daily work efficiency",
|
||||
"version": "1.37.0"
|
||||
"version": "1.38.0"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
|
||||
@@ -2,6 +2,22 @@
|
||||
|
||||
English | [中文](./CHANGELOG.zh.md)
|
||||
|
||||
## 1.38.0 - 2026-02-28
|
||||
|
||||
### Features
|
||||
- `baoyu-danger-x-to-markdown`: render embedded tweets in articles as blockquotes with author info and text summary
|
||||
- `baoyu-danger-x-to-markdown`: reuse existing markdown when `--download-media` targets already-converted URLs
|
||||
- `baoyu-danger-x-to-markdown`: upgrade Twitter image downloads to 4096x4096 high resolution
|
||||
|
||||
### Fixes
|
||||
- `baoyu-danger-x-to-markdown`: improve entity resolution with logical key lookup for reliable media and link mapping
|
||||
- `baoyu-danger-x-to-markdown`: support trailing media for all block types (headings, lists, blockquotes)
|
||||
|
||||
## 1.37.1 - 2026-02-27
|
||||
|
||||
### Fixes
|
||||
- `baoyu-danger-gemini-web`: sync model headers with upstream and update model list (by @xkcoding)
|
||||
|
||||
## 1.37.0 - 2026-02-27
|
||||
|
||||
### Features
|
||||
|
||||
@@ -2,6 +2,22 @@
|
||||
|
||||
[English](./CHANGELOG.md) | 中文
|
||||
|
||||
## 1.38.0 - 2026-02-28
|
||||
|
||||
### 新功能
|
||||
- `baoyu-danger-x-to-markdown`:支持文章内嵌推文渲染,以引用块形式显示作者信息和推文摘要
|
||||
- `baoyu-danger-x-to-markdown`:`--download-media` 复用已转换的 Markdown 文件,跳过重复抓取
|
||||
- `baoyu-danger-x-to-markdown`:推特图片下载升级至 4096x4096 高分辨率
|
||||
|
||||
### 修复
|
||||
- `baoyu-danger-x-to-markdown`:改进实体解析逻辑,通过逻辑键查找提升媒体和链接映射准确性
|
||||
- `baoyu-danger-x-to-markdown`:所有区块类型(标题、列表、引用块)支持尾随媒体展示
|
||||
|
||||
## 1.37.1 - 2026-02-27
|
||||
|
||||
### 修复
|
||||
- `baoyu-danger-gemini-web`:同步上游模型请求头并更新模型列表 (by @xkcoding)
|
||||
|
||||
## 1.37.0 - 2026-02-27
|
||||
|
||||
### 新功能
|
||||
|
||||
@@ -76,7 +76,7 @@ test -f "$HOME/.baoyu-skills/baoyu-danger-gemini-web/EXTEND.md" && echo "user"
|
||||
```bash
|
||||
# Text generation
|
||||
npx -y bun ${SKILL_DIR}/scripts/main.ts "Your prompt"
|
||||
npx -y bun ${SKILL_DIR}/scripts/main.ts --prompt "Your prompt" --model gemini-2.5-pro
|
||||
npx -y bun ${SKILL_DIR}/scripts/main.ts --prompt "Your prompt" --model gemini-3-flash
|
||||
|
||||
# Image generation
|
||||
npx -y bun ${SKILL_DIR}/scripts/main.ts --prompt "A cute cat" --image cat.png
|
||||
@@ -100,7 +100,7 @@ npx -y bun ${SKILL_DIR}/scripts/main.ts "Hello" --json
|
||||
|--------|-------------|
|
||||
| `--prompt`, `-p` | Prompt text |
|
||||
| `--promptfiles` | Read prompt from files (concatenated) |
|
||||
| `--model`, `-m` | Model: gemini-3-pro (default), gemini-2.5-pro, gemini-2.5-flash |
|
||||
| `--model`, `-m` | Model: gemini-3-pro (default), gemini-3-flash, gemini-3-flash-thinking, gemini-3.1-pro-preview |
|
||||
| `--image [path]` | Generate image (default: generated.png) |
|
||||
| `--reference`, `--ref` | Reference images for vision input |
|
||||
| `--sessionId` | Session ID for multi-turn conversation |
|
||||
@@ -114,9 +114,10 @@ npx -y bun ${SKILL_DIR}/scripts/main.ts "Hello" --json
|
||||
|
||||
| Model | Description |
|
||||
|-------|-------------|
|
||||
| `gemini-3-pro` | Default, latest |
|
||||
| `gemini-2.5-pro` | Previous pro |
|
||||
| `gemini-2.5-flash` | Fast, lightweight |
|
||||
| `gemini-3-pro` | Default, latest 3.0 Pro |
|
||||
| `gemini-3-flash` | Fast, lightweight 3.0 Flash |
|
||||
| `gemini-3-flash-thinking` | 3.0 Flash with thinking |
|
||||
| `gemini-3.1-pro-preview` | 3.1 Pro preview (empty header, auto-routed) |
|
||||
|
||||
## Authentication
|
||||
|
||||
|
||||
@@ -47,17 +47,22 @@ export class Model {
|
||||
static readonly UNSPECIFIED = new Model('unspecified', {}, false);
|
||||
static readonly G_3_0_PRO = new Model(
|
||||
'gemini-3.0-pro',
|
||||
{ 'x-goog-ext-525001261-jspb': '[1,null,null,null,"9d8ca3786ebdfbea",null,null,0,[4]]' },
|
||||
{ 'x-goog-ext-525001261-jspb': '[1,null,null,null,"9d8ca3786ebdfbea",null,null,0,[4],null,null,1]' },
|
||||
false,
|
||||
);
|
||||
static readonly G_2_5_PRO = new Model(
|
||||
'gemini-2.5-pro',
|
||||
{ 'x-goog-ext-525001261-jspb': '[1,null,null,null,"4af6c7f5da75d65d",null,null,0,[4]]' },
|
||||
static readonly G_3_0_FLASH = new Model(
|
||||
'gemini-3.0-flash',
|
||||
{ 'x-goog-ext-525001261-jspb': '[1,null,null,null,"fbb127bbb056c959",null,null,0,[4],null,null,1]' },
|
||||
false,
|
||||
);
|
||||
static readonly G_2_5_FLASH = new Model(
|
||||
'gemini-2.5-flash',
|
||||
{ 'x-goog-ext-525001261-jspb': '[1,null,null,null,"9ec249fc9ad08861",null,null,0,[4]]' },
|
||||
static readonly G_3_0_FLASH_THINKING = new Model(
|
||||
'gemini-3.0-flash-thinking',
|
||||
{ 'x-goog-ext-525001261-jspb': '[1,null,null,null,"5bf011840784117a",null,null,0,[4],null,null,1]' },
|
||||
false,
|
||||
);
|
||||
static readonly G_3_1_PRO_PREVIEW = new Model(
|
||||
'gemini-3.1-pro-preview',
|
||||
{},
|
||||
false,
|
||||
);
|
||||
|
||||
@@ -68,12 +73,12 @@ export class Model {
|
||||
) {}
|
||||
|
||||
static from_name(name: string): Model {
|
||||
for (const model of [Model.UNSPECIFIED, Model.G_3_0_PRO, Model.G_2_5_PRO, Model.G_2_5_FLASH]) {
|
||||
for (const model of [Model.UNSPECIFIED, Model.G_3_0_PRO, Model.G_3_0_FLASH, Model.G_3_0_FLASH_THINKING, Model.G_3_1_PRO_PREVIEW]) {
|
||||
if (model.model_name === name) return model;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Unknown model name: ${name}. Available models: ${[Model.UNSPECIFIED, Model.G_3_0_PRO, Model.G_2_5_PRO, Model.G_2_5_FLASH]
|
||||
`Unknown model name: ${name}. Available models: ${[Model.UNSPECIFIED, Model.G_3_0_PRO, Model.G_3_0_FLASH, Model.G_3_0_FLASH_THINKING, Model.G_3_1_PRO_PREVIEW]
|
||||
.map((m) => m.model_name)
|
||||
.join(', ')}`,
|
||||
);
|
||||
|
||||
@@ -73,7 +73,7 @@ Multi-turn conversation (agent generates unique sessionId):
|
||||
Options:
|
||||
-p, --prompt <text> Prompt text
|
||||
--promptfiles <files...> Read prompt from one or more files (concatenated in order)
|
||||
-m, --model <id> gemini-3-pro | gemini-2.5-pro | gemini-2.5-flash (default: gemini-3-pro)
|
||||
-m, --model <id> gemini-3-pro | gemini-3-flash | gemini-3-flash-thinking | gemini-3.1-pro-preview (default: gemini-3-pro)
|
||||
--json Output JSON
|
||||
--image [path] Generate an image and save it (default: ./generated.png)
|
||||
--reference <files...> Reference images for vision input
|
||||
@@ -227,8 +227,11 @@ function resolveModel(id: string): Model {
|
||||
const k = id.trim();
|
||||
if (k === 'gemini-3-pro') return Model.G_3_0_PRO;
|
||||
if (k === 'gemini-3.0-pro') return Model.G_3_0_PRO;
|
||||
if (k === 'gemini-2.5-pro') return Model.G_2_5_PRO;
|
||||
if (k === 'gemini-2.5-flash') return Model.G_2_5_FLASH;
|
||||
if (k === 'gemini-3-flash') return Model.G_3_0_FLASH;
|
||||
if (k === 'gemini-3.0-flash') return Model.G_3_0_FLASH;
|
||||
if (k === 'gemini-3-flash-thinking') return Model.G_3_0_FLASH_THINKING;
|
||||
if (k === 'gemini-3.0-flash-thinking') return Model.G_3_0_FLASH_THINKING;
|
||||
if (k === 'gemini-3.1-pro-preview') return Model.G_3_1_PRO_PREVIEW;
|
||||
return Model.from_name(k);
|
||||
}
|
||||
|
||||
|
||||
@@ -170,7 +170,7 @@ coverImage: "https://pbs.twimg.com/media/example.jpg"
|
||||
Content...
|
||||
```
|
||||
|
||||
**File structure**: `x-to-markdown/{username}/{tweet-id}.md`
|
||||
**File structure**: `x-to-markdown/{username}/{tweet-id}/{content-slug}.md`
|
||||
|
||||
When `--download-media` is enabled:
|
||||
- Images are saved to `imgs/` next to the markdown file
|
||||
|
||||
@@ -7,6 +7,7 @@ import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { fetchXArticle } from "./graphql.js";
|
||||
import { formatArticleMarkdown } from "./markdown.js";
|
||||
import { localizeMarkdownMedia, type LocalizeMarkdownMediaResult } from "./media-localizer.js";
|
||||
import { resolveReferencedTweetsFromArticle } from "./referenced-tweets.js";
|
||||
import { hasRequiredXCookies, loadXCookies, refreshXCookies } from "./cookies.js";
|
||||
import { resolveXToMarkdownConsentPath } from "./paths.js";
|
||||
import { tweetToMarkdown } from "./tweet-to-markdown.js";
|
||||
@@ -203,6 +204,134 @@ function extractContentSlug(markdown: string): string {
|
||||
return "untitled";
|
||||
}
|
||||
|
||||
function resolveSlugAndId(normalizedUrl: string, kind: "tweet" | "article"): { slug: string; idPart: string } {
|
||||
const articleId = kind === "article" ? parseArticleId(normalizedUrl) : null;
|
||||
const tweetId = kind === "tweet" ? parseTweetId(normalizedUrl) : null;
|
||||
const username = kind === "tweet" ? parseTweetUsername(normalizedUrl) : null;
|
||||
|
||||
const idPart = articleId ?? tweetId ?? String(Date.now());
|
||||
const userSlug = username ? sanitizeSlug(username) : null;
|
||||
const slug = userSlug ?? idPart;
|
||||
return { slug, idPart };
|
||||
}
|
||||
|
||||
function extractFrontmatterUrls(markdown: string): string[] {
|
||||
const match = markdown.match(/^---\n([\s\S]*?)\n---/);
|
||||
if (!match?.[1]) return [];
|
||||
|
||||
const lines = match[1].split("\n");
|
||||
const urls: string[] = [];
|
||||
for (const line of lines) {
|
||||
const m = line.match(/^(url|requestedUrl):\s*["']([^"']+)["']\s*$/);
|
||||
if (m?.[2]) {
|
||||
urls.push(m[2]);
|
||||
}
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
function frontmatterMatchesTarget(
|
||||
markdown: string,
|
||||
normalizedUrl: string,
|
||||
kind: "tweet" | "article"
|
||||
): boolean {
|
||||
const urls = extractFrontmatterUrls(markdown);
|
||||
if (urls.length === 0) return false;
|
||||
|
||||
const targetId = kind === "article" ? parseArticleId(normalizedUrl) : parseTweetId(normalizedUrl);
|
||||
if (!targetId) return false;
|
||||
|
||||
for (const url of urls) {
|
||||
const candidateId = kind === "article" ? parseArticleId(url) : parseTweetId(url);
|
||||
if (candidateId && candidateId === targetId) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function listMarkdownFiles(dirPath: string): string[] {
|
||||
try {
|
||||
return fs
|
||||
.readdirSync(dirPath)
|
||||
.filter((name) => name.toLowerCase().endsWith(".md"))
|
||||
.map((name) => path.join(dirPath, name))
|
||||
.sort();
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function resolveExistingMarkdownPath(
|
||||
normalizedUrl: string,
|
||||
kind: "tweet" | "article",
|
||||
argsOutput: string | null
|
||||
): string | null {
|
||||
const { slug, idPart } = resolveSlugAndId(normalizedUrl, kind);
|
||||
const candidateDirs = new Set<string>();
|
||||
const candidateFiles = new Set<string>();
|
||||
|
||||
if (argsOutput) {
|
||||
const resolved = path.resolve(argsOutput);
|
||||
const looksDir = argsOutput.endsWith("/") || argsOutput.endsWith("\\");
|
||||
try {
|
||||
if (fs.existsSync(resolved)) {
|
||||
const stat = fs.statSync(resolved);
|
||||
if (stat.isFile()) {
|
||||
candidateFiles.add(resolved);
|
||||
} else if (stat.isDirectory()) {
|
||||
candidateDirs.add(path.join(resolved, slug, idPart));
|
||||
candidateDirs.add(resolved);
|
||||
}
|
||||
} else if (looksDir) {
|
||||
candidateDirs.add(path.join(resolved, slug, idPart));
|
||||
}
|
||||
} catch {
|
||||
// ignore and continue
|
||||
}
|
||||
} else {
|
||||
candidateDirs.add(path.resolve(process.cwd(), "x-to-markdown", slug, idPart));
|
||||
}
|
||||
|
||||
for (const filePath of candidateFiles) {
|
||||
if (!filePath.toLowerCase().endsWith(".md")) continue;
|
||||
try {
|
||||
const markdown = fs.readFileSync(filePath, "utf8");
|
||||
if (frontmatterMatchesTarget(markdown, normalizedUrl, kind)) {
|
||||
return filePath;
|
||||
}
|
||||
} catch {
|
||||
// ignore and continue
|
||||
}
|
||||
}
|
||||
|
||||
for (const dirPath of candidateDirs) {
|
||||
if (!fs.existsSync(dirPath)) continue;
|
||||
let stat: fs.Stats;
|
||||
try {
|
||||
stat = fs.statSync(dirPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!stat.isDirectory()) continue;
|
||||
|
||||
const markdownFiles = listMarkdownFiles(dirPath);
|
||||
for (const filePath of markdownFiles) {
|
||||
try {
|
||||
const markdown = fs.readFileSync(filePath, "utf8");
|
||||
if (frontmatterMatchesTarget(markdown, normalizedUrl, kind)) {
|
||||
return filePath;
|
||||
}
|
||||
} catch {
|
||||
// ignore and continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function resolveOutputPath(
|
||||
normalizedUrl: string,
|
||||
kind: "tweet" | "article",
|
||||
@@ -218,14 +347,14 @@ async function resolveOutputPath(
|
||||
const idPart = articleId ?? tweetId ?? String(Date.now());
|
||||
const slug = userSlug ?? idPart;
|
||||
|
||||
const defaultFileName = `${idPart}.md`;
|
||||
const defaultFileName = `${contentSlug}.md`;
|
||||
|
||||
if (argsOutput) {
|
||||
const wantsDir = argsOutput.endsWith("/") || argsOutput.endsWith("\\");
|
||||
const resolved = path.resolve(argsOutput);
|
||||
try {
|
||||
if (wantsDir || (fs.existsSync(resolved) && fs.statSync(resolved).isDirectory())) {
|
||||
const outputDir = path.join(resolved, slug, contentSlug);
|
||||
const outputDir = path.join(resolved, slug, idPart);
|
||||
await mkdir(outputDir, { recursive: true });
|
||||
return { outputDir, markdownPath: path.join(outputDir, defaultFileName), slug };
|
||||
}
|
||||
@@ -238,7 +367,7 @@ async function resolveOutputPath(
|
||||
return { outputDir, markdownPath: resolved, slug };
|
||||
}
|
||||
|
||||
const outputDir = path.resolve(process.cwd(), "x-to-markdown", slug, contentSlug);
|
||||
const outputDir = path.resolve(process.cwd(), "x-to-markdown", slug, idPart);
|
||||
await mkdir(outputDir, { recursive: true });
|
||||
return { outputDir, markdownPath: path.join(outputDir, defaultFileName), slug };
|
||||
}
|
||||
@@ -349,7 +478,8 @@ async function convertArticleToMarkdown(
|
||||
|
||||
log(`[x-to-markdown] Fetching article ${articleId}...`);
|
||||
const article = await fetchXArticle(articleId, cookieMap, false);
|
||||
const { markdown: body, coverUrl } = formatArticleMarkdown(article);
|
||||
const referencedTweets = await resolveReferencedTweetsFromArticle(article, cookieMap, { log });
|
||||
const { markdown: body, coverUrl } = formatArticleMarkdown(article, { referencedTweets });
|
||||
|
||||
const title = typeof (article as any)?.title === "string" ? String((article as any).title).trim() : "";
|
||||
const meta = formatMetaMarkdown({
|
||||
@@ -389,6 +519,58 @@ async function main(): Promise<void> {
|
||||
|
||||
const kind = articleId ? ("article" as const) : ("tweet" as const);
|
||||
|
||||
if (args.downloadMedia) {
|
||||
const existingMarkdownPath = resolveExistingMarkdownPath(normalizedUrl, kind, args.output);
|
||||
if (existingMarkdownPath) {
|
||||
log(`[x-to-markdown] Reusing existing markdown: ${existingMarkdownPath}`);
|
||||
const existingMarkdown = await readFile(existingMarkdownPath, "utf8");
|
||||
const mediaResult = await localizeMarkdownMedia(existingMarkdown, {
|
||||
markdownPath: existingMarkdownPath,
|
||||
log,
|
||||
});
|
||||
const didLocalize =
|
||||
mediaResult.downloadedImages > 0 ||
|
||||
mediaResult.downloadedVideos > 0 ||
|
||||
mediaResult.markdown !== existingMarkdown;
|
||||
|
||||
if (didLocalize) {
|
||||
await writeFile(existingMarkdownPath, mediaResult.markdown, "utf8");
|
||||
log(
|
||||
`[x-to-markdown] Media localized: images=${mediaResult.downloadedImages}, videos=${mediaResult.downloadedVideos}`
|
||||
);
|
||||
log(`[x-to-markdown] Saved: ${existingMarkdownPath}`);
|
||||
|
||||
const { slug } = resolveSlugAndId(normalizedUrl, kind);
|
||||
if (args.json) {
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
url: articleId ? `https://x.com/i/article/${articleId}` : normalizedUrl,
|
||||
requestedUrl: normalizedUrl,
|
||||
type: kind,
|
||||
slug,
|
||||
outputDir: path.dirname(existingMarkdownPath),
|
||||
markdownPath: existingMarkdownPath,
|
||||
downloadMedia: true,
|
||||
downloadedImages: mediaResult.downloadedImages,
|
||||
downloadedVideos: mediaResult.downloadedVideos,
|
||||
imageDir: mediaResult.imageDir,
|
||||
videoDir: mediaResult.videoDir,
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
} else {
|
||||
console.log(existingMarkdownPath);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
log("[x-to-markdown] Existing markdown already localized; rebuilding content to refresh placement.");
|
||||
}
|
||||
}
|
||||
|
||||
let markdown =
|
||||
kind === "article" && articleId
|
||||
? await convertArticleToMarkdown(normalizedUrl, articleId, log)
|
||||
|
||||
@@ -2,9 +2,22 @@ import type {
|
||||
ArticleBlock,
|
||||
ArticleContentState,
|
||||
ArticleEntity,
|
||||
ArticleEntityMapEntry,
|
||||
ArticleMediaInfo,
|
||||
} from "./types.js";
|
||||
|
||||
export type ReferencedTweetInfo = {
|
||||
id: string;
|
||||
url: string;
|
||||
authorName?: string;
|
||||
authorUsername?: string;
|
||||
text?: string;
|
||||
};
|
||||
|
||||
export type FormatArticleOptions = {
|
||||
referencedTweets?: Map<string, ReferencedTweetInfo>;
|
||||
};
|
||||
|
||||
function coerceArticleEntity(value: unknown): ArticleEntity | null {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const candidate = value as ArticleEntity;
|
||||
@@ -29,6 +42,73 @@ function normalizeCaption(caption?: string): string {
|
||||
return trimmed.replace(/\s+/g, " ");
|
||||
}
|
||||
|
||||
function summarizeTweetText(text?: string): string {
|
||||
const trimmed = text?.trim();
|
||||
if (!trimmed) return "";
|
||||
const normalized = trimmed
|
||||
.split(/\r?\n+/)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.join(" ");
|
||||
if (normalized.length <= 280) return normalized;
|
||||
return `${normalized.slice(0, 277)}...`;
|
||||
}
|
||||
|
||||
function buildTweetUrl(tweetId?: string, username?: string): string | null {
|
||||
if (!tweetId) return null;
|
||||
if (username) {
|
||||
return `https://x.com/${username}/status/${tweetId}`;
|
||||
}
|
||||
return `https://x.com/i/web/status/${tweetId}`;
|
||||
}
|
||||
|
||||
type EntityLookup = {
|
||||
byIndex: Map<number, ArticleEntityMapEntry>;
|
||||
byLogicalKey: Map<number, ArticleEntityMapEntry>;
|
||||
};
|
||||
|
||||
function buildEntityLookup(
|
||||
entityMap: ArticleContentState["entityMap"] | undefined
|
||||
): EntityLookup {
|
||||
const lookup: EntityLookup = {
|
||||
byIndex: new Map<number, ArticleEntityMapEntry>(),
|
||||
byLogicalKey: new Map<number, ArticleEntityMapEntry>(),
|
||||
};
|
||||
|
||||
if (!entityMap) return lookup;
|
||||
|
||||
for (const [idx, entry] of Object.entries(entityMap)) {
|
||||
const idxNum = Number(idx);
|
||||
if (Number.isFinite(idxNum)) {
|
||||
lookup.byIndex.set(idxNum, entry);
|
||||
}
|
||||
|
||||
const logicalKey = parseInt(entry?.key ?? "", 10);
|
||||
if (Number.isFinite(logicalKey) && !lookup.byLogicalKey.has(logicalKey)) {
|
||||
lookup.byLogicalKey.set(logicalKey, entry);
|
||||
}
|
||||
}
|
||||
|
||||
return lookup;
|
||||
}
|
||||
|
||||
function resolveEntityEntry(
|
||||
entityKey: number | undefined,
|
||||
entityMap: ArticleContentState["entityMap"] | undefined,
|
||||
lookup: EntityLookup
|
||||
): ArticleEntityMapEntry | undefined {
|
||||
if (entityKey === undefined) return undefined;
|
||||
|
||||
const byLogicalKey = lookup.byLogicalKey.get(entityKey);
|
||||
if (byLogicalKey) return byLogicalKey;
|
||||
|
||||
const byIndex = lookup.byIndex.get(entityKey);
|
||||
if (byIndex) return byIndex;
|
||||
|
||||
if (!entityMap) return undefined;
|
||||
return entityMap[String(entityKey)];
|
||||
}
|
||||
|
||||
function resolveMediaUrl(info?: ArticleMediaInfo): string | undefined {
|
||||
if (!info) return undefined;
|
||||
if (info.original_img_url) return info.original_img_url;
|
||||
@@ -79,11 +159,12 @@ function collectMediaUrls(
|
||||
function resolveEntityMediaLines(
|
||||
entityKey: number | undefined,
|
||||
entityMap: ArticleContentState["entityMap"] | undefined,
|
||||
entityLookup: EntityLookup,
|
||||
mediaById: Map<string, string>,
|
||||
usedUrls: Set<string>
|
||||
): string[] {
|
||||
if (entityKey === undefined || !entityMap) return [];
|
||||
const entry = entityMap[String(entityKey)];
|
||||
if (entityKey === undefined) return [];
|
||||
const entry = resolveEntityEntry(entityKey, entityMap, entityLookup);
|
||||
const value = entry?.value;
|
||||
if (!value) return [];
|
||||
const type = value.type;
|
||||
@@ -117,6 +198,45 @@ function resolveEntityMediaLines(
|
||||
return lines;
|
||||
}
|
||||
|
||||
function resolveEntityTweetLines(
|
||||
entityKey: number | undefined,
|
||||
entityMap: ArticleContentState["entityMap"] | undefined,
|
||||
entityLookup: EntityLookup,
|
||||
referencedTweets?: Map<string, ReferencedTweetInfo>
|
||||
): string[] {
|
||||
if (entityKey === undefined) return [];
|
||||
const entry = resolveEntityEntry(entityKey, entityMap, entityLookup);
|
||||
const value = entry?.value;
|
||||
if (!value || value.type !== "TWEET") return [];
|
||||
|
||||
const tweetId = typeof value.data?.tweetId === "string" ? value.data.tweetId : "";
|
||||
if (!tweetId) return [];
|
||||
|
||||
const referenced = referencedTweets?.get(tweetId);
|
||||
const url =
|
||||
referenced?.url ??
|
||||
buildTweetUrl(tweetId, referenced?.authorUsername) ??
|
||||
`https://x.com/i/web/status/${tweetId}`;
|
||||
|
||||
const authorText =
|
||||
referenced?.authorName && referenced?.authorUsername
|
||||
? `${referenced.authorName} (@${referenced.authorUsername})`
|
||||
: referenced?.authorUsername
|
||||
? `@${referenced.authorUsername}`
|
||||
: referenced?.authorName;
|
||||
|
||||
const lines: string[] = [];
|
||||
lines.push(`> 引用推文${authorText ? `:${authorText}` : ""}`);
|
||||
|
||||
const summary = summarizeTweetText(referenced?.text);
|
||||
if (summary) {
|
||||
lines.push(`> ${summary}`);
|
||||
}
|
||||
|
||||
lines.push(`> ${url}`);
|
||||
return lines;
|
||||
}
|
||||
|
||||
function buildMediaLinkMap(
|
||||
entityMap: ArticleContentState["entityMap"] | undefined
|
||||
): Map<number, string> {
|
||||
@@ -151,6 +271,7 @@ function buildMediaLinkMap(
|
||||
if (linkIdx === -1) linkIdx = 0;
|
||||
const link = pool.splice(linkIdx, 1)[0]!;
|
||||
map.set(media.idx, link.url);
|
||||
map.set(media.key, link.url);
|
||||
}
|
||||
|
||||
return map;
|
||||
@@ -160,6 +281,7 @@ function renderInlineLinks(
|
||||
text: string,
|
||||
entityRanges: Array<{ key?: number; offset?: number; length?: number }>,
|
||||
entityMap: ArticleContentState["entityMap"] | undefined,
|
||||
entityLookup: EntityLookup,
|
||||
mediaLinkMap: Map<number, string>
|
||||
): string {
|
||||
if (!entityMap || entityRanges.length === 0) return text;
|
||||
@@ -181,7 +303,7 @@ function renderInlineLinks(
|
||||
const length = range.length!;
|
||||
const key = range.key!;
|
||||
|
||||
const entry = entityMap[String(key)];
|
||||
const entry = resolveEntityEntry(key, entityMap, entityLookup);
|
||||
const value = entry?.value;
|
||||
if (!value) continue;
|
||||
|
||||
@@ -204,27 +326,14 @@ function renderInlineLinks(
|
||||
return result;
|
||||
}
|
||||
|
||||
function buildAtomicMediaQueue(
|
||||
article: ArticleEntity,
|
||||
usedUrls: Set<string>
|
||||
): string[] {
|
||||
const queue: string[] = [];
|
||||
for (const entity of article.media_entities ?? []) {
|
||||
const url = resolveMediaUrl(entity?.media_info);
|
||||
if (url && !usedUrls.has(url)) {
|
||||
queue.push(url);
|
||||
}
|
||||
}
|
||||
return queue;
|
||||
}
|
||||
|
||||
function renderContentBlocks(
|
||||
blocks: ArticleBlock[],
|
||||
entityMap: ArticleContentState["entityMap"] | undefined,
|
||||
entityLookup: EntityLookup,
|
||||
mediaById: Map<string, string>,
|
||||
usedUrls: Set<string>,
|
||||
atomicMediaQueue: string[],
|
||||
mediaLinkMap: Map<number, string>
|
||||
mediaLinkMap: Map<number, string>,
|
||||
referencedTweets?: Map<string, ReferencedTweetInfo>
|
||||
): string[] {
|
||||
const lines: string[] = [];
|
||||
let previousKind: "list" | "quote" | "heading" | "text" | "code" | "media" | null = null;
|
||||
@@ -253,18 +362,54 @@ function renderContentBlocks(
|
||||
const mediaLines: string[] = [];
|
||||
for (const range of ranges) {
|
||||
if (typeof range?.key !== "number") continue;
|
||||
mediaLines.push(...resolveEntityMediaLines(range.key, entityMap, mediaById, usedUrls));
|
||||
mediaLines.push(
|
||||
...resolveEntityMediaLines(range.key, entityMap, entityLookup, mediaById, usedUrls)
|
||||
);
|
||||
}
|
||||
return mediaLines;
|
||||
};
|
||||
|
||||
const collectTweetLines = (block: ArticleBlock): string[] => {
|
||||
const ranges = Array.isArray(block.entityRanges) ? block.entityRanges : [];
|
||||
const tweetLines: string[] = [];
|
||||
for (const range of ranges) {
|
||||
if (typeof range?.key !== "number") continue;
|
||||
tweetLines.push(
|
||||
...resolveEntityTweetLines(range.key, entityMap, entityLookup, referencedTweets)
|
||||
);
|
||||
}
|
||||
return tweetLines;
|
||||
};
|
||||
|
||||
const collectLinkLines = (block: ArticleBlock): string[] => {
|
||||
const ranges = Array.isArray(block.entityRanges) ? block.entityRanges : [];
|
||||
const linkLines: string[] = [];
|
||||
for (const range of ranges) {
|
||||
if (typeof range?.key !== "number") continue;
|
||||
const entry = resolveEntityEntry(range.key, entityMap, entityLookup);
|
||||
const value = entry?.value;
|
||||
if (value?.type !== "LINK") continue;
|
||||
const url = typeof value.data?.url === "string" ? value.data.url : "";
|
||||
if (url) {
|
||||
linkLines.push(url);
|
||||
}
|
||||
}
|
||||
return [...new Set(linkLines)];
|
||||
};
|
||||
|
||||
const pushTrailingMedia = (mediaLines: string[]) => {
|
||||
if (mediaLines.length > 0) {
|
||||
pushBlock(mediaLines, "media");
|
||||
}
|
||||
};
|
||||
|
||||
for (const block of blocks) {
|
||||
const type = typeof block?.type === "string" ? block.type : "unstyled";
|
||||
const rawText = typeof block?.text === "string" ? block.text : "";
|
||||
const ranges = Array.isArray(block.entityRanges) ? block.entityRanges : [];
|
||||
const text =
|
||||
type !== "atomic" && type !== "code-block"
|
||||
? renderInlineLinks(rawText, ranges, entityMap, mediaLinkMap)
|
||||
? renderInlineLinks(rawText, ranges, entityMap, entityLookup, mediaLinkMap)
|
||||
: rawText;
|
||||
|
||||
if (type === "code-block") {
|
||||
@@ -290,16 +435,22 @@ function renderContentBlocks(
|
||||
}
|
||||
listKind = null;
|
||||
orderedIndex = 0;
|
||||
|
||||
const tweetLines = collectTweetLines(block);
|
||||
if (tweetLines.length > 0) {
|
||||
pushBlock(tweetLines, "quote");
|
||||
}
|
||||
|
||||
const mediaLines = collectMediaLines(block);
|
||||
if (mediaLines.length > 0) {
|
||||
pushBlock(mediaLines, "media");
|
||||
} else if (atomicMediaQueue.length > 0) {
|
||||
const url = atomicMediaQueue.shift()!;
|
||||
if (!usedUrls.has(url)) {
|
||||
usedUrls.add(url);
|
||||
pushBlock([``], "media");
|
||||
}
|
||||
}
|
||||
|
||||
const linkLines = collectLinkLines(block);
|
||||
if (linkLines.length > 0) {
|
||||
pushBlock(linkLines, "text");
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -313,6 +464,7 @@ function renderContentBlocks(
|
||||
listKind = "unordered";
|
||||
orderedIndex = 0;
|
||||
pushBlock([`- ${text}`], "list");
|
||||
pushTrailingMedia(collectMediaLines(block));
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -323,6 +475,7 @@ function renderContentBlocks(
|
||||
listKind = "ordered";
|
||||
orderedIndex += 1;
|
||||
pushBlock([`${orderedIndex}. ${text}`], "list");
|
||||
pushTrailingMedia(collectMediaLines(block));
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -332,29 +485,41 @@ function renderContentBlocks(
|
||||
switch (type) {
|
||||
case "header-one":
|
||||
pushBlock([`# ${text}`], "heading");
|
||||
pushTrailingMedia(collectMediaLines(block));
|
||||
break;
|
||||
case "header-two":
|
||||
pushBlock([`## ${text}`], "heading");
|
||||
pushTrailingMedia(collectMediaLines(block));
|
||||
break;
|
||||
case "header-three":
|
||||
pushBlock([`### ${text}`], "heading");
|
||||
pushTrailingMedia(collectMediaLines(block));
|
||||
break;
|
||||
case "header-four":
|
||||
pushBlock([`#### ${text}`], "heading");
|
||||
pushTrailingMedia(collectMediaLines(block));
|
||||
break;
|
||||
case "header-five":
|
||||
pushBlock([`##### ${text}`], "heading");
|
||||
pushTrailingMedia(collectMediaLines(block));
|
||||
break;
|
||||
case "header-six":
|
||||
pushBlock([`###### ${text}`], "heading");
|
||||
pushTrailingMedia(collectMediaLines(block));
|
||||
break;
|
||||
case "blockquote": {
|
||||
const quoteLines = text.length > 0 ? text.split("\n") : [""];
|
||||
pushBlock(quoteLines.map((line) => `> ${line}`), "quote");
|
||||
pushTrailingMedia(collectMediaLines(block));
|
||||
break;
|
||||
}
|
||||
default:
|
||||
if (/^XIMGPH_\d+$/.test(text.trim())) {
|
||||
pushTrailingMedia(collectMediaLines(block));
|
||||
break;
|
||||
}
|
||||
pushBlock([text], "text");
|
||||
pushTrailingMedia(collectMediaLines(block));
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -371,7 +536,28 @@ export type FormatArticleResult = {
|
||||
coverUrl: string | null;
|
||||
};
|
||||
|
||||
export function formatArticleMarkdown(article: unknown): FormatArticleResult {
|
||||
export function extractReferencedTweetIds(article: unknown): string[] {
|
||||
const candidate = coerceArticleEntity(article);
|
||||
const entityMap = candidate?.content_state?.entityMap;
|
||||
if (!entityMap) return [];
|
||||
|
||||
const ids: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const entry of Object.values(entityMap)) {
|
||||
const value = entry?.value;
|
||||
if (value?.type !== "TWEET") continue;
|
||||
const tweetId = typeof value.data?.tweetId === "string" ? value.data.tweetId : "";
|
||||
if (!tweetId || seen.has(tweetId)) continue;
|
||||
seen.add(tweetId);
|
||||
ids.push(tweetId);
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
export function formatArticleMarkdown(
|
||||
article: unknown,
|
||||
options: FormatArticleOptions = {}
|
||||
): FormatArticleResult {
|
||||
const candidate = coerceArticleEntity(article);
|
||||
if (!candidate) {
|
||||
return { markdown: `\`\`\`json\n${JSON.stringify(article, null, 2)}\n\`\`\``, coverUrl: null };
|
||||
@@ -392,10 +578,18 @@ export function formatArticleMarkdown(article: unknown): FormatArticleResult {
|
||||
|
||||
const blocks = candidate.content_state?.blocks;
|
||||
const entityMap = candidate.content_state?.entityMap;
|
||||
const entityLookup = buildEntityLookup(entityMap);
|
||||
if (Array.isArray(blocks) && blocks.length > 0) {
|
||||
const atomicMediaQueue = buildAtomicMediaQueue(candidate, usedUrls);
|
||||
const mediaLinkMap = buildMediaLinkMap(entityMap);
|
||||
const rendered = renderContentBlocks(blocks, entityMap, mediaById, usedUrls, atomicMediaQueue, mediaLinkMap);
|
||||
const rendered = renderContentBlocks(
|
||||
blocks,
|
||||
entityMap,
|
||||
entityLookup,
|
||||
mediaById,
|
||||
usedUrls,
|
||||
mediaLinkMap,
|
||||
options.referencedTweets
|
||||
);
|
||||
if (rendered.length > 0) {
|
||||
if (lines.length > 0) lines.push("");
|
||||
lines.push(...rendered);
|
||||
|
||||
@@ -187,11 +187,35 @@ function buildFileName(kind: MediaKind, index: number, sourceUrl: string, extens
|
||||
|
||||
const FRONTMATTER_COVER_RE = /^(coverImage:\s*")(https?:\/\/[^"]+)(")/m;
|
||||
|
||||
function toHighResUrl(rawUrl: string): string {
|
||||
try {
|
||||
const parsed = new URL(rawUrl);
|
||||
if (parsed.hostname !== "pbs.twimg.com") return rawUrl;
|
||||
const ext = path.posix.extname(parsed.pathname).replace(/^\./, "").toLowerCase();
|
||||
if (!ext || !IMAGE_EXTENSIONS.has(ext)) return rawUrl;
|
||||
parsed.pathname = parsed.pathname.replace(new RegExp(`\\.${ext}$`), "");
|
||||
parsed.searchParams.set("format", ext === "jpeg" ? "jpg" : ext);
|
||||
parsed.searchParams.set("name", "4096x4096");
|
||||
return parsed.toString();
|
||||
} catch {
|
||||
return rawUrl;
|
||||
}
|
||||
}
|
||||
|
||||
function collectMarkdownLinkCandidates(markdown: string): MarkdownLinkCandidate[] {
|
||||
MARKDOWN_LINK_RE.lastIndex = 0;
|
||||
const candidates: MarkdownLinkCandidate[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
const fmMatch = markdown.match(/^---\n([\s\S]*?)\n---/);
|
||||
if (fmMatch) {
|
||||
const coverMatch = fmMatch[1]?.match(FRONTMATTER_COVER_RE);
|
||||
if (coverMatch?.[2] && !seen.has(coverMatch[2])) {
|
||||
seen.add(coverMatch[2]);
|
||||
candidates.push({ url: coverMatch[2], hint: "image" });
|
||||
}
|
||||
}
|
||||
|
||||
MARKDOWN_LINK_RE.lastIndex = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = MARKDOWN_LINK_RE.exec(markdown))) {
|
||||
const label = match[1] ?? "";
|
||||
@@ -204,15 +228,6 @@ function collectMarkdownLinkCandidates(markdown: string): MarkdownLinkCandidate[
|
||||
});
|
||||
}
|
||||
|
||||
const fmMatch = markdown.match(/^---\n([\s\S]*?)\n---/);
|
||||
if (fmMatch) {
|
||||
const coverMatch = fmMatch[1]?.match(FRONTMATTER_COVER_RE);
|
||||
if (coverMatch?.[2] && !seen.has(coverMatch[2])) {
|
||||
seen.add(coverMatch[2]);
|
||||
candidates.push({ url: coverMatch[2], hint: "image" });
|
||||
}
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
@@ -259,7 +274,8 @@ export async function localizeMarkdownMedia(
|
||||
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
const response = await fetch(candidate.url, {
|
||||
const downloadUrl = toHighResUrl(candidate.url);
|
||||
const response = await fetch(downloadUrl, {
|
||||
method: "GET",
|
||||
redirect: "follow",
|
||||
headers: {
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import { fetchXTweet } from "./graphql.js";
|
||||
import {
|
||||
extractReferencedTweetIds,
|
||||
type ReferencedTweetInfo,
|
||||
} from "./markdown.js";
|
||||
|
||||
type ResolveReferencedTweetsOptions = {
|
||||
log?: (message: string) => void;
|
||||
};
|
||||
|
||||
function extractReferencedTweetInfo(tweet: any, fallbackTweetId: string): ReferencedTweetInfo {
|
||||
const userCore = tweet?.core?.user_results?.result?.core;
|
||||
const userLegacy = tweet?.core?.user_results?.result?.legacy;
|
||||
|
||||
const authorName =
|
||||
typeof userCore?.name === "string"
|
||||
? userCore.name
|
||||
: typeof userLegacy?.name === "string"
|
||||
? userLegacy.name
|
||||
: undefined;
|
||||
|
||||
const authorUsername =
|
||||
typeof userCore?.screen_name === "string"
|
||||
? userCore.screen_name
|
||||
: typeof userLegacy?.screen_name === "string"
|
||||
? userLegacy.screen_name
|
||||
: undefined;
|
||||
|
||||
const text =
|
||||
tweet?.note_tweet?.note_tweet_results?.result?.text ??
|
||||
tweet?.legacy?.full_text ??
|
||||
tweet?.legacy?.text ??
|
||||
undefined;
|
||||
|
||||
const tweetId =
|
||||
typeof tweet?.rest_id === "string" && tweet.rest_id.length > 0
|
||||
? tweet.rest_id
|
||||
: fallbackTweetId;
|
||||
|
||||
const url = authorUsername
|
||||
? `https://x.com/${authorUsername}/status/${tweetId}`
|
||||
: `https://x.com/i/web/status/${tweetId}`;
|
||||
|
||||
return {
|
||||
id: tweetId,
|
||||
url,
|
||||
authorName,
|
||||
authorUsername,
|
||||
text: typeof text === "string" ? text : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export async function resolveReferencedTweetsFromArticle(
|
||||
article: unknown,
|
||||
cookieMap: Record<string, string>,
|
||||
options: ResolveReferencedTweetsOptions = {}
|
||||
): Promise<Map<string, ReferencedTweetInfo>> {
|
||||
const log = options.log ?? (() => {});
|
||||
const ids = extractReferencedTweetIds(article);
|
||||
const referencedTweets = new Map<string, ReferencedTweetInfo>();
|
||||
|
||||
for (const id of ids) {
|
||||
try {
|
||||
const tweet = await fetchXTweet(id, cookieMap, false);
|
||||
const info = extractReferencedTweetInfo(tweet, id);
|
||||
referencedTweets.set(id, info);
|
||||
} catch (error) {
|
||||
log(
|
||||
`[x-to-markdown] Failed to fetch referenced tweet ${id}: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
);
|
||||
referencedTweets.set(id, {
|
||||
id,
|
||||
url: `https://x.com/i/web/status/${id}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return referencedTweets;
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url";
|
||||
import { hasRequiredXCookies, loadXCookies } from "./cookies.js";
|
||||
import { fetchTweetThread } from "./thread.js";
|
||||
import { formatArticleMarkdown } from "./markdown.js";
|
||||
import { resolveReferencedTweetsFromArticle } from "./referenced-tweets.js";
|
||||
import { formatThreadTweetsMarkdown } from "./thread-markdown.js";
|
||||
import { resolveArticleEntityFromTweet } from "./tweet-article.js";
|
||||
|
||||
@@ -129,7 +130,8 @@ export async function tweetToMarkdown(
|
||||
const parts: string[] = [];
|
||||
|
||||
if (articleEntity) {
|
||||
const articleResult = formatArticleMarkdown(articleEntity);
|
||||
const referencedTweets = await resolveReferencedTweetsFromArticle(articleEntity, cookieMap, { log });
|
||||
const articleResult = formatArticleMarkdown(articleEntity, { referencedTweets });
|
||||
coverImage = articleResult.coverUrl;
|
||||
const articleMarkdown = articleResult.markdown.trimEnd();
|
||||
if (articleMarkdown) {
|
||||
|
||||
@@ -40,6 +40,7 @@ export type ArticleEntityMapEntry = {
|
||||
caption?: string;
|
||||
mediaItems?: ArticleEntityMapMediaItem[];
|
||||
url?: string;
|
||||
tweetId?: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user