mirror of
https://github.com/JimLiu/baoyu-skills.git
synced 2026-07-26 03:49:47 +08:00
feat(baoyu-fetch): add URL reader CLI with Chrome CDP and site adapters
This commit is contained in:
Jim Liu 宝玉
@@ -0,0 +1,25 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { resolveAdapter } from "../adapters";
|
||||
|
||||
describe("adapter registry", () => {
|
||||
test("matches x adapter for x.com status URLs", () => {
|
||||
const adapter = resolveAdapter({
|
||||
url: new URL("https://x.com/openai/status/1234567890"),
|
||||
});
|
||||
expect(adapter.name).toBe("x");
|
||||
});
|
||||
|
||||
test("matches hn adapter for item URLs", () => {
|
||||
const adapter = resolveAdapter({
|
||||
url: new URL("https://news.ycombinator.com/item?id=47534848"),
|
||||
});
|
||||
expect(adapter.name).toBe("hn");
|
||||
});
|
||||
|
||||
test("falls back to generic adapter", () => {
|
||||
const adapter = resolveAdapter({
|
||||
url: new URL("https://example.com/post"),
|
||||
});
|
||||
expect(adapter.name).toBe("generic");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { HELP_TEXT, parseArgs } from "../cli";
|
||||
|
||||
describe("parseArgs", () => {
|
||||
test("defaults to markdown output", () => {
|
||||
const options = parseArgs(["bun", "src/cli.ts", "https://example.com"]);
|
||||
expect(options.format).toBe("markdown");
|
||||
});
|
||||
|
||||
test("parses explicit json output format", () => {
|
||||
const options = parseArgs(["bun", "src/cli.ts", "https://example.com", "--format", "json"]);
|
||||
expect(options.format).toBe("json");
|
||||
});
|
||||
|
||||
test("maps --json to json output format", () => {
|
||||
const options = parseArgs(["bun", "src/cli.ts", "https://example.com", "--json"]);
|
||||
expect(options.format).toBe("json");
|
||||
});
|
||||
|
||||
test("parses --wait-for interaction", () => {
|
||||
const options = parseArgs(["bun", "src/cli.ts", "https://example.com", "--wait-for", "interaction"]);
|
||||
expect(options.waitMode).toBe("interaction");
|
||||
});
|
||||
|
||||
test("parses --wait-for force", () => {
|
||||
const options = parseArgs(["bun", "src/cli.ts", "https://example.com", "--wait-for", "force"]);
|
||||
expect(options.waitMode).toBe("force");
|
||||
});
|
||||
|
||||
test("maps legacy wait flags to interaction mode", () => {
|
||||
const options = parseArgs(["bun", "src/cli.ts", "https://example.com", "--wait-for-interaction"]);
|
||||
expect(options.waitMode).toBe("interaction");
|
||||
});
|
||||
|
||||
test("parses media download options", () => {
|
||||
const options = parseArgs([
|
||||
"bun",
|
||||
"src/cli.ts",
|
||||
"https://example.com",
|
||||
"--download-media",
|
||||
"--media-dir",
|
||||
"./assets",
|
||||
]);
|
||||
|
||||
expect(options.downloadMedia).toBe(true);
|
||||
expect(options.mediaDir).toBe("./assets");
|
||||
});
|
||||
|
||||
test("rejects invalid wait modes", () => {
|
||||
expect(() =>
|
||||
parseArgs(["bun", "src/cli.ts", "https://example.com", "--wait-for", "unknown"]),
|
||||
).toThrow("Invalid wait mode");
|
||||
});
|
||||
|
||||
test("rejects invalid output formats", () => {
|
||||
expect(() =>
|
||||
parseArgs(["bun", "src/cli.ts", "https://example.com", "--format", "xml"]),
|
||||
).toThrow("Invalid output format");
|
||||
});
|
||||
|
||||
test("documents wait modes in help text", () => {
|
||||
expect(HELP_TEXT).toContain("baoyu-fetch");
|
||||
expect(HELP_TEXT).toContain("--format <type>");
|
||||
expect(HELP_TEXT).toContain("--wait-for <mode>");
|
||||
expect(HELP_TEXT).toContain("--download-media");
|
||||
expect(HELP_TEXT).toContain("force: start visible Chrome, then auto-continue");
|
||||
expect(HELP_TEXT).toContain("or continue immediately when you press Enter");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { formatOutputContent } from "../commands/convert";
|
||||
|
||||
describe("formatOutputContent", () => {
|
||||
test("returns raw markdown for markdown output", () => {
|
||||
expect(
|
||||
formatOutputContent("markdown", {
|
||||
adapter: "generic",
|
||||
status: "ok",
|
||||
media: [],
|
||||
downloads: null,
|
||||
document: {
|
||||
url: "https://example.com",
|
||||
content: [],
|
||||
},
|
||||
markdown: "# Example",
|
||||
}),
|
||||
).toBe("# Example");
|
||||
});
|
||||
|
||||
test("returns structured json for json output", () => {
|
||||
const parsed = JSON.parse(
|
||||
formatOutputContent("json", {
|
||||
adapter: "generic",
|
||||
status: "ok",
|
||||
media: [],
|
||||
downloads: null,
|
||||
document: {
|
||||
url: "https://example.com",
|
||||
content: [],
|
||||
},
|
||||
markdown: "# Example",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(parsed.status).toBe("ok");
|
||||
expect(parsed.markdown).toBe("# Example");
|
||||
expect(parsed.document.url).toBe("https://example.com");
|
||||
});
|
||||
|
||||
test("rejects markdown output for interaction-required payloads", () => {
|
||||
expect(() =>
|
||||
formatOutputContent("markdown", {
|
||||
adapter: "x",
|
||||
status: "needs_interaction",
|
||||
interaction: {
|
||||
type: "wait_for_interaction",
|
||||
kind: "login",
|
||||
provider: "x",
|
||||
prompt: "Login required",
|
||||
},
|
||||
}),
|
||||
).toThrow("Markdown output is only available");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,167 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { renderMarkdown } from "../extract/markdown-renderer";
|
||||
import {
|
||||
buildHnDocument,
|
||||
buildHnThreadMarkdown,
|
||||
extractHnThreadFromHtml,
|
||||
parseHnItemId,
|
||||
type HnCommentNode,
|
||||
type HnItem,
|
||||
} from "../adapters/hn";
|
||||
|
||||
describe("hn adapter helpers", () => {
|
||||
test("parses item id from hn item url", () => {
|
||||
expect(parseHnItemId(new URL("https://news.ycombinator.com/item?id=47534848"))).toBe(47534848);
|
||||
expect(parseHnItemId(new URL("https://news.ycombinator.com/newest"))).toBeNull();
|
||||
});
|
||||
|
||||
test("renders threaded comments with author, time, and nested indentation", () => {
|
||||
const story: HnItem = {
|
||||
id: 47534848,
|
||||
type: "story",
|
||||
by: "mmcclure",
|
||||
time: 1774554485,
|
||||
title: "Example & Title",
|
||||
url: "https://example.com/post",
|
||||
score: 257,
|
||||
descendants: 2,
|
||||
};
|
||||
|
||||
const comments: HnCommentNode[] = [
|
||||
{
|
||||
item: {
|
||||
id: 47535377,
|
||||
type: "comment",
|
||||
by: "jackfruitpeel",
|
||||
time: 1774557334,
|
||||
text: "Root comment<p>With two paragraphs.",
|
||||
},
|
||||
children: [
|
||||
{
|
||||
item: {
|
||||
id: 47535469,
|
||||
type: "comment",
|
||||
by: "__MatrixMan__",
|
||||
time: 1774557848,
|
||||
text: "Nested reply with a <a href=\"item?id=1\">relative link</a>.",
|
||||
},
|
||||
children: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const body = buildHnThreadMarkdown(story, comments, "https://news.ycombinator.com/item?id=47534848");
|
||||
expect(body).toContain("Source: [https://example.com/post](https://example.com/post)");
|
||||
expect(body).toContain("Submitted by mmcclure at 2026-03-26 19:48:05 UTC");
|
||||
expect(body).toContain("- jackfruitpeel · [2026-03-26 20:35:34 UTC](https://news.ycombinator.com/item?id=47534848#47535377)");
|
||||
expect(body).toContain(" Root comment");
|
||||
expect(body).toContain(" With two paragraphs.");
|
||||
expect(body).toContain(" - __MatrixMan__ · [2026-03-26 20:44:08 UTC](https://news.ycombinator.com/item?id=47534848#47535469)");
|
||||
expect(body).toContain(" Nested reply with a [relative link](https://news.ycombinator.com/item?id=1).");
|
||||
});
|
||||
|
||||
test("extracts story metadata and nested comments from hn html", () => {
|
||||
const parsed = extractHnThreadFromHtml(
|
||||
`
|
||||
<html>
|
||||
<body>
|
||||
<table class="fatitem">
|
||||
<tr class="athing submission" id="47534848">
|
||||
<td class="title">
|
||||
<span class="titleline">
|
||||
<a href="https://example.com/post">Example story</a>
|
||||
</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="subtext">
|
||||
<span class="subline">
|
||||
<span class="score">257 points</span>
|
||||
by <a href="user?id=mmcclure" class="hnuser">mmcclure</a>
|
||||
<span class="age" title="2026-03-26T19:48:05 1774554485">
|
||||
<a href="item?id=47534848">1 hour ago</a>
|
||||
</span>
|
||||
<a href="item?id=47534848">152 comments</a>
|
||||
</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><div class="toptext">Story <p>body</p></div></td>
|
||||
</tr>
|
||||
</table>
|
||||
<table class="comment-tree">
|
||||
<tr class="athing comtr" id="47535377">
|
||||
<td class="ind" indent="0"></td>
|
||||
<td class="default">
|
||||
<span class="comhead">
|
||||
<a href="user?id=jackfruitpeel" class="hnuser">jackfruitpeel</a>
|
||||
<span class="age" title="2026-03-26T20:35:34 1774557334">
|
||||
<a href="item?id=47535377">36 minutes ago</a>
|
||||
</span>
|
||||
</span>
|
||||
<div class="comment"><div class="commtext c00">Root</div></div>
|
||||
</td>
|
||||
</tr>
|
||||
<tr class="athing comtr" id="47535469">
|
||||
<td class="ind" indent="1"></td>
|
||||
<td class="default">
|
||||
<span class="comhead">
|
||||
<a href="user?id=willio58" class="hnuser">willio58</a>
|
||||
<span class="age" title="2026-03-26T20:44:08 1774557848">
|
||||
<a href="item?id=47535469">27 minutes ago</a>
|
||||
</span>
|
||||
</span>
|
||||
<div class="comment"><div class="commtext c00">Child</div></div>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
`,
|
||||
"https://news.ycombinator.com/item?id=47534848",
|
||||
);
|
||||
|
||||
expect(parsed).not.toBeNull();
|
||||
expect(parsed?.story.title).toBe("Example story");
|
||||
expect(parsed?.story.url).toBe("https://example.com/post");
|
||||
expect(parsed?.story.by).toBe("mmcclure");
|
||||
expect(parsed?.story.time).toBe(1774554485);
|
||||
expect(parsed?.story.score).toBe(257);
|
||||
expect(parsed?.story.descendants).toBe(152);
|
||||
expect(parsed?.story.text).toContain("Story");
|
||||
expect(parsed?.comments).toHaveLength(1);
|
||||
expect(parsed?.comments[0]?.item.by).toBe("jackfruitpeel");
|
||||
expect(parsed?.comments[0]?.children).toHaveLength(1);
|
||||
expect(parsed?.comments[0]?.children[0]?.item.by).toBe("willio58");
|
||||
});
|
||||
|
||||
test("builds hn document with metadata and markdown body", () => {
|
||||
const document = buildHnDocument(
|
||||
{
|
||||
id: 123,
|
||||
type: "story",
|
||||
by: "pg",
|
||||
time: 1175714200,
|
||||
title: "Ask HN: Example",
|
||||
text: "What are you working on?",
|
||||
score: 111,
|
||||
descendants: 0,
|
||||
},
|
||||
[],
|
||||
"https://news.ycombinator.com/item?id=123",
|
||||
);
|
||||
|
||||
const markdown = renderMarkdown(document);
|
||||
expect(document.adapter).toBe("hn");
|
||||
expect(document.siteName).toBe("Hacker News");
|
||||
expect(document.publishedAt).toBe("2007-04-04T19:16:40.000Z");
|
||||
expect(markdown).toContain('adapter: "hn"');
|
||||
expect(markdown).toContain('siteName: "Hacker News"');
|
||||
expect(markdown).toContain("# Ask HN: Example");
|
||||
expect(markdown).toContain("## Post");
|
||||
expect(markdown).toContain("What are you working on?");
|
||||
expect(markdown).toContain("## Comments");
|
||||
expect(markdown).toContain("No comments.");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
import { afterEach, describe, expect, test } from "bun:test";
|
||||
import {
|
||||
convertHtmlToMarkdown,
|
||||
extractTitleFromMarkdownDocument,
|
||||
} from "../extract/html-to-markdown";
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
describe("extractTitleFromMarkdownDocument", () => {
|
||||
test("prefers frontmatter title when present", () => {
|
||||
const title = extractTitleFromMarkdownDocument(`---
|
||||
title: "Frontmatter Title"
|
||||
---
|
||||
|
||||
# Heading Title
|
||||
`);
|
||||
|
||||
expect(title).toBe("Frontmatter Title");
|
||||
});
|
||||
|
||||
test("falls back to the first markdown heading", () => {
|
||||
const title = extractTitleFromMarkdownDocument(`
|
||||
Intro text
|
||||
|
||||
# Heading Title
|
||||
|
||||
Body text.
|
||||
`);
|
||||
|
||||
expect(title).toBe("Heading Title");
|
||||
});
|
||||
});
|
||||
|
||||
describe("convertHtmlToMarkdown remote fallback", () => {
|
||||
test("does not call defuddle.md when the remote fallback option is disabled", async () => {
|
||||
let fetchCalls = 0;
|
||||
globalThis.fetch = Object.assign(
|
||||
async () => {
|
||||
fetchCalls += 1;
|
||||
return new Response("# Remote Title\n\nRemote body.", {
|
||||
headers: {
|
||||
"content-type": "text/markdown",
|
||||
},
|
||||
});
|
||||
},
|
||||
{
|
||||
preconnect: originalFetch.preconnect,
|
||||
},
|
||||
) as typeof fetch;
|
||||
|
||||
const result = await convertHtmlToMarkdown(
|
||||
"<!doctype html><html><head><title>Local Title</title></head><body></body></html>",
|
||||
"https://example.com/post",
|
||||
);
|
||||
|
||||
expect(fetchCalls).toBe(0);
|
||||
expect(result.conversionMethod).not.toBe("defuddle-api");
|
||||
});
|
||||
|
||||
test("uses defuddle.md markdown when local extraction is empty", async () => {
|
||||
const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];
|
||||
globalThis.fetch = Object.assign(
|
||||
async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
fetchCalls.push({ input, init });
|
||||
return new Response(`---
|
||||
title: "Remote Title"
|
||||
---
|
||||
|
||||
# Remote Title
|
||||
|
||||
Remote body.
|
||||
`, {
|
||||
headers: {
|
||||
"content-type": "text/markdown",
|
||||
},
|
||||
});
|
||||
},
|
||||
{
|
||||
preconnect: originalFetch.preconnect,
|
||||
},
|
||||
) as typeof fetch;
|
||||
|
||||
const result = await convertHtmlToMarkdown(
|
||||
"<!doctype html><html><head><title>Local Title</title></head><body></body></html>",
|
||||
"https://example.com/post",
|
||||
{ enableRemoteMarkdownFallback: true },
|
||||
);
|
||||
|
||||
expect(fetchCalls).toHaveLength(1);
|
||||
expect(String(fetchCalls[0]?.input)).toBe(
|
||||
"https://defuddle.md/https%3A%2F%2Fexample.com%2Fpost",
|
||||
);
|
||||
expect(fetchCalls[0]?.init?.headers).toEqual({
|
||||
accept: "text/markdown,text/plain;q=0.9,*/*;q=0.1",
|
||||
});
|
||||
expect(result.conversionMethod).toBe("defuddle-api");
|
||||
expect(result.metadata.title).toBe("Remote Title");
|
||||
expect(result.markdown).toBe("# Remote Title\n\nRemote body.");
|
||||
expect(result.fallbackReason).toContain("defuddle.md");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { detectInteractionGateFromSnapshot } from "../browser/interaction-gates";
|
||||
|
||||
describe("detectInteractionGateFromSnapshot", () => {
|
||||
test("detects cloudflare challenge", () => {
|
||||
const gate = detectInteractionGateFromSnapshot({
|
||||
title: "Just a moment...",
|
||||
currentUrl: "https://example.com/cdn-cgi/challenge-platform/h/b",
|
||||
bodyText: "Checking your browser before accessing example.com",
|
||||
hasCloudflareTurnstile: true,
|
||||
hasCloudflareChallenge: true,
|
||||
hasRecaptcha: false,
|
||||
hasRecaptchaIframe: false,
|
||||
hasHcaptcha: false,
|
||||
hasHcaptchaIframe: false,
|
||||
});
|
||||
|
||||
expect(gate?.kind).toBe("cloudflare");
|
||||
expect(gate?.provider).toBe("cloudflare");
|
||||
});
|
||||
|
||||
test("detects google recaptcha", () => {
|
||||
const gate = detectInteractionGateFromSnapshot({
|
||||
title: "Protected page",
|
||||
currentUrl: "https://example.com/form",
|
||||
bodyText: "Please verify that you're not a robot via reCAPTCHA",
|
||||
hasCloudflareTurnstile: false,
|
||||
hasCloudflareChallenge: false,
|
||||
hasRecaptcha: true,
|
||||
hasRecaptchaIframe: true,
|
||||
hasHcaptcha: false,
|
||||
hasHcaptchaIframe: false,
|
||||
});
|
||||
|
||||
expect(gate?.kind).toBe("recaptcha");
|
||||
expect(gate?.provider).toBe("google_recaptcha");
|
||||
});
|
||||
|
||||
test("returns null when no challenge is present", () => {
|
||||
const gate = detectInteractionGateFromSnapshot({
|
||||
title: "Example",
|
||||
currentUrl: "https://example.com/article",
|
||||
bodyText: "Normal article body",
|
||||
hasCloudflareTurnstile: false,
|
||||
hasCloudflareChallenge: false,
|
||||
hasRecaptcha: false,
|
||||
hasRecaptchaIframe: false,
|
||||
hasHcaptcha: false,
|
||||
hasHcaptchaIframe: false,
|
||||
});
|
||||
|
||||
expect(gate).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,138 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
collectMediaFromDocument,
|
||||
collectMediaFromMarkdown,
|
||||
normalizeMarkdownMediaLinks,
|
||||
rewriteMarkdownMediaLinks,
|
||||
} from "../media/markdown-media";
|
||||
|
||||
describe("markdown media helpers", () => {
|
||||
test("collects cover, image markdown, and plain media urls from a document", () => {
|
||||
const media = collectMediaFromDocument({
|
||||
url: "https://example.com/post",
|
||||
metadata: {
|
||||
coverImage: "https://cdn.example.com/cover.jpg",
|
||||
},
|
||||
content: [
|
||||
{ type: "paragraph", text: "Poster: https://cdn.example.com/poster.png" },
|
||||
{ type: "markdown", markdown: "\n\n[video](https://cdn.example.com/clip.mp4)" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(media).toEqual([
|
||||
{ url: "https://cdn.example.com/cover.jpg", kind: "image", role: "cover" },
|
||||
{ url: "https://cdn.example.com/poster.png", kind: "image", role: "inline" },
|
||||
{ url: "https://cdn.example.com/body.webp", kind: "image", role: "inline" },
|
||||
{ url: "https://cdn.example.com/clip.mp4", kind: "video", role: "inline" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("rewrites markdown links, frontmatter cover images, and plain url mentions", () => {
|
||||
const markdown = `---
|
||||
coverImage: "https://cdn.example.com/cover.jpg"
|
||||
---
|
||||
|
||||
|
||||
|
||||
Poster: https://cdn.example.com/poster.png
|
||||
`;
|
||||
|
||||
const rewritten = rewriteMarkdownMediaLinks(markdown, [
|
||||
{
|
||||
url: "https://cdn.example.com/cover.jpg",
|
||||
localPath: "imgs/img-001-cover.jpg",
|
||||
absolutePath: "/tmp/imgs/img-001-cover.jpg",
|
||||
kind: "image",
|
||||
},
|
||||
{
|
||||
url: "https://cdn.example.com/body.webp",
|
||||
localPath: "imgs/img-002-body.webp",
|
||||
absolutePath: "/tmp/imgs/img-002-body.webp",
|
||||
kind: "image",
|
||||
},
|
||||
{
|
||||
url: "https://cdn.example.com/poster.png",
|
||||
localPath: "imgs/img-003-poster.png",
|
||||
absolutePath: "/tmp/imgs/img-003-poster.png",
|
||||
kind: "image",
|
||||
},
|
||||
]);
|
||||
|
||||
expect(rewritten).toContain('coverImage: "imgs/img-001-cover.jpg"');
|
||||
expect(rewritten).toContain("");
|
||||
expect(rewritten).toContain("Poster: imgs/img-003-poster.png");
|
||||
});
|
||||
|
||||
test("normalizes and dedupes linked Substack CDN image variants", () => {
|
||||
const resizedUrl =
|
||||
"https://substackcdn.com/image/fetch/$s_!wORh!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fb83f9d2f-711f-4edd-bc8a-303b8de422e5_1600x1300.png";
|
||||
const linkedUrl =
|
||||
"https://substackcdn.com/image/fetch/$s_!wORh!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fb83f9d2f-711f-4edd-bc8a-303b8de422e5_1600x1300.png";
|
||||
const canonicalUrl =
|
||||
"https://substack-post-media.s3.amazonaws.com/public/images/b83f9d2f-711f-4edd-bc8a-303b8de422e5_1600x1300.png";
|
||||
const markdown = `[](${linkedUrl})`;
|
||||
|
||||
expect(normalizeMarkdownMediaLinks(markdown)).toBe(``);
|
||||
expect(collectMediaFromMarkdown(markdown)).toEqual([
|
||||
{
|
||||
url: canonicalUrl,
|
||||
kind: "image",
|
||||
role: "inline",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("collapses linked images when href equals image url after normalization", () => {
|
||||
const resizedUrl =
|
||||
"https://substackcdn.com/image/fetch/$s_!wORh!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fb83f9d2f-711f-4edd-bc8a-303b8de422e5_1600x1300.png";
|
||||
const linkedUrl =
|
||||
"https://substackcdn.com/image/fetch/$s_!wORh!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fb83f9d2f-711f-4edd-bc8a-303b8de422e5_1600x1300.png";
|
||||
const canonicalUrl =
|
||||
"https://substack-post-media.s3.amazonaws.com/public/images/b83f9d2f-711f-4edd-bc8a-303b8de422e5_1600x1300.png";
|
||||
const markdown = `[
|
||||
|
||||
|
||||
|
||||
](${linkedUrl})`;
|
||||
|
||||
expect(normalizeMarkdownMediaLinks(markdown)).toBe(``);
|
||||
});
|
||||
|
||||
test("compacts linked images when href differs from the image url", () => {
|
||||
const markdown = `[
|
||||
|
||||
|
||||
|
||||
](https://example.com/source)`;
|
||||
|
||||
expect(normalizeMarkdownMediaLinks(markdown)).toBe(
|
||||
"[](https://example.com/source)",
|
||||
);
|
||||
});
|
||||
|
||||
test("keeps single-line linked images on one line after parser-based normalization", () => {
|
||||
const markdown = `[](https://example.com/source)`;
|
||||
|
||||
expect(normalizeMarkdownMediaLinks(markdown)).toBe(
|
||||
"[](https://example.com/source)",
|
||||
);
|
||||
});
|
||||
|
||||
test("repairs broken linked image blocks without disturbing surrounding paragraphs", () => {
|
||||
const markdown = `Before
|
||||
|
||||
[
|
||||
|
||||
|
||||
|
||||
](https://example.com/source)
|
||||
|
||||
After`;
|
||||
|
||||
expect(normalizeMarkdownMediaLinks(markdown)).toBe(`Before
|
||||
|
||||
[](https://example.com/source)
|
||||
|
||||
After`);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { renderMarkdown } from "../extract/markdown-renderer";
|
||||
|
||||
describe("renderMarkdown", () => {
|
||||
test("renders frontmatter and content blocks", () => {
|
||||
const markdown = renderMarkdown({
|
||||
url: "https://example.com/post",
|
||||
requestedUrl: "https://example.com/post?ref=test",
|
||||
title: "Example Title",
|
||||
author: "Alice",
|
||||
siteName: "Example",
|
||||
publishedAt: "2026-03-25",
|
||||
adapter: "generic",
|
||||
metadata: {
|
||||
authorName: "Alice Example",
|
||||
authorUsername: "alice",
|
||||
authorUrl: "https://example.com/@alice",
|
||||
kind: "generic/article",
|
||||
},
|
||||
content: [
|
||||
{ type: "paragraph", text: "First paragraph." },
|
||||
{ type: "list", ordered: false, items: ["One", "Two"] },
|
||||
],
|
||||
});
|
||||
|
||||
expect(markdown).toContain("---");
|
||||
expect(markdown).toContain('title: "Example Title"');
|
||||
expect(markdown).toContain('url: "https://example.com/post"');
|
||||
expect(markdown).toContain('requestedUrl: "https://example.com/post?ref=test"');
|
||||
expect(markdown).toContain('author: "Alice"');
|
||||
expect(markdown).toContain('authorName: "Alice Example"');
|
||||
expect(markdown).toContain('authorUsername: "alice"');
|
||||
expect(markdown).toContain('authorUrl: "https://example.com/@alice"');
|
||||
expect(markdown).toContain("# Example Title");
|
||||
expect(markdown).toContain("First paragraph.");
|
||||
expect(markdown).toContain("- One");
|
||||
});
|
||||
|
||||
test("avoids duplicating the title when body already starts with it", () => {
|
||||
const markdown = renderMarkdown({
|
||||
url: "https://example.com/post",
|
||||
title: "Example Title",
|
||||
content: [{ type: "markdown", markdown: "# Example Title\n\nBody text." }],
|
||||
});
|
||||
|
||||
expect(markdown.match(/# Example Title/g)?.length).toBe(1);
|
||||
expect(markdown).toContain("Body text.");
|
||||
});
|
||||
|
||||
test("normalizes Substack CDN image links in rendered markdown", () => {
|
||||
const resizedUrl =
|
||||
"https://substackcdn.com/image/fetch/$s_!wORh!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fb83f9d2f-711f-4edd-bc8a-303b8de422e5_1600x1300.png";
|
||||
const linkedUrl =
|
||||
"https://substackcdn.com/image/fetch/$s_!wORh!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fb83f9d2f-711f-4edd-bc8a-303b8de422e5_1600x1300.png";
|
||||
const canonicalUrl =
|
||||
"https://substack-post-media.s3.amazonaws.com/public/images/b83f9d2f-711f-4edd-bc8a-303b8de422e5_1600x1300.png";
|
||||
|
||||
const markdown = renderMarkdown({
|
||||
url: "https://example.com/post",
|
||||
metadata: {
|
||||
coverImage: resizedUrl,
|
||||
},
|
||||
content: [
|
||||
{
|
||||
type: "markdown",
|
||||
markdown: `[
|
||||
|
||||
|
||||
|
||||
](${linkedUrl})`,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(markdown).toContain(`coverImage: "${canonicalUrl}"`);
|
||||
expect(markdown).toContain(``);
|
||||
expect(markdown).not.toContain(`[](${canonicalUrl})`);
|
||||
expect(markdown).not.toContain("substackcdn.com/image/fetch");
|
||||
});
|
||||
|
||||
test("renders linked images on a single line when href differs from the image url", () => {
|
||||
const markdown = renderMarkdown({
|
||||
url: "https://example.com/post",
|
||||
content: [
|
||||
{
|
||||
type: "markdown",
|
||||
markdown: `[
|
||||
|
||||
|
||||
|
||||
](https://example.com/source)`,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(markdown).toContain("[](https://example.com/source)");
|
||||
expect(markdown).not.toContain("](https://example.com/source)\n");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
import { afterEach, describe, expect, test } from "bun:test";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import { ensureChromeProfileDir, resolveChromeProfileDir } from "../browser/profile";
|
||||
|
||||
const originalProfile = process.env.BAOYU_CHROME_PROFILE_DIR;
|
||||
|
||||
afterEach(() => {
|
||||
if (originalProfile === undefined) {
|
||||
delete process.env.BAOYU_CHROME_PROFILE_DIR;
|
||||
} else {
|
||||
process.env.BAOYU_CHROME_PROFILE_DIR = originalProfile;
|
||||
}
|
||||
});
|
||||
|
||||
describe("resolveChromeProfileDir", () => {
|
||||
test("uses BAOYU_CHROME_PROFILE_DIR when set", () => {
|
||||
process.env.BAOYU_CHROME_PROFILE_DIR = "/tmp/baoyu-profile";
|
||||
expect(resolveChromeProfileDir()).toBe("/tmp/baoyu-profile");
|
||||
});
|
||||
|
||||
test("falls back to shared baoyu-skills profile path", () => {
|
||||
delete process.env.BAOYU_CHROME_PROFILE_DIR;
|
||||
const resolved = resolveChromeProfileDir();
|
||||
if (process.platform === "darwin") {
|
||||
expect(resolved).toBe(path.join(os.homedir(), "Library", "Application Support", "baoyu-skills", "chrome-profile"));
|
||||
} else if (process.platform === "win32") {
|
||||
expect(resolved.endsWith(path.join("baoyu-skills", "chrome-profile"))).toBe(true);
|
||||
} else {
|
||||
expect(resolved.endsWith(path.join("baoyu-skills", "chrome-profile"))).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("ensureChromeProfileDir", () => {
|
||||
test("creates the profile directory when missing", () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "baoyu-fetch-profile-"));
|
||||
const profileDir = path.join(tempRoot, "nested", "chrome-profile");
|
||||
|
||||
try {
|
||||
expect(fs.existsSync(profileDir)).toBe(false);
|
||||
expect(ensureChromeProfileDir(profileDir)).toBe(profileDir);
|
||||
expect(fs.statSync(profileDir).isDirectory()).toBe(true);
|
||||
} finally {
|
||||
fs.rmSync(tempRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { shouldAutoContinueForceWait } from "../commands/convert";
|
||||
|
||||
describe("shouldAutoContinueForceWait", () => {
|
||||
test("continues when a challenge disappears", () => {
|
||||
expect(
|
||||
shouldAutoContinueForceWait(
|
||||
{
|
||||
url: "https://example.com/challenge",
|
||||
hasGate: true,
|
||||
loginState: "unknown",
|
||||
},
|
||||
{
|
||||
url: "https://example.com/article",
|
||||
hasGate: false,
|
||||
loginState: "unknown",
|
||||
},
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("continues when login state improves from logged out", () => {
|
||||
expect(
|
||||
shouldAutoContinueForceWait(
|
||||
{
|
||||
url: "https://x.com/i/flow/login",
|
||||
hasGate: false,
|
||||
loginState: "logged_out",
|
||||
},
|
||||
{
|
||||
url: "https://x.com/home",
|
||||
hasGate: false,
|
||||
loginState: "logged_in",
|
||||
},
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("does not continue when nothing changed yet", () => {
|
||||
expect(
|
||||
shouldAutoContinueForceWait(
|
||||
{
|
||||
url: "https://x.com/lennysan/status/2036483059407810640",
|
||||
hasGate: false,
|
||||
loginState: "unknown",
|
||||
},
|
||||
{
|
||||
url: "https://x.com/lennysan/status/2036483059407810640",
|
||||
hasGate: false,
|
||||
loginState: "unknown",
|
||||
},
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,342 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { extractArticleDocumentFromPayload } from "../adapters/x/article";
|
||||
|
||||
describe("x article extraction", () => {
|
||||
test("renders markdown entities referenced by atomic blocks", () => {
|
||||
const payload = {
|
||||
data: {
|
||||
tweetResult: {
|
||||
result: {
|
||||
rest_id: "2036762680401223946",
|
||||
legacy: {
|
||||
full_text: "Fallback text",
|
||||
favorite_count: 12,
|
||||
retweet_count: 3,
|
||||
reply_count: 1,
|
||||
created_at: "Wed Mar 25 11:10:38 +0000 2026",
|
||||
},
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: {
|
||||
name: "Eric Zakariasson",
|
||||
screen_name: "ericzakariasson",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
article: {
|
||||
article_results: {
|
||||
result: {
|
||||
title: "Building CLIs for agents",
|
||||
content_state: {
|
||||
blocks: [
|
||||
{
|
||||
type: "unstyled",
|
||||
text: "Make it non-interactive.",
|
||||
data: {},
|
||||
entityRanges: [],
|
||||
inlineStyleRanges: [],
|
||||
},
|
||||
{
|
||||
type: "atomic",
|
||||
text: " ",
|
||||
data: {},
|
||||
entityRanges: [{ key: 0, length: 1, offset: 0 }],
|
||||
inlineStyleRanges: [],
|
||||
},
|
||||
{
|
||||
type: "unstyled",
|
||||
text: "Return data on success.",
|
||||
data: {},
|
||||
entityRanges: [],
|
||||
inlineStyleRanges: [],
|
||||
},
|
||||
],
|
||||
entityMap: [
|
||||
{
|
||||
key: "0",
|
||||
value: {
|
||||
type: "MARKDOWN",
|
||||
mutability: "Mutable",
|
||||
data: {
|
||||
markdown: "```bash\n$ mycli deploy --env production --dry-run\n```",
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const document = extractArticleDocumentFromPayload(
|
||||
payload,
|
||||
"2036762680401223946",
|
||||
"https://x.com/ericzakariasson/status/2036762680401223946",
|
||||
);
|
||||
|
||||
expect(document).not.toBeNull();
|
||||
expect(document?.metadata?.kind).toBe("x/article");
|
||||
|
||||
const content = document?.content[0];
|
||||
expect(content?.type).toBe("markdown");
|
||||
if (!content || content.type !== "markdown") {
|
||||
throw new Error("Expected markdown content");
|
||||
}
|
||||
|
||||
expect(content.markdown).toContain("```bash");
|
||||
expect(content.markdown).toContain("$ mycli deploy --env production --dry-run");
|
||||
expect(content.markdown).toContain("Make it non-interactive.");
|
||||
expect(content.markdown).toContain("Return data on success.");
|
||||
});
|
||||
|
||||
test("renders media, embedded tweets, and cover image from article entities", () => {
|
||||
const embeddedTweetPayload = {
|
||||
data: {
|
||||
tweetResult: {
|
||||
result: {
|
||||
rest_id: "999",
|
||||
legacy: {
|
||||
full_text: "Embedded tweet text",
|
||||
favorite_count: 4,
|
||||
retweet_count: 2,
|
||||
reply_count: 1,
|
||||
created_at: "Wed Mar 25 11:10:38 +0000 2026",
|
||||
extended_entities: {
|
||||
media: [
|
||||
{
|
||||
type: "photo",
|
||||
media_url_https: "https://pbs.twimg.com/media/embedded.jpg",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
core: {
|
||||
name: "Embedded Author",
|
||||
screen_name: "embedded_author",
|
||||
},
|
||||
legacy: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const articlePayload = {
|
||||
data: {
|
||||
tweetResult: {
|
||||
result: {
|
||||
rest_id: "2036670816344064290",
|
||||
legacy: {
|
||||
full_text: "Fallback text",
|
||||
favorite_count: 12,
|
||||
retweet_count: 3,
|
||||
reply_count: 1,
|
||||
created_at: "Wed Mar 25 11:10:38 +0000 2026",
|
||||
},
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: {
|
||||
name: "Eric Zakariasson",
|
||||
screen_name: "ericzakariasson",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
article: {
|
||||
article_results: {
|
||||
result: {
|
||||
title: "Article with media",
|
||||
cover_media: {
|
||||
media_info: {
|
||||
original_img_url: "https://pbs.twimg.com/media/cover?format=jpeg&name=small",
|
||||
},
|
||||
},
|
||||
media_entities: [
|
||||
{
|
||||
media_id: "42",
|
||||
media_info: {
|
||||
original_img_url: "https://pbs.twimg.com/media/body.jpg",
|
||||
},
|
||||
},
|
||||
],
|
||||
content_state: {
|
||||
blocks: [
|
||||
{
|
||||
type: "unstyled",
|
||||
text: "Read more: https://t.co/example",
|
||||
data: {},
|
||||
entityRanges: [{ key: 2, length: 20, offset: 11 }],
|
||||
inlineStyleRanges: [],
|
||||
},
|
||||
{
|
||||
type: "atomic",
|
||||
text: " ",
|
||||
data: {},
|
||||
entityRanges: [{ key: 0, length: 1, offset: 0 }],
|
||||
inlineStyleRanges: [],
|
||||
},
|
||||
{
|
||||
type: "atomic",
|
||||
text: " ",
|
||||
data: {},
|
||||
entityRanges: [{ key: 1, length: 1, offset: 0 }],
|
||||
inlineStyleRanges: [],
|
||||
},
|
||||
],
|
||||
entityMap: [
|
||||
{
|
||||
key: "0",
|
||||
value: {
|
||||
type: "MEDIA",
|
||||
mutability: "Immutable",
|
||||
data: {
|
||||
mediaItems: [{ mediaId: "42" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
key: "1",
|
||||
value: {
|
||||
type: "TWEET",
|
||||
mutability: "Immutable",
|
||||
data: {
|
||||
tweetId: "999",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
key: "2",
|
||||
value: {
|
||||
type: "LINK",
|
||||
mutability: "Mutable",
|
||||
data: {
|
||||
url: "https://example.com/report",
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const document = extractArticleDocumentFromPayload(
|
||||
articlePayload,
|
||||
"2036670816344064290",
|
||||
"https://x.com/ericzakariasson/status/2036670816344064290",
|
||||
[articlePayload, embeddedTweetPayload],
|
||||
);
|
||||
|
||||
expect(document).not.toBeNull();
|
||||
expect(document?.metadata?.coverImage).toBe(
|
||||
"https://pbs.twimg.com/media/cover?format=jpg&name=4096x4096",
|
||||
);
|
||||
|
||||
const content = document?.content[0];
|
||||
expect(content?.type).toBe("markdown");
|
||||
if (!content || content.type !== "markdown") {
|
||||
throw new Error("Expected markdown content");
|
||||
}
|
||||
|
||||
expect(content.markdown).toContain("https://example.com/report");
|
||||
expect(content.markdown).toContain("");
|
||||
expect(content.markdown).toContain("> Embedded Author (@embedded_author)");
|
||||
expect(content.markdown).toContain("> Embedded tweet text");
|
||||
expect(content.markdown).toContain(
|
||||
"> ",
|
||||
);
|
||||
});
|
||||
|
||||
test("prefers expanded link entity urls in article blocks", () => {
|
||||
const payload = {
|
||||
data: {
|
||||
tweetResult: {
|
||||
result: {
|
||||
rest_id: "2036670816344064290",
|
||||
legacy: {
|
||||
full_text: "Fallback text",
|
||||
favorite_count: 12,
|
||||
retweet_count: 3,
|
||||
reply_count: 1,
|
||||
created_at: "Wed Mar 25 11:10:38 +0000 2026",
|
||||
},
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: {
|
||||
name: "Eric Zakariasson",
|
||||
screen_name: "ericzakariasson",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
article: {
|
||||
article_results: {
|
||||
result: {
|
||||
title: "Article with expanded links",
|
||||
content_state: {
|
||||
blocks: [
|
||||
{
|
||||
type: "unstyled",
|
||||
text: "Read more: https://t.co/example",
|
||||
data: {},
|
||||
entityRanges: [{ key: 0, length: 20, offset: 11 }],
|
||||
inlineStyleRanges: [],
|
||||
},
|
||||
],
|
||||
entityMap: [
|
||||
{
|
||||
key: "0",
|
||||
value: {
|
||||
type: "LINK",
|
||||
mutability: "Mutable",
|
||||
data: {
|
||||
expanded_url: "https://example.com/report",
|
||||
url: "https://t.co/example",
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const document = extractArticleDocumentFromPayload(
|
||||
payload,
|
||||
"2036670816344064290",
|
||||
"https://x.com/ericzakariasson/status/2036670816344064290",
|
||||
);
|
||||
|
||||
expect(document).not.toBeNull();
|
||||
|
||||
const content = document?.content[0];
|
||||
expect(content?.type).toBe("markdown");
|
||||
if (!content || content.type !== "markdown") {
|
||||
throw new Error("Expected markdown content");
|
||||
}
|
||||
|
||||
expect(content.markdown).toContain("https://example.com/report");
|
||||
expect(content.markdown).not.toContain("https://t.co/example");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,187 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { extractSingleTweetDocumentFromPayload } from "../adapters/x/single";
|
||||
|
||||
describe("x single tweet extraction", () => {
|
||||
test("replaces t.co links in note tweets with expanded urls", () => {
|
||||
const payload = {
|
||||
data: {
|
||||
tweetResult: {
|
||||
result: {
|
||||
rest_id: "2036483061635039711",
|
||||
legacy: {
|
||||
full_text:
|
||||
"First, some context:\n\n1. This analysis is based on data from @trueupio, one of my favorite collaborators and sources of data. They track job openings at tech companies and top startups around the world (over 9,000 companies) and make it easy to browse open gigs. Their data looks",
|
||||
favorite_count: 43,
|
||||
retweet_count: 1,
|
||||
reply_count: 1,
|
||||
created_at: "Tue Mar 24 16:39:32 +0000 2026",
|
||||
entities: {
|
||||
hashtags: [],
|
||||
symbols: [],
|
||||
timestamps: [],
|
||||
urls: [],
|
||||
user_mentions: [
|
||||
{
|
||||
id_str: "1407256023547613193",
|
||||
indices: [61, 70],
|
||||
name: "TrueUp",
|
||||
screen_name: "trueupio",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
note_tweet: {
|
||||
note_tweet_results: {
|
||||
result: {
|
||||
text:
|
||||
"First, some context:\n\n1. This analysis is based on data from @trueupio, one of my favorite collaborators and sources of data. They track job openings at tech companies and top startups around the world (over 9,000 companies) and make it easy to browse open gigs. Their data looks at roles at tech companies—the most sought-after and lucrative jobs. (It doesn’t include roles at non-tech companies and consulting agencies.) Browse open roles here: https://t.co/x7ff2NjpP1\n\n2. Keep reading for highlights, or jump straight to the full report: https://t.co/AbqPp2TEde",
|
||||
entity_set: {
|
||||
hashtags: [],
|
||||
symbols: [],
|
||||
urls: [
|
||||
{
|
||||
display_url: "trueup.io/jobs",
|
||||
expanded_url: "https://trueup.io/jobs",
|
||||
indices: [447, 470],
|
||||
url: "https://t.co/x7ff2NjpP1",
|
||||
},
|
||||
{
|
||||
display_url: "lennysnewsletter.com/i/191595250/if…",
|
||||
expanded_url:
|
||||
"https://www.lennysnewsletter.com/i/191595250/if-youre-having-trouble-finding-a-job",
|
||||
indices: [541, 564],
|
||||
url: "https://t.co/AbqPp2TEde",
|
||||
},
|
||||
],
|
||||
user_mentions: [
|
||||
{
|
||||
id_str: "1407256023547613193",
|
||||
indices: [61, 70],
|
||||
name: "TrueUp",
|
||||
screen_name: "trueupio",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: {
|
||||
name: "Lenny Rachitsky",
|
||||
screen_name: "lennysan",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const document = extractSingleTweetDocumentFromPayload(
|
||||
payload,
|
||||
"2036483061635039711",
|
||||
"https://x.com/lennysan/status/2036483061635039711",
|
||||
);
|
||||
|
||||
expect(document).not.toBeNull();
|
||||
|
||||
const paragraphBlock = document?.content.find((block) => block.type === "paragraph");
|
||||
expect(paragraphBlock).toEqual({
|
||||
type: "paragraph",
|
||||
text:
|
||||
"First, some context:\n\n1. This analysis is based on data from @trueupio, one of my favorite collaborators and sources of data. They track job openings at tech companies and top startups around the world (over 9,000 companies) and make it easy to browse open gigs. Their data looks at roles at tech companies—the most sought-after and lucrative jobs. (It doesn’t include roles at non-tech companies and consulting agencies.) Browse open roles here: https://trueup.io/jobs\n\n2. Keep reading for highlights, or jump straight to the full report: https://www.lennysnewsletter.com/i/191595250/if-youre-having-trouble-finding-a-job",
|
||||
});
|
||||
});
|
||||
|
||||
test("upgrades image urls to high resolution for tweet and quoted tweet media", () => {
|
||||
const payload = {
|
||||
data: {
|
||||
tweetResult: {
|
||||
result: {
|
||||
rest_id: "2036762680401223946",
|
||||
legacy: {
|
||||
full_text: "Main tweet text https://t.co/media",
|
||||
favorite_count: 12,
|
||||
retweet_count: 3,
|
||||
reply_count: 1,
|
||||
created_at: "Wed Mar 25 11:10:38 +0000 2026",
|
||||
extended_entities: {
|
||||
media: [
|
||||
{
|
||||
type: "photo",
|
||||
media_url_https: "https://pbs.twimg.com/media/main-image.png",
|
||||
url: "https://t.co/media",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: {
|
||||
name: "Eric Zakariasson",
|
||||
screen_name: "ericzakariasson",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
quoted_status_result: {
|
||||
result: {
|
||||
rest_id: "999",
|
||||
legacy: {
|
||||
full_text: "Quoted tweet text",
|
||||
favorite_count: 4,
|
||||
retweet_count: 2,
|
||||
reply_count: 1,
|
||||
created_at: "Wed Mar 25 10:10:38 +0000 2026",
|
||||
extended_entities: {
|
||||
media: [
|
||||
{
|
||||
type: "photo",
|
||||
media_url_https: "https://pbs.twimg.com/media/quoted?format=jpeg&name=small",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: {
|
||||
name: "Quoted Author",
|
||||
screen_name: "quoted_author",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const document = extractSingleTweetDocumentFromPayload(
|
||||
payload,
|
||||
"2036762680401223946",
|
||||
"https://x.com/ericzakariasson/status/2036762680401223946",
|
||||
);
|
||||
|
||||
expect(document).not.toBeNull();
|
||||
|
||||
const imageBlock = document?.content.find((block) => block.type === "image");
|
||||
expect(imageBlock).toEqual({
|
||||
type: "image",
|
||||
url: "https://pbs.twimg.com/media/main-image?format=png&name=4096x4096",
|
||||
});
|
||||
|
||||
const quoteBlock = document?.content.find((block) => block.type === "quote");
|
||||
expect(quoteBlock).toEqual({
|
||||
type: "quote",
|
||||
text:
|
||||
"Quoted Author (@quoted_author)\n\nQuoted tweet text\n\nphoto: https://pbs.twimg.com/media/quoted?format=jpg&name=4096x4096",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,305 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { extractThreadDocumentFromPayloads, extractThreadTweetsFromPayloads } from "../adapters/x/thread";
|
||||
|
||||
function buildTweet(options: {
|
||||
id: string;
|
||||
text: string;
|
||||
createdAt: string;
|
||||
userId?: string;
|
||||
screenName?: string;
|
||||
name?: string;
|
||||
conversationId?: string;
|
||||
inReplyToStatusId?: string;
|
||||
inReplyToUserId?: string;
|
||||
quotedTweet?: unknown;
|
||||
}) {
|
||||
const userId = options.userId ?? "3178231";
|
||||
const screenName = options.screenName ?? "dotey";
|
||||
const name = options.name ?? "宝玉";
|
||||
|
||||
return {
|
||||
__typename: "Tweet",
|
||||
rest_id: options.id,
|
||||
legacy: {
|
||||
id_str: options.id,
|
||||
full_text: options.text,
|
||||
favorite_count: 0,
|
||||
retweet_count: 0,
|
||||
reply_count: 0,
|
||||
created_at: options.createdAt,
|
||||
user_id_str: userId,
|
||||
conversation_id_str: options.conversationId ?? options.id,
|
||||
in_reply_to_status_id_str: options.inReplyToStatusId,
|
||||
in_reply_to_user_id_str: options.inReplyToUserId,
|
||||
},
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
core: {
|
||||
name,
|
||||
screen_name: screenName,
|
||||
},
|
||||
legacy: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
quoted_status_result: options.quotedTweet
|
||||
? {
|
||||
result: options.quotedTweet,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function tweetEntry(tweet: unknown) {
|
||||
return {
|
||||
content: {
|
||||
itemContent: {
|
||||
tweet_results: {
|
||||
result: tweet,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function moduleTweetItem(tweet: unknown) {
|
||||
return {
|
||||
item: {
|
||||
itemContent: {
|
||||
tweet_results: {
|
||||
result: tweet,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("x thread extraction", () => {
|
||||
test("keeps only the continuous same-author reply chain", () => {
|
||||
const rootId = "1996285439867556304";
|
||||
const reply1Id = "1996285442275340783";
|
||||
const reply2Id = "1996285444582146559";
|
||||
const quotedId = "1993729800922341810";
|
||||
const quotedInsideThreadId = "1993729800922341811";
|
||||
const otherAuthorReplyId = "2000000000000000001";
|
||||
const sameAuthorAfterOtherId = "2000000000000000002";
|
||||
|
||||
const root = buildTweet({
|
||||
id: rootId,
|
||||
text: "A thread for my nana banana pro prompts 🧵",
|
||||
createdAt: "Wed Dec 03 18:28:32 +0000 2025",
|
||||
conversationId: rootId,
|
||||
});
|
||||
const reply1 = buildTweet({
|
||||
id: reply1Id,
|
||||
text: "Prompt 1",
|
||||
createdAt: "Wed Dec 03 18:28:33 +0000 2025",
|
||||
conversationId: rootId,
|
||||
inReplyToStatusId: rootId,
|
||||
inReplyToUserId: "3178231",
|
||||
quotedTweet: buildTweet({
|
||||
id: quotedInsideThreadId,
|
||||
text: "Quoted inside the thread body",
|
||||
createdAt: "Tue Nov 25 18:28:35 +0000 2025",
|
||||
screenName: "quoted_author",
|
||||
name: "Quoted Author",
|
||||
}),
|
||||
});
|
||||
const reply2 = buildTweet({
|
||||
id: reply2Id,
|
||||
text: "Prompt 2",
|
||||
createdAt: "Wed Dec 03 18:28:34 +0000 2025",
|
||||
conversationId: rootId,
|
||||
inReplyToStatusId: reply1Id,
|
||||
inReplyToUserId: "3178231",
|
||||
});
|
||||
const quotedSameAuthor = buildTweet({
|
||||
id: quotedId,
|
||||
text: "Quoted standalone tweet",
|
||||
createdAt: "Tue Nov 25 18:28:34 +0000 2025",
|
||||
});
|
||||
const otherAuthorReply = buildTweet({
|
||||
id: otherAuthorReplyId,
|
||||
text: "Another author joined the conversation",
|
||||
createdAt: "Wed Dec 03 18:28:35 +0000 2025",
|
||||
userId: "42",
|
||||
screenName: "someone_else",
|
||||
name: "Someone Else",
|
||||
conversationId: rootId,
|
||||
inReplyToStatusId: reply2Id,
|
||||
inReplyToUserId: "3178231",
|
||||
});
|
||||
const sameAuthorAfterOther = buildTweet({
|
||||
id: sameAuthorAfterOtherId,
|
||||
text: "This should not be part of the continuous author chain",
|
||||
createdAt: "Wed Dec 03 18:28:36 +0000 2025",
|
||||
conversationId: rootId,
|
||||
inReplyToStatusId: otherAuthorReplyId,
|
||||
inReplyToUserId: "42",
|
||||
});
|
||||
|
||||
const payloads = [
|
||||
{
|
||||
data: {
|
||||
threaded_conversation_with_injections_v2: {
|
||||
instructions: [
|
||||
{
|
||||
type: "TimelineAddEntries",
|
||||
entries: [
|
||||
tweetEntry(root),
|
||||
tweetEntry(reply1),
|
||||
tweetEntry(quotedSameAuthor),
|
||||
{
|
||||
content: {
|
||||
items: [moduleTweetItem(reply2)],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "TimelineAddToModule",
|
||||
moduleItems: [moduleTweetItem(otherAuthorReply), moduleTweetItem(sameAuthorAfterOther)],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const tweets = extractThreadTweetsFromPayloads(
|
||||
payloads,
|
||||
rootId,
|
||||
"https://x.com/dotey/status/1996285439867556304",
|
||||
);
|
||||
|
||||
expect(tweets.map((tweet) => tweet.id)).toEqual([rootId, reply1Id, reply2Id]);
|
||||
|
||||
const document = extractThreadDocumentFromPayloads(
|
||||
payloads,
|
||||
rootId,
|
||||
"https://x.com/dotey/status/1996285439867556304",
|
||||
);
|
||||
|
||||
expect(document).not.toBeNull();
|
||||
expect(document?.metadata?.tweetCount).toBe(3);
|
||||
expect(document?.metadata?.lastTweetId).toBe(reply2Id);
|
||||
|
||||
const content = document?.content[0];
|
||||
expect(content?.type).toBe("markdown");
|
||||
if (!content || content.type !== "markdown") {
|
||||
throw new Error("Expected markdown content");
|
||||
}
|
||||
|
||||
expect(content.markdown).toContain("Prompt 1");
|
||||
expect(content.markdown).toContain("Prompt 2");
|
||||
expect(content.markdown).toContain("Quoted inside the thread body");
|
||||
expect(content.markdown).not.toContain("Quoted standalone tweet");
|
||||
expect(content.markdown).not.toContain("This should not be part of the continuous author chain");
|
||||
});
|
||||
|
||||
test("returns null when there is no same-author reply chain", () => {
|
||||
const rootId = "1996285439867556304";
|
||||
const root = buildTweet({
|
||||
id: rootId,
|
||||
text: "Root tweet",
|
||||
createdAt: "Wed Dec 03 18:28:32 +0000 2025",
|
||||
conversationId: rootId,
|
||||
});
|
||||
const quotedSameAuthor = buildTweet({
|
||||
id: "1993729800922341810",
|
||||
text: "Quoted standalone tweet",
|
||||
createdAt: "Tue Nov 25 18:28:34 +0000 2025",
|
||||
});
|
||||
|
||||
const payloads = [
|
||||
{
|
||||
data: {
|
||||
threaded_conversation_with_injections_v2: {
|
||||
instructions: [
|
||||
{
|
||||
type: "TimelineAddEntries",
|
||||
entries: [tweetEntry(root), tweetEntry(quotedSameAuthor)],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
expect(
|
||||
extractThreadDocumentFromPayloads(
|
||||
payloads,
|
||||
rootId,
|
||||
"https://x.com/dotey/status/1996285439867556304",
|
||||
),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
test("restores ancestors when the requested tweet is in the middle of a thread", () => {
|
||||
const rootId = "1996285439867556304";
|
||||
const reply1Id = "1996285442275340783";
|
||||
const reply2Id = "1996285444582146559";
|
||||
|
||||
const root = buildTweet({
|
||||
id: rootId,
|
||||
text: "Root tweet",
|
||||
createdAt: "Wed Dec 03 18:28:32 +0000 2025",
|
||||
conversationId: rootId,
|
||||
});
|
||||
const reply1 = buildTweet({
|
||||
id: reply1Id,
|
||||
text: "Middle tweet",
|
||||
createdAt: "Wed Dec 03 18:28:33 +0000 2025",
|
||||
conversationId: rootId,
|
||||
inReplyToStatusId: rootId,
|
||||
inReplyToUserId: "3178231",
|
||||
});
|
||||
const reply2 = buildTweet({
|
||||
id: reply2Id,
|
||||
text: "Last tweet",
|
||||
createdAt: "Wed Dec 03 18:28:34 +0000 2025",
|
||||
conversationId: rootId,
|
||||
inReplyToStatusId: reply1Id,
|
||||
inReplyToUserId: "3178231",
|
||||
});
|
||||
|
||||
const payloads = [
|
||||
{
|
||||
data: {
|
||||
threaded_conversation_with_injections_v2: {
|
||||
instructions: [
|
||||
{
|
||||
type: "TimelineAddEntries",
|
||||
entries: [
|
||||
tweetEntry(root),
|
||||
tweetEntry(reply1),
|
||||
tweetEntry(reply2),
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const tweets = extractThreadTweetsFromPayloads(
|
||||
payloads,
|
||||
reply1Id,
|
||||
"https://x.com/dotey/status/1996285442275340783",
|
||||
);
|
||||
|
||||
expect(tweets.map((tweet) => tweet.id)).toEqual([rootId, reply1Id, reply2Id]);
|
||||
|
||||
const document = extractThreadDocumentFromPayloads(
|
||||
payloads,
|
||||
reply1Id,
|
||||
"https://x.com/dotey/status/1996285442275340783",
|
||||
);
|
||||
|
||||
expect(document).not.toBeNull();
|
||||
expect(document?.metadata?.tweetId).toBe(rootId);
|
||||
expect(document?.metadata?.lastTweetId).toBe(reply2Id);
|
||||
expect(document?.metadata?.tweetCount).toBe(3);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,97 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
buildYouTubeThumbnailCandidates,
|
||||
formatTimestampRange,
|
||||
parseYouTubeDescriptionChapters,
|
||||
parseYouTubeVideoId,
|
||||
renderYouTubeTranscriptMarkdown,
|
||||
} from "../adapters/youtube/utils";
|
||||
|
||||
describe("parseYouTubeVideoId", () => {
|
||||
test("parses watch URLs", () => {
|
||||
expect(parseYouTubeVideoId(new URL("https://www.youtube.com/watch?v=abc123"))).toBe("abc123");
|
||||
});
|
||||
|
||||
test("parses youtu.be URLs", () => {
|
||||
expect(parseYouTubeVideoId(new URL("https://youtu.be/abc123"))).toBe("abc123");
|
||||
});
|
||||
|
||||
test("parses shorts URLs", () => {
|
||||
expect(parseYouTubeVideoId(new URL("https://www.youtube.com/shorts/abc123"))).toBe("abc123");
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseYouTubeDescriptionChapters", () => {
|
||||
test("extracts chapter timestamps from description lines", () => {
|
||||
expect(
|
||||
parseYouTubeDescriptionChapters(`0:00 Intro
|
||||
2:15 What is a product engineer?
|
||||
10:05 Career paths`),
|
||||
).toEqual([
|
||||
{ title: "Intro", time: 0 },
|
||||
{ title: "What is a product engineer?", time: 135 },
|
||||
{ title: "Career paths", time: 605 },
|
||||
]);
|
||||
});
|
||||
|
||||
test("ignores isolated timestamps that do not look like chapters", () => {
|
||||
expect(parseYouTubeDescriptionChapters("Published on 2026-03-26\nSee you at 1:23")).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderYouTubeTranscriptMarkdown", () => {
|
||||
test("renders description before chapters and keeps every segment on its own line", () => {
|
||||
const markdown = renderYouTubeTranscriptMarkdown({
|
||||
description: "Line one\nLine two",
|
||||
chapters: [
|
||||
{ title: "Intro", time: 0 },
|
||||
{ title: "Deep Dive", time: 4 },
|
||||
],
|
||||
segments: [
|
||||
{ start: 0, end: 2, text: "Hello everyone." },
|
||||
{ start: 2, end: 4, text: "Welcome back." },
|
||||
{ start: 4, end: 7, text: "Now the details." },
|
||||
],
|
||||
});
|
||||
|
||||
expect(markdown).toContain("## Description");
|
||||
expect(markdown).toContain("Line one \nLine two");
|
||||
expect(markdown).toContain("## Chapters");
|
||||
expect(markdown).toContain("### Intro [0:00 -> 0:04]");
|
||||
expect(markdown).toContain("[0:00 -> 0:02] Hello everyone.");
|
||||
expect(markdown).toContain("[0:02 -> 0:04] Welcome back.");
|
||||
expect(markdown).toContain("### Deep Dive [0:04 -> 0:07]");
|
||||
expect(markdown).toContain("[0:04 -> 0:07] Now the details.");
|
||||
});
|
||||
|
||||
test("falls back to a transcript section when chapters are unavailable", () => {
|
||||
const markdown = renderYouTubeTranscriptMarkdown({
|
||||
segments: [{ start: 65, end: 70, text: "Single line." }],
|
||||
chapters: [],
|
||||
});
|
||||
|
||||
expect(markdown).toContain("## Transcript");
|
||||
expect(markdown).toContain("[1:05 -> 1:10] Single line.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("thumbnail helpers", () => {
|
||||
test("prefers max resolution thumbnail candidates before listed fallbacks", () => {
|
||||
expect(
|
||||
buildYouTubeThumbnailCandidates("abc123", [
|
||||
"https://i.ytimg.com/vi/abc123/hqdefault.jpg",
|
||||
"https://i.ytimg.com/vi/abc123/mqdefault.jpg?foo=bar",
|
||||
]),
|
||||
).toEqual([
|
||||
"https://i.ytimg.com/vi/abc123/maxresdefault.jpg",
|
||||
"https://i.ytimg.com/vi/abc123/sddefault.jpg",
|
||||
"https://i.ytimg.com/vi/abc123/hqdefault.jpg",
|
||||
"https://i.ytimg.com/vi/abc123/mqdefault.jpg",
|
||||
"https://i.ytimg.com/vi/abc123/default.jpg",
|
||||
]);
|
||||
});
|
||||
|
||||
test("renders timestamp ranges with start and end values", () => {
|
||||
expect(formatTimestampRange(3661, 3675)).toBe("[1:01:01 -> 1:01:15]");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user