(`
+ (async () => {
+ const response = await fetch(${JSON.stringify(captionInfo.captionUrl)});
+ const xml = await response.text();
+ if (!xml) {
+ return { error: 'Caption XML is empty' };
+ }
+
+ function getAttr(tag, name) {
+ const needle = name + '="';
+ const index = tag.indexOf(needle);
+ if (index === -1) return '';
+ const valueStart = index + needle.length;
+ const valueEnd = tag.indexOf('"', valueStart);
+ if (valueEnd === -1) return '';
+ return tag.substring(valueStart, valueEnd);
+ }
+
+ function decodeEntities(value) {
+ return value
+ .replaceAll('&', '&')
+ .replaceAll('<', '<')
+ .replaceAll('>', '>')
+ .replaceAll('"', '"')
+ .replaceAll(''', "'");
+ }
+
+ const marker = xml.includes('(`
+ (() => {
+ const data = window.ytInitialData;
+ const markers = data?.playerOverlays?.playerOverlayRenderer
+ ?.decoratedPlayerBarRenderer?.decoratedPlayerBarRenderer
+ ?.playerBar?.multiMarkersPlayerBarRenderer?.markersMap || [];
+ const results = [];
+
+ for (const marker of markers) {
+ const chapters = marker?.value?.chapters;
+ if (!Array.isArray(chapters)) continue;
+ for (const chapter of chapters) {
+ const renderer = chapter?.chapterRenderer;
+ const title = renderer?.title?.simpleText;
+ const timeRangeStartMillis = renderer?.timeRangeStartMillis;
+ if (title && typeof timeRangeStartMillis === 'number') {
+ results.push({ title, time: Math.floor(timeRangeStartMillis / 1000) });
+ }
+ }
+ }
+
+ return results;
+ })()
+ `).catch(() => []);
+
+ const descriptionChapters = parseYouTubeDescriptionChapters(captionInfo.description);
+ const chapters = extractedChapters.length > 0 ? extractedChapters : descriptionChapters;
+ const markdown = renderYouTubeTranscriptMarkdown({
+ description: captionInfo.description,
+ segments,
+ chapters,
+ });
+
+ if (!markdown) {
+ return null;
+ }
+
+ const pageUrl = await context.browser.getURL();
+ const coverImage = await resolveBestCoverImage(videoId, captionInfo.coverImages);
+ const summary = buildSummary(captionInfo.description, segments);
+
+ return {
+ url: pageUrl,
+ canonicalUrl: pageUrl,
+ title: captionInfo.title || "YouTube Transcript",
+ author: captionInfo.author,
+ publishedAt: captionInfo.publishedAt,
+ siteName: "YouTube",
+ summary,
+ adapter: "youtube",
+ metadata: {
+ kind: "youtube/transcript",
+ videoId,
+ authorUrl: normalizeUrl(captionInfo.authorUrl),
+ channelId: captionInfo.channelId,
+ coverImage,
+ description: captionInfo.description,
+ durationSeconds: captionInfo.durationSeconds,
+ language: captionInfo.language,
+ captionKind: captionInfo.kind,
+ availableLanguages: captionInfo.available,
+ viewCount: captionInfo.viewCount,
+ keywords: captionInfo.keywords,
+ category: captionInfo.category,
+ isLiveContent: captionInfo.isLiveContent,
+ chapterCount: chapters.length,
+ },
+ content: [{ type: "markdown", markdown }],
+ };
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/utils.ts b/skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/utils.ts
new file mode 100644
index 0000000..3770df9
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/adapters/youtube/utils.ts
@@ -0,0 +1,253 @@
+export interface YouTubeTranscriptSegment {
+ start: number;
+ end: number;
+ text: string;
+}
+
+export interface YouTubeChapter {
+ title: string;
+ time: number;
+}
+
+interface RenderYouTubeTranscriptMarkdownInput {
+ description?: string;
+ segments: YouTubeTranscriptSegment[];
+ chapters: YouTubeChapter[];
+}
+
+const DESCRIPTION_CHAPTER_RE = /^((?:\d{1,2}:)?\d{1,2}:\d{2})(?:\s+[-|:]\s+|\s+)(.+)$/;
+const YOUTUBE_THUMBNAIL_VARIANTS = [
+ "maxresdefault.jpg",
+ "sddefault.jpg",
+ "hqdefault.jpg",
+ "mqdefault.jpg",
+ "default.jpg",
+];
+
+export function isYouTubeHost(hostname: string): boolean {
+ return [
+ "youtube.com",
+ "www.youtube.com",
+ "m.youtube.com",
+ "youtu.be",
+ ].includes(hostname);
+}
+
+export function parseYouTubeVideoId(url: URL): string | null {
+ if (url.hostname === "youtu.be") {
+ return url.pathname.split("/").filter(Boolean)[0] ?? null;
+ }
+
+ if (url.pathname === "/watch") {
+ return url.searchParams.get("v");
+ }
+
+ const shortsMatch = url.pathname.match(/^\/shorts\/([^/?#]+)/);
+ if (shortsMatch) {
+ return shortsMatch[1];
+ }
+
+ const liveMatch = url.pathname.match(/^\/live\/([^/?#]+)/);
+ if (liveMatch) {
+ return liveMatch[1];
+ }
+
+ return null;
+}
+
+function parseTimestampValue(raw: string): number | null {
+ const parts = raw
+ .split(":")
+ .map((part) => Number.parseInt(part, 10))
+ .filter((part) => Number.isFinite(part));
+
+ if (parts.length < 2 || parts.length > 3) {
+ return null;
+ }
+
+ if (parts.some((part) => part < 0)) {
+ return null;
+ }
+
+ if (parts.length === 2) {
+ const [minutes, seconds] = parts;
+ return minutes * 60 + seconds;
+ }
+
+ const [hours, minutes, seconds] = parts;
+ return hours * 3600 + minutes * 60 + seconds;
+}
+
+export function formatTimestamp(totalSeconds: number): string {
+ const rounded = Math.max(0, Math.floor(totalSeconds));
+ const hours = Math.floor(rounded / 3600);
+ const minutes = Math.floor((rounded % 3600) / 60);
+ const seconds = rounded % 60;
+
+ if (hours > 0) {
+ return `${hours}:${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}`;
+ }
+ return `${minutes}:${String(seconds).padStart(2, "0")}`;
+}
+
+export function formatTimestampRange(start: number, end: number): string {
+ const safeStart = Math.max(0, start);
+ const safeEnd = Math.max(safeStart, end);
+ return `[${formatTimestamp(safeStart)} -> ${formatTimestamp(safeEnd)}]`;
+}
+
+export function normalizeYouTubeChapters(chapters: YouTubeChapter[]): YouTubeChapter[] {
+ const seenTimes = new Set();
+
+ return chapters
+ .map((chapter) => ({
+ title: chapter.title.trim(),
+ time: Math.max(0, Math.floor(chapter.time)),
+ }))
+ .filter((chapter) => chapter.title)
+ .sort((left, right) => left.time - right.time)
+ .filter((chapter) => {
+ if (seenTimes.has(chapter.time)) {
+ return false;
+ }
+ seenTimes.add(chapter.time);
+ return true;
+ });
+}
+
+export function parseYouTubeDescriptionChapters(description?: string | null): YouTubeChapter[] {
+ if (!description) {
+ return [];
+ }
+
+ const chapters: YouTubeChapter[] = [];
+ const seen = new Set();
+
+ for (const rawLine of description.replace(/\r\n/g, "\n").split("\n")) {
+ const line = rawLine.trim();
+ if (!line) {
+ continue;
+ }
+
+ const match = line.match(DESCRIPTION_CHAPTER_RE);
+ if (!match) {
+ continue;
+ }
+
+ const time = parseTimestampValue(match[1]);
+ const title = match[2]?.trim();
+ if (time === null || !title) {
+ continue;
+ }
+
+ const key = `${time}:${title.toLowerCase()}`;
+ if (seen.has(key)) {
+ continue;
+ }
+
+ seen.add(key);
+ chapters.push({ title, time });
+ }
+
+ const normalized = normalizeYouTubeChapters(chapters);
+ if (normalized.length >= 2) {
+ return normalized;
+ }
+
+ if (normalized.length === 1 && normalized[0]?.time === 0) {
+ return normalized;
+ }
+
+ return [];
+}
+
+function renderDescriptionMarkdown(description: string): string {
+ return description
+ .replace(/\r\n/g, "\n")
+ .trim()
+ .split(/\n{2,}/)
+ .map((block) => block.split("\n").map((line) => line.trimEnd()).join(" \n"))
+ .join("\n\n")
+ .trim();
+}
+
+function renderSegmentLine(segment: YouTubeTranscriptSegment): string {
+ return `${formatTimestampRange(segment.start, segment.end)} ${segment.text}`;
+}
+
+export function renderYouTubeTranscriptMarkdown({
+ description,
+ segments,
+ chapters,
+}: RenderYouTubeTranscriptMarkdownInput): string {
+ if (segments.length === 0) {
+ return "";
+ }
+
+ const parts: string[] = [];
+ const normalizedDescription = description?.trim();
+ const transcriptEnd = segments.reduce((maxEnd, segment) => Math.max(maxEnd, segment.end, segment.start), 0);
+ const normalizedChapters = normalizeYouTubeChapters(chapters).filter(
+ (chapter) => transcriptEnd <= 0 || chapter.time < transcriptEnd,
+ );
+
+ if (normalizedDescription) {
+ parts.push("## Description");
+ parts.push(renderDescriptionMarkdown(normalizedDescription));
+ }
+
+ if (normalizedChapters.length > 0) {
+ parts.push("## Chapters");
+
+ for (let index = 0; index < normalizedChapters.length; index += 1) {
+ const chapter = normalizedChapters[index];
+ const nextChapter = normalizedChapters[index + 1];
+ const chapterEnd = nextChapter ? nextChapter.time : transcriptEnd;
+ const chapterSegments = segments.filter(
+ (segment) => segment.start >= chapter.time && segment.start < chapterEnd,
+ );
+
+ parts.push(`### ${chapter.title} ${formatTimestampRange(chapter.time, chapterEnd)}`);
+
+ if (chapterSegments.length > 0) {
+ parts.push(chapterSegments.map(renderSegmentLine).join("\n"));
+ }
+ }
+ } else {
+ parts.push("## Transcript");
+ parts.push(segments.map(renderSegmentLine).join("\n"));
+ }
+
+ return parts.filter(Boolean).join("\n\n").trim();
+}
+
+function normalizeThumbnailKey(url: string): string {
+ try {
+ const parsed = new URL(url);
+ return `${parsed.origin}${parsed.pathname}`;
+ } catch {
+ return url;
+ }
+}
+
+export function buildYouTubeThumbnailCandidates(videoId: string, listedUrls: string[]): string[] {
+ const candidates = [
+ ...YOUTUBE_THUMBNAIL_VARIANTS.map((variant) => `https://i.ytimg.com/vi/${videoId}/${variant}`),
+ ...listedUrls,
+ ];
+
+ const seen = new Set();
+ return candidates.filter((candidate) => {
+ if (!candidate) {
+ return false;
+ }
+
+ const key = normalizeThumbnailKey(candidate);
+ if (seen.has(key)) {
+ return false;
+ }
+
+ seen.add(key);
+ return true;
+ });
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/cdp-client.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/cdp-client.ts
new file mode 100644
index 0000000..8c7eaef
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/cdp-client.ts
@@ -0,0 +1,258 @@
+import { EventEmitter } from "node:events";
+import WebSocket from "ws";
+
+type JsonObject = Record;
+
+interface CdpPendingCommand {
+ resolve(value: unknown): void;
+ reject(error: unknown): void;
+ method: string;
+}
+
+interface CdpErrorShape {
+ message?: string;
+}
+
+interface CdpCommandResult {
+ result?: T;
+ error?: CdpErrorShape;
+}
+
+interface CreatePageSessionOptions {
+ initialUrl?: string;
+ visible?: boolean;
+}
+
+export class TargetSession extends EventEmitter {
+ constructor(
+ private readonly client: CdpClient,
+ public readonly targetId: string,
+ public readonly sessionId: string,
+ ) {
+ super();
+ }
+
+ async send(method: string, params: JsonObject = {}): Promise {
+ return this.client.sendSessionCommand(this.sessionId, method, params);
+ }
+
+ handleEvent(method: string, params: JsonObject): void {
+ this.emit(method, params);
+ this.emit("event", { method, params });
+ }
+
+ async waitForEvent(
+ method: string,
+ predicate?: (params: T) => boolean,
+ timeoutMs = 30_000,
+ ): Promise {
+ return new Promise((resolve, reject) => {
+ const timeout = setTimeout(() => {
+ this.off(method, listener);
+ reject(new Error(`Timed out waiting for ${method}`));
+ }, timeoutMs);
+
+ const listener = (params: T): void => {
+ if (predicate && !predicate(params)) {
+ return;
+ }
+ clearTimeout(timeout);
+ this.off(method, listener);
+ resolve(params);
+ };
+
+ this.on(method, listener);
+ });
+ }
+}
+
+export class CdpClient {
+ private readonly ws: WebSocket;
+ private readonly pending = new Map();
+ private readonly sessions = new Map();
+ private nextId = 1;
+
+ private constructor(ws: WebSocket) {
+ this.ws = ws;
+ this.ws.on("message", (raw) => {
+ this.handleMessage(raw.toString());
+ });
+ }
+
+ static async connect(browserWsUrl: string): Promise {
+ const ws = await new Promise((resolve, reject) => {
+ const socket = new WebSocket(browserWsUrl);
+ socket.once("open", () => resolve(socket));
+ socket.once("error", (error) => reject(error));
+ });
+
+ return new CdpClient(ws);
+ }
+
+ private handleMessage(rawMessage: string): void {
+ const message = JSON.parse(rawMessage) as {
+ id?: number;
+ sessionId?: string;
+ method?: string;
+ params?: JsonObject;
+ result?: unknown;
+ error?: CdpErrorShape;
+ };
+
+ if (typeof message.id === "number") {
+ const pending = this.pending.get(message.id);
+ if (!pending) {
+ return;
+ }
+ this.pending.delete(message.id);
+ if (message.error) {
+ pending.reject(new Error(`${pending.method}: ${message.error.message ?? "Unknown CDP error"}`));
+ return;
+ }
+ pending.resolve(message.result);
+ return;
+ }
+
+ if (typeof message.sessionId === "string" && typeof message.method === "string") {
+ const session = this.sessions.get(message.sessionId);
+ if (session) {
+ session.handleEvent(message.method, (message.params ?? {}) as JsonObject);
+ }
+ }
+ }
+
+ private async sendCommand(
+ method: string,
+ params: JsonObject = {},
+ sessionId?: string,
+ ): Promise {
+ const id = this.nextId;
+ this.nextId += 1;
+
+ const payload = sessionId ? { id, method, params, sessionId } : { id, method, params };
+
+ const result = new Promise((resolve, reject) => {
+ this.pending.set(id, {
+ resolve: (value) => resolve(value as T),
+ reject,
+ method,
+ });
+ });
+
+ this.ws.send(JSON.stringify(payload));
+ return result;
+ }
+
+ async sendBrowserCommand(method: string, params: JsonObject = {}): Promise {
+ return this.sendCommand(method, params);
+ }
+
+ async sendSessionCommand(sessionId: string, method: string, params: JsonObject = {}): Promise {
+ return this.sendCommand(method, params, sessionId);
+ }
+
+ private async createPageTarget(initialUrl: string, visible = false): Promise<{ targetId: string }> {
+ const attempts: JsonObject[] = visible
+ ? [
+ {
+ url: initialUrl,
+ newWindow: true,
+ focus: true,
+ },
+ {
+ url: initialUrl,
+ focus: true,
+ },
+ {
+ url: initialUrl,
+ },
+ ]
+ : [
+ {
+ url: initialUrl,
+ hidden: true,
+ },
+ {
+ url: initialUrl,
+ background: true,
+ focus: false,
+ },
+ {
+ url: initialUrl,
+ },
+ ];
+
+ let lastError: unknown;
+
+ for (const params of attempts) {
+ try {
+ return await this.sendBrowserCommand<{ targetId: string }>("Target.createTarget", params);
+ } catch (error) {
+ lastError = error;
+ }
+ }
+
+ throw lastError instanceof Error ? lastError : new Error("Target.createTarget failed");
+ }
+
+ async createPageSession(options: CreatePageSessionOptions = {}): Promise {
+ const initialUrl = options.initialUrl ?? "about:blank";
+ const created = await this.createPageTarget(initialUrl, Boolean(options.visible));
+ const attached = await this.sendBrowserCommand<{ sessionId: string }>("Target.attachToTarget", {
+ targetId: created.targetId,
+ flatten: true,
+ });
+
+ const session = new TargetSession(this, created.targetId, attached.sessionId);
+ this.sessions.set(attached.sessionId, session);
+
+ if (options.visible) {
+ await this.sendBrowserCommand("Target.activateTarget", {
+ targetId: created.targetId,
+ }).catch(() => {});
+ }
+
+ await session.send("Page.enable");
+ await session.send("Runtime.enable");
+ await session.send("DOM.enable");
+
+ if (options.visible) {
+ await session.send("Page.bringToFront").catch(() => {});
+ }
+
+ return session;
+ }
+
+ async closeTarget(targetId: string): Promise {
+ try {
+ await this.sendBrowserCommand("Target.closeTarget", { targetId });
+ } catch {
+ // Target may already be gone.
+ }
+ }
+
+ async close(): Promise {
+ await new Promise((resolve) => {
+ if (this.ws.readyState === WebSocket.CLOSED) {
+ resolve();
+ return;
+ }
+ this.ws.once("close", () => resolve());
+ this.ws.close();
+ });
+ }
+}
+
+export async function evaluateRuntime(session: TargetSession, expression: string): Promise {
+ const response = await session.send>("Runtime.evaluate", {
+ expression,
+ awaitPromise: true,
+ returnByValue: true,
+ });
+
+ if (response.error) {
+ throw new Error(response.error.message ?? "Runtime.evaluate failed");
+ }
+
+ return (response.result?.value as T | undefined) ?? (undefined as T);
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/chrome-launcher.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/chrome-launcher.ts
new file mode 100644
index 0000000..1efe4c5
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/chrome-launcher.ts
@@ -0,0 +1,187 @@
+import { launch, type LaunchedChrome } from "chrome-launcher";
+import WebSocket from "ws";
+import type { Logger } from "../utils/logger";
+import {
+ cleanChromeLockArtifacts,
+ ensureChromeProfileDir,
+ findChromeProcessUsingProfile,
+ findExistingChromeDebugPort,
+ hasChromeLockArtifacts,
+ listChromeProfileEntries,
+ resolveChromeProfileDir,
+ shouldRetryChromeLaunchRecovery,
+} from "./profile";
+
+interface ChromeVersionResponse {
+ webSocketDebuggerUrl: string;
+}
+
+export interface ChromeConnectOptions {
+ cdpUrl?: string;
+ browserPath?: string;
+ headless?: boolean;
+ logger?: Logger;
+ profileDir?: string;
+}
+
+export interface ChromeConnection {
+ browserWsUrl: string;
+ origin?: string;
+ port?: number;
+ profileDir?: string;
+ launched: boolean;
+ close(): Promise;
+}
+
+async function fetchJson(url: string): Promise {
+ const response = await fetch(url);
+ if (!response.ok) {
+ throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`);
+ }
+ return (await response.json()) as T;
+}
+
+async function connectToHttpEndpoint(origin: string): Promise {
+ const normalizedOrigin = origin.replace(/\/$/, "");
+ const version = await fetchJson(`${normalizedOrigin}/json/version`);
+ return {
+ browserWsUrl: version.webSocketDebuggerUrl,
+ origin: normalizedOrigin,
+ port: Number(new URL(normalizedOrigin).port || 80),
+ launched: false,
+ async close() {
+ // Reused external Chrome, nothing to close here.
+ },
+ };
+}
+
+async function tryReuseChrome(profileDir: string, logger?: Logger): Promise {
+ const port = await findExistingChromeDebugPort({ profileDir });
+ if (!port) {
+ return null;
+ }
+
+ const origin = `http://127.0.0.1:${port}`;
+ try {
+ const connection = await connectToHttpEndpoint(origin);
+ logger?.info(`Reusing Chrome debugger at ${origin} for profile ${profileDir}`);
+ return {
+ ...connection,
+ profileDir,
+ };
+ } catch {
+ // Debugger disappeared between detection and connect.
+ }
+ return null;
+}
+
+async function launchFreshChrome(
+ profileDir: string,
+ options: Pick,
+): Promise {
+ let launchedChrome: LaunchedChrome | null = null;
+ try {
+ launchedChrome = await launch({
+ chromePath: options.browserPath,
+ userDataDir: profileDir,
+ chromeFlags: [
+ "--disable-background-networking",
+ "--disable-default-apps",
+ "--disable-popup-blocking",
+ "--disable-sync",
+ "--no-first-run",
+ "--no-default-browser-check",
+ "--remote-allow-origins=*",
+ ...(!options.headless ? ["--no-startup-window"] : []),
+ ...(options.headless ? ["--headless=new"] : []),
+ ],
+ });
+
+ const origin = `http://127.0.0.1:${launchedChrome.port}`;
+ const version = await fetchJson(`${origin}/json/version`);
+
+ const chrome = launchedChrome;
+ return {
+ browserWsUrl: version.webSocketDebuggerUrl,
+ origin,
+ port: launchedChrome.port,
+ profileDir,
+ launched: true,
+ async close() {
+ if (!chrome) return;
+ await gracefulCloseChrome(chrome, origin);
+ },
+ };
+ } catch (error) {
+ launchedChrome?.kill();
+ throw error;
+ }
+}
+
+async function gracefulCloseChrome(chrome: LaunchedChrome, origin: string): Promise {
+ try {
+ const resp = await fetch(`${origin}/json/version`);
+ const { webSocketDebuggerUrl } = (await resp.json()) as ChromeVersionResponse;
+ if (webSocketDebuggerUrl) {
+ const ws = await new Promise((resolve, reject) => {
+ const socket = new WebSocket(webSocketDebuggerUrl);
+ socket.once("open", () => resolve(socket));
+ socket.once("error", reject);
+ });
+ const id = 1;
+ ws.send(JSON.stringify({ id, method: "Browser.close" }));
+ await new Promise((resolve) => {
+ const timer = setTimeout(() => { ws.close(); resolve(); }, 5_000);
+ ws.once("close", () => { clearTimeout(timer); resolve(); });
+ });
+ const exited = await new Promise((resolve) => {
+ if (chrome.pid && !isProcessAlive(chrome.pid)) { resolve(true); return; }
+ const timer = setTimeout(() => resolve(false), 3_000);
+ chrome.process.once("exit", () => { clearTimeout(timer); resolve(true); });
+ });
+ if (exited) return;
+ }
+ } catch {}
+ chrome.kill();
+}
+
+function isProcessAlive(pid: number): boolean {
+ try { process.kill(pid, 0); return true; } catch { return false; }
+}
+
+export async function connectChrome(options: ChromeConnectOptions): Promise {
+ if (options.cdpUrl) {
+ if (options.cdpUrl.startsWith("ws://") || options.cdpUrl.startsWith("wss://")) {
+ return {
+ browserWsUrl: options.cdpUrl,
+ launched: false,
+ async close() {},
+ };
+ }
+ return connectToHttpEndpoint(options.cdpUrl);
+ }
+
+ const profileDir = ensureChromeProfileDir(resolveChromeProfileDir(options.profileDir));
+ const reused = await tryReuseChrome(profileDir, options.logger);
+ if (reused) {
+ return reused;
+ }
+
+ options.logger?.warn(`No running Chrome debugger found for profile ${profileDir}. Launching Chrome with that profile.`);
+ try {
+ return await launchFreshChrome(profileDir, options);
+ } catch (error) {
+ const entries = await listChromeProfileEntries(profileDir);
+ const shouldRetry = shouldRetryChromeLaunchRecovery({
+ hasLockArtifacts: hasChromeLockArtifacts(entries),
+ hasLiveOwner: findChromeProcessUsingProfile(profileDir),
+ });
+ if (!shouldRetry) {
+ throw error;
+ }
+
+ options.logger?.warn(`Chrome launch failed with stale profile locks. Cleaning ${profileDir} and retrying once.`);
+ cleanChromeLockArtifacts(profileDir);
+ return await launchFreshChrome(profileDir, options);
+ }
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/cookie-sidecar.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/cookie-sidecar.ts
new file mode 100644
index 0000000..4b80302
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/cookie-sidecar.ts
@@ -0,0 +1,100 @@
+import { readFile, writeFile, mkdir } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { resolveChromeProfileDir } from "./profile";
+import type { TargetSession } from "./cdp-client";
+
+export interface CdpCookie {
+ name: string;
+ value: string;
+ domain: string;
+ path: string;
+ expires: number;
+ size: number;
+ httpOnly: boolean;
+ secure: boolean;
+ session: boolean;
+ sameSite?: string;
+ priority?: string;
+ sameParty?: boolean;
+ sourceScheme?: string;
+ sourcePort?: number;
+ partitionKey?: string;
+}
+
+interface SidecarData {
+ savedAt: string;
+ cookies: CdpCookie[];
+}
+
+export interface CookieSidecarConfig {
+ urls: readonly string[];
+ filename: string;
+ requiredCookieNames: readonly string[];
+ filterCookie?: (cookie: CdpCookie) => boolean;
+}
+
+function sidecarPath(filename: string, profileDir?: string): string {
+ return join(resolveChromeProfileDir(profileDir), filename);
+}
+
+function hasRequired(cookies: CdpCookie[], names: readonly string[]): boolean {
+ return names.every((name) =>
+ cookies.some((c) => c.name === name && Boolean(c.value)),
+ );
+}
+
+async function getCookies(session: TargetSession, urls: readonly string[]): Promise {
+ const { cookies } = await session.send<{ cookies: CdpCookie[] }>(
+ "Network.getCookies",
+ { urls: [...urls] },
+ );
+ return cookies ?? [];
+}
+
+export async function exportCookies(
+ session: TargetSession,
+ config: CookieSidecarConfig,
+ profileDir?: string,
+): Promise {
+ const all = await getCookies(session, config.urls);
+ const filtered = config.filterCookie ? all.filter(config.filterCookie) : all;
+ if (!hasRequired(filtered, config.requiredCookieNames)) return false;
+
+ const filePath = sidecarPath(config.filename, profileDir);
+ await mkdir(dirname(filePath), { recursive: true });
+ const data: SidecarData = { savedAt: new Date().toISOString(), cookies: filtered };
+ await writeFile(filePath, JSON.stringify(data, null, 2));
+ return true;
+}
+
+export async function restoreCookies(
+ session: TargetSession,
+ config: CookieSidecarConfig,
+ profileDir?: string,
+): Promise {
+ const live = await getCookies(session, config.urls);
+ if (hasRequired(live, config.requiredCookieNames)) return false;
+
+ const filePath = sidecarPath(config.filename, profileDir);
+ const raw = await readFile(filePath, "utf8");
+ const data = JSON.parse(raw) as SidecarData;
+ if (!data.cookies?.length) return false;
+
+ const now = Date.now() / 1000;
+ const valid = data.cookies.filter((c) => c.session || !c.expires || c.expires > now);
+ if (!hasRequired(valid, config.requiredCookieNames)) return false;
+
+ await session.send("Network.setCookies", {
+ cookies: valid.map((c) => ({
+ name: c.name,
+ value: c.value,
+ domain: c.domain,
+ path: c.path,
+ httpOnly: c.httpOnly,
+ secure: c.secure,
+ sameSite: c.sameSite,
+ expires: c.expires,
+ })),
+ });
+ return true;
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/interaction-gates.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/interaction-gates.ts
new file mode 100644
index 0000000..123e2d9
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/interaction-gates.ts
@@ -0,0 +1,123 @@
+import type { WaitForInteractionRequest } from "../adapters/types";
+import type { BrowserSession } from "./session";
+
+interface GateSnapshot {
+ title: string;
+ currentUrl: string;
+ bodyText: string;
+ hasCloudflareTurnstile: boolean;
+ hasCloudflareChallenge: boolean;
+ hasRecaptcha: boolean;
+ hasRecaptchaIframe: boolean;
+ hasHcaptcha: boolean;
+ hasHcaptchaIframe: boolean;
+}
+
+export function detectInteractionGateFromSnapshot(snapshot: GateSnapshot): WaitForInteractionRequest | null {
+ const text = snapshot.bodyText.toLowerCase();
+ const title = snapshot.title.toLowerCase();
+ const url = snapshot.currentUrl.toLowerCase();
+
+ if (
+ snapshot.hasCloudflareTurnstile ||
+ snapshot.hasCloudflareChallenge ||
+ title.includes("just a moment") ||
+ text.includes("verify you are human") ||
+ text.includes("checking your browser before accessing") ||
+ text.includes("enable javascript and cookies to continue") ||
+ url.includes("/cdn-cgi/challenge-platform/")
+ ) {
+ return {
+ type: "wait_for_interaction",
+ kind: "cloudflare",
+ provider: "cloudflare",
+ reason: "Cloudflare human verification detected",
+ prompt: "Please complete the Cloudflare verification in the opened Chrome window. Extraction will continue automatically once the challenge disappears.",
+ requiresVisibleBrowser: true,
+ };
+ }
+
+ if (
+ snapshot.hasRecaptcha ||
+ snapshot.hasRecaptchaIframe ||
+ text.includes("i'm not a robot") ||
+ text.includes("recaptcha")
+ ) {
+ return {
+ type: "wait_for_interaction",
+ kind: "recaptcha",
+ provider: "google_recaptcha",
+ reason: "Google reCAPTCHA detected",
+ prompt: "Please complete the reCAPTCHA verification in the opened Chrome window. Extraction will continue automatically once the challenge disappears.",
+ requiresVisibleBrowser: true,
+ };
+ }
+
+ if (
+ snapshot.hasHcaptcha ||
+ snapshot.hasHcaptchaIframe ||
+ text.includes("hcaptcha")
+ ) {
+ return {
+ type: "wait_for_interaction",
+ kind: "hcaptcha",
+ provider: "hcaptcha",
+ reason: "hCaptcha verification detected",
+ prompt: "Please complete the hCaptcha verification in the opened Chrome window. Extraction will continue automatically once the challenge disappears.",
+ requiresVisibleBrowser: true,
+ };
+ }
+
+ return null;
+}
+
+export async function detectInteractionGate(browser: BrowserSession): Promise {
+ const snapshot = await browser.evaluate(`
+ (() => {
+ const bodyText = (document.body?.innerText ?? "").slice(0, 4000);
+ return {
+ title: document.title ?? "",
+ currentUrl: window.location.href,
+ bodyText,
+ hasCloudflareTurnstile: Boolean(
+ document.querySelector(
+ '.cf-turnstile, [name="cf-turnstile-response"], iframe[src*="challenges.cloudflare.com"]'
+ )
+ ),
+ hasCloudflareChallenge: Boolean(
+ document.querySelector(
+ '#challenge-running, #cf-challenge-running, .challenge-platform, [data-ray], [data-translate="checking_browser"]'
+ )
+ ),
+ hasRecaptcha: Boolean(
+ document.querySelector(
+ '.g-recaptcha, textarea[name="g-recaptcha-response"], iframe[title*="reCAPTCHA"]'
+ )
+ ),
+ hasRecaptchaIframe: Boolean(
+ document.querySelector('iframe[src*="google.com/recaptcha"], iframe[src*="recaptcha/api2"]')
+ ),
+ hasHcaptcha: Boolean(
+ document.querySelector(
+ '.h-captcha, textarea[name="h-captcha-response"], iframe[title*="hCaptcha"]'
+ )
+ ),
+ hasHcaptchaIframe: Boolean(
+ document.querySelector('iframe[src*="hcaptcha.com"]')
+ ),
+ };
+ })()
+ `).catch(() => ({
+ title: "",
+ currentUrl: "",
+ bodyText: "",
+ hasCloudflareTurnstile: false,
+ hasCloudflareChallenge: false,
+ hasRecaptcha: false,
+ hasRecaptchaIframe: false,
+ hasHcaptcha: false,
+ hasHcaptchaIframe: false,
+ }));
+
+ return detectInteractionGateFromSnapshot(snapshot);
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/network-journal.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/network-journal.ts
new file mode 100644
index 0000000..8107910
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/network-journal.ts
@@ -0,0 +1,235 @@
+import type { TargetSession } from "./cdp-client";
+import type { Logger } from "../utils/logger";
+
+type JsonObject = Record;
+
+export interface NetworkEntry {
+ requestId: string;
+ url: string;
+ method: string;
+ resourceType: string;
+ timestamp: number;
+ requestHeaders?: Record;
+ requestBody?: string;
+ status?: number;
+ statusText?: string;
+ responseHeaders?: Record;
+ mimeType?: string;
+ body?: string;
+ bodyBase64?: boolean;
+ bodyError?: string;
+ failed?: boolean;
+ failureReason?: string;
+ finished: boolean;
+}
+
+function normalizeHeaders(headers: unknown): Record | undefined {
+ if (!headers || typeof headers !== "object") {
+ return undefined;
+ }
+ return Object.fromEntries(
+ Object.entries(headers as Record).map(([key, value]) => [key, String(value)]),
+ );
+}
+
+function sleep(ms: number): Promise {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export class NetworkJournal {
+ private readonly entries = new Map();
+ private lastActivityAt = Date.now();
+ private started = false;
+
+ constructor(
+ private readonly session: TargetSession,
+ private readonly log: Logger,
+ ) {}
+
+ async start(): Promise {
+ if (this.started) {
+ return;
+ }
+
+ this.started = true;
+ this.session.on("Network.requestWillBeSent", this.handleRequestWillBeSent);
+ this.session.on("Network.responseReceived", this.handleResponseReceived);
+ this.session.on("Network.loadingFinished", this.handleLoadingFinished);
+ this.session.on("Network.loadingFailed", this.handleLoadingFailed);
+ await this.session.send("Network.enable");
+ }
+
+ stop(): void {
+ if (!this.started) {
+ return;
+ }
+ this.session.off("Network.requestWillBeSent", this.handleRequestWillBeSent);
+ this.session.off("Network.responseReceived", this.handleResponseReceived);
+ this.session.off("Network.loadingFinished", this.handleLoadingFinished);
+ this.session.off("Network.loadingFailed", this.handleLoadingFailed);
+ this.started = false;
+ }
+
+ private touch(): void {
+ this.lastActivityAt = Date.now();
+ }
+
+ private readonly handleRequestWillBeSent = (params: JsonObject): void => {
+ const requestId = typeof params.requestId === "string" ? params.requestId : undefined;
+ const request = params.request as JsonObject | undefined;
+ if (!requestId || !request) {
+ return;
+ }
+
+ this.touch();
+ this.entries.set(requestId, {
+ requestId,
+ url: String(request.url ?? ""),
+ method: String(request.method ?? "GET"),
+ resourceType: String(params.type ?? "Other"),
+ timestamp: Date.now(),
+ requestHeaders: normalizeHeaders(request.headers),
+ requestBody: typeof request.postData === "string" ? request.postData : undefined,
+ finished: false,
+ });
+ };
+
+ private readonly handleResponseReceived = (params: JsonObject): void => {
+ const requestId = typeof params.requestId === "string" ? params.requestId : undefined;
+ const response = params.response as JsonObject | undefined;
+ if (!requestId || !response) {
+ return;
+ }
+
+ this.touch();
+ const existing = this.entries.get(requestId);
+ if (!existing) {
+ return;
+ }
+
+ existing.status = typeof response.status === "number" ? response.status : undefined;
+ existing.statusText = typeof response.statusText === "string" ? response.statusText : undefined;
+ existing.responseHeaders = normalizeHeaders(response.headers);
+ existing.mimeType = typeof response.mimeType === "string" ? response.mimeType : undefined;
+ this.entries.set(requestId, existing);
+ };
+
+ private readonly handleLoadingFinished = (params: JsonObject): void => {
+ const requestId = typeof params.requestId === "string" ? params.requestId : undefined;
+ if (!requestId) {
+ return;
+ }
+
+ this.touch();
+ const existing = this.entries.get(requestId);
+ if (!existing) {
+ return;
+ }
+ existing.finished = true;
+ this.entries.set(requestId, existing);
+ };
+
+ private readonly handleLoadingFailed = (params: JsonObject): void => {
+ const requestId = typeof params.requestId === "string" ? params.requestId : undefined;
+ if (!requestId) {
+ return;
+ }
+
+ this.touch();
+ const existing = this.entries.get(requestId);
+ if (!existing) {
+ return;
+ }
+ existing.finished = true;
+ existing.failed = true;
+ existing.failureReason = typeof params.errorText === "string" ? params.errorText : "Unknown error";
+ this.entries.set(requestId, existing);
+ };
+
+ getEntries(): NetworkEntry[] {
+ return Array.from(this.entries.values());
+ }
+
+ findEntries(predicate: (entry: NetworkEntry) => boolean): NetworkEntry[] {
+ return this.getEntries().filter(predicate);
+ }
+
+ async waitForIdle(options: { idleMs?: number; timeoutMs?: number } = {}): Promise {
+ const idleMs = options.idleMs ?? 1_200;
+ const timeoutMs = options.timeoutMs ?? 15_000;
+ const startedAt = Date.now();
+
+ while (Date.now() - startedAt < timeoutMs) {
+ if (Date.now() - this.lastActivityAt >= idleMs) {
+ return;
+ }
+ await sleep(Math.min(150, idleMs));
+ }
+
+ throw new Error("Timed out waiting for network idle");
+ }
+
+ async waitForResponse(
+ predicate: (entry: NetworkEntry) => boolean,
+ options: { timeoutMs?: number } = {},
+ ): Promise {
+ const timeoutMs = options.timeoutMs ?? 10_000;
+ const startedAt = Date.now();
+
+ while (Date.now() - startedAt < timeoutMs) {
+ const matched = this.getEntries().find((entry) => entry.finished && predicate(entry));
+ if (matched) {
+ return matched;
+ }
+ await sleep(150);
+ }
+
+ throw new Error("Timed out waiting for matching network response");
+ }
+
+ async ensureBody(entry: NetworkEntry): Promise {
+ if (entry.body !== undefined) {
+ return entry.body;
+ }
+ if (entry.bodyError || entry.failed || !entry.finished) {
+ return undefined;
+ }
+
+ try {
+ const result = await this.session.send<{ body: string; base64Encoded: boolean }>("Network.getResponseBody", {
+ requestId: entry.requestId,
+ });
+ entry.bodyBase64 = result.base64Encoded;
+ entry.body = result.base64Encoded ? Buffer.from(result.body, "base64").toString("utf8") : result.body;
+ return entry.body;
+ } catch (error) {
+ entry.bodyError = error instanceof Error ? error.message : String(error);
+ this.log.debug(`Failed to fetch response body for ${entry.url}: ${entry.bodyError}`);
+ return undefined;
+ }
+ }
+
+ async getJsonBody(entry: NetworkEntry): Promise {
+ const body = await this.ensureBody(entry);
+ if (!body) {
+ return null;
+ }
+
+ try {
+ return JSON.parse(body);
+ } catch {
+ return null;
+ }
+ }
+
+ async toJSON(options: { includeBodies?: boolean } = {}): Promise {
+ const entries = this.getEntries();
+ if (!options.includeBodies) {
+ return entries;
+ }
+
+ await Promise.all(entries.map((entry) => this.ensureBody(entry)));
+ return entries;
+ }
+}
+
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/page-snapshot.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/page-snapshot.ts
new file mode 100644
index 0000000..4b55958
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/page-snapshot.ts
@@ -0,0 +1,105 @@
+import type { BrowserSession } from "./session";
+
+export interface CapturedPageSnapshot {
+ html: string;
+ finalUrl: string;
+}
+
+export const CAPTURE_NORMALIZED_PAGE_SCRIPT = String.raw`
+(() => {
+ const baseUrl = document.baseURI || location.href;
+ const htmlClone = document.documentElement.cloneNode(true);
+
+ function materializeShadowDom(sourceRoot, cloneRoot) {
+ const sourceElements = Array.from(sourceRoot.querySelectorAll("*"));
+ const cloneElements = Array.from(cloneRoot.querySelectorAll("*"));
+
+ for (let index = sourceElements.length - 1; index >= 0; index -= 1) {
+ const sourceElement = sourceElements[index];
+ const cloneElement = cloneElements[index];
+ const shadowRoot = sourceElement && sourceElement.shadowRoot;
+ if (!shadowRoot || !cloneElement || !shadowRoot.innerHTML) {
+ continue;
+ }
+
+ if (cloneElement.tagName && cloneElement.tagName.includes("-")) {
+ const wrapper = document.createElement("div");
+ wrapper.setAttribute("data-shadow-host", cloneElement.tagName.toLowerCase());
+ wrapper.innerHTML = shadowRoot.innerHTML;
+ cloneElement.replaceWith(wrapper);
+ } else {
+ cloneElement.innerHTML = shadowRoot.innerHTML;
+ }
+ }
+ }
+
+ function toAbsolute(url) {
+ if (!url) return url;
+ try {
+ return new URL(url, baseUrl).href;
+ } catch {
+ return url;
+ }
+ }
+
+ function absolutizeAttribute(root, selector, attribute) {
+ root.querySelectorAll(selector).forEach((element) => {
+ const value = element.getAttribute(attribute);
+ if (!value) return;
+ const absolute = toAbsolute(value);
+ if (absolute) {
+ element.setAttribute(attribute, absolute);
+ }
+ });
+ }
+
+ function absolutizeSrcset(root, selector) {
+ root.querySelectorAll(selector).forEach((element) => {
+ const srcset = element.getAttribute("srcset");
+ if (!srcset) return;
+ element.setAttribute(
+ "srcset",
+ srcset
+ .split(",")
+ .map((part) => {
+ const trimmed = part.trim();
+ if (!trimmed) return "";
+ const [url, ...descriptor] = trimmed.split(/\s+/);
+ const absolute = toAbsolute(url);
+ return descriptor.length > 0 ? absolute + " " + descriptor.join(" ") : absolute;
+ })
+ .filter(Boolean)
+ .join(", "),
+ );
+ });
+ }
+
+ materializeShadowDom(document.documentElement, htmlClone);
+
+ htmlClone
+ .querySelectorAll("img[data-src], video[data-src], audio[data-src], source[data-src]")
+ .forEach((element) => {
+ const dataSource = element.getAttribute("data-src");
+ const current = element.getAttribute("src");
+ if (dataSource && (!current || current === "" || current.startsWith("data:"))) {
+ element.setAttribute("src", dataSource);
+ }
+ });
+
+ absolutizeAttribute(htmlClone, "a[href]", "href");
+ absolutizeAttribute(htmlClone, "img[src], video[src], audio[src], source[src], iframe[src]", "src");
+ absolutizeAttribute(htmlClone, "video[poster]", "poster");
+ absolutizeSrcset(htmlClone, "img[srcset], source[srcset]");
+
+ return {
+ html: "\n" + htmlClone.outerHTML,
+ finalUrl: location.href,
+ };
+})()
+`;
+
+export async function captureNormalizedPageSnapshot(
+ browser: BrowserSession,
+): Promise {
+ return browser.evaluate(CAPTURE_NORMALIZED_PAGE_SCRIPT);
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/profile.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/profile.ts
new file mode 100644
index 0000000..986fb32
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/profile.ts
@@ -0,0 +1,201 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import process from "node:process";
+import { spawnSync } from "node:child_process";
+
+export interface ResolveSharedChromeProfileDirOptions {
+ envNames?: string[];
+ appDataDirName?: string;
+ profileDirName?: string;
+}
+
+export interface FindExistingChromeDebugPortOptions {
+ profileDir: string;
+ timeoutMs?: number;
+}
+
+interface ChromeVersionResponse {
+ webSocketDebuggerUrl?: string;
+}
+
+const CHROME_LOCK_FILE_NAMES = ["SingletonLock", "SingletonSocket", "SingletonCookie", "chrome.pid"] as const;
+
+function resolveDataBaseDir(): string {
+ if (process.platform === "darwin") {
+ return path.join(os.homedir(), "Library", "Application Support");
+ }
+ if (process.platform === "win32") {
+ return process.env.APPDATA ?? path.join(os.homedir(), "AppData", "Roaming");
+ }
+ return process.env.XDG_DATA_HOME ?? path.join(os.homedir(), ".local", "share");
+}
+
+export function resolveSharedChromeProfileDir(
+ options: ResolveSharedChromeProfileDirOptions = {},
+): string {
+ for (const envName of options.envNames ?? []) {
+ const override = process.env[envName]?.trim();
+ if (override) {
+ return path.resolve(override);
+ }
+ }
+
+ const appDataDirName = options.appDataDirName ?? "baoyu-skills";
+ const profileDirName = options.profileDirName ?? "chrome-profile";
+ return path.join(resolveDataBaseDir(), appDataDirName, profileDirName);
+}
+
+export function resolveChromeProfileDir(profileDir?: string): string {
+ if (profileDir?.trim()) {
+ return path.resolve(profileDir.trim());
+ }
+
+ return resolveSharedChromeProfileDir({
+ envNames: ["BAOYU_CHROME_PROFILE_DIR"],
+ appDataDirName: "baoyu-skills",
+ profileDirName: "chrome-profile",
+ });
+}
+
+export function ensureChromeProfileDir(profileDir: string): string {
+ fs.mkdirSync(profileDir, { recursive: true });
+ return profileDir;
+}
+
+export function hasChromeLockArtifacts(entries: readonly string[]): boolean {
+ return CHROME_LOCK_FILE_NAMES.some((name) => entries.includes(name));
+}
+
+export function shouldRetryChromeLaunchRecovery(options: {
+ hasLockArtifacts: boolean;
+ hasLiveOwner: boolean;
+}): boolean {
+ return options.hasLockArtifacts && !options.hasLiveOwner;
+}
+
+export function findChromeProcessUsingProfile(profileDir: string): boolean {
+ if (process.platform === "win32") {
+ return false;
+ }
+
+ try {
+ const result = spawnSync("ps", ["aux"], {
+ encoding: "utf8",
+ timeout: 5_000,
+ });
+ if (result.status !== 0 || !result.stdout) {
+ return false;
+ }
+
+ return result.stdout
+ .split("\n")
+ .some((line) => line.includes(`--user-data-dir=${profileDir}`));
+ } catch {
+ return false;
+ }
+}
+
+export function cleanChromeLockArtifacts(profileDir: string): void {
+ for (const name of CHROME_LOCK_FILE_NAMES) {
+ try {
+ fs.unlinkSync(path.join(profileDir, name));
+ } catch {
+ // Ignore missing files and continue cleaning the remaining artifacts.
+ }
+ }
+}
+
+export async function listChromeProfileEntries(profileDir: string): Promise {
+ try {
+ return await fs.promises.readdir(profileDir);
+ } catch {
+ return [];
+ }
+}
+
+async function fetchWithTimeout(url: string, timeoutMs = 3_000): Promise {
+ const controller = new AbortController();
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
+ try {
+ return await fetch(url, {
+ redirect: "follow",
+ signal: controller.signal,
+ });
+ } finally {
+ clearTimeout(timer);
+ }
+}
+
+async function fetchJson(url: string, timeoutMs = 3_000): Promise {
+ const response = await fetchWithTimeout(url, timeoutMs);
+ if (!response.ok) {
+ throw new Error(`Request failed: ${response.status} ${response.statusText}`);
+ }
+ return (await response.json()) as T;
+}
+
+async function isDebugPortReady(port: number, timeoutMs = 3_000): Promise {
+ try {
+ const version = await fetchJson(`http://127.0.0.1:${port}/json/version`, timeoutMs);
+ return Boolean(version.webSocketDebuggerUrl);
+ } catch {
+ return false;
+ }
+}
+
+function parseDevToolsActivePort(filePath: string): { port: number; wsPath: string } | null {
+ try {
+ const content = fs.readFileSync(filePath, "utf8");
+ const lines = content.split(/\r?\n/);
+ const port = Number.parseInt(lines[0]?.trim() ?? "", 10);
+ const wsPath = lines[1]?.trim() ?? "";
+ if (port > 0 && wsPath) {
+ return { port, wsPath };
+ }
+ } catch {
+ // Ignore and fall back to process inspection.
+ }
+
+ return null;
+}
+
+export async function findExistingChromeDebugPort(
+ options: FindExistingChromeDebugPortOptions,
+): Promise {
+ const timeoutMs = options.timeoutMs ?? 3_000;
+ const activePort = parseDevToolsActivePort(path.join(options.profileDir, "DevToolsActivePort"));
+ if (activePort && await isDebugPortReady(activePort.port, timeoutMs)) {
+ return activePort.port;
+ }
+
+ if (process.platform === "win32") {
+ return null;
+ }
+
+ try {
+ const result = spawnSync("ps", ["aux"], {
+ encoding: "utf8",
+ timeout: 5_000,
+ });
+ if (result.status !== 0 || !result.stdout) {
+ return null;
+ }
+
+ const lines = result.stdout
+ .split("\n")
+ .filter((line) => line.includes(options.profileDir) && line.includes("--remote-debugging-port="));
+
+ for (const line of lines) {
+ const match = line.match(/--remote-debugging-port=(\d+)/);
+ const port = Number.parseInt(match?.[1] ?? "", 10);
+ if (port > 0 && await isDebugPortReady(port, timeoutMs)) {
+ return port;
+ }
+ }
+ } catch {
+ // Ignore and report no reusable debugger.
+ }
+
+ return null;
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/browser/session.ts b/skills/baoyu-url-to-markdown/scripts/lib/browser/session.ts
new file mode 100644
index 0000000..7f66e8a
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/browser/session.ts
@@ -0,0 +1,155 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { CdpClient, TargetSession, evaluateRuntime } from "./cdp-client";
+
+interface NavigationResult {
+ errorText?: string;
+}
+
+const execFileAsync = promisify(execFile);
+const MACOS_BROWSER_APP_IDS = [
+ "com.google.Chrome",
+ "org.chromium.Chromium",
+ "com.brave.Browser",
+ "com.microsoft.edgemac",
+];
+
+function sleep(ms: number): Promise {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function activateBrowserApp(): Promise {
+ if (process.platform !== "darwin") {
+ return;
+ }
+
+ for (const appId of MACOS_BROWSER_APP_IDS) {
+ try {
+ await execFileAsync("osascript", ["-e", `tell application id "${appId}" to activate`]);
+ return;
+ } catch {
+ // Try the next installed browser bundle id.
+ }
+ }
+}
+
+export class BrowserSession {
+ private constructor(
+ private readonly cdp: CdpClient,
+ public readonly targetSession: TargetSession,
+ public readonly interactive: boolean,
+ ) {}
+
+ static async open(
+ cdp: CdpClient,
+ options: {
+ initialUrl?: string;
+ interactive?: boolean;
+ } = {},
+ ): Promise {
+ const targetSession = await cdp.createPageSession({
+ initialUrl: options.initialUrl,
+ visible: options.interactive,
+ });
+ const browser = new BrowserSession(cdp, targetSession, Boolean(options.interactive));
+ if (browser.interactive) {
+ await browser.bringToFront().catch(() => {});
+ }
+ return browser;
+ }
+
+ async goto(url: string, timeoutMs = 30_000): Promise {
+ const loadPromise = this.targetSession.waitForEvent("Page.loadEventFired", undefined, timeoutMs).catch(() => null);
+ const result = await this.targetSession.send("Page.navigate", { url });
+ if (result.errorText) {
+ throw new Error(`Navigation failed: ${result.errorText}`);
+ }
+ await loadPromise;
+ await this.waitForReadyState(timeoutMs);
+ }
+
+ async waitForReadyState(timeoutMs = 30_000): Promise {
+ const startedAt = Date.now();
+ while (Date.now() - startedAt < timeoutMs) {
+ const state = await this.evaluate("document.readyState");
+ if (state === "interactive" || state === "complete") {
+ return;
+ }
+ await sleep(150);
+ }
+ throw new Error("Timed out waiting for document.readyState");
+ }
+
+ async evaluate(expression: string): Promise {
+ return evaluateRuntime(this.targetSession, expression);
+ }
+
+ async getHTML(): Promise {
+ return this.evaluate("document.documentElement.outerHTML");
+ }
+
+ async getTitle(): Promise {
+ return this.evaluate("document.title");
+ }
+
+ async getURL(): Promise {
+ return this.evaluate("window.location.href");
+ }
+
+ async bringToFront(): Promise {
+ await this.targetSession.send("Page.bringToFront").catch(async () => {
+ await this.cdp.sendBrowserCommand("Target.activateTarget", {
+ targetId: this.targetSession.targetId,
+ });
+ });
+ if (this.interactive) {
+ await activateBrowserApp().catch(() => {});
+ }
+ }
+
+ async click(selector: string): Promise {
+ const result = await this.evaluate<{ ok: boolean; error?: string }>(`
+ (() => {
+ const element = document.querySelector(${JSON.stringify(selector)});
+ if (!element) {
+ return { ok: false, error: "Element not found" };
+ }
+ element.scrollIntoView({ block: "center", inline: "center" });
+ if (element instanceof HTMLElement) {
+ element.click();
+ return { ok: true };
+ }
+ return { ok: false, error: "Element is not clickable" };
+ })()
+ `);
+
+ if (!result.ok) {
+ throw new Error(result.error ?? `Failed to click ${selector}`);
+ }
+ }
+
+ async scrollToEnd(options: { stepPx?: number; delayMs?: number; maxSteps?: number } = {}): Promise {
+ const stepPx = options.stepPx ?? 1_400;
+ const delayMs = options.delayMs ?? 250;
+ const maxSteps = options.maxSteps ?? 6;
+
+ for (let step = 0; step < maxSteps; step += 1) {
+ const done = await this.evaluate(`
+ (() => {
+ const before = window.scrollY;
+ window.scrollBy(0, ${stepPx});
+ const atBottom = window.innerHeight + window.scrollY >= document.body.scrollHeight - 4;
+ return atBottom || window.scrollY === before;
+ })()
+ `);
+ if (done) {
+ break;
+ }
+ await sleep(delayMs);
+ }
+ }
+
+ async close(): Promise {
+ await this.cdp.closeTarget(this.targetSession.targetId);
+ }
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/cli.ts b/skills/baoyu-url-to-markdown/scripts/lib/cli.ts
new file mode 100755
index 0000000..abde93f
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/cli.ts
@@ -0,0 +1,227 @@
+#!/usr/bin/env bun
+
+import {
+ runConvertCommand,
+ type ConvertCommandOptions,
+ type OutputFormat,
+ type WaitMode,
+} from "./commands/convert";
+
+export const HELP_TEXT = `
+baoyu-fetch - Read a URL into Markdown or JSON with Chrome CDP
+
+Usage:
+ baoyu-fetch [options]
+
+Options:
+ --output Save output to file
+ --format Output format: markdown | json
+ --json Alias for --format json
+ --adapter Force an adapter (e.g. x, generic)
+ --download-media Download adapter-reported media into ./imgs and ./videos, then rewrite markdown links
+ --media-dir Base directory for downloaded media. Defaults to the output directory
+ --debug-dir Write debug artifacts
+ --cdp-url Reuse an existing Chrome DevTools endpoint
+ --browser-path Explicit Chrome binary path
+ --chrome-profile-dir
+ Chrome user data dir. Defaults to BAOYU_CHROME_PROFILE_DIR
+ or baoyu-skills/chrome-profile.
+ --headless Launch a temporary headless Chrome if needed
+ --wait-for Wait mode: interaction | force
+ interaction: start visible Chrome and auto-wait only when login or verification is required
+ force: start visible Chrome, then auto-continue after it detects login/challenge progress
+ or continue immediately when you press Enter
+ --wait-for-interaction
+ Alias for --wait-for interaction
+ --wait-for-login Alias for --wait-for interaction
+ --interaction-timeout
+ How long to wait for manual interaction before failing (default: 600000)
+ --interaction-poll-interval
+ How often to poll interaction state while waiting (default: 1500)
+ --login-timeout Alias for --interaction-timeout
+ --login-poll-interval
+ Alias for --interaction-poll-interval
+ --timeout Page timeout in milliseconds (default: 30000)
+ --help Show help
+
+Examples:
+ baoyu-fetch https://example.com
+ baoyu-fetch https://example.com --format markdown --output article.md --download-media
+ baoyu-fetch https://example.com --format json --output article.json
+ baoyu-fetch https://x.com/lennysan/status/2036483059407810640 --wait-for interaction
+ baoyu-fetch https://x.com/lennysan/status/2036483059407810640 --wait-for force
+`.trim();
+
+interface CliOptions extends ConvertCommandOptions {
+ url?: string;
+ help: boolean;
+}
+
+function normalizeWaitMode(raw: string): WaitMode {
+ const value = raw.toLowerCase();
+ if (value === "interaction" || value === "auto") {
+ return "interaction";
+ }
+ if (value === "force" || value === "manual" || value === "always") {
+ return "force";
+ }
+ throw new Error(`Invalid wait mode: ${raw}. Expected interaction or force.`);
+}
+
+function normalizeOutputFormat(raw: string): OutputFormat {
+ const value = raw.toLowerCase();
+ if (value === "markdown" || value === "json") {
+ return value;
+ }
+
+ throw new Error(`Invalid output format: ${raw}. Expected markdown or json.`);
+}
+
+export function parseArgs(argv: string[]): CliOptions {
+ const options: CliOptions = {
+ format: "markdown",
+ headless: false,
+ downloadMedia: false,
+ waitMode: "none",
+ interactionTimeoutMs: 600_000,
+ interactionPollIntervalMs: 1_500,
+ timeoutMs: 30_000,
+ help: false,
+ };
+
+ const args = argv.slice(2);
+ for (let index = 0; index < args.length; index += 1) {
+ const value = args[index];
+
+ if (value === "--help" || value === "-h") {
+ options.help = true;
+ continue;
+ }
+ if (value === "--format") {
+ const format = args[index + 1];
+ if (!format) {
+ throw new Error("--format requires a value");
+ }
+ options.format = normalizeOutputFormat(format);
+ index += 1;
+ continue;
+ }
+ if (value === "--json") {
+ options.format = "json";
+ continue;
+ }
+ if (value === "--download-media") {
+ options.downloadMedia = true;
+ continue;
+ }
+ if (value === "--headless") {
+ options.headless = true;
+ continue;
+ }
+ if (value === "--wait-for") {
+ const mode = args[index + 1];
+ if (!mode) {
+ throw new Error("--wait-for requires a mode");
+ }
+ options.waitMode = normalizeWaitMode(mode);
+ index += 1;
+ continue;
+ }
+ if (value === "--wait-for-interaction" || value === "--wait-for-login") {
+ options.waitMode = "interaction";
+ continue;
+ }
+ if (value === "--output") {
+ options.output = args[index + 1];
+ index += 1;
+ continue;
+ }
+ if (value === "--adapter") {
+ options.adapter = args[index + 1];
+ index += 1;
+ continue;
+ }
+ if (value === "--debug-dir") {
+ options.debugDir = args[index + 1];
+ index += 1;
+ continue;
+ }
+ if (value === "--media-dir") {
+ options.mediaDir = args[index + 1];
+ index += 1;
+ continue;
+ }
+ if (value === "--cdp-url") {
+ options.cdpUrl = args[index + 1];
+ index += 1;
+ continue;
+ }
+ if (value === "--browser-path") {
+ options.browserPath = args[index + 1];
+ index += 1;
+ continue;
+ }
+ if (value === "--chrome-profile-dir") {
+ options.chromeProfileDir = args[index + 1];
+ index += 1;
+ continue;
+ }
+ if (value === "--timeout") {
+ const parsed = Number(args[index + 1]);
+ if (!Number.isFinite(parsed) || parsed <= 0) {
+ throw new Error(`Invalid timeout: ${args[index + 1]}`);
+ }
+ options.timeoutMs = parsed;
+ index += 1;
+ continue;
+ }
+ if (value === "--interaction-timeout" || value === "--login-timeout") {
+ const parsed = Number(args[index + 1]);
+ if (!Number.isFinite(parsed) || parsed <= 0) {
+ throw new Error(`Invalid interaction timeout: ${args[index + 1]}`);
+ }
+ options.interactionTimeoutMs = parsed;
+ index += 1;
+ continue;
+ }
+ if (value === "--interaction-poll-interval" || value === "--login-poll-interval") {
+ const parsed = Number(args[index + 1]);
+ if (!Number.isFinite(parsed) || parsed <= 0) {
+ throw new Error(`Invalid interaction poll interval: ${args[index + 1]}`);
+ }
+ options.interactionPollIntervalMs = parsed;
+ index += 1;
+ continue;
+ }
+ if (value.startsWith("-")) {
+ throw new Error(`Unknown option: ${value}`);
+ }
+ if (!options.url) {
+ options.url = value;
+ continue;
+ }
+ throw new Error(`Unexpected argument: ${value}`);
+ }
+
+ return options;
+}
+
+async function main(): Promise {
+ try {
+ const options = parseArgs(process.argv);
+ if (options.help || !options.url) {
+ console.log(HELP_TEXT);
+ return;
+ }
+
+ await runConvertCommand(options);
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ console.error(message);
+ process.exitCode = 1;
+ }
+}
+
+if (import.meta.main) {
+ void main();
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/commands/convert.ts b/skills/baoyu-url-to-markdown/scripts/lib/commands/convert.ts
new file mode 100644
index 0000000..659e18e
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/commands/convert.ts
@@ -0,0 +1,580 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { createInterface } from "node:readline";
+import { connectChrome, type ChromeConnection } from "../browser/chrome-launcher";
+import { CdpClient } from "../browser/cdp-client";
+import { detectInteractionGate } from "../browser/interaction-gates";
+import { NetworkJournal } from "../browser/network-journal";
+import { BrowserSession } from "../browser/session";
+import { genericAdapter, resolveAdapter } from "../adapters";
+import { isXSessionReady } from "../adapters/x/session";
+import type { ExtractedDocument } from "../extract/document";
+import { renderMarkdown } from "../extract/markdown-renderer";
+import { downloadMediaAssets } from "../media/default-downloader";
+import { rewriteMarkdownMediaLinks } from "../media/markdown-media";
+import { createLogger } from "../utils/logger";
+import { normalizeUrl } from "../utils/url";
+import type {
+ Adapter,
+ AdapterContext,
+ AdapterLoginInfo,
+ LoginState,
+ MediaAsset,
+ WaitForInteractionRequest,
+} from "../adapters/types";
+
+export type WaitMode = "none" | "interaction" | "force";
+export type OutputFormat = "markdown" | "json";
+
+export interface ConvertCommandOptions {
+ url?: string;
+ output?: string;
+ format: OutputFormat;
+ adapter?: string;
+ debugDir?: string;
+ cdpUrl?: string;
+ browserPath?: string;
+ chromeProfileDir?: string;
+ headless: boolean;
+ downloadMedia: boolean;
+ mediaDir?: string;
+ waitMode: WaitMode;
+ interactionTimeoutMs: number;
+ interactionPollIntervalMs: number;
+ timeoutMs: number;
+}
+
+interface RuntimeResources {
+ chrome: ChromeConnection;
+ cdp: CdpClient;
+ browser: BrowserSession;
+ network: NetworkJournal;
+ interactive: boolean;
+}
+
+interface ForceWaitSnapshot {
+ url: string;
+ hasGate: boolean;
+ loginState: LoginState | "unavailable";
+ sessionReady: boolean;
+}
+
+interface SuccessfulConvertOutput {
+ adapter: string;
+ status: "ok";
+ login?: AdapterLoginInfo;
+ media: MediaAsset[];
+ downloads: Awaited> | null;
+ document: ExtractedDocument;
+ markdown: string;
+}
+
+interface InteractionRequiredOutput {
+ adapter: string;
+ status: "needs_interaction";
+ login?: AdapterLoginInfo;
+ interaction: WaitForInteractionRequest;
+}
+
+function sleep(ms: number): Promise {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function isForceWaitSessionReady(snapshot: ForceWaitSnapshot): boolean {
+ return snapshot.sessionReady;
+}
+
+export function shouldKeepBrowserOpenAfterInteraction(options: {
+ launched: boolean;
+ interaction: Pick;
+}): boolean {
+ return options.launched && options.interaction.kind === "login" && options.interaction.provider === "x";
+}
+
+export function shouldAutoContinueForceWait(
+ initial: ForceWaitSnapshot,
+ current: ForceWaitSnapshot,
+): boolean {
+ if (initial.hasGate && !current.hasGate) {
+ return true;
+ }
+
+ if (initial.loginState === "logged_out" && current.loginState !== "logged_out" && isForceWaitSessionReady(current)) {
+ return true;
+ }
+
+ if (initial.loginState !== "logged_in" && current.loginState === "logged_in" && isForceWaitSessionReady(current)) {
+ return true;
+ }
+
+ if (
+ current.url !== initial.url &&
+ !current.hasGate &&
+ current.loginState !== "logged_out" &&
+ isForceWaitSessionReady(current)
+ ) {
+ return true;
+ }
+
+ return false;
+}
+
+async function writeOutput(path: string, content: string): Promise {
+ const directory = path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : "";
+ if (directory) {
+ await mkdir(directory, { recursive: true });
+ }
+ await writeFile(path, content, "utf8");
+}
+
+async function writeDebugArtifacts(
+ debugDir: string,
+ document: ExtractedDocument,
+ markdown: string,
+ browser: BrowserSession,
+ network: NetworkJournal,
+): Promise {
+ await mkdir(debugDir, { recursive: true });
+
+ const html = await browser.getHTML().catch(() => "");
+ const networkDump = await network.toJSON({ includeBodies: true });
+
+ await Promise.all([
+ writeFile(join(debugDir, "document.json"), JSON.stringify(document, null, 2), "utf8"),
+ writeFile(join(debugDir, "markdown.md"), markdown, "utf8"),
+ writeFile(join(debugDir, "page.html"), html, "utf8"),
+ writeFile(join(debugDir, "network.json"), JSON.stringify(networkDump, null, 2), "utf8"),
+ ]);
+}
+
+async function openRuntime(
+ options: ConvertCommandOptions,
+ interactive: boolean,
+ debugEnabled: boolean,
+): Promise {
+ const logger = createLogger(debugEnabled);
+ if (interactive) {
+ logger.info("Opening Chrome in interactive mode.");
+ }
+ const chrome = await connectChrome({
+ cdpUrl: options.cdpUrl,
+ browserPath: options.browserPath,
+ profileDir: options.chromeProfileDir,
+ headless: interactive ? false : options.headless,
+ logger,
+ });
+
+ const cdp = await CdpClient.connect(chrome.browserWsUrl);
+ const browser = await BrowserSession.open(cdp, { interactive });
+ if (interactive) {
+ await browser.bringToFront().catch(() => {});
+ }
+ const network = new NetworkJournal(browser.targetSession, logger);
+ await network.start();
+
+ return {
+ chrome,
+ cdp,
+ browser,
+ network,
+ interactive,
+ };
+}
+
+async function closeRuntime(runtime: RuntimeResources | null | undefined): Promise {
+ if (!runtime) {
+ return;
+ }
+ runtime.network.stop();
+ await runtime.browser.close().catch(() => {});
+ await runtime.cdp.close().catch(() => {});
+ await runtime.chrome.close().catch(() => {});
+}
+
+async function isInteractionSessionReady(
+ context: AdapterContext,
+ interaction: WaitForInteractionRequest,
+): Promise {
+ if (interaction.provider !== "x") {
+ return true;
+ }
+ return await isXSessionReady(context).catch(() => false);
+}
+
+async function reopenInteractiveRuntime(
+ runtime: RuntimeResources,
+ options: ConvertCommandOptions,
+ debugEnabled: boolean,
+): Promise {
+ if (runtime.interactive) {
+ return runtime;
+ }
+
+ await closeRuntime(runtime);
+ return openRuntime(options, true, debugEnabled);
+}
+
+async function captureForceWaitSnapshot(
+ adapter: Adapter,
+ context: AdapterContext,
+): Promise {
+ const [gate, url, login] = await Promise.all([
+ detectInteractionGate(context.browser).catch(() => null),
+ context.browser.getURL().catch(() => context.input.url.toString()),
+ adapter.checkLogin?.(context).catch(() => ({
+ provider: adapter.name,
+ state: "unknown" as const,
+ })),
+ ]);
+
+ return {
+ url,
+ hasGate: Boolean(gate),
+ loginState: login?.state ?? "unavailable",
+ sessionReady: adapter.name === "x" ? await isXSessionReady(context).catch(() => false) : true,
+ };
+}
+
+async function waitForForceResume(
+ adapter: Adapter,
+ context: AdapterContext,
+ options: ConvertCommandOptions,
+): Promise {
+ if (context.interactive) {
+ await context.browser.bringToFront().catch(() => {});
+ }
+
+ const prompt =
+ "Chrome is ready. Complete any manual login or verification. Extraction will continue automatically after it detects progress, or press Enter to continue immediately.";
+ context.log.info(prompt);
+
+ const rl = createInterface({
+ input: process.stdin,
+ output: process.stderr,
+ });
+
+ let manualContinue = false;
+ let closed = false;
+ const closeReadline = (): void => {
+ if (!closed) {
+ closed = true;
+ rl.close();
+ }
+ };
+
+ rl.once("line", () => {
+ manualContinue = true;
+ closeReadline();
+ });
+
+ const initial = await captureForceWaitSnapshot(adapter, context);
+ const startedAt = Date.now();
+
+ try {
+ while (Date.now() - startedAt < options.interactionTimeoutMs) {
+ if (manualContinue) {
+ return;
+ }
+
+ const current = await captureForceWaitSnapshot(adapter, context);
+ if (shouldAutoContinueForceWait(initial, current)) {
+ return;
+ }
+
+ await sleep(options.interactionPollIntervalMs);
+ }
+ } finally {
+ closeReadline();
+ }
+
+ throw new Error("Timed out waiting for force-mode interaction to complete");
+}
+
+async function waitForInteraction(
+ adapter: Adapter,
+ context: AdapterContext,
+ interaction: WaitForInteractionRequest,
+ options: ConvertCommandOptions,
+): Promise {
+ const timeoutMs = interaction.timeoutMs ?? options.interactionTimeoutMs;
+ const pollIntervalMs = interaction.pollIntervalMs ?? options.interactionPollIntervalMs;
+ if (context.interactive) {
+ await context.browser.bringToFront().catch(() => {});
+ }
+ context.log.info(interaction.prompt);
+
+ const startedAt = Date.now();
+ let lastLogin: AdapterLoginInfo | null = null;
+
+ while (Date.now() - startedAt < timeoutMs) {
+ if (interaction.kind === "login" && adapter.checkLogin) {
+ lastLogin = await adapter.checkLogin(context);
+ if (lastLogin.state === "logged_in" && await isInteractionSessionReady(context, interaction)) {
+ return lastLogin;
+ }
+ }
+
+ const gate = await detectInteractionGate(context.browser);
+ if (!gate) {
+ if (interaction.kind !== "login") {
+ return lastLogin ?? {
+ provider: interaction.provider,
+ state: "unknown",
+ reason: `${interaction.provider} challenge cleared`,
+ };
+ }
+
+ if (!adapter.checkLogin) {
+ return {
+ provider: interaction.provider,
+ state: "unknown",
+ };
+ }
+
+ lastLogin = await adapter.checkLogin(context);
+ if (lastLogin.state !== "logged_out" && await isInteractionSessionReady(context, interaction)) {
+ return lastLogin;
+ }
+ }
+ await sleep(pollIntervalMs);
+ }
+
+ const reason = lastLogin?.reason ? ` (${lastLogin.reason})` : "";
+ throw new Error(`Timed out waiting for ${interaction.provider} interaction${reason}`);
+}
+
+export function formatOutputContent(
+ format: OutputFormat,
+ payload: SuccessfulConvertOutput | InteractionRequiredOutput,
+): string {
+ if (format === "json") {
+ return JSON.stringify(payload, null, 2);
+ }
+
+ if (payload.status !== "ok") {
+ throw new Error("Markdown output is only available for successful extraction results");
+ }
+
+ return payload.markdown;
+}
+
+function printOutput(content: string): void {
+ process.stdout.write(content);
+ if (!content.endsWith("\n")) {
+ process.stdout.write("\n");
+ }
+}
+
+export async function runConvertCommand(options: ConvertCommandOptions): Promise {
+ if (!options.url) {
+ throw new Error("URL is required");
+ }
+ if (options.downloadMedia && !options.output) {
+ throw new Error("--download-media requires --output so media paths can be rewritten relative to the saved output file");
+ }
+
+ const url = normalizeUrl(options.url);
+ let runtime = await openRuntime(options, options.waitMode !== "none", Boolean(options.debugDir));
+ const logger = createLogger(Boolean(options.debugDir));
+ let didLogin = false;
+ let adapter: Adapter | null = null;
+ let context: AdapterContext | null = null;
+
+ try {
+ adapter = resolveAdapter({ url }, options.adapter);
+ context = {
+ input: { url },
+ browser: runtime.browser,
+ network: runtime.network,
+ cdp: runtime.cdp,
+ log: logger,
+ outputFormat: options.format,
+ timeoutMs: options.timeoutMs,
+ interactive: runtime.interactive,
+ downloadMedia: options.downloadMedia,
+ };
+
+ if (adapter.restoreCookies) {
+ const restored = await adapter.restoreCookies(context, runtime.chrome.profileDir).catch(() => false);
+ if (restored) logger.info(`Restored ${adapter.name} session cookies from sidecar.`);
+ }
+
+ if (options.waitMode === "interaction" && adapter.checkLogin) {
+ await context.browser.goto(url.toString(), options.timeoutMs).catch(() => {});
+ const preLogin = await adapter.checkLogin(context);
+ if (preLogin.state !== "logged_in") {
+ didLogin = true;
+ await waitForInteraction(adapter, context, {
+ type: "wait_for_interaction",
+ kind: "login",
+ provider: preLogin.provider ?? adapter.name,
+ prompt: `Please sign in to ${adapter.name === "x" ? "X" : adapter.name} in the opened Chrome window. Extraction will continue automatically once login is detected.`,
+ reason: preLogin.reason ?? `Not logged in to ${adapter.name}`,
+ requiresVisibleBrowser: true,
+ }, options);
+ }
+ }
+
+ if (options.waitMode === "force") {
+ await context.browser.goto(url.toString(), options.timeoutMs).catch(() => {});
+ await waitForForceResume(adapter, context, options);
+ }
+
+ let result = await adapter.process(context);
+
+ if (result.status === "no_document") {
+ const interaction = await detectInteractionGate(context.browser);
+ if (interaction) {
+ result = {
+ status: "needs_interaction",
+ interaction,
+ login: result.login,
+ };
+ }
+ }
+
+ while (result.status === "needs_interaction") {
+ if (options.waitMode === "none") {
+ if (options.format === "json") {
+ printOutput(
+ formatOutputContent(options.format, {
+ adapter: adapter.name,
+ status: result.status,
+ login: result.login,
+ interaction: result.interaction,
+ }),
+ );
+ return;
+ }
+
+ throw new Error(`${adapter.name} requires manual interaction. Re-run with --wait-for interaction to continue after completing it.`);
+ }
+
+ if (result.interaction.requiresVisibleBrowser !== false) {
+ runtime = await reopenInteractiveRuntime(runtime, options, Boolean(options.debugDir));
+ }
+
+ context = {
+ input: { url },
+ browser: runtime.browser,
+ network: runtime.network,
+ cdp: runtime.cdp,
+ log: logger,
+ outputFormat: options.format,
+ timeoutMs: options.timeoutMs,
+ interactive: runtime.interactive,
+ downloadMedia: options.downloadMedia,
+ };
+
+ await context.browser.goto(url.toString(), options.timeoutMs).catch(() => {});
+ if (result.interaction.kind === "login") {
+ didLogin = true;
+ }
+ await waitForInteraction(adapter, context, result.interaction, options);
+ result = await adapter.process(context);
+
+ if (result.status === "no_document") {
+ const interaction = await detectInteractionGate(context.browser);
+ if (interaction) {
+ result = {
+ status: "needs_interaction",
+ interaction,
+ login: result.login,
+ };
+ }
+ }
+ }
+
+ let document: ExtractedDocument | null = result.status === "ok" ? result.document : null;
+ let media: MediaAsset[] = result.status === "ok" ? (result.media ?? []) : [];
+ let login = result.login;
+ let mediaAdapter = adapter;
+
+ if (!document && adapter.name !== genericAdapter.name && result.status === "no_document") {
+ logger.info(`Adapter ${adapter.name} returned no structured document; falling back to generic extraction`);
+ const fallback = await genericAdapter.process(context);
+ if (fallback.status === "ok") {
+ document = fallback.document;
+ media = fallback.media ?? [];
+ mediaAdapter = genericAdapter;
+ }
+ }
+
+ if (!document) {
+ throw new Error("Failed to extract a document from the target URL");
+ }
+
+ document.requestedUrl ??= url.toString();
+
+ let markdown = renderMarkdown(document);
+ let downloadResult:
+ | Awaited>
+ | null = null;
+
+ if (options.downloadMedia && options.output) {
+ downloadResult = mediaAdapter.downloadMedia
+ ? await mediaAdapter.downloadMedia({
+ media,
+ outputPath: options.output,
+ mediaDir: options.mediaDir,
+ log: logger,
+ })
+ : await downloadMediaAssets({
+ media,
+ outputPath: options.output,
+ mediaDir: options.mediaDir,
+ log: logger,
+ });
+
+ markdown = rewriteMarkdownMediaLinks(markdown, downloadResult.replacements);
+ if (downloadResult.downloadedImages > 0 || downloadResult.downloadedVideos > 0) {
+ logger.info(
+ `Downloaded ${downloadResult.downloadedImages} images and ${downloadResult.downloadedVideos} videos`,
+ );
+ }
+ }
+
+ if (options.output) {
+ await writeOutput(
+ options.output,
+ formatOutputContent(options.format, {
+ adapter: document.adapter ?? adapter.name,
+ status: "ok",
+ login,
+ media,
+ downloads: downloadResult,
+ document,
+ markdown,
+ }),
+ );
+ logger.info(`Saved ${options.format} to ${options.output}`);
+ }
+
+ if (options.debugDir) {
+ await writeDebugArtifacts(options.debugDir, document, markdown, runtime.browser, runtime.network);
+ logger.info(`Wrote debug artifacts to ${options.debugDir}`);
+ }
+
+ if (options.format === "json") {
+ printOutput(
+ formatOutputContent(options.format, {
+ adapter: document.adapter ?? adapter.name,
+ status: "ok",
+ login,
+ media,
+ downloads: downloadResult,
+ document,
+ markdown,
+ }),
+ );
+ return;
+ }
+
+ printOutput(markdown);
+ } finally {
+ if (adapter?.exportCookies && context) {
+ await adapter.exportCookies(context, runtime.chrome.profileDir).catch(() => {});
+ }
+ await closeRuntime(runtime);
+ }
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/extract/document.ts b/skills/baoyu-url-to-markdown/scripts/lib/extract/document.ts
new file mode 100644
index 0000000..baaee51
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/extract/document.ts
@@ -0,0 +1,51 @@
+export type ContentBlock =
+ | {
+ type: "paragraph";
+ text: string;
+ }
+ | {
+ type: "heading";
+ depth: number;
+ text: string;
+ }
+ | {
+ type: "list";
+ ordered: boolean;
+ items: string[];
+ }
+ | {
+ type: "quote";
+ text: string;
+ }
+ | {
+ type: "code";
+ code: string;
+ language?: string;
+ }
+ | {
+ type: "image";
+ url: string;
+ alt?: string;
+ }
+ | {
+ type: "html";
+ html: string;
+ }
+ | {
+ type: "markdown";
+ markdown: string;
+ };
+
+export interface ExtractedDocument {
+ url: string;
+ requestedUrl?: string;
+ canonicalUrl?: string;
+ title?: string;
+ author?: string;
+ siteName?: string;
+ publishedAt?: string;
+ summary?: string;
+ content: ContentBlock[];
+ metadata?: Record;
+ adapter?: string;
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/extract/html-cleaner.ts b/skills/baoyu-url-to-markdown/scripts/lib/extract/html-cleaner.ts
new file mode 100644
index 0000000..f7034be
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/extract/html-cleaner.ts
@@ -0,0 +1,467 @@
+import { JSDOM } from "jsdom";
+
+export interface CleanHtmlOptions {
+ removeAds?: boolean;
+ removeBase64Images?: boolean;
+ onlyMainContent?: boolean;
+ includeSelectors?: string[];
+ excludeSelectors?: string[];
+}
+
+const ALWAYS_REMOVE_SELECTORS = [
+ "script",
+ "style",
+ "noscript",
+ "link[rel='stylesheet']",
+ "[hidden]",
+ "[aria-hidden='true']",
+ "[style*='display: none']",
+ "[style*='display:none']",
+ "[style*='visibility: hidden']",
+ "[style*='visibility:hidden']",
+ "svg[aria-hidden='true']",
+ "svg.icon",
+ "svg[class*='icon']",
+ "template",
+ "meta",
+ "iframe",
+ "canvas",
+ "object",
+ "embed",
+ "form",
+ "input",
+ "select",
+ "textarea",
+ "button",
+];
+
+const OVERLAY_SELECTORS = [
+ "[class*='modal']",
+ "[class*='popup']",
+ "[class*='overlay']",
+ "[class*='dialog']",
+ "[role='dialog']",
+ "[role='alertdialog']",
+ "[class*='cookie']",
+ "[class*='consent']",
+ "[class*='gdpr']",
+ "[class*='privacy-banner']",
+ "[class*='notification-bar']",
+ "[id*='cookie']",
+ "[id*='consent']",
+ "[id*='gdpr']",
+ "[style*='position: fixed']",
+ "[style*='position:fixed']",
+ "[style*='position: sticky']",
+ "[style*='position:sticky']",
+];
+
+const NAVIGATION_SELECTORS = [
+ "header",
+ "footer",
+ "nav",
+ "aside",
+ ".header",
+ ".top",
+ ".navbar",
+ "#header",
+ ".footer",
+ ".bottom",
+ "#footer",
+ ".sidebar",
+ ".side",
+ ".aside",
+ "#sidebar",
+ ".modal",
+ ".popup",
+ "#modal",
+ ".overlay",
+ ".ad",
+ ".ads",
+ ".advert",
+ "#ad",
+ ".lang-selector",
+ ".language",
+ "#language-selector",
+ ".social",
+ ".social-media",
+ ".social-links",
+ "#social",
+ ".menu",
+ ".navigation",
+ "#nav",
+ ".breadcrumbs",
+ "#breadcrumbs",
+ ".share",
+ "#share",
+ ".widget",
+ "#widget",
+ ".cookie",
+ "#cookie",
+];
+
+const FORCE_INCLUDE_SELECTORS = [
+ "#main",
+ "#content",
+ "#main-content",
+ "#article",
+ "#post",
+ "#page-content",
+ "main",
+ "article",
+ "[role='main']",
+ ".main-content",
+ ".content",
+ ".post-content",
+ ".article-content",
+ ".entry-content",
+ ".page-content",
+ ".article-body",
+ ".post-body",
+ ".story-content",
+ ".blog-content",
+];
+
+const AD_SELECTORS = [
+ "ins.adsbygoogle",
+ ".google-ad",
+ ".adsense",
+ "[data-ad]",
+ "[data-ads]",
+ "[data-ad-slot]",
+ "[data-ad-client]",
+ ".ad-container",
+ ".ad-wrapper",
+ ".advertisement",
+ ".sponsored-content",
+ "img[width='1'][height='1']",
+ "img[src*='pixel']",
+ "img[src*='tracking']",
+ "img[src*='analytics']",
+];
+
+function getLinkDensity(element: Element): number {
+ const text = element.textContent || "";
+ const textLength = text.trim().length;
+ if (textLength === 0) {
+ return 1;
+ }
+
+ let linkLength = 0;
+ element.querySelectorAll("a").forEach((link) => {
+ linkLength += (link.textContent || "").trim().length;
+ });
+
+ return linkLength / textLength;
+}
+
+function getContentScore(element: Element): number {
+ let score = 0;
+ const text = element.textContent || "";
+ const textLength = text.trim().length;
+
+ score += Math.min(textLength / 100, 50);
+ score += element.querySelectorAll("p").length * 3;
+ score += element.querySelectorAll("h1, h2, h3, h4, h5, h6").length * 2;
+ score += element.querySelectorAll("img").length;
+
+ score -= element.querySelectorAll("a").length * 0.5;
+ score -= element.querySelectorAll("li").length * 0.2;
+
+ const linkDensity = getLinkDensity(element);
+ if (linkDensity > 0.5) {
+ score -= 30;
+ } else if (linkDensity > 0.3) {
+ score -= 15;
+ }
+
+ const className = typeof element.className === "string" ? element.className : "";
+ const classAndId = `${className} ${element.id || ""}`;
+ if (/article|content|post|body|main|entry/i.test(classAndId)) {
+ score += 25;
+ }
+ if (/comment|sidebar|footer|nav|menu|header|widget|ad/i.test(classAndId)) {
+ score -= 25;
+ }
+
+ return score;
+}
+
+function looksLikeNavigation(element: Element): boolean {
+ const linkDensity = getLinkDensity(element);
+ if (linkDensity > 0.5) {
+ return true;
+ }
+
+ const listItems = element.querySelectorAll("li");
+ const links = element.querySelectorAll("a");
+ return listItems.length > 5 && links.length > listItems.length * 0.8;
+}
+
+function removeElements(document: Document, selectors: string[]): void {
+ for (const selector of selectors) {
+ try {
+ document.querySelectorAll(selector).forEach((element) => element.remove());
+ } catch {
+ // Ignore unsupported selectors.
+ }
+ }
+}
+
+function removeWithProtection(
+ document: Document,
+ selectorsToRemove: string[],
+ protectedSelectors: string[],
+): void {
+ for (const selector of selectorsToRemove) {
+ try {
+ document.querySelectorAll(selector).forEach((element) => {
+ const isProtected = protectedSelectors.some((protectedSelector) => {
+ try {
+ return element.matches(protectedSelector);
+ } catch {
+ return false;
+ }
+ });
+
+ if (isProtected) {
+ return;
+ }
+
+ const containsProtected = protectedSelectors.some((protectedSelector) => {
+ try {
+ return element.querySelector(protectedSelector) !== null;
+ } catch {
+ return false;
+ }
+ });
+
+ if (containsProtected) {
+ return;
+ }
+
+ element.remove();
+ });
+ } catch {
+ // Ignore unsupported selectors.
+ }
+ }
+}
+
+function isValidContent(element: Element | null): element is Element {
+ if (!element) {
+ return false;
+ }
+ const text = element.textContent || "";
+ if (text.trim().length < 100) {
+ return false;
+ }
+ return !looksLikeNavigation(element);
+}
+
+function findMainContent(document: Document): Element | null {
+ const main = document.querySelector("main");
+ if (isValidContent(main) && getLinkDensity(main) < 0.4) {
+ return main;
+ }
+
+ const roleMain = document.querySelector('[role="main"]');
+ if (isValidContent(roleMain) && getLinkDensity(roleMain) < 0.4) {
+ return roleMain;
+ }
+
+ const articles = document.querySelectorAll("article");
+ if (articles.length === 1 && isValidContent(articles[0] ?? null)) {
+ return articles[0] ?? null;
+ }
+
+ const contentSelectors = [
+ "#content",
+ "#main-content",
+ "#main",
+ ".content",
+ ".main-content",
+ ".post-content",
+ ".article-content",
+ ".entry-content",
+ ".page-content",
+ ".article-body",
+ ".post-body",
+ ".story-content",
+ ".blog-content",
+ ];
+
+ for (const selector of contentSelectors) {
+ try {
+ const element = document.querySelector(selector);
+ if (isValidContent(element) && getLinkDensity(element) < 0.4) {
+ return element;
+ }
+ } catch {
+ // Ignore invalid selectors.
+ }
+ }
+
+ const candidates: Array<{ element: Element; score: number }> = [];
+ document.querySelectorAll("div, section, article").forEach((element) => {
+ const text = element.textContent || "";
+ if (text.trim().length < 200) {
+ return;
+ }
+
+ const score = getContentScore(element);
+ if (score > 0) {
+ candidates.push({ element, score });
+ }
+ });
+
+ candidates.sort((left, right) => right.score - left.score);
+ if ((candidates[0]?.score ?? 0) > 20) {
+ return candidates[0]?.element ?? null;
+ }
+
+ return null;
+}
+
+function removeBase64ImagesFromDocument(document: Document): void {
+ document.querySelectorAll("img[src^='data:']").forEach((element) => element.remove());
+
+ document.querySelectorAll("[style*='data:image']").forEach((element) => {
+ const style = element.getAttribute("style");
+ if (!style) {
+ return;
+ }
+
+ const cleanedStyle = style.replace(
+ /background(-image)?:\s*url\([^)]*data:image[^)]*\)[^;]*;?/gi,
+ "",
+ );
+
+ if (cleanedStyle.trim()) {
+ element.setAttribute("style", cleanedStyle);
+ } else {
+ element.removeAttribute("style");
+ }
+ });
+
+ document
+ .querySelectorAll("source[src^='data:'], source[srcset*='data:']")
+ .forEach((element) => element.remove());
+}
+
+function makeAbsoluteUrl(value: string, baseUrl: string): string | null {
+ try {
+ return new URL(value, baseUrl).toString();
+ } catch {
+ return null;
+ }
+}
+
+function convertRelativeUrls(document: Document, baseUrl: string): void {
+ document.querySelectorAll("[src]").forEach((element) => {
+ const src = element.getAttribute("src");
+ if (!src || src.startsWith("http") || src.startsWith("//") || src.startsWith("data:")) {
+ return;
+ }
+
+ const absolute = makeAbsoluteUrl(src, baseUrl);
+ if (absolute) {
+ element.setAttribute("src", absolute);
+ }
+ });
+
+ document.querySelectorAll("[href]").forEach((element) => {
+ const href = element.getAttribute("href");
+ if (
+ !href ||
+ href.startsWith("http") ||
+ href.startsWith("//") ||
+ href.startsWith("#") ||
+ href.startsWith("mailto:") ||
+ href.startsWith("tel:") ||
+ href.startsWith("javascript:")
+ ) {
+ return;
+ }
+
+ const absolute = makeAbsoluteUrl(href, baseUrl);
+ if (absolute) {
+ element.setAttribute("href", absolute);
+ }
+ });
+}
+
+function removeComments(document: Document): void {
+ const walker = document.createTreeWalker(document, document.defaultView?.NodeFilter.SHOW_COMMENT ?? 128);
+ const comments: Comment[] = [];
+ while (walker.nextNode()) {
+ comments.push(walker.currentNode as Comment);
+ }
+ comments.forEach((comment) => comment.parentNode?.removeChild(comment));
+}
+
+export function cleanHtml(
+ html: string,
+ baseUrl: string,
+ options: CleanHtmlOptions = {},
+): string {
+ const {
+ removeAds = true,
+ removeBase64Images = true,
+ onlyMainContent = true,
+ includeSelectors,
+ excludeSelectors,
+ } = options;
+
+ const dom = new JSDOM(html, { url: baseUrl });
+ const { document } = dom.window;
+
+ removeElements(document, ALWAYS_REMOVE_SELECTORS);
+ removeElements(document, OVERLAY_SELECTORS);
+
+ if (removeAds) {
+ removeElements(document, AD_SELECTORS);
+ }
+
+ if (excludeSelectors?.length) {
+ removeElements(document, excludeSelectors);
+ }
+
+ if (onlyMainContent) {
+ removeWithProtection(document, NAVIGATION_SELECTORS, FORCE_INCLUDE_SELECTORS);
+
+ const mainContent = findMainContent(document);
+ if (mainContent && document.body) {
+ const clone = mainContent.cloneNode(true);
+ document.body.innerHTML = "";
+ document.body.appendChild(clone);
+ }
+ }
+
+ if (includeSelectors?.length && document.body) {
+ const matchedElements: Element[] = [];
+ for (const selector of includeSelectors) {
+ try {
+ document.querySelectorAll(selector).forEach((element) => {
+ matchedElements.push(element.cloneNode(true) as Element);
+ });
+ } catch {
+ // Ignore invalid selectors.
+ }
+ }
+
+ if (matchedElements.length > 0) {
+ document.body.innerHTML = "";
+ matchedElements.forEach((element) => document.body?.appendChild(element));
+ }
+ }
+
+ if (removeBase64Images) {
+ removeBase64ImagesFromDocument(document);
+ }
+
+ removeComments(document);
+ convertRelativeUrls(document, baseUrl);
+
+ return document.documentElement.outerHTML || html;
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/extract/html-extractor.ts b/skills/baoyu-url-to-markdown/scripts/lib/extract/html-extractor.ts
new file mode 100644
index 0000000..cbd7ef4
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/extract/html-extractor.ts
@@ -0,0 +1,83 @@
+import { Readability } from "@mozilla/readability";
+import { JSDOM } from "jsdom";
+import type { ExtractedDocument } from "./document";
+
+function getMetaContent(document: Document, selectors: string[]): string | undefined {
+ for (const selector of selectors) {
+ const value = document.querySelector(selector)?.getAttribute("content")?.trim();
+ if (value) {
+ return value;
+ }
+ }
+ return undefined;
+}
+
+export function extractDocumentFromHtml(input: {
+ url: string;
+ html: string;
+ adapter?: string;
+}): ExtractedDocument {
+ const dom = new JSDOM(input.html, { url: input.url });
+ const document = dom.window.document;
+
+ const canonicalUrl =
+ document.querySelector('link[rel="canonical"]')?.getAttribute("href")?.trim() ??
+ getMetaContent(document, ['meta[property="og:url"]']);
+
+ const siteName = getMetaContent(document, [
+ 'meta[property="og:site_name"]',
+ 'meta[name="application-name"]',
+ ]);
+
+ const metadataAuthor = getMetaContent(document, [
+ 'meta[name="author"]',
+ 'meta[property="article:author"]',
+ 'meta[name="twitter:creator"]',
+ ]);
+
+ const publishedAt = getMetaContent(document, [
+ 'meta[property="article:published_time"]',
+ 'meta[name="pubdate"]',
+ 'meta[name="date"]',
+ 'meta[itemprop="datePublished"]',
+ ]);
+
+ const article = new Readability(document).parse();
+ const title =
+ article?.title?.trim() ||
+ getMetaContent(document, ['meta[property="og:title"]']) ||
+ document.title.trim() ||
+ undefined;
+
+ const summary =
+ article?.excerpt?.trim() ||
+ getMetaContent(document, [
+ 'meta[name="description"]',
+ 'meta[property="og:description"]',
+ 'meta[name="twitter:description"]',
+ ]);
+
+ const contentHtml =
+ article?.content?.trim() ||
+ document.querySelector("main")?.innerHTML?.trim() ||
+ document.body?.innerHTML?.trim() ||
+ "";
+
+ const author = article?.byline?.trim() || metadataAuthor;
+
+ return {
+ url: input.url,
+ canonicalUrl,
+ title,
+ author,
+ siteName,
+ publishedAt,
+ summary,
+ adapter: input.adapter ?? "generic",
+ metadata: {
+ language: document.documentElement.lang || undefined,
+ },
+ content: contentHtml ? [{ type: "html", html: contentHtml }] : [],
+ };
+}
+
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/extract/html-to-markdown.ts b/skills/baoyu-url-to-markdown/scripts/lib/extract/html-to-markdown.ts
new file mode 100644
index 0000000..56aad98
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/extract/html-to-markdown.ts
@@ -0,0 +1,758 @@
+import { Readability } from "@mozilla/readability";
+import { Defuddle } from "defuddle/node";
+import { JSDOM, VirtualConsole } from "jsdom";
+import TurndownService from "turndown";
+import { gfm } from "turndown-plugin-gfm";
+import { collectMediaFromMarkdown } from "../media/markdown-media";
+import type { MediaAsset } from "../media/types";
+import { cleanHtml } from "./html-cleaner";
+
+export interface HtmlConversionMetadata {
+ url: string;
+ canonicalUrl?: string;
+ siteName?: string;
+ title?: string;
+ summary?: string;
+ author?: string;
+ publishedAt?: string;
+ coverImage?: string;
+ language?: string;
+ capturedAt: string;
+}
+
+export interface ConvertHtmlToMarkdownOptions {
+ enableRemoteMarkdownFallback?: boolean;
+ preserveBase64Images?: boolean;
+}
+
+export interface HtmlToMarkdownResult {
+ metadata: HtmlConversionMetadata;
+ markdown: string;
+ rawHtml: string;
+ cleanedHtml: string;
+ media: MediaAsset[];
+ conversionMethod: string;
+ fallbackReason?: string;
+}
+
+type JsonObject = Record;
+
+const MIN_CONTENT_LENGTH = 120;
+const DEFUDDLE_API_ORIGIN = "https://defuddle.md";
+const LOCAL_FALLBACK_SCORE_DELTA = 120;
+const REMOTE_FALLBACK_SCORE_DELTA = 20;
+const LOW_QUALITY_MARKERS = [
+ /Join The Conversation/i,
+ /One Community\. Many Voices/i,
+ /Read our community guidelines/i,
+ /Create a free account to share your thoughts/i,
+ /Become a Forbes Member/i,
+ /Subscribe to trusted journalism/i,
+ /\bComments\b/i,
+];
+
+const ARTICLE_TYPES = new Set([
+ "Article",
+ "NewsArticle",
+ "BlogPosting",
+ "WebPage",
+ "ReportageNewsArticle",
+]);
+
+const turndown = new TurndownService({
+ headingStyle: "atx",
+ bulletListMarker: "-",
+ codeBlockStyle: "fenced",
+}) as TurndownService & {
+ remove(selectors: string[]): void;
+ addRule(
+ key: string,
+ rule: {
+ filter: string | ((node: Node) => boolean);
+ replacement: (content: string) => string;
+ },
+ ): void;
+};
+
+turndown.use(gfm);
+turndown.remove(["script", "style", "iframe", "noscript", "template", "svg", "path"]);
+turndown.addRule("collapseFigure", {
+ filter: "figure",
+ replacement(content: string) {
+ return `\n\n${content.trim()}\n\n`;
+ },
+});
+turndown.addRule("dropInvisibleAnchors", {
+ filter(node: Node) {
+ return (
+ node.nodeName === "A" &&
+ !(node as Element).textContent?.trim() &&
+ !(node as Element).querySelector("img, video, picture, source")
+ );
+ },
+ replacement() {
+ return "";
+ },
+});
+
+function pickString(...values: unknown[]): string | undefined {
+ for (const value of values) {
+ if (typeof value !== "string") {
+ continue;
+ }
+ const trimmed = value.trim();
+ if (trimmed) {
+ return trimmed;
+ }
+ }
+ return undefined;
+}
+
+function normalizeMarkdown(markdown: string): string {
+ return markdown
+ .replace(/\r\n/g, "\n")
+ .replace(/[ \t]+\n/g, "\n")
+ .replace(/\n{3,}/g, "\n\n")
+ .trim();
+}
+
+function stripWrappingQuotes(value: string): string {
+ const trimmed = value.trim();
+ if (
+ (trimmed.startsWith('"') && trimmed.endsWith('"')) ||
+ (trimmed.startsWith("'") && trimmed.endsWith("'"))
+ ) {
+ return trimmed.slice(1, -1).trim();
+ }
+ return trimmed;
+}
+
+function stripMarkdownFrontmatter(markdown: string): string {
+ return markdown.replace(/^\uFEFF?---\n[\s\S]*?\n---(?:\n|$)/, "").trim();
+}
+
+function cleanMarkdownTitle(value: string): string | undefined {
+ const cleaned = stripWrappingQuotes(
+ value
+ .replace(/\s+#+\s*$/, "")
+ .replace(/!\[[^\]]*\]\([^)]+\)/g, "")
+ .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
+ .replace(/[*_`~]/g, "")
+ .trim(),
+ );
+
+ return cleaned || undefined;
+}
+
+export function extractTitleFromMarkdownDocument(markdown: string): string | undefined {
+ const normalized = markdown.replace(/\r\n/g, "\n").trim();
+ if (!normalized) {
+ return undefined;
+ }
+
+ const frontmatterMatch = normalized.match(/^\uFEFF?---\n([\s\S]*?)\n---(?:\n|$)/);
+ if (frontmatterMatch) {
+ for (const line of frontmatterMatch[1].split("\n")) {
+ const match = line.match(/^title:\s*(.+?)\s*$/i);
+ if (!match) {
+ continue;
+ }
+
+ const title = cleanMarkdownTitle(match[1]);
+ if (title) {
+ return title;
+ }
+ }
+ }
+
+ const body = stripMarkdownFrontmatter(normalized);
+ const headingMatch = body.match(/^#{1,6}\s+(.+)$/m);
+ if (!headingMatch) {
+ return undefined;
+ }
+
+ return cleanMarkdownTitle(headingMatch[1]);
+}
+
+function trimKnownBoilerplate(markdown: string): string {
+ const normalized = normalizeMarkdown(markdown);
+ const lines = normalized.split("\n");
+
+ while (lines.length > 0) {
+ const lastLine = lines[lines.length - 1]?.trim();
+ if (!lastLine) {
+ lines.pop();
+ continue;
+ }
+
+ if (/^继续滑动看下一个$/.test(lastLine) || /^轻触阅读原文$/.test(lastLine)) {
+ lines.pop();
+ continue;
+ }
+
+ break;
+ }
+
+ return normalizeMarkdown(lines.join("\n"));
+}
+
+function buildDefuddleApiUrl(targetUrl: string): string {
+ return `${DEFUDDLE_API_ORIGIN}/${encodeURIComponent(targetUrl)}`;
+}
+
+async function fetchDefuddleApiMarkdown(
+ targetUrl: string,
+): Promise<{ markdown: string; title?: string }> {
+ const response = await fetch(buildDefuddleApiUrl(targetUrl), {
+ headers: {
+ accept: "text/markdown,text/plain;q=0.9,*/*;q=0.1",
+ },
+ redirect: "follow",
+ });
+
+ if (!response.ok) {
+ throw new Error(`defuddle.md returned ${response.status} ${response.statusText}`);
+ }
+
+ const rawMarkdown = (await response.text()).replace(/\r\n/g, "\n").trim();
+ if (!rawMarkdown) {
+ throw new Error("defuddle.md returned empty markdown");
+ }
+
+ const title = extractTitleFromMarkdownDocument(rawMarkdown);
+ const markdown = trimKnownBoilerplate(stripMarkdownFrontmatter(rawMarkdown));
+ if (!markdown) {
+ throw new Error("defuddle.md returned empty markdown");
+ }
+
+ return {
+ markdown,
+ title,
+ };
+}
+
+function sanitizeHtmlFragment(html: string): string {
+ const dom = new JSDOM(`${html}
`);
+ const root = dom.window.document.querySelector("#__root");
+ if (!root) {
+ return html;
+ }
+
+ for (const selector of ["script", "style", "iframe", "noscript", "template", "svg", "path"]) {
+ root.querySelectorAll(selector).forEach((element) => element.remove());
+ }
+
+ return root.innerHTML;
+}
+
+function extractTextFromHtml(html: string): string {
+ const dom = new JSDOM(`${html}`);
+ const { document } = dom.window;
+ for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) {
+ document.querySelectorAll(selector).forEach((element) => element.remove());
+ }
+ return document.body?.textContent?.replace(/\s+/g, " ").trim() ?? "";
+}
+
+function getMetaContent(document: Document, names: string[]): string | undefined {
+ for (const name of names) {
+ const element =
+ document.querySelector(`meta[name="${name}"]`) ??
+ document.querySelector(`meta[property="${name}"]`);
+ const content = element?.getAttribute("content")?.trim();
+ if (content) {
+ return content;
+ }
+ }
+ return undefined;
+}
+
+function normalizeLanguageTag(value: string | null | undefined): string | undefined {
+ if (!value) {
+ return undefined;
+ }
+
+ const trimmed = value.trim();
+ if (!trimmed) {
+ return undefined;
+ }
+
+ const primary = trimmed.split(/[,\s;]/, 1)[0]?.trim();
+ if (!primary) {
+ return undefined;
+ }
+
+ return primary.replace(/_/g, "-");
+}
+
+function flattenJsonLdItems(data: unknown): JsonObject[] {
+ if (!data || typeof data !== "object") {
+ return [];
+ }
+
+ if (Array.isArray(data)) {
+ return data.flatMap(flattenJsonLdItems);
+ }
+
+ const item = data as JsonObject;
+ if (Array.isArray(item["@graph"])) {
+ return (item["@graph"] as unknown[]).flatMap(flattenJsonLdItems);
+ }
+
+ return [item];
+}
+
+function parseJsonLdScripts(document: Document): JsonObject[] {
+ const results: JsonObject[] = [];
+ document.querySelectorAll("script[type='application/ld+json']").forEach((script) => {
+ try {
+ const data = JSON.parse(script.textContent ?? "");
+ results.push(...flattenJsonLdItems(data));
+ } catch {
+ // Ignore malformed json-ld blocks.
+ }
+ });
+ return results;
+}
+
+function extractAuthorFromJsonLd(authorData: unknown): string | undefined {
+ if (typeof authorData === "string") {
+ return authorData.trim() || undefined;
+ }
+
+ if (!authorData || typeof authorData !== "object") {
+ return undefined;
+ }
+
+ if (Array.isArray(authorData)) {
+ return authorData
+ .map((author) => extractAuthorFromJsonLd(author))
+ .filter((value): value is string => Boolean(value))
+ .join(", ") || undefined;
+ }
+
+ const author = authorData as JsonObject;
+ return pickString(author.name);
+}
+
+function extractPrimaryJsonLdMeta(document: Document): Partial {
+ for (const item of parseJsonLdScripts(document)) {
+ const type = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"];
+ if (typeof type !== "string" || !ARTICLE_TYPES.has(type)) {
+ continue;
+ }
+
+ return {
+ title: pickString(item.headline, item.name),
+ summary: pickString(item.description),
+ author: extractAuthorFromJsonLd(item.author),
+ publishedAt: pickString(item.datePublished, item.dateCreated),
+ coverImage: pickString(
+ item.image,
+ (item.image as JsonObject | undefined)?.url,
+ Array.isArray(item.image) ? item.image[0] : undefined,
+ ),
+ };
+ }
+
+ return {};
+}
+
+function extractPageMetadata(
+ html: string,
+ url: string,
+ capturedAt: string,
+): HtmlConversionMetadata {
+ const dom = new JSDOM(html, { url });
+ const { document } = dom.window;
+ const jsonLd = extractPrimaryJsonLdMeta(document);
+
+ return {
+ url,
+ canonicalUrl:
+ document.querySelector('link[rel="canonical"]')?.getAttribute("href")?.trim() ??
+ getMetaContent(document, ["og:url"]),
+ siteName: pickString(
+ getMetaContent(document, ["og:site_name"]),
+ document.querySelector('meta[name="application-name"]')?.getAttribute("content"),
+ ),
+ title: pickString(
+ getMetaContent(document, ["og:title", "twitter:title"]),
+ jsonLd.title,
+ document.querySelector("h1")?.textContent,
+ document.title,
+ ),
+ summary: pickString(
+ getMetaContent(document, ["description", "og:description", "twitter:description"]),
+ jsonLd.summary,
+ ),
+ author: pickString(
+ getMetaContent(document, ["author", "article:author", "twitter:creator"]),
+ jsonLd.author,
+ ),
+ publishedAt: pickString(
+ document.querySelector("time[datetime]")?.getAttribute("datetime"),
+ getMetaContent(document, ["article:published_time", "datePublished", "publishdate", "date"]),
+ jsonLd.publishedAt,
+ ),
+ coverImage: pickString(
+ getMetaContent(document, ["og:image", "twitter:image", "twitter:image:src"]),
+ jsonLd.coverImage,
+ ),
+ language: pickString(
+ normalizeLanguageTag(document.documentElement.getAttribute("lang")),
+ normalizeLanguageTag(
+ pickString(
+ getMetaContent(document, ["language", "content-language", "og:locale"]),
+ document.querySelector("meta[http-equiv='content-language']")?.getAttribute("content"),
+ ),
+ ),
+ ),
+ capturedAt,
+ };
+}
+
+function isMarkdownUsable(markdown: string, html: string): boolean {
+ const normalized = normalizeMarkdown(markdown);
+ if (!normalized) {
+ return false;
+ }
+
+ const htmlTextLength = extractTextFromHtml(html).length;
+ if (htmlTextLength < MIN_CONTENT_LENGTH) {
+ return true;
+ }
+
+ if (normalized.length >= 80) {
+ return true;
+ }
+
+ return normalized.length >= Math.min(200, Math.floor(htmlTextLength * 0.2));
+}
+
+function countMarkerHits(markdown: string, markers: RegExp[]): number {
+ let hits = 0;
+ for (const marker of markers) {
+ if (marker.test(markdown)) {
+ hits += 1;
+ }
+ }
+ return hits;
+}
+
+function countUsefulParagraphs(markdown: string): number {
+ const paragraphs = normalizeMarkdown(markdown).split(/\n{2,}/);
+ let count = 0;
+
+ for (const paragraph of paragraphs) {
+ const trimmed = paragraph.trim();
+ if (!trimmed) {
+ continue;
+ }
+ if (/^!?\[[^\]]*\]\([^)]+\)$/.test(trimmed)) {
+ continue;
+ }
+ if (/^#{1,6}\s+/.test(trimmed)) {
+ continue;
+ }
+ if ((trimmed.match(/\b[\p{L}\p{N}']+\b/gu) || []).length < 8) {
+ continue;
+ }
+ count += 1;
+ }
+
+ return count;
+}
+
+function scoreMarkdownQuality(markdown: string): number {
+ const normalized = normalizeMarkdown(markdown);
+ const wordCount = (normalized.match(/\b[\p{L}\p{N}']+\b/gu) || []).length;
+ const usefulParagraphs = countUsefulParagraphs(normalized);
+ const headingCount = (normalized.match(/^#{1,6}\s+/gm) || []).length;
+ const markerHits = countMarkerHits(normalized, LOW_QUALITY_MARKERS);
+ return Math.min(wordCount, 4000) + usefulParagraphs * 40 + headingCount * 10 - markerHits * 180;
+}
+
+function shouldCompareWithFallback(markdown: string): boolean {
+ const normalized = normalizeMarkdown(markdown);
+ return countMarkerHits(normalized, LOW_QUALITY_MARKERS) > 0 || countUsefulParagraphs(normalized) < 6;
+}
+
+function hasMeaningfulMarkdownStructure(markdown: string): boolean {
+ const normalized = normalizeMarkdown(markdown);
+ if (!normalized) {
+ return false;
+ }
+
+ return (
+ countUsefulParagraphs(normalized) > 0 ||
+ /^#{1,6}\s+/m.test(normalized) ||
+ /^[-*]\s+/m.test(normalized) ||
+ /^\d+\.\s+/m.test(normalized) ||
+ /!\[[^\]]*\]\([^)]+\)/.test(normalized)
+ );
+}
+
+function shouldTryRemoteMarkdownFallback(
+ markdown: string,
+ html: string,
+ options: ConvertHtmlToMarkdownOptions,
+): boolean {
+ if (!options.enableRemoteMarkdownFallback) {
+ return false;
+ }
+
+ return !isMarkdownUsable(markdown, html) || shouldCompareWithFallback(markdown);
+}
+
+function shouldPreferRemoteMarkdown(
+ current: HtmlToMarkdownResult,
+ remote: HtmlToMarkdownResult,
+ html: string,
+): boolean {
+ if (!isMarkdownUsable(current.markdown, html)) {
+ return true;
+ }
+
+ if (!hasMeaningfulMarkdownStructure(current.markdown) && hasMeaningfulMarkdownStructure(remote.markdown)) {
+ return true;
+ }
+
+ return scoreMarkdownQuality(remote.markdown) > scoreMarkdownQuality(current.markdown) + REMOTE_FALLBACK_SCORE_DELTA;
+}
+
+function buildRemoteFallbackReason(current: HtmlToMarkdownResult, html: string): string {
+ if (!isMarkdownUsable(current.markdown, html)) {
+ return current.fallbackReason
+ ? `Used defuddle.md markdown fallback after local extraction failed: ${current.fallbackReason}`
+ : "Used defuddle.md markdown fallback after local extraction returned empty or incomplete markdown";
+ }
+
+ return "defuddle.md produced higher-quality markdown than local extraction";
+}
+
+async function tryDefuddleConversion(
+ html: string,
+ url: string,
+ baseMetadata: HtmlConversionMetadata,
+): Promise<{ ok: true; result: HtmlToMarkdownResult } | { ok: false; reason: string }> {
+ try {
+ const virtualConsole = new VirtualConsole();
+ virtualConsole.on("jsdomError", (error: Error & { type?: string }) => {
+ if (error.type === "css parsing" || /Could not parse CSS stylesheet/i.test(error.message)) {
+ return;
+ }
+ });
+
+ const dom = new JSDOM(html, { url, virtualConsole });
+ const result = await Defuddle(dom, url, { markdown: true });
+ const markdown = trimKnownBoilerplate(result.content || "");
+
+ if (!isMarkdownUsable(markdown, html)) {
+ return { ok: false, reason: "Defuddle returned empty or incomplete markdown" };
+ }
+
+ const metadata: HtmlConversionMetadata = {
+ ...baseMetadata,
+ title: pickString(result.title, baseMetadata.title),
+ summary: pickString(result.description, baseMetadata.summary),
+ author: pickString(result.author, baseMetadata.author),
+ publishedAt: pickString(result.published, baseMetadata.publishedAt),
+ coverImage: pickString(result.image, baseMetadata.coverImage),
+ language: pickString(result.language, baseMetadata.language),
+ };
+
+ return {
+ ok: true,
+ result: {
+ metadata,
+ markdown,
+ rawHtml: html,
+ cleanedHtml: html,
+ media: collectMediaFromMarkdown(markdown).concat(
+ metadata.coverImage
+ ? [{ url: metadata.coverImage, kind: "image", role: "cover" as const }]
+ : [],
+ ),
+ conversionMethod: "defuddle",
+ },
+ };
+ } catch (error) {
+ return {
+ ok: false,
+ reason: error instanceof Error ? error.message : String(error),
+ };
+ }
+}
+
+async function tryDefuddleApiConversion(
+ html: string,
+ url: string,
+ baseMetadata: HtmlConversionMetadata,
+): Promise<{ ok: true; result: HtmlToMarkdownResult } | { ok: false; reason: string }> {
+ try {
+ const result = await fetchDefuddleApiMarkdown(url);
+ const markdown = result.markdown;
+
+ if (!isMarkdownUsable(markdown, html) && scoreMarkdownQuality(markdown) < 80) {
+ return { ok: false, reason: "defuddle.md returned empty or incomplete markdown" };
+ }
+
+ const metadata: HtmlConversionMetadata = {
+ ...baseMetadata,
+ title: pickString(result.title, baseMetadata.title),
+ };
+
+ return {
+ ok: true,
+ result: {
+ metadata,
+ markdown,
+ rawHtml: html,
+ cleanedHtml: html,
+ media: collectMediaFromMarkdown(markdown).concat(
+ metadata.coverImage
+ ? [{ url: metadata.coverImage, kind: "image", role: "cover" as const }]
+ : [],
+ ),
+ conversionMethod: "defuddle-api",
+ },
+ };
+ } catch (error) {
+ return {
+ ok: false,
+ reason: error instanceof Error ? error.message : String(error),
+ };
+ }
+}
+
+function convertHtmlFragmentToMarkdown(html: string): string {
+ if (!html.trim()) {
+ return "";
+ }
+
+ try {
+ return turndown.turndown(sanitizeHtmlFragment(html));
+ } catch {
+ return "";
+ }
+}
+
+function fallbackPlainText(html: string): string {
+ return trimKnownBoilerplate(extractTextFromHtml(html));
+}
+
+function convertWithReadability(
+ rawHtml: string,
+ cleanedHtml: string,
+ url: string,
+ baseMetadata: HtmlConversionMetadata,
+): HtmlToMarkdownResult {
+ const dom = new JSDOM(cleanedHtml, { url });
+ const document = dom.window.document;
+ const article = new Readability(document).parse();
+
+ const contentHtml =
+ article?.content?.trim() ??
+ document.querySelector("main")?.innerHTML?.trim() ??
+ document.body?.innerHTML?.trim() ??
+ "";
+
+ let markdown = contentHtml ? convertHtmlFragmentToMarkdown(contentHtml) : "";
+ if (!markdown) {
+ markdown = fallbackPlainText(cleanedHtml);
+ }
+
+ const metadata: HtmlConversionMetadata = {
+ ...baseMetadata,
+ title: pickString(article?.title, baseMetadata.title),
+ summary: pickString(article?.excerpt, baseMetadata.summary),
+ author: pickString(article?.byline, baseMetadata.author),
+ };
+
+ const media = collectMediaFromMarkdown(markdown);
+ if (metadata.coverImage) {
+ media.unshift({
+ url: metadata.coverImage,
+ kind: "image",
+ role: "cover",
+ });
+ }
+
+ return {
+ metadata,
+ markdown: trimKnownBoilerplate(markdown),
+ rawHtml,
+ cleanedHtml,
+ media,
+ conversionMethod: article?.content ? "legacy:readability" : "legacy:body",
+ };
+}
+
+export async function convertHtmlToMarkdown(
+ html: string,
+ url: string,
+ options: ConvertHtmlToMarkdownOptions = {},
+): Promise {
+ const capturedAt = new Date().toISOString();
+ const baseMetadata = extractPageMetadata(html, url, capturedAt);
+
+ let cleanedHtml = html;
+ try {
+ cleanedHtml = cleanHtml(html, url, {
+ removeBase64Images: !options.preserveBase64Images,
+ });
+ } catch {
+ cleanedHtml = html;
+ }
+
+ let selectedResult: HtmlToMarkdownResult;
+ const defuddleResult = await tryDefuddleConversion(cleanedHtml, url, baseMetadata);
+ if (defuddleResult.ok) {
+ if (shouldCompareWithFallback(defuddleResult.result.markdown)) {
+ const fallbackResult = convertWithReadability(html, cleanedHtml, url, baseMetadata);
+ if (
+ scoreMarkdownQuality(fallbackResult.markdown) >
+ scoreMarkdownQuality(defuddleResult.result.markdown) + LOCAL_FALLBACK_SCORE_DELTA
+ ) {
+ selectedResult = {
+ ...fallbackResult,
+ fallbackReason: "Readability/Turndown produced higher-quality markdown than Defuddle",
+ };
+ } else {
+ selectedResult = {
+ ...defuddleResult.result,
+ rawHtml: html,
+ cleanedHtml,
+ };
+ }
+ } else {
+ selectedResult = {
+ ...defuddleResult.result,
+ rawHtml: html,
+ cleanedHtml,
+ };
+ }
+ } else {
+ selectedResult = {
+ ...convertWithReadability(html, cleanedHtml, url, baseMetadata),
+ fallbackReason: defuddleResult.reason,
+ };
+ }
+
+ if (!shouldTryRemoteMarkdownFallback(selectedResult.markdown, cleanedHtml, options)) {
+ return selectedResult;
+ }
+
+ const remoteDefuddleResult = await tryDefuddleApiConversion(cleanedHtml, url, baseMetadata);
+ if (!remoteDefuddleResult.ok || !shouldPreferRemoteMarkdown(selectedResult, remoteDefuddleResult.result, cleanedHtml)) {
+ return selectedResult;
+ }
+
+ return {
+ ...remoteDefuddleResult.result,
+ rawHtml: html,
+ cleanedHtml,
+ fallbackReason: buildRemoteFallbackReason(selectedResult, cleanedHtml),
+ };
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/extract/markdown-renderer.ts b/skills/baoyu-url-to-markdown/scripts/lib/extract/markdown-renderer.ts
new file mode 100644
index 0000000..e843583
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/extract/markdown-renderer.ts
@@ -0,0 +1,169 @@
+import TurndownService from "turndown";
+import { gfm } from "turndown-plugin-gfm";
+import { normalizeMarkdownMediaLinks } from "../media/markdown-media";
+import type { ContentBlock, ExtractedDocument } from "./document";
+
+const turndownService = new TurndownService({
+ codeBlockStyle: "fenced",
+ headingStyle: "atx",
+ bulletListMarker: "-",
+});
+
+turndownService.use(gfm);
+
+function renderBlock(block: ContentBlock): string {
+ switch (block.type) {
+ case "paragraph":
+ return block.text.trim();
+ case "heading":
+ return `${"#".repeat(Math.min(Math.max(block.depth, 1), 6))} ${block.text.trim()}`;
+ case "list":
+ return block.items
+ .map((item, index) => (block.ordered ? `${index + 1}. ${item.trim()}` : `- ${item.trim()}`))
+ .join("\n");
+ case "quote":
+ return block.text
+ .split("\n")
+ .map((line) => `> ${line}`)
+ .join("\n");
+ case "code":
+ return `\`\`\`${block.language ?? ""}\n${block.code.trimEnd()}\n\`\`\``;
+ case "image":
+ return ``;
+ case "html":
+ return turndownService.turndown(block.html).trim();
+ case "markdown":
+ return block.markdown.trim();
+ }
+}
+
+function isDefinedValue(value: unknown): boolean {
+ return value !== undefined && value !== null && value !== "";
+}
+
+function renderFrontmatterValue(value: unknown): string {
+ if (typeof value === "string") {
+ if (value.includes("\n")) {
+ return `|-\n${value
+ .replace(/\r\n/g, "\n")
+ .split("\n")
+ .map((line) => ` ${line}`)
+ .join("\n")}`;
+ }
+ return JSON.stringify(value);
+ }
+ if (typeof value === "number" || typeof value === "boolean") {
+ return String(value);
+ }
+ return JSON.stringify(value);
+}
+
+function renderFrontmatter(document: ExtractedDocument): string {
+ const fields = new Map();
+ const preferredOrder = [
+ "title",
+ "url",
+ "requestedUrl",
+ "author",
+ "authorName",
+ "authorUsername",
+ "authorUrl",
+ "coverImage",
+ "siteName",
+ "publishedAt",
+ "summary",
+ "adapter",
+ ];
+
+ fields.set("title", document.title);
+ fields.set("url", document.canonicalUrl ?? document.url);
+ fields.set("requestedUrl", document.requestedUrl ?? document.url);
+ fields.set("author", document.author);
+ fields.set("siteName", document.siteName);
+ fields.set("publishedAt", document.publishedAt);
+ fields.set("summary", document.summary);
+ fields.set("adapter", document.adapter);
+
+ for (const [key, value] of Object.entries(document.metadata ?? {})) {
+ if (!fields.has(key)) {
+ fields.set(key, value);
+ }
+ }
+
+ const orderedKeys = [
+ ...preferredOrder.filter((key) => fields.has(key)),
+ ...Array.from(fields.keys()).filter((key) => !preferredOrder.includes(key)).sort(),
+ ];
+
+ const lines = orderedKeys
+ .map((key) => [key, fields.get(key)] as const)
+ .filter(([, value]) => isDefinedValue(value))
+ .map(([key, value]) => `${key}: ${renderFrontmatterValue(value)}`);
+
+ if (lines.length === 0) {
+ return "";
+ }
+
+ return `---\n${lines.join("\n")}\n---`;
+}
+
+function cleanMarkdown(markdown: string): string {
+ return normalizeMarkdownMediaLinks(markdown.replace(/\n{3,}/g, "\n\n").trim());
+}
+
+function normalizeComparableTitle(value: string): string {
+ return value
+ .trim()
+ .toLowerCase()
+ .replace(/^>\s*/, "")
+ .replace(/^#+\s+/, "")
+ .replace(/(?:\.{3}|…)\s*$/, "");
+}
+
+function bodyStartsWithTitle(body: string, title: string): boolean {
+ const firstMeaningfulLine = body
+ .replace(/\r\n/g, "\n")
+ .split("\n")
+ .map((line) => line.trim())
+ .find((line) => line && !/^!?\[[^\]]*\]\([^)]+\)$/.test(line));
+
+ if (!firstMeaningfulLine) {
+ return false;
+ }
+
+ const comparableTitle = normalizeComparableTitle(title);
+ const comparableFirstLine = normalizeComparableTitle(firstMeaningfulLine);
+ if (!comparableTitle || !comparableFirstLine) {
+ return false;
+ }
+
+ return (
+ comparableFirstLine === comparableTitle ||
+ comparableFirstLine.startsWith(comparableTitle) ||
+ comparableTitle.startsWith(comparableFirstLine)
+ );
+}
+
+export function renderMarkdown(document: ExtractedDocument): string {
+ const sections: string[] = [];
+ const frontmatter = renderFrontmatter(document);
+
+ if (frontmatter) {
+ sections.push(frontmatter);
+ }
+
+ const body = document.content
+ .map((block) => renderBlock(block))
+ .filter(Boolean)
+ .join("\n\n");
+
+ if (document.title && !bodyStartsWithTitle(body, document.title)) {
+ sections.push(`# ${document.title}`);
+ }
+
+ if (body) {
+ sections.push(body);
+ }
+
+ return cleanMarkdown(sections.join("\n\n"));
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/media/default-downloader.ts b/skills/baoyu-url-to-markdown/scripts/lib/media/default-downloader.ts
new file mode 100644
index 0000000..3d11e0e
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/media/default-downloader.ts
@@ -0,0 +1,161 @@
+import path from "node:path";
+import { mkdir, writeFile } from "node:fs/promises";
+import {
+ buildFileName,
+ isDataUri,
+ normalizeContentType,
+ normalizeMediaUrl,
+ resolveExtensionFromContentType,
+ resolveExtensionFromUrl,
+ resolveMediaKind,
+ resolveOutputExtension,
+ toPosixPath,
+} from "./media-utils";
+import type { MediaAsset, MediaDownloadRequest, MediaDownloadResult, MediaKind } from "./types";
+
+const DOWNLOAD_USER_AGENT =
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
+
+function parseBase64DataUri(rawUrl: string): { contentType: string; bytes: Buffer } | null {
+ const match = rawUrl.match(/^data:([^;,]+);base64,([A-Za-z0-9+/=\s]+)$/i);
+ if (!match?.[1] || !match[2]) {
+ return null;
+ }
+
+ const contentType = normalizeContentType(match[1]);
+ if (!contentType) {
+ return null;
+ }
+
+ try {
+ const bytes = Buffer.from(match[2].replace(/\s+/g, ""), "base64");
+ if (bytes.length === 0) {
+ return null;
+ }
+ return { contentType, bytes };
+ } catch {
+ return null;
+ }
+}
+
+function dedupeMedia(media: MediaAsset[]): MediaAsset[] {
+ const deduped: MediaAsset[] = [];
+ const seen = new Set();
+ for (const item of media) {
+ const normalizedUrl = normalizeMediaUrl(item.url);
+ if (!normalizedUrl || seen.has(normalizedUrl)) {
+ continue;
+ }
+ seen.add(normalizedUrl);
+ deduped.push({
+ ...item,
+ url: normalizedUrl,
+ });
+ }
+ return deduped;
+}
+
+function toRelativePath(fromDir: string, absoluteTarget: string): string {
+ const relative = path.relative(fromDir, absoluteTarget) || path.basename(absoluteTarget);
+ return toPosixPath(relative);
+}
+
+export async function downloadMediaAssets(
+ request: MediaDownloadRequest,
+): Promise {
+ const dedupedMedia = dedupeMedia(request.media);
+ const absoluteOutputPath = path.resolve(request.outputPath);
+ const markdownDir = path.dirname(absoluteOutputPath);
+ const baseDir = request.mediaDir ? path.resolve(request.mediaDir) : markdownDir;
+ const replacements: MediaDownloadResult["replacements"] = [];
+
+ let downloadedImages = 0;
+ let downloadedVideos = 0;
+
+ for (const asset of dedupedMedia) {
+ try {
+ let sourceUrl = normalizeMediaUrl(asset.url);
+ let contentType = "";
+ let extension: string | undefined;
+ let kind: MediaKind | undefined;
+ let bytes: Buffer | null = null;
+
+ if (isDataUri(asset.url)) {
+ const parsed = parseBase64DataUri(asset.url);
+ if (!parsed) {
+ request.log.warn(`Skipping unsupported embedded media: ${asset.url.slice(0, 32)}...`);
+ continue;
+ }
+
+ contentType = parsed.contentType;
+ extension =
+ resolveExtensionFromContentType(contentType) ??
+ resolveExtensionFromUrl(asset.fileNameHint ?? "");
+ kind = resolveMediaKind(sourceUrl, contentType, extension, asset.kind);
+ bytes = parsed.bytes;
+ } else {
+ const response = await fetch(sourceUrl, {
+ method: "GET",
+ redirect: "follow",
+ headers: {
+ "user-agent": DOWNLOAD_USER_AGENT,
+ ...(asset.headers ?? {}),
+ },
+ });
+
+ if (!response.ok) {
+ request.log.warn(`Skipping media (${response.status}): ${asset.url}`);
+ continue;
+ }
+
+ sourceUrl = normalizeMediaUrl(response.url || sourceUrl);
+ contentType = normalizeContentType(response.headers.get("content-type"));
+ extension =
+ resolveExtensionFromUrl(sourceUrl) ??
+ resolveExtensionFromUrl(asset.url) ??
+ resolveExtensionFromUrl(asset.fileNameHint ?? "");
+ kind = resolveMediaKind(sourceUrl, contentType, extension, asset.kind);
+ bytes = Buffer.from(await response.arrayBuffer());
+ }
+
+ if (!kind || !bytes) {
+ request.log.debug(`Skipping media with unresolved kind: ${asset.url}`);
+ continue;
+ }
+
+ const outputExtension = resolveOutputExtension(contentType, extension, kind);
+ const nextIndex = kind === "image" ? downloadedImages + 1 : downloadedVideos + 1;
+ const dirName = kind === "image" ? "imgs" : "videos";
+ const targetDir = path.join(baseDir, dirName);
+ await mkdir(targetDir, { recursive: true });
+
+ const fileName = buildFileName(kind, nextIndex, sourceUrl, outputExtension, asset.fileNameHint);
+ const absolutePath = path.join(targetDir, fileName);
+ await writeFile(absolutePath, bytes);
+
+ replacements.push({
+ url: asset.url,
+ localPath: toRelativePath(markdownDir, absolutePath),
+ absolutePath,
+ kind,
+ });
+
+ if (kind === "image") {
+ downloadedImages = nextIndex;
+ } else {
+ downloadedVideos = nextIndex;
+ }
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ request.log.warn(`Failed to download media ${asset.url}: ${message}`);
+ }
+ }
+
+ return {
+ replacements,
+ downloadedImages,
+ downloadedVideos,
+ imageDir: downloadedImages > 0 ? path.join(baseDir, "imgs") : null,
+ videoDir: downloadedVideos > 0 ? path.join(baseDir, "videos") : null,
+ };
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/media/markdown-media.ts b/skills/baoyu-url-to-markdown/scripts/lib/media/markdown-media.ts
new file mode 100644
index 0000000..a930671
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/media/markdown-media.ts
@@ -0,0 +1,458 @@
+import remarkGfm from "remark-gfm";
+import remarkParse from "remark-parse";
+import { unified } from "unified";
+import type { ContentBlock, ExtractedDocument } from "../extract/document";
+import {
+ isDataUri,
+ normalizeContentType,
+ normalizeMediaUrl,
+ resolveExtensionFromContentType,
+ resolveExtensionFromUrl,
+ resolveKindFromExtension,
+} from "./media-utils";
+import type { MediaAsset, MediaReplacement } from "./types";
+
+const MARKDOWN_LINK_RE =
+ /(!?\[[^\]\n]*\])\((<)?((?:https?:\/\/[^)\s>]+)|(?:data:[^)>\s]+))(>)?\)/g;
+const FRONTMATTER_COVER_RE = /^(coverImage:\s*")((?:https?:\/\/[^"]+)|(?:data:[^"]+))(")/m;
+const RAW_URL_RE = /(?:https?:\/\/[^\s<>"')\]]+|data:[^\s<>"')\]]+)/g;
+
+interface MarkdownAstNode {
+ type: string;
+ url?: string | null;
+ alt?: string | null;
+ title?: string | null;
+ value?: string | null;
+ children?: MarkdownAstNode[];
+ position?: {
+ start?: { offset?: number | null };
+ end?: { offset?: number | null };
+ };
+}
+
+interface MarkdownReplacementRange {
+ start: number;
+ end: number;
+ value: string;
+}
+
+function inferMediaKindFromLabel(label: string, rawUrl: string): "image" | "video" | undefined {
+ if (label.startsWith("![")) {
+ return "image";
+ }
+
+ const normalizedLabel = label.replace(/[!\[\]]/g, "").trim().toLowerCase();
+ if (/\b(video|animated[_ -]?gif|gif)\b/.test(normalizedLabel)) {
+ return "video";
+ }
+
+ if (isDataUri(rawUrl)) {
+ const contentType = normalizeContentType(rawUrl.slice(5, rawUrl.indexOf(";")));
+ return contentType.startsWith("image/") ? "image" : contentType.startsWith("video/") ? "video" : undefined;
+ }
+
+ return resolveKindFromExtension(resolveExtensionFromUrl(rawUrl));
+}
+
+function inferMediaKindFromRawUrl(rawUrl: string): "image" | "video" | undefined {
+ if (isDataUri(rawUrl)) {
+ const contentType = normalizeContentType(rawUrl.slice(5, rawUrl.indexOf(";")));
+ return contentType.startsWith("image/") ? "image" : contentType.startsWith("video/") ? "video" : undefined;
+ }
+
+ return resolveKindFromExtension(resolveExtensionFromUrl(rawUrl));
+}
+
+function pushMedia(assets: MediaAsset[], seen: Set, media: MediaAsset): void {
+ const normalizedUrl = normalizeMediaUrl(media.url);
+ if (!normalizedUrl || seen.has(normalizedUrl)) {
+ return;
+ }
+ seen.add(normalizedUrl);
+ assets.push({
+ ...media,
+ url: normalizedUrl,
+ });
+}
+
+function getNodeOffsets(node: MarkdownAstNode): { start: number; end: number } | null {
+ const start = node.position?.start?.offset;
+ const end = node.position?.end?.offset;
+ if (typeof start !== "number" || typeof end !== "number" || start < 0 || end < start) {
+ return null;
+ }
+ return { start, end };
+}
+
+function escapeMarkdownLabel(value: string): string {
+ return value.replace(/\\/g, "\\\\").replace(/\[/g, "\\[").replace(/\]/g, "\\]");
+}
+
+function escapeMarkdownTitle(value: string): string {
+ return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
+}
+
+function formatMarkdownDestination(url: string): string {
+ return /[\s()<>]/.test(url) ? `<${url}>` : url;
+}
+
+function serializeImageNode(node: MarkdownAstNode): string {
+ const rawUrl = node.url ?? "";
+ const normalizedUrl = normalizeMediaUrl(rawUrl);
+ const alt = escapeMarkdownLabel(node.alt ?? "");
+ const title = node.title ? ` "${escapeMarkdownTitle(node.title)}"` : "";
+ return `}${title})`;
+}
+
+function serializeLinkedImageNode(linkNode: MarkdownAstNode, imageNode: MarkdownAstNode): string {
+ const imageMarkdown = serializeImageNode(imageNode);
+ const imageUrl = normalizeMediaUrl(imageNode.url ?? "");
+ const linkUrl = normalizeMediaUrl(linkNode.url ?? "");
+
+ if (!linkUrl || linkUrl === imageUrl) {
+ return imageMarkdown;
+ }
+
+ const title = linkNode.title ? ` "${escapeMarkdownTitle(linkNode.title)}"` : "";
+ return `[${imageMarkdown}](${formatMarkdownDestination(linkUrl)}${title})`;
+}
+
+function isParagraphWithSingleText(node: MarkdownAstNode | undefined, expectedValue: string): boolean {
+ if (node?.type !== "paragraph" || node.children?.length !== 1) {
+ return false;
+ }
+
+ const child = node.children[0];
+ return child?.type === "text" && child.value?.trim() === expectedValue;
+}
+
+function getSingleImageFromParagraph(node: MarkdownAstNode | undefined): MarkdownAstNode | null {
+ if (node?.type !== "paragraph" || node.children?.length !== 1) {
+ return null;
+ }
+
+ return node.children[0]?.type === "image" ? node.children[0] : null;
+}
+
+function extractBrokenLinkedImageDestination(node: MarkdownAstNode | undefined): string | null {
+ if (node?.type !== "paragraph") {
+ return null;
+ }
+
+ const children = node.children ?? [];
+ if (children.length !== 3) {
+ return null;
+ }
+
+ const [prefix, linkNode, suffix] = children;
+ if (prefix?.type !== "text" || prefix.value?.trim() !== "](") {
+ return null;
+ }
+ if (linkNode?.type !== "link" || !linkNode.url) {
+ return null;
+ }
+ if (suffix?.type !== "text" || suffix.value?.trim() !== ")") {
+ return null;
+ }
+
+ return linkNode.url;
+}
+
+function collectLinkedImageReplacements(
+ node: MarkdownAstNode,
+ replacements: MarkdownReplacementRange[],
+): void {
+ const children = node.children ?? [];
+
+ if (node.type === "link" && children.length === 1 && children[0]?.type === "image") {
+ const offsets = getNodeOffsets(node);
+ if (offsets) {
+ replacements.push({
+ start: offsets.start,
+ end: offsets.end,
+ value: serializeLinkedImageNode(node, children[0]),
+ });
+ }
+ return;
+ }
+
+ for (const child of children) {
+ collectLinkedImageReplacements(child, replacements);
+ }
+}
+
+function collectBrokenLinkedImageReplacements(
+ node: MarkdownAstNode,
+ replacements: MarkdownReplacementRange[],
+): void {
+ const children = node.children ?? [];
+ for (let index = 0; index <= children.length - 3; index += 1) {
+ const openParagraph = children[index];
+ const imageParagraph = children[index + 1];
+ const closeParagraph = children[index + 2];
+
+ if (!isParagraphWithSingleText(openParagraph, "[")) {
+ continue;
+ }
+
+ const imageNode = getSingleImageFromParagraph(imageParagraph);
+ if (!imageNode) {
+ continue;
+ }
+
+ const linkUrl = extractBrokenLinkedImageDestination(closeParagraph);
+ if (!linkUrl) {
+ continue;
+ }
+
+ const start = openParagraph.position?.start?.offset;
+ const end = closeParagraph.position?.end?.offset;
+ if (typeof start !== "number" || typeof end !== "number" || end < start) {
+ continue;
+ }
+
+ replacements.push({
+ start,
+ end,
+ value: serializeLinkedImageNode({ type: "link", url: linkUrl }, imageNode),
+ });
+
+ index += 2;
+ }
+
+ for (const child of children) {
+ collectBrokenLinkedImageReplacements(child, replacements);
+ }
+}
+
+function applyReplacements(source: string, replacements: MarkdownReplacementRange[]): string {
+ if (replacements.length === 0) {
+ return source;
+ }
+
+ let result = source;
+ const sorted = [...replacements].sort((left, right) => right.start - left.start);
+ for (const replacement of sorted) {
+ result = `${result.slice(0, replacement.start)}${replacement.value}${result.slice(replacement.end)}`;
+ }
+ return result;
+}
+
+function normalizeLinkedImageMarkdown(markdown: string): string {
+ let tree: MarkdownAstNode;
+ try {
+ tree = unified().use(remarkParse).use(remarkGfm).parse(markdown) as MarkdownAstNode;
+ } catch {
+ return markdown;
+ }
+
+ const replacements: MarkdownReplacementRange[] = [];
+ collectLinkedImageReplacements(tree, replacements);
+ collectBrokenLinkedImageReplacements(tree, replacements);
+ return applyReplacements(markdown, replacements);
+}
+
+export function normalizeMarkdownMediaLinks(markdown: string): string {
+ MARKDOWN_LINK_RE.lastIndex = 0;
+ let result = markdown.replace(MARKDOWN_LINK_RE, (full, label, openAngle, rawUrl, closeAngle) => {
+ const normalizedUrl = normalizeMediaUrl(rawUrl);
+ if (normalizedUrl === rawUrl) {
+ return full;
+ }
+ return `${label}(${openAngle ?? ""}${normalizedUrl}${closeAngle ?? ""})`;
+ });
+
+ result = result.replace(FRONTMATTER_COVER_RE, (full, prefix, rawUrl, suffix) => {
+ const normalizedUrl = normalizeMediaUrl(rawUrl);
+ if (normalizedUrl === rawUrl) {
+ return full;
+ }
+ return `${prefix}${normalizedUrl}${suffix}`;
+ });
+
+ RAW_URL_RE.lastIndex = 0;
+ result = result.replace(RAW_URL_RE, (rawUrl) => normalizeMediaUrl(rawUrl));
+ return normalizeLinkedImageMarkdown(result);
+}
+
+export function collectMediaFromText(
+ text: string,
+ options: {
+ role?: MediaAsset["role"];
+ defaultKind?: MediaAsset["kind"];
+ seen?: Set;
+ into?: MediaAsset[];
+ } = {},
+): MediaAsset[] {
+ const assets = options.into ?? [];
+ const seen = options.seen ?? new Set();
+
+ MARKDOWN_LINK_RE.lastIndex = 0;
+ let linkMatch: RegExpExecArray | null;
+ while ((linkMatch = MARKDOWN_LINK_RE.exec(text))) {
+ const label = linkMatch[1] ?? "";
+ const rawUrl = linkMatch[3] ?? "";
+ const kind = inferMediaKindFromLabel(label, rawUrl) ?? options.defaultKind;
+ if (!kind) {
+ continue;
+ }
+ pushMedia(assets, seen, {
+ url: rawUrl,
+ kind,
+ role: options.role ?? "inline",
+ });
+ }
+
+ RAW_URL_RE.lastIndex = 0;
+ let rawMatch: RegExpExecArray | null;
+ while ((rawMatch = RAW_URL_RE.exec(text))) {
+ const rawUrl = rawMatch[0] ?? "";
+ const kind = inferMediaKindFromRawUrl(rawUrl) ?? options.defaultKind;
+ if (!kind) {
+ continue;
+ }
+ pushMedia(assets, seen, {
+ url: rawUrl,
+ kind,
+ role: options.role ?? "inline",
+ });
+ }
+
+ return assets;
+}
+
+function collectMediaFromBlock(
+ block: ContentBlock,
+ assets: MediaAsset[],
+ seen: Set,
+): void {
+ switch (block.type) {
+ case "image":
+ pushMedia(assets, seen, {
+ url: block.url,
+ kind: "image",
+ role: "inline",
+ alt: block.alt,
+ });
+ return;
+ case "html":
+ case "markdown":
+ collectMediaFromText(block.type === "html" ? block.html : block.markdown, {
+ role: "inline",
+ seen,
+ into: assets,
+ });
+ return;
+ case "paragraph":
+ case "quote":
+ collectMediaFromText(block.text, {
+ role: "inline",
+ seen,
+ into: assets,
+ });
+ return;
+ case "list":
+ for (const item of block.items) {
+ collectMediaFromText(item, {
+ role: "attachment",
+ seen,
+ into: assets,
+ });
+ }
+ return;
+ case "heading":
+ case "code":
+ return;
+ }
+}
+
+export function collectMediaFromDocument(document: ExtractedDocument): MediaAsset[] {
+ const assets: MediaAsset[] = [];
+ const seen = new Set();
+ const coverImage =
+ typeof document.metadata?.coverImage === "string" ? document.metadata.coverImage : undefined;
+
+ if (coverImage) {
+ pushMedia(assets, seen, {
+ url: coverImage,
+ kind: "image",
+ role: "cover",
+ });
+ }
+
+ for (const block of document.content) {
+ collectMediaFromBlock(block, assets, seen);
+ }
+
+ return assets;
+}
+
+export function collectMediaFromMarkdown(markdown: string): MediaAsset[] {
+ const assets: MediaAsset[] = [];
+ const seen = new Set();
+ const fmMatch = markdown.match(/^---\n([\s\S]*?)\n---/);
+ if (fmMatch) {
+ const coverMatch = fmMatch[1]?.match(FRONTMATTER_COVER_RE);
+ if (coverMatch?.[2]) {
+ pushMedia(assets, seen, {
+ url: coverMatch[2],
+ kind: "image",
+ role: "cover",
+ });
+ }
+ }
+
+ collectMediaFromText(markdown, { seen, into: assets });
+ return assets;
+}
+
+export function rewriteMarkdownMediaLinks(
+ markdown: string,
+ replacements: MediaReplacement[],
+): string {
+ if (replacements.length === 0) {
+ return markdown;
+ }
+
+ const replacementMap = new Map();
+ for (const item of replacements) {
+ replacementMap.set(item.url, item.localPath);
+ replacementMap.set(normalizeMediaUrl(item.url), item.localPath);
+ }
+
+ MARKDOWN_LINK_RE.lastIndex = 0;
+ let result = markdown.replace(MARKDOWN_LINK_RE, (full, label, _openAngle, rawUrl) => {
+ const replacement = replacementMap.get(rawUrl) ?? replacementMap.get(normalizeMediaUrl(rawUrl));
+ if (!replacement) {
+ return full;
+ }
+ return `${label}(${replacement})`;
+ });
+
+ result = result.replace(FRONTMATTER_COVER_RE, (full, prefix, rawUrl, suffix) => {
+ const replacement = replacementMap.get(rawUrl) ?? replacementMap.get(normalizeMediaUrl(rawUrl));
+ if (!replacement) {
+ return full;
+ }
+ return `${prefix}${replacement}${suffix}`;
+ });
+
+ for (const { url, localPath } of replacements) {
+ result = result.split(url).join(localPath);
+ const normalizedUrl = normalizeMediaUrl(url);
+ if (normalizedUrl !== url) {
+ result = result.split(normalizedUrl).join(localPath);
+ }
+ }
+
+ return result;
+}
+
+export function resolveDataUriExtension(rawUrl: string): string | undefined {
+ if (!isDataUri(rawUrl)) {
+ return undefined;
+ }
+ const separatorIndex = rawUrl.indexOf(";");
+ const contentType = normalizeContentType(rawUrl.slice(5, separatorIndex === -1 ? undefined : separatorIndex));
+ return resolveExtensionFromContentType(contentType);
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/media/media-utils.ts b/skills/baoyu-url-to-markdown/scripts/lib/media/media-utils.ts
new file mode 100644
index 0000000..30a7447
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/media/media-utils.ts
@@ -0,0 +1,261 @@
+import path from "node:path";
+import type { MediaKind } from "./types";
+
+const IMAGE_EXTENSIONS = new Set([
+ "jpg",
+ "jpeg",
+ "png",
+ "webp",
+ "gif",
+ "bmp",
+ "avif",
+ "heic",
+ "heif",
+ "svg",
+]);
+
+const VIDEO_EXTENSIONS = new Set(["mp4", "m4v", "mov", "webm", "mkv"]);
+
+const MIME_EXTENSION_MAP: Record = {
+ "image/jpeg": "jpg",
+ "image/jpg": "jpg",
+ "image/png": "png",
+ "image/webp": "webp",
+ "image/gif": "gif",
+ "image/bmp": "bmp",
+ "image/avif": "avif",
+ "image/heic": "heic",
+ "image/heif": "heif",
+ "image/svg+xml": "svg",
+ "video/mp4": "mp4",
+ "video/webm": "webm",
+ "video/quicktime": "mov",
+ "video/x-m4v": "m4v",
+};
+
+export function normalizeContentType(raw: string | null): string {
+ return raw?.split(";")[0]?.trim().toLowerCase() ?? "";
+}
+
+export function normalizeExtension(raw: string | undefined | null): string | undefined {
+ if (!raw) {
+ return undefined;
+ }
+ const trimmed = raw.replace(/^\./, "").trim().toLowerCase();
+ if (!trimmed) {
+ return undefined;
+ }
+ if (trimmed === "jpeg" || trimmed === "jpg") {
+ return "jpg";
+ }
+ return trimmed;
+}
+
+export function resolveExtensionFromUrl(rawUrl: string): string | undefined {
+ try {
+ const parsed = new URL(rawUrl);
+ const extFromPath = normalizeExtension(path.posix.extname(parsed.pathname));
+ if (extFromPath) {
+ return extFromPath;
+ }
+ const extFromFormat = normalizeExtension(parsed.searchParams.get("format"));
+ if (extFromFormat) {
+ return extFromFormat;
+ }
+ } catch {
+ return undefined;
+ }
+ return undefined;
+}
+
+export function resolveExtensionFromContentType(contentType: string): string | undefined {
+ return normalizeExtension(MIME_EXTENSION_MAP[contentType]);
+}
+
+export function resolveKindFromContentType(contentType: string): MediaKind | undefined {
+ if (!contentType) {
+ return undefined;
+ }
+ if (contentType.startsWith("image/")) {
+ return "image";
+ }
+ if (contentType.startsWith("video/")) {
+ return "video";
+ }
+ return undefined;
+}
+
+export function resolveKindFromExtension(extension: string | undefined): MediaKind | undefined {
+ if (!extension) {
+ return undefined;
+ }
+ if (IMAGE_EXTENSIONS.has(extension)) {
+ return "image";
+ }
+ if (VIDEO_EXTENSIONS.has(extension)) {
+ return "video";
+ }
+ return undefined;
+}
+
+export function resolveMediaKind(
+ rawUrl: string,
+ contentType: string,
+ extension: string | undefined,
+ hint?: MediaKind,
+): MediaKind | undefined {
+ const kindFromType = resolveKindFromContentType(contentType);
+ if (kindFromType) {
+ return kindFromType;
+ }
+
+ const kindFromExtension = resolveKindFromExtension(extension);
+ if (kindFromExtension) {
+ return kindFromExtension;
+ }
+
+ if (contentType && contentType !== "application/octet-stream") {
+ return undefined;
+ }
+
+ if (hint) {
+ return hint;
+ }
+
+ if (rawUrl.startsWith("data:image/")) {
+ return "image";
+ }
+
+ if (rawUrl.startsWith("data:video/")) {
+ return "video";
+ }
+
+ return undefined;
+}
+
+export function resolveOutputExtension(
+ contentType: string,
+ extension: string | undefined,
+ kind: MediaKind,
+): string {
+ const fromMime = resolveExtensionFromContentType(contentType);
+ if (fromMime) {
+ return fromMime;
+ }
+ const normalized = normalizeExtension(extension);
+ if (normalized) {
+ return normalized;
+ }
+ return kind === "video" ? "mp4" : "jpg";
+}
+
+export function isDataUri(value: string): boolean {
+ return value.startsWith("data:");
+}
+
+export function safeDecodeURIComponent(value: string): string {
+ try {
+ return decodeURIComponent(value);
+ } catch {
+ return value;
+ }
+}
+
+function extractEmbeddedUrl(value: string): string | undefined {
+ const encodedMatch = value.match(/https?%3A%2F%2F.+$/i)?.[0];
+ if (encodedMatch) {
+ const decoded = safeDecodeURIComponent(encodedMatch);
+ try {
+ return new URL(decoded).href;
+ } catch {
+ return undefined;
+ }
+ }
+
+ const literalMatch = value.match(/https?:\/\/.+$/i)?.[0];
+ if (!literalMatch) {
+ return undefined;
+ }
+
+ try {
+ return new URL(literalMatch).href;
+ } catch {
+ return undefined;
+ }
+}
+
+export function normalizeMediaUrl(rawUrl: string): string {
+ if (isDataUri(rawUrl)) {
+ return rawUrl;
+ }
+
+ try {
+ const parsed = new URL(rawUrl);
+ const hostname = parsed.hostname.toLowerCase();
+
+ if (hostname === "substackcdn.com" || hostname.endsWith(".substackcdn.com")) {
+ const embeddedUrl = extractEmbeddedUrl(`${parsed.pathname}${parsed.search}`);
+ if (embeddedUrl) {
+ return embeddedUrl;
+ }
+ }
+
+ return parsed.href;
+ } catch {
+ return rawUrl;
+ }
+}
+
+export function sanitizeFileSegment(input: string): string {
+ return input
+ .replace(/[^a-zA-Z0-9_-]+/g, "-")
+ .replace(/-+/g, "-")
+ .replace(/^[-_]+|[-_]+$/g, "")
+ .slice(0, 48);
+}
+
+export function resolveFileStem(rawUrl: string, extension: string, fileNameHint?: string): string {
+ const hintBase = fileNameHint?.trim();
+ if (hintBase) {
+ const parsed = path.posix.parse(hintBase);
+ const stem = parsed.name || parsed.base;
+ return sanitizeFileSegment(stem);
+ }
+
+ if (isDataUri(rawUrl)) {
+ return "";
+ }
+
+ try {
+ const parsed = new URL(rawUrl);
+ const base = path.posix.basename(parsed.pathname);
+ if (!base) {
+ return "";
+ }
+ const decodedBase = safeDecodeURIComponent(base);
+ const normalizedExtension = normalizeExtension(extension);
+ const stripExtension = normalizedExtension ? new RegExp(`\\.${normalizedExtension}$`, "i") : null;
+ const rawStem = stripExtension ? decodedBase.replace(stripExtension, "") : decodedBase;
+ return sanitizeFileSegment(rawStem);
+ } catch {
+ return "";
+ }
+}
+
+export function buildFileName(
+ kind: MediaKind,
+ index: number,
+ sourceUrl: string,
+ extension: string,
+ fileNameHint?: string,
+): string {
+ const stem = resolveFileStem(sourceUrl, extension, fileNameHint);
+ const prefix = kind === "image" ? "img" : "video";
+ const serial = String(index).padStart(3, "0");
+ const suffix = stem ? `-${stem}` : "";
+ return `${prefix}-${serial}${suffix}.${extension}`;
+}
+
+export function toPosixPath(value: string): string {
+ return value.split(path.sep).join(path.posix.sep);
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/media/types.ts b/skills/baoyu-url-to-markdown/scripts/lib/media/types.ts
new file mode 100644
index 0000000..00cae8f
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/media/types.ts
@@ -0,0 +1,34 @@
+import type { Logger } from "../utils/logger";
+
+export type MediaKind = "image" | "video";
+
+export interface MediaAsset {
+ url: string;
+ kind?: MediaKind;
+ role?: "cover" | "inline" | "attachment";
+ alt?: string;
+ fileNameHint?: string;
+ headers?: Record;
+}
+
+export interface MediaReplacement {
+ url: string;
+ localPath: string;
+ absolutePath: string;
+ kind: MediaKind;
+}
+
+export interface MediaDownloadRequest {
+ media: MediaAsset[];
+ outputPath: string;
+ mediaDir?: string;
+ log: Logger;
+}
+
+export interface MediaDownloadResult {
+ replacements: MediaReplacement[];
+ downloadedImages: number;
+ downloadedVideos: number;
+ imageDir: string | null;
+ videoDir: string | null;
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/types/defuddle-node.d.ts b/skills/baoyu-url-to-markdown/scripts/lib/types/defuddle-node.d.ts
new file mode 100644
index 0000000..ec23b3a
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/types/defuddle-node.d.ts
@@ -0,0 +1,31 @@
+declare module "defuddle/node" {
+ export interface DefuddleResponse {
+ content?: string;
+ title?: string;
+ description?: string;
+ author?: string;
+ published?: string;
+ image?: string;
+ language?: string;
+ }
+
+ export interface DefuddleOptions {
+ markdown?: boolean;
+ }
+
+ export function Defuddle(
+ input:
+ | Document
+ | string
+ | {
+ window: {
+ document: Document;
+ location: {
+ href: string;
+ };
+ };
+ },
+ url?: string,
+ options?: DefuddleOptions,
+ ): Promise;
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/types/shims.d.ts b/skills/baoyu-url-to-markdown/scripts/lib/types/shims.d.ts
new file mode 100644
index 0000000..c29bbe9
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/types/shims.d.ts
@@ -0,0 +1,17 @@
+declare module "turndown" {
+ export interface TurndownOptions {
+ codeBlockStyle?: "indented" | "fenced";
+ headingStyle?: "setext" | "atx";
+ bulletListMarker?: "-" | "*" | "+";
+ }
+
+ export default class TurndownService {
+ constructor(options?: TurndownOptions);
+ use(plugin: unknown): void;
+ turndown(input: string): string;
+ }
+}
+
+declare module "turndown-plugin-gfm" {
+ export const gfm: unknown;
+}
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/utils/logger.ts b/skills/baoyu-url-to-markdown/scripts/lib/utils/logger.ts
new file mode 100644
index 0000000..cde9e19
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/utils/logger.ts
@@ -0,0 +1,30 @@
+export interface Logger {
+ info(message: string): void;
+ warn(message: string): void;
+ error(message: string): void;
+ debug(message: string): void;
+}
+
+export function createLogger(debugEnabled = false): Logger {
+ const print = (level: string, message: string): void => {
+ console.error(`[${level}] ${message}`);
+ };
+
+ return {
+ info(message: string) {
+ print("info", message);
+ },
+ warn(message: string) {
+ print("warn", message);
+ },
+ error(message: string) {
+ print("error", message);
+ },
+ debug(message: string) {
+ if (debugEnabled) {
+ print("debug", message);
+ }
+ },
+ };
+}
+
diff --git a/skills/baoyu-url-to-markdown/scripts/lib/utils/url.ts b/skills/baoyu-url-to-markdown/scripts/lib/utils/url.ts
new file mode 100644
index 0000000..578e97a
--- /dev/null
+++ b/skills/baoyu-url-to-markdown/scripts/lib/utils/url.ts
@@ -0,0 +1,12 @@
+export function normalizeUrl(input: string): URL {
+ try {
+ return new URL(input);
+ } catch {
+ throw new Error(`Invalid URL: ${input}`);
+ }
+}
+
+export function sanitizeFilename(input: string): string {
+ return input.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "document";
+}
+
diff --git a/skills/baoyu-url-to-markdown/scripts/package.json b/skills/baoyu-url-to-markdown/scripts/package.json
index 183f70a..850d688 100644
--- a/skills/baoyu-url-to-markdown/scripts/package.json
+++ b/skills/baoyu-url-to-markdown/scripts/package.json
@@ -2,7 +2,25 @@
"name": "baoyu-url-to-markdown-scripts",
"private": true,
"type": "module",
+ "bin": {
+ "baoyu-fetch": "./baoyu-fetch"
+ },
+ "scripts": {
+ "reader": "bun ./lib/cli.ts"
+ },
"dependencies": {
- "baoyu-fetch": "^0.1.2"
+ "@mozilla/readability": "^0.6.0",
+ "chrome-launcher": "^1.2.1",
+ "defuddle": "^0.17.0",
+ "jsdom": "^29.0.2",
+ "remark-gfm": "^4.0.1",
+ "remark-parse": "^11.0.0",
+ "turndown": "^7.2.0",
+ "turndown-plugin-gfm": "^1.0.2",
+ "unified": "^11.0.5",
+ "ws": "^8.18.3"
+ },
+ "overrides": {
+ "@xmldom/xmldom": "0.8.13"
}
}