chore: release v1.45.0

Add automatic check after article images are inserted: verifies remaining
XIMGPH placeholders and expected vs actual image count. Increase CDP timeout
to 60s and add 3s DOM stabilization delay between image insertions.

.claude-plugin/marketplace.json

@@ -6,7 +6,7 @@
   },
   "metadata": {
     "description": "Skills shared by Baoyu for improving daily work efficiency",
-    "version": "1.42.3"
+    "version": "1.45.0"
   },
   "plugins": [
     {

CHANGELOG.md

@@ -2,6 +2,39 @@
 
 English | [中文](./CHANGELOG.zh.md)
 
+## 1.45.0 - 2026-03-05
+
+### Features
+- `baoyu-post-to-x`: add post-composition verification for X Articles — automatically checks remaining placeholders and image count after all images are inserted
+- `baoyu-post-to-x`: increase CDP timeout to 60s and add 3s DOM stabilization delay between image insertions for long articles
+
+## 1.44.0 - 2026-03-05
+
+### Features
+- `baoyu-url-to-markdown`: add `--download-media` flag to download images and videos to local directories, rewriting markdown links to local paths
+- `baoyu-url-to-markdown`: extract cover image from page meta (og:image) into YAML front matter `coverImage` field
+- `baoyu-url-to-markdown`: handle `data-src` lazy loading for WeChat and similar sites
+- `baoyu-url-to-markdown`: add EXTEND.md preferences with first-time setup for media download behavior
+
+## 1.43.2 - 2026-03-05
+
+### Refactor
+- `baoyu-url-to-markdown`: replace custom HTML extraction (linkedom + Readability + Turndown) with defuddle library for cleaner content extraction and markdown conversion
+
+## 1.43.1 - 2026-03-02
+
+### Features
+- `baoyu-post-to-x`: auto-detect WSL environment and resolve Chrome profile to Windows-native path for stable login persistence
+- `baoyu-post-to-wechat`: auto-detect WSL environment and resolve Chrome profile to Windows-native path for stable login persistence
+- `baoyu-danger-gemini-web`: WSL auto-detection for Chrome profile path; add `GEMINI_WEB_DEBUG_PORT` env var for fixed debug port
+- `baoyu-danger-x-to-markdown`: WSL auto-detection for Chrome profile path; add `X_DEBUG_PORT` env var for fixed debug port
+
+## 1.43.0 - 2026-03-02
+
+### Features
+- `baoyu-post-to-wechat`: support env var overrides for browser debug port (`WECHAT_BROWSER_DEBUG_PORT`) and profile directory (`WECHAT_BROWSER_PROFILE_DIR`)
+- `baoyu-post-to-x`: support env var overrides for browser debug port (`X_BROWSER_DEBUG_PORT`) and profile directory (`X_BROWSER_PROFILE_DIR`)
+
 ## 1.42.3 - 2026-03-02
 
 ### Fixes

CHANGELOG.zh.md

@@ -2,6 +2,39 @@
 
 [English](./CHANGELOG.md) | 中文
 
+## 1.45.0 - 2026-03-05
+
+### 新功能
+- `baoyu-post-to-x`：X 文章发布后自动验证——检查残留占位符和图片数量是否正确
+- `baoyu-post-to-x`：增加 CDP 超时至 60 秒，图片插入间隔增加 3 秒 DOM 稳定等待，改善长文章发布稳定性
+
+## 1.44.0 - 2026-03-05
+
+### 新功能
+- `baoyu-url-to-markdown`：新增 `--download-media` 参数，支持下载图片和视频到本地目录，并将 Markdown 中的链接改写为本地路径
+- `baoyu-url-to-markdown`：从页面 meta 信息（og:image）提取封面图，写入 YAML front matter 的 `coverImage` 字段
+- `baoyu-url-to-markdown`：支持 `data-src` 懒加载图片提取（兼容微信公众号等站点）
+- `baoyu-url-to-markdown`：新增 EXTEND.md 偏好设置，支持首次使用引导配置媒体下载行为
+
+## 1.43.2 - 2026-03-05
+
+### 重构
+- `baoyu-url-to-markdown`：使用 defuddle 库替换自定义 HTML 提取逻辑（linkedom + Readability + Turndown），简化内容提取和 Markdown 转换
+
+## 1.43.1 - 2026-03-02
+
+### 新功能
+- `baoyu-post-to-x`：自动检测 WSL 环境，将 Chrome profile 路径解析为 Windows 本地路径，解决登录态丢失问题
+- `baoyu-post-to-wechat`：自动检测 WSL 环境，将 Chrome profile 路径解析为 Windows 本地路径，解决登录态丢失问题
+- `baoyu-danger-gemini-web`：WSL 自动检测 Chrome profile 路径；新增 `GEMINI_WEB_DEBUG_PORT` 环境变量支持固定调试端口
+- `baoyu-danger-x-to-markdown`：WSL 自动检测 Chrome profile 路径；新增 `X_DEBUG_PORT` 环境变量支持固定调试端口
+
+## 1.43.0 - 2026-03-02
+
+### 新功能
+- `baoyu-post-to-wechat`：支持通过环境变量覆盖浏览器调试端口（`WECHAT_BROWSER_DEBUG_PORT`）和配置目录（`WECHAT_BROWSER_PROFILE_DIR`）
+- `baoyu-post-to-x`：支持通过环境变量覆盖浏览器调试端口（`X_BROWSER_DEBUG_PORT`）和配置目录（`X_BROWSER_PROFILE_DIR`）
+
 ## 1.42.3 - 2026-03-02
 
 ### 修复

skills/baoyu-danger-gemini-web/scripts/gemini-webapi/utils/load-browser-cookies.ts

@@ -91,6 +91,8 @@ class CdpConnection {
 }
 
 async function get_free_port(): Promise<number> {
+  const fixed = parseInt(process.env.GEMINI_WEB_DEBUG_PORT || '', 10);
+  if (fixed > 0) return fixed;
   return await new Promise((resolve, reject) => {
     const srv = net.createServer();
     srv.unref();

skills/baoyu-danger-gemini-web/scripts/gemini-webapi/utils/paths.ts

@@ -1,3 +1,4 @@
+import { execSync } from 'node:child_process';
 import os from 'node:os';
 import path from 'node:path';
 import process from 'node:process';
@@ -29,9 +30,22 @@ export function resolveGeminiWebCookiePath(): string {
   return path.join(resolveGeminiWebDataDir(), COOKIE_FILE_NAME);
 }
 
+let _wslHome: string | null | undefined;
+function getWslWindowsHome(): string | null {
+  if (_wslHome !== undefined) return _wslHome;
+  if (!process.env.WSL_DISTRO_NAME) { _wslHome = null; return null; }
+  try {
+    const raw = execSync('cmd.exe /C "echo %USERPROFILE%"', { encoding: 'utf-8', timeout: 5000 }).trim().replace(/\r/g, '');
+    _wslHome = execSync(`wslpath -u "${raw}"`, { encoding: 'utf-8', timeout: 5000 }).trim() || null;
+  } catch { _wslHome = null; }
+  return _wslHome;
+}
+
 export function resolveGeminiWebChromeProfileDir(): string {
   const override = process.env.GEMINI_WEB_CHROME_PROFILE_DIR?.trim();
   if (override) return path.resolve(override);
+  const wslHome = getWslWindowsHome();
+  if (wslHome) return path.join(wslHome, '.local', 'share', APP_DATA_DIR, GEMINI_DATA_DIR, PROFILE_DIR_NAME);
   return path.join(resolveGeminiWebDataDir(), PROFILE_DIR_NAME);
 }
 

skills/baoyu-danger-x-to-markdown/scripts/cookies.ts

@@ -117,6 +117,8 @@ class CdpConnection {
 }
 
 async function getFreePort(): Promise<number> {
+  const fixed = parseInt(process.env.X_DEBUG_PORT || "", 10);
+  if (fixed > 0) return fixed;
   return await new Promise((resolve, reject) => {
     const srv = net.createServer();
     srv.unref();

skills/baoyu-danger-x-to-markdown/scripts/paths.ts

@@ -1,3 +1,4 @@
+import { execSync } from "node:child_process";
 import os from "node:os";
 import path from "node:path";
 import process from "node:process";
@@ -30,9 +31,22 @@ export function resolveXToMarkdownCookiePath(): string {
   return path.join(resolveXToMarkdownDataDir(), COOKIE_FILE_NAME);
 }
 
+let _wslHome: string | null | undefined;
+function getWslWindowsHome(): string | null {
+  if (_wslHome !== undefined) return _wslHome;
+  if (!process.env.WSL_DISTRO_NAME) { _wslHome = null; return null; }
+  try {
+    const raw = execSync('cmd.exe /C "echo %USERPROFILE%"', { encoding: 'utf-8', timeout: 5000 }).trim().replace(/\r/g, '');
+    _wslHome = execSync(`wslpath -u "${raw}"`, { encoding: 'utf-8', timeout: 5000 }).trim() || null;
+  } catch { _wslHome = null; }
+  return _wslHome;
+}
+
 export function resolveXToMarkdownChromeProfileDir(): string {
   const override = process.env.X_CHROME_PROFILE_DIR?.trim();
   if (override) return path.resolve(override);
+  const wslHome = getWslWindowsHome();
+  if (wslHome) return path.join(wslHome, ".local", "share", APP_DATA_DIR, X_TO_MARKDOWN_DATA_DIR, PROFILE_DIR_NAME);
   return path.join(resolveXToMarkdownDataDir(), PROFILE_DIR_NAME);
 }
 

skills/baoyu-post-to-wechat/scripts/wechat-browser.ts

@@ -1,4 +1,4 @@
-import { spawn } from 'node:child_process';
+import { execSync, spawn } from 'node:child_process';
 import fs from 'node:fs';
 import { mkdir, readdir } from 'node:fs/promises';
 import net from 'node:net';
@@ -109,6 +109,8 @@ function sleep(ms: number): Promise<void> {
 }
 
 async function getFreePort(): Promise<number> {
+  const fixed = parseInt(process.env.WECHAT_BROWSER_DEBUG_PORT || '', 10);
+  if (fixed > 0) return fixed;
   return new Promise((resolve, reject) => {
     const server = net.createServer();
     server.unref();
@@ -169,8 +171,22 @@ function findChromeExecutable(): string | undefined {
   return undefined;
 }
 
+let _wslHome: string | null | undefined;
+function getWslWindowsHome(): string | null {
+  if (_wslHome !== undefined) return _wslHome;
+  if (!process.env.WSL_DISTRO_NAME) { _wslHome = null; return null; }
+  try {
+    const raw = execSync('cmd.exe /C "echo %USERPROFILE%"', { encoding: 'utf-8', timeout: 5000 }).trim().replace(/\r/g, '');
+    _wslHome = execSync(`wslpath -u "${raw}"`, { encoding: 'utf-8', timeout: 5000 }).trim() || null;
+  } catch { _wslHome = null; }
+  return _wslHome;
+}
+
 function getDefaultProfileDir(): string {
-  const base = process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
+  const override = process.env.WECHAT_BROWSER_PROFILE_DIR?.trim();
+  if (override) return path.resolve(override);
+  const home = getWslWindowsHome() ?? os.homedir();
+  const base = process.env.XDG_DATA_HOME || path.join(home, '.local', 'share');
   return path.join(base, 'wechat-browser-profile');
 }
 

skills/baoyu-post-to-x/SKILL.md

@@ -172,6 +172,12 @@ npx -y bun ${SKILL_DIR}/scripts/x-article.ts article.md --cover ./cover.jpg
 
 **Note**: Script opens browser with article filled in. User reviews and publishes manually.
 
+**Post-Composition Check**: The script automatically verifies after all images are inserted:
+- Remaining `XIMGPH_` placeholders in editor content
+- Expected vs actual image count
+
+If the check fails (warnings in output), alert the user with the specific issues before they publish.
+
 ---
 
 ## Troubleshooting

skills/baoyu-post-to-x/references/articles.md

@@ -132,8 +132,12 @@ JSON output:
    - Select the placeholder
    - Copy image to clipboard
    - Paste to replace selection
-9. **Review**: Browser stays open for 60s preview
+9. **Post-Composition Check** (automatic):
-10. **Publish**: Only with `--submit` flag
+   - Scan editor for remaining `XIMGPH_` placeholders
+   - Compare expected vs actual image count
+   - Warn if issues found
+10. **Review**: Browser stays open for 60s preview
+11. **Publish**: Only with `--submit` flag
 
 ## Example Session
 

skills/baoyu-post-to-x/scripts/x-article.ts

@@ -130,7 +130,7 @@ export async function publishArticle(options: ArticleOptions): Promise<void> {
 
   try {
     const wsUrl = await waitForChromeDebugPort(port, 30_000, { includeLastError: true });
-    cdp = await CdpConnection.connect(wsUrl, 30_000, { defaultTimeoutMs: 30_000 });
+    cdp = await CdpConnection.connect(wsUrl, 30_000, { defaultTimeoutMs: 60_000 });
 
     // Get page target
     const targets = await cdp.send<{ targetInfos: Array<{ targetId: string; url: string; type: string }> }>('Target.getTargets');
@@ -148,7 +148,7 @@ export async function publishArticle(options: ArticleOptions): Promise<void> {
     await cdp.send('DOM.enable', {}, { sessionId });
 
     console.log('[x-article] Waiting for articles page...');
-    await sleep(3000);
+    await sleep(1000);
 
     // Wait for and click "create" button
     const waitForElement = async (selector: string, timeoutMs = 60_000): Promise<boolean> => {
@@ -644,6 +644,8 @@ export async function publishArticle(options: ArticleOptions): Promise<void> {
 
         if (imgUploadOk) {
           console.log(`[x-article] Image upload verified (${expectedImgCount} image block(s))`);
+          // Wait for DraftEditor DOM to stabilize after image insertion
+          await sleep(3000);
         } else {
           console.warn(`[x-article] Image upload not detected after 15s`);
           if (i === 0) {
@@ -653,6 +655,42 @@ export async function publishArticle(options: ArticleOptions): Promise<void> {
       }
 
       console.log('[x-article] All images processed.');
+
+      // Final verification: check placeholder residue and image count
+      console.log('[x-article] Running post-composition verification...');
+      const finalEditorContent = await cdp.send<{ result: { value: string } }>('Runtime.evaluate', {
+        expression: `document.querySelector('.DraftEditor-editorContainer [data-contents="true"]')?.innerText || ''`,
+        returnByValue: true,
+      }, { sessionId });
+
+      const remainingPlaceholders: string[] = [];
+      for (const img of parsed.contentImages) {
+        const regex = new RegExp(img.placeholder + '(?!\\d)');
+        if (regex.test(finalEditorContent.result.value)) {
+          remainingPlaceholders.push(img.placeholder);
+        }
+      }
+
+      const finalImgCount = await cdp.send<{ result: { value: number } }>('Runtime.evaluate', {
+        expression: `document.querySelectorAll('section[data-block="true"][contenteditable="false"] img[src^="blob:"]').length`,
+        returnByValue: true,
+      }, { sessionId });
+
+      const expectedCount = parsed.contentImages.length;
+      const actualCount = finalImgCount.result.value;
+
+      if (remainingPlaceholders.length > 0 || actualCount < expectedCount) {
+        console.warn('[x-article] ⚠ POST-COMPOSITION CHECK FAILED:');
+        if (remainingPlaceholders.length > 0) {
+          console.warn(`[x-article]   Remaining placeholders: ${remainingPlaceholders.join(', ')}`);
+        }
+        if (actualCount < expectedCount) {
+          console.warn(`[x-article]   Image count: expected ${expectedCount}, found ${actualCount}`);
+        }
+        console.warn('[x-article]   Please check the article before publishing.');
+      } else {
+        console.log(`[x-article] ✓ Verification passed: ${actualCount} image(s), no remaining placeholders.`);
+      }
     }
 
     // Before preview: blur editor to trigger save

skills/baoyu-post-to-x/scripts/x-utils.ts

@@ -1,4 +1,4 @@
-import { spawnSync } from 'node:child_process';
+import { execSync, spawnSync } from 'node:child_process';
 import fs from 'node:fs';
 import net from 'node:net';
 import os from 'node:os';
@@ -69,8 +69,22 @@ export function findChromeExecutable(candidates: PlatformCandidates): string | u
   return undefined;
 }
 
+let _wslHome: string | null | undefined;
+function getWslWindowsHome(): string | null {
+  if (_wslHome !== undefined) return _wslHome;
+  if (!process.env.WSL_DISTRO_NAME) { _wslHome = null; return null; }
+  try {
+    const raw = execSync('cmd.exe /C "echo %USERPROFILE%"', { encoding: 'utf-8', timeout: 5000 }).trim().replace(/\r/g, '');
+    _wslHome = execSync(`wslpath -u "${raw}"`, { encoding: 'utf-8', timeout: 5000 }).trim() || null;
+  } catch { _wslHome = null; }
+  return _wslHome;
+}
+
 export function getDefaultProfileDir(): string {
-  const base = process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
+  const override = process.env.X_BROWSER_PROFILE_DIR?.trim();
+  if (override) return path.resolve(override);
+  const home = getWslWindowsHome() ?? os.homedir();
+  const base = process.env.XDG_DATA_HOME || path.join(home, '.local', 'share');
   return path.join(base, 'x-browser-profile');
 }
 
@@ -79,6 +93,8 @@ export function sleep(ms: number): Promise<void> {
 }
 
 export async function getFreePort(): Promise<number> {
+  const fixed = parseInt(process.env.X_BROWSER_DEBUG_PORT || '', 10);
+  if (fixed > 0) return fixed;
   return new Promise((resolve, reject) => {
     const server = net.createServer();
     server.unref();

skills/baoyu-url-to-markdown/SKILL.md

@@ -46,10 +46,45 @@ test -f "$HOME/.baoyu-skills/baoyu-url-to-markdown/EXTEND.md" && echo "user"
 ├───────────┼───────────────────────────────────────────────────────────────────────────┤
 │ Found     │ Read, parse, apply settings                                               │
 ├───────────┼───────────────────────────────────────────────────────────────────────────┤
-│ Not found │ Use defaults                                                              │
+│ Not found │ **MUST** run first-time setup (see below) — do NOT silently create defaults │
 └───────────┴───────────────────────────────────────────────────────────────────────────┘
 
-**EXTEND.md Supports**: Default output directory | Default capture mode | Timeout settings
+**EXTEND.md Supports**: Download media by default | Default output directory | Default capture mode | Timeout settings
+
+### First-Time Setup (BLOCKING)
+
+**CRITICAL**: When EXTEND.md is not found, you **MUST use `AskUserQuestion`** to ask the user for their preferences before creating EXTEND.md. **NEVER** create EXTEND.md with defaults without asking. This is a **BLOCKING** operation — do NOT proceed with any conversion until setup is complete.
+
+Use `AskUserQuestion` with ALL questions in ONE call:
+
+**Question 1** — header: "Media", question: "How to handle images and videos in pages?"
+- "Ask each time (Recommended)" — After saving markdown, ask whether to download media
+- "Always download" — Always download media to local imgs/ and videos/ directories
+- "Never download" — Keep original remote URLs in markdown
+
+**Question 2** — header: "Output", question: "Default output directory?"
+- "url-to-markdown (Recommended)" — Save to ./url-to-markdown/{domain}/{slug}.md
+- (User may choose "Other" to type a custom path)
+
+**Question 3** — header: "Save", question: "Where to save preferences?"
+- "User (Recommended)" — ~/.baoyu-skills/ (all projects)
+- "Project" — .baoyu-skills/ (this project only)
+
+After user answers, create EXTEND.md at the chosen location, confirm "Preferences saved to [path]", then continue.
+
+Full reference: [references/config/first-time-setup.md](references/config/first-time-setup.md)
+
+### Supported Keys
+
+| Key | Default | Values | Description |
+|-----|---------|--------|-------------|
+| `download_media` | `ask` | `ask` / `1` / `0` | `ask` = prompt each time, `1` = always download, `0` = never |
+| `default_output_dir` | empty | path or empty | Default output directory (empty = `./url-to-markdown/`) |
+
+**Value priority**:
+1. CLI arguments (`--download-media`, `-o`)
+2. EXTEND.md
+3. Skill defaults
 
 ## Features
 
@@ -57,6 +92,7 @@ test -f "$HOME/.baoyu-skills/baoyu-url-to-markdown/EXTEND.md" && echo "user"
 - Two capture modes: auto or wait-for-user
 - Clean markdown output with metadata
 - Handles login-required pages via wait mode
+- Download images and videos to local directories
 
 ## Usage
 
@@ -69,6 +105,9 @@ npx -y bun ${SKILL_DIR}/scripts/main.ts <url> --wait
 
 # Save to specific file
 npx -y bun ${SKILL_DIR}/scripts/main.ts <url> -o output.md
+
+# Download images and videos to local directories
+npx -y bun ${SKILL_DIR}/scripts/main.ts <url> --download-media
 ```
 
 ## Options
@@ -79,6 +118,7 @@ npx -y bun ${SKILL_DIR}/scripts/main.ts <url> -o output.md
 | `-o <path>` | Output file path (default: auto-generated) |
 | `--wait` | Wait for user signal before capturing |
 | `--timeout <ms>` | Page load timeout (default: 30000) |
+| `--download-media` | Download image/video assets to local `imgs/` and `videos/`, and rewrite markdown links to local relative paths |
 
 ## Capture Modes
 
@@ -105,6 +145,32 @@ url-to-markdown/<domain>/<slug>.md
 - `<slug>`: From page title or URL path (kebab-case, 2-6 words)
 - Conflict resolution: Append timestamp `<slug>-YYYYMMDD-HHMMSS.md`
 
+When `--download-media` is enabled:
+- Images are saved to `imgs/` next to the markdown file
+- Videos are saved to `videos/` next to the markdown file
+- Markdown media links are rewritten to local relative paths
+
+## Media Download Workflow
+
+Based on `download_media` setting in EXTEND.md:
+
+| Setting | Behavior |
+|---------|----------|
+| `1` (always) | Run script with `--download-media` flag |
+| `0` (never) | Run script without `--download-media` flag |
+| `ask` (default) | Follow the ask-each-time flow below |
+
+### Ask-Each-Time Flow
+
+1. Run script **without** `--download-media` → markdown saved
+2. Check saved markdown for remote media URLs (`https://` in image/video links)
+3. **If no remote media found** → done, no prompt needed
+4. **If remote media found** → use `AskUserQuestion`:
+   - header: "Media", question: "Download N images/videos to local files?"
+   - "Yes" — Download to local directories
+   - "No" — Keep remote URLs
+5. If user confirms → run script **again** with `--download-media` (overwrites markdown with localized links)
+
 ## Environment Variables
 
 | Variable | Description |

skills/baoyu-url-to-markdown/references/config/first-time-setup.md

@@ -0,0 +1,106 @@
+---
+name: first-time-setup
+description: First-time setup flow for baoyu-url-to-markdown preferences
+---
+
+# First-Time Setup
+
+## Overview
+
+When no EXTEND.md is found, guide user through preference setup.
+
+**BLOCKING OPERATION**: This setup MUST complete before ANY other workflow steps. Do NOT:
+- Start converting URLs
+- Ask about URLs or output paths
+- Proceed to any conversion
+
+ONLY ask the questions in this setup flow, save EXTEND.md, then continue.
+
+## Setup Flow
+
+```
+No EXTEND.md found
+        |
+        v
++---------------------+
+| AskUserQuestion     |
+| (all questions)     |
++---------------------+
+        |
+        v
++---------------------+
+| Create EXTEND.md    |
++---------------------+
+        |
+        v
+    Continue conversion
+```
+
+## Questions
+
+**Language**: Use user's input language or saved language preference.
+
+Use AskUserQuestion with ALL questions in ONE call:
+
+### Question 1: Download Media
+
+```yaml
+header: "Media"
+question: "How to handle images and videos in pages?"
+options:
+  - label: "Ask each time (Recommended)"
+    description: "After saving markdown, ask whether to download media"
+  - label: "Always download"
+    description: "Always download media to local imgs/ and videos/ directories"
+  - label: "Never download"
+    description: "Keep original remote URLs in markdown"
+```
+
+### Question 2: Default Output Directory
+
+```yaml
+header: "Output"
+question: "Default output directory?"
+options:
+  - label: "url-to-markdown (Recommended)"
+    description: "Save to ./url-to-markdown/{domain}/{slug}.md"
+```
+
+Note: User will likely choose "Other" to type a custom path.
+
+### Question 3: Save Location
+
+```yaml
+header: "Save"
+question: "Where to save preferences?"
+options:
+  - label: "User (Recommended)"
+    description: "~/.baoyu-skills/ (all projects)"
+  - label: "Project"
+    description: ".baoyu-skills/ (this project only)"
+```
+
+## Save Locations
+
+| Choice | Path | Scope |
+|--------|------|-------|
+| User | `~/.baoyu-skills/baoyu-url-to-markdown/EXTEND.md` | All projects |
+| Project | `.baoyu-skills/baoyu-url-to-markdown/EXTEND.md` | Current project |
+
+## After Setup
+
+1. Create directory if needed
+2. Write EXTEND.md
+3. Confirm: "Preferences saved to [path]"
+4. Continue with conversion using saved preferences
+
+## EXTEND.md Template
+
+```md
+download_media: [ask/1/0]
+default_output_dir: [path or empty]
+```
+
+## Modifying Preferences Later
+
+Users can edit EXTEND.md directly or delete it to trigger setup again.

skills/baoyu-url-to-markdown/scripts/html-to-markdown.ts

@@ -1,7 +1,5 @@
-import { parseHTML } from "linkedom";
+import { JSDOM } from "jsdom";
-import { Readability } from "@mozilla/readability";
+import { Defuddle } from "defuddle/node";
-import TurndownService from "turndown";
-import { gfm } from "turndown-plugin-gfm";
 
 export interface PageMetadata {
   url: string;
@@ -9,6 +7,7 @@ export interface PageMetadata {
   description?: string;
   author?: string;
   published?: string;
+  coverImage?: string;
   captured_at: string;
 }
 
@@ -17,812 +16,58 @@ export interface ConversionResult {
   markdown: string;
 }
 
-interface ExtractionCandidate {
+export const absolutizeUrlsScript = String.raw`
-  title: string | null;
+(function() {
-  byline: string | null;
-  excerpt: string | null;
-  published: string | null;
-  html: string | null;
-  textContent: string;
-  method: string;
-}
-
-type AnyRecord = Record<string, unknown>;
-
-const MIN_CONTENT_LENGTH = 120;
-const GOOD_CONTENT_LENGTH = 900;
-
-const CONTENT_SELECTORS = [
-  "article",
-  "main article",
-  "[role='main'] article",
-  "[itemprop='articleBody']",
-  ".article-content",
-  ".article-body",
-  ".post-content",
-  ".entry-content",
-  ".story-body",
-  "main",
-  "[role='main']",
-  "#content",
-  ".content",
-];
-
-const REMOVE_SELECTORS = [
-  "script",
-  "style",
-  "noscript",
-  "template",
-  "iframe",
-  "svg",
-  "path",
-  "nav",
-  "aside",
-  "footer",
-  "header",
-  "form",
-  ".advertisement",
-  ".ads",
-  ".social-share",
-  ".related-articles",
-  ".comments",
-  ".newsletter",
-  ".cookie-banner",
-  ".cookie-consent",
-  "[role='navigation']",
-  "[aria-label*='cookie' i]",
-];
-
-const PUBLISHED_TIME_SELECTORS = [
-  "meta[property='article:published_time']",
-  "meta[name='pubdate']",
-  "meta[name='publishdate']",
-  "meta[name='date']",
-  "time[datetime]",
-];
-
-const ARTICLE_TYPES = new Set([
-  "Article",
-  "NewsArticle",
-  "BlogPosting",
-  "WebPage",
-  "ReportageNewsArticle",
-]);
-
-const NEXT_DATA_CONTENT_PATHS = [
-  "props.pageProps.content.body",
-  "props.pageProps.article.body",
-  "props.pageProps.article.content",
-  "props.pageProps.post.body",
-  "props.pageProps.post.content",
-  "props.pageProps.data.body",
-  "props.pageProps.story.body.content",
-];
-
-const cleanupAndExtractScriptBody = String.raw`
-(function () {
   const baseUrl = document.baseURI || location.href;
-
   function toAbsolute(url) {
     if (!url) return url;
-    try {
+    try { return new URL(url, baseUrl).href; } catch { return url; }
-      return new URL(url, baseUrl).href;
-    } catch {
-      return url;
   }
-  }
+  function absAttr(sel, attr) {
-
+    document.querySelectorAll(sel).forEach(el => {
-  function absolutizeAttr(selector, attr) {
+      const v = el.getAttribute(attr);
-    document.querySelectorAll(selector).forEach((el) => {
+      if (v) { const a = toAbsolute(v); if (a) el.setAttribute(attr, a); }
-      const value = el.getAttribute(attr);
-      if (!value) return;
-      const abs = toAbsolute(value);
-      if (abs) el.setAttribute(attr, abs);
     });
   }
-
+  function absSrcset(sel) {
-  function absolutizeSrcset(selector) {
+    document.querySelectorAll(sel).forEach(el => {
-    document.querySelectorAll(selector).forEach((el) => {
+      const s = el.getAttribute("srcset");
-      const srcset = el.getAttribute("srcset");
+      if (!s) return;
-      if (!srcset) return;
+      el.setAttribute("srcset", s.split(",").map(p => {
-
+        const t = p.trim(); if (!t) return "";
-      const normalized = srcset
+        const [url, ...d] = t.split(/\s+/);
-        .split(",")
+        return d.length ? toAbsolute(url) + " " + d.join(" ") : toAbsolute(url);
-        .map((part) => {
+      }).filter(Boolean).join(", "));
-          const trimmed = part.trim();
-          if (!trimmed) return "";
-          const pieces = trimmed.split(/\s+/);
-          const url = pieces[0];
-          const descriptor = pieces.slice(1).join(" ");
-          const absoluteUrl = toAbsolute(url);
-          return descriptor ? absoluteUrl + " " + descriptor : absoluteUrl;
-        })
-        .filter(Boolean)
-        .join(", ");
-
-      if (normalized) {
-        el.setAttribute("srcset", normalized);
-      }
     });
   }
-
+  document.querySelectorAll("img[data-src], video[data-src], audio[data-src], source[data-src]").forEach(el => {
-  absolutizeAttr("a[href]", "href");
+    const ds = el.getAttribute("data-src");
-  absolutizeAttr("img[src], video[src], audio[src], source[src]", "src");
+    if (ds && (!el.getAttribute("src") || el.getAttribute("src") === "" || el.getAttribute("src")?.startsWith("data:"))) {
-  absolutizeSrcset("img[srcset], source[srcset]");
+      el.setAttribute("src", ds);
-
-  const removeSelectors = [
-    "noscript",
-    "template",
-    ".cookie-banner",
-    ".cookie-consent",
-    ".consent-banner",
-    "[aria-label*='cookie' i]",
-    ".advertisement",
-    ".ads"
-  ];
-
-  for (const sel of removeSelectors) {
-    try {
-      document.querySelectorAll(sel).forEach((el) => el.remove());
-    } catch {}
     }
-
+  });
-  function getMeta(names) {
+  absAttr("a[href]", "href");
-    for (const name of names) {
+  absAttr("img[src], video[src], audio[src], source[src]", "src");
-      const el = document.querySelector('meta[name="' + name + '"]') || document.querySelector('meta[property="' + name + '"]');
+  absSrcset("img[srcset], source[srcset]");
-      const content = el && el.getAttribute("content");
+  return { html: document.documentElement.outerHTML };
-      if (content && content.trim()) return content.trim();
-    }
-    return undefined;
-  }
-
-  function flattenJsonLdItems(value) {
-    if (!value || typeof value !== "object") return [];
-    if (Array.isArray(value)) {
-      return value.flatMap(flattenJsonLdItems);
-    }
-
-    const obj = value;
-    if (Array.isArray(obj["@graph"])) {
-      return obj["@graph"].flatMap(flattenJsonLdItems);
-    }
-
-    return [obj];
-  }
-
-  function extractJsonLdMeta() {
-    const scripts = document.querySelectorAll('script[type="application/ld+json"]');
-    for (const script of scripts) {
-      try {
-        const parsed = JSON.parse(script.textContent || "");
-        const items = flattenJsonLdItems(parsed);
-        for (const item of items) {
-          const rawType = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"];
-          if (typeof rawType !== "string") continue;
-          if (!["Article", "NewsArticle", "BlogPosting", "WebPage"].includes(rawType)) continue;
-
-          const author = (() => {
-            if (typeof item.author === "string") return item.author;
-            if (Array.isArray(item.author) && item.author.length > 0) {
-              const first = item.author[0];
-              return first && typeof first === "object" ? first.name : undefined;
-            }
-            if (item.author && typeof item.author === "object") {
-              return item.author.name;
-            }
-            return undefined;
-          })();
-
-          return {
-            title: item.headline || item.name,
-            description: item.description,
-            author: typeof author === "string" ? author : undefined,
-            published: item.datePublished || item.dateCreated,
-          };
-        }
-      } catch {}
-    }
-    return {};
-  }
-
-  const jsonLd = extractJsonLdMeta();
-
-  const timeEl = document.querySelector("time[datetime]");
-  const title =
-    getMeta(["og:title", "twitter:title"]) ||
-    (typeof jsonLd.title === "string" ? jsonLd.title : undefined) ||
-    document.querySelector("h1")?.textContent?.trim() ||
-    document.title?.trim() ||
-    "";
-
-  const description =
-    getMeta(["description", "og:description", "twitter:description"]) ||
-    (typeof jsonLd.description === "string" ? jsonLd.description : undefined);
-
-  const author =
-    getMeta(["author", "article:author", "twitter:creator"]) ||
-    (typeof jsonLd.author === "string" ? jsonLd.author : undefined);
-
-  const published =
-    timeEl?.getAttribute("datetime") ||
-    getMeta(["article:published_time", "datePublished", "publishdate", "date"]) ||
-    (typeof jsonLd.published === "string" ? jsonLd.published : undefined);
-
-  return {
-    title,
-    description,
-    author,
-    published,
-    html: document.documentElement.outerHTML,
-  };
 })()
 `;
 
-export const cleanupAndExtractScript = cleanupAndExtractScriptBody;
+export async function extractContent(html: string, url: string): Promise<ConversionResult> {
+  const dom = new JSDOM(html, { url });
+  const result = await Defuddle(dom, url, { markdown: true });
 
-function pickString(...values: unknown[]): string | null {
+  const metadata: PageMetadata = {
-  for (const value of values) {
+    url,
-    if (typeof value === "string") {
+    title: result.title || "",
-      const trimmed = value.trim();
+    description: result.description || undefined,
-      if (trimmed) return trimmed;
+    author: result.author || undefined,
-    }
+    published: result.published || undefined,
-  }
+    coverImage: result.image || undefined,
-  return null;
+    captured_at: new Date().toISOString(),
-}
-
-function generateExcerpt(excerpt: string | null, textContent: string | null): string | null {
-  if (excerpt) return excerpt;
-  if (!textContent) return null;
-  const trimmed = textContent.trim();
-  if (!trimmed) return null;
-  return trimmed.length > 200 ? `${trimmed.slice(0, 200)}...` : trimmed;
-}
-
-function extractPublishedTime(document: any): string | null {
-  for (const selector of PUBLISHED_TIME_SELECTORS) {
-    const el = document.querySelector(selector);
-    if (!el) continue;
-    const value = el.getAttribute("content") ?? el.getAttribute("datetime");
-    if (value && value.trim()) return value.trim();
-  }
-  return null;
-}
-
-function extractTitle(document: any): string | null {
-  const ogTitle = document.querySelector("meta[property='og:title']")?.getAttribute("content");
-  if (ogTitle && ogTitle.trim()) return ogTitle.trim();
-
-  const twitterTitle = document
-    .querySelector("meta[name='twitter:title']")
-    ?.getAttribute("content");
-  if (twitterTitle && twitterTitle.trim()) return twitterTitle.trim();
-
-  const title = document.querySelector("title")?.textContent?.trim();
-  if (title) {
-    const cleaned = title.split(/\s*[-|–—]\s*/)[0]?.trim();
-    if (cleaned) return cleaned;
-  }
-
-  const h1 = document.querySelector("h1")?.textContent?.trim();
-  return h1 || null;
-}
-
-function extractTextFromHtml(html: string): string {
-  const { document } = parseHTML(`<!doctype html><html><body>${html}</body></html>`);
-  for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) {
-    for (const el of document.querySelectorAll(selector)) {
-      el.remove();
-    }
-  }
-  return document.body?.textContent?.replace(/\s+/g, " ").trim() ?? "";
-}
-
-function parseDocument(html: string): any {
-  const normalized = /<\s*html[\s>]/i.test(html)
-    ? html
-    : `<!doctype html><html><body>${html}</body></html>`;
-  return parseHTML(normalized).document;
-}
-
-function sanitizeHtml(html: string): string {
-  const { document } = parseHTML(`<div id="__root">${html}</div>`);
-  const root = document.querySelector("#__root");
-  if (!root) return html;
-
-  for (const selector of ["script", "style", "iframe", "noscript", "template", "svg", "path"]) {
-    for (const el of root.querySelectorAll(selector)) {
-      el.remove();
-    }
-  }
-
-  return root.innerHTML;
-}
-
-function flattenJsonLdItems(data: unknown): AnyRecord[] {
-  if (!data || typeof data !== "object") return [];
-  if (Array.isArray(data)) return data.flatMap(flattenJsonLdItems);
-
-  const item = data as AnyRecord;
-  if (Array.isArray(item["@graph"])) {
-    return (item["@graph"] as unknown[]).flatMap(flattenJsonLdItems);
-  }
-
-  return [item];
-}
-
-function parseJsonLdScripts(document: any): AnyRecord[] {
-  const results: AnyRecord[] = [];
-  const scripts = document.querySelectorAll("script[type='application/ld+json']");
-
-  for (const script of scripts) {
-    try {
-      const data = JSON.parse(script.textContent ?? "");
-      results.push(...flattenJsonLdItems(data));
-    } catch {
-      // ignore malformed blocks
-    }
-  }
-
-  return results;
-}
-
-function isArticleType(item: AnyRecord): boolean {
-  const value = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"];
-  return typeof value === "string" && ARTICLE_TYPES.has(value);
-}
-
-function extractAuthorFromJsonLd(authorData: unknown): string | null {
-  if (typeof authorData === "string") return authorData;
-  if (!authorData || typeof authorData !== "object") return null;
-
-  if (Array.isArray(authorData)) {
-    const names = authorData
-      .map((author) => extractAuthorFromJsonLd(author))
-      .filter((name): name is string => Boolean(name));
-    return names.length > 0 ? names.join(", ") : null;
-  }
-
-  const author = authorData as AnyRecord;
-  return typeof author.name === "string" ? author.name : null;
-}
-
-function parseJsonLdItem(item: AnyRecord): ExtractionCandidate | null {
-  if (!isArticleType(item)) return null;
-
-  const rawContent =
-    (typeof item.articleBody === "string" && item.articleBody) ||
-    (typeof item.text === "string" && item.text) ||
-    (typeof item.description === "string" && item.description) ||
-    null;
-
-  if (!rawContent) return null;
-
-  const content = rawContent.trim();
-  const htmlLike = /<\/?[a-z][\s\S]*>/i.test(content);
-  const textContent = htmlLike ? extractTextFromHtml(content) : content;
-
-  if (textContent.length < MIN_CONTENT_LENGTH) return null;
-
-  return {
-    title: pickString(item.headline, item.name),
-    byline: extractAuthorFromJsonLd(item.author),
-    excerpt: pickString(item.description),
-    published: pickString(item.datePublished, item.dateCreated),
-    html: htmlLike ? content : null,
-    textContent,
-    method: "json-ld",
-  };
-}
-
-function tryJsonLdExtraction(document: any): ExtractionCandidate | null {
-  for (const item of parseJsonLdScripts(document)) {
-    const extracted = parseJsonLdItem(item);
-    if (extracted) return extracted;
-  }
-  return null;
-}
-
-function getByPath(value: unknown, path: string): unknown {
-  let current = value;
-  for (const part of path.split(".")) {
-    if (!current || typeof current !== "object") return undefined;
-    current = (current as AnyRecord)[part];
-  }
-  return current;
-}
-
-function isContentBlockArray(value: unknown): value is AnyRecord[] {
-  if (!Array.isArray(value) || value.length === 0) return false;
-  return value.slice(0, 5).some((item) => {
-    if (!item || typeof item !== "object") return false;
-    const obj = item as AnyRecord;
-    return "type" in obj || "text" in obj || "textHtml" in obj || "content" in obj;
-  });
-}
-
-function extractTextFromContentBlocks(blocks: AnyRecord[]): string {
-  const parts: string[] = [];
-
-  function pushParagraph(text: string): void {
-    const trimmed = text.trim();
-    if (!trimmed) return;
-    parts.push(trimmed, "\n\n");
-  }
-
-  function walk(node: unknown): void {
-    if (!node || typeof node !== "object") return;
-    const block = node as AnyRecord;
-
-    if (typeof block.text === "string") {
-      pushParagraph(block.text);
-      return;
-    }
-
-    if (typeof block.textHtml === "string") {
-      pushParagraph(extractTextFromHtml(block.textHtml));
-      return;
-    }
-
-    if (Array.isArray(block.items)) {
-      for (const item of block.items) {
-        if (item && typeof item === "object") {
-          const text = pickString((item as AnyRecord).text);
-          if (text) parts.push(`- ${text}\n`);
-        }
-      }
-      parts.push("\n");
-    }
-
-    if (Array.isArray(block.components)) {
-      for (const component of block.components) {
-        walk(component);
-      }
-    }
-
-    if (Array.isArray(block.content)) {
-      for (const child of block.content) {
-        walk(child);
-      }
-    }
-  }
-
-  for (const block of blocks) {
-    walk(block);
-  }
-
-  return parts.join("").replace(/\n{3,}/g, "\n\n").trim();
-}
-
-function tryStringBodyExtraction(
-  content: string,
-  meta: AnyRecord,
-  document: any,
-  method: string
-): ExtractionCandidate | null {
-  if (!content || content.length < MIN_CONTENT_LENGTH) return null;
-
-  const isHtml = /<\/?[a-z][\s\S]*>/i.test(content);
-  const html = isHtml ? sanitizeHtml(content) : null;
-  const textContent = isHtml ? extractTextFromHtml(html) : content.trim();
-
-  if (textContent.length < MIN_CONTENT_LENGTH) return null;
-
-  return {
-    title: pickString(meta.headline, meta.title, extractTitle(document)),
-    byline: pickString(meta.byline, meta.author),
-    excerpt: pickString(meta.description, meta.excerpt, generateExcerpt(null, textContent)),
-    published: pickString(meta.datePublished, meta.publishedAt, extractPublishedTime(document)),
-    html,
-    textContent,
-    method,
-  };
-}
-
-function tryNextDataExtraction(document: any): ExtractionCandidate | null {
-  try {
-    const script = document.querySelector("script#__NEXT_DATA__");
-    if (!script?.textContent) return null;
-
-    const data = JSON.parse(script.textContent) as AnyRecord;
-    const pageProps = (getByPath(data, "props.pageProps") ?? {}) as AnyRecord;
-
-    for (const path of NEXT_DATA_CONTENT_PATHS) {
-      const value = getByPath(data, path);
-
-      if (typeof value === "string") {
-        const parentPath = path.split(".").slice(0, -1).join(".");
-        const parent = (getByPath(data, parentPath) ?? {}) as AnyRecord;
-        const meta = {
-          ...pageProps,
-          ...parent,
-          title: parent.title ?? (pageProps.title as string | undefined),
   };
 
-        const candidate = tryStringBodyExtraction(value, meta, document, "next-data");
+  return { metadata, markdown: result.content || "" };
-        if (candidate) return candidate;
-      }
-
-      if (isContentBlockArray(value)) {
-        const textContent = extractTextFromContentBlocks(value);
-        if (textContent.length < MIN_CONTENT_LENGTH) continue;
-
-        return {
-          title: pickString(
-            getByPath(data, "props.pageProps.content.headline"),
-            getByPath(data, "props.pageProps.article.headline"),
-            getByPath(data, "props.pageProps.article.title"),
-            getByPath(data, "props.pageProps.post.title"),
-            pageProps.title,
-            extractTitle(document)
-          ),
-          byline: pickString(
-            getByPath(data, "props.pageProps.author.name"),
-            getByPath(data, "props.pageProps.article.author.name")
-          ),
-          excerpt: pickString(
-            getByPath(data, "props.pageProps.content.description"),
-            getByPath(data, "props.pageProps.article.description"),
-            pageProps.description,
-            generateExcerpt(null, textContent)
-          ),
-          published: pickString(
-            getByPath(data, "props.pageProps.content.datePublished"),
-            getByPath(data, "props.pageProps.article.datePublished"),
-            getByPath(data, "props.pageProps.publishedAt"),
-            extractPublishedTime(document)
-          ),
-          html: null,
-          textContent,
-          method: "next-data",
-        };
-      }
-    }
-  } catch {
-    return null;
-  }
-
-  return null;
-}
-
-function buildReadabilityCandidate(
-  article: ReturnType<Readability["parse"]>,
-  document: any,
-  method: string
-): ExtractionCandidate | null {
-  const textContent = article?.textContent?.trim() ?? "";
-  if (textContent.length < MIN_CONTENT_LENGTH) return null;
-
-  return {
-    title: pickString(article?.title, extractTitle(document)),
-    byline: pickString(article?.byline),
-    excerpt: pickString(article?.excerpt, generateExcerpt(null, textContent)),
-    published: pickString(article?.publishedTime, extractPublishedTime(document)),
-    html: article?.content ? sanitizeHtml(article.content) : null,
-    textContent,
-    method,
-  };
-}
-
-function tryReadability(document: any): ExtractionCandidate | null {
-  try {
-    const strictClone = document.cloneNode(true) as Document;
-    const strictResult = buildReadabilityCandidate(
-      new Readability(strictClone).parse(),
-      document,
-      "readability"
-    );
-    if (strictResult) return strictResult;
-
-    const relaxedClone = document.cloneNode(true) as Document;
-    return buildReadabilityCandidate(
-      new Readability(relaxedClone, { charThreshold: 120 }).parse(),
-      document,
-      "readability-relaxed"
-    );
-  } catch {
-    return null;
-  }
-}
-
-function trySelectorExtraction(document: any): ExtractionCandidate | null {
-  for (const selector of CONTENT_SELECTORS) {
-    const element = document.querySelector(selector);
-    if (!element) continue;
-
-    const clone = element.cloneNode(true) as Element;
-    for (const removeSelector of REMOVE_SELECTORS) {
-      for (const node of clone.querySelectorAll(removeSelector)) {
-        node.remove();
-      }
-    }
-
-    const html = sanitizeHtml(clone.innerHTML);
-    const textContent = extractTextFromHtml(html);
-    if (textContent.length < MIN_CONTENT_LENGTH) continue;
-
-    return {
-      title: extractTitle(document),
-      byline: null,
-      excerpt: generateExcerpt(null, textContent),
-      published: extractPublishedTime(document),
-      html,
-      textContent,
-      method: `selector:${selector}`,
-    };
-  }
-
-  return null;
-}
-
-function tryBodyExtraction(document: any): ExtractionCandidate | null {
-  const body = document.body;
-  if (!body) return null;
-
-  const clone = body.cloneNode(true) as Element;
-  for (const removeSelector of REMOVE_SELECTORS) {
-    for (const node of clone.querySelectorAll(removeSelector)) {
-      node.remove();
-    }
-  }
-
-  const html = sanitizeHtml(clone.innerHTML);
-  const textContent = extractTextFromHtml(html);
-  if (!textContent) return null;
-
-  return {
-    title: extractTitle(document),
-    byline: null,
-    excerpt: generateExcerpt(null, textContent),
-    published: extractPublishedTime(document),
-    html,
-    textContent,
-    method: "body-fallback",
-  };
-}
-
-function pickBestCandidate(candidates: ExtractionCandidate[]): ExtractionCandidate | null {
-  if (candidates.length === 0) return null;
-
-  const methodOrder = [
-    "readability",
-    "readability-relaxed",
-    "next-data",
-    "json-ld",
-    "selector:",
-    "body-fallback",
-  ];
-
-  function methodRank(method: string): number {
-    const idx = methodOrder.findIndex((entry) =>
-      entry.endsWith(":") ? method.startsWith(entry) : method === entry
-    );
-    return idx === -1 ? methodOrder.length : idx;
-  }
-
-  const ranked = [...candidates].sort((a, b) => {
-    const rankA = methodRank(a.method);
-    const rankB = methodRank(b.method);
-    if (rankA !== rankB) return rankA - rankB;
-    return (b.textContent.length ?? 0) - (a.textContent.length ?? 0);
-  });
-
-  for (const candidate of ranked) {
-    if (candidate.textContent.length >= GOOD_CONTENT_LENGTH) {
-      return candidate;
-    }
-  }
-
-  for (const candidate of ranked) {
-    if (candidate.textContent.length >= MIN_CONTENT_LENGTH) {
-      return candidate;
-    }
-  }
-
-  return ranked[0];
-}
-
-function extractFromHtml(html: string): ExtractionCandidate | null {
-  const document = parseDocument(html);
-
-  const readabilityCandidate = tryReadability(document);
-  const nextDataCandidate = tryNextDataExtraction(document);
-  const jsonLdCandidate = tryJsonLdExtraction(document);
-  const selectorCandidate = trySelectorExtraction(document);
-  const bodyCandidate = tryBodyExtraction(document);
-
-  const candidates = [
-    readabilityCandidate,
-    nextDataCandidate,
-    jsonLdCandidate,
-    selectorCandidate,
-    bodyCandidate,
-  ].filter((candidate): candidate is ExtractionCandidate => Boolean(candidate));
-
-  const winner = pickBestCandidate(candidates);
-  if (!winner) return null;
-
-  return {
-    ...winner,
-    title: winner.title ?? extractTitle(document),
-    published: winner.published ?? extractPublishedTime(document),
-    excerpt: winner.excerpt ?? generateExcerpt(null, winner.textContent),
-  };
-}
-
-const turndown = new TurndownService({
-  headingStyle: "atx",
-  hr: "---",
-  bulletListMarker: "-",
-  codeBlockStyle: "fenced",
-  emDelimiter: "*",
-  strongDelimiter: "**",
-  linkStyle: "inlined",
-});
-
-turndown.use(gfm);
-
-turndown.remove(["script", "style", "iframe", "noscript", "template", "svg", "path"]);
-
-turndown.addRule("collapseFigure", {
-  filter: "figure",
-  replacement(content) {
-    return `\n\n${content.trim()}\n\n`;
-  },
-});
-
-turndown.addRule("dropInvisibleAnchors", {
-  filter(node) {
-    return node.nodeName === "A" && !(node as Element).textContent?.trim();
-  },
-  replacement() {
-    return "";
-  },
-});
-
-function normalizeMarkdown(markdown: string): string {
-  return markdown
-    .replace(/\r\n/g, "\n")
-    .replace(/[ \t]+\n/g, "\n")
-    .replace(/\n{3,}/g, "\n\n")
-    .trim();
-}
-
-function convertHtmlToMarkdown(html: string): string {
-  if (!html || !html.trim()) return "";
-
-  try {
-    const sanitized = sanitizeHtml(html);
-    return turndown.turndown(sanitized);
-  } catch {
-    return "";
-  }
-}
-
-function fallbackPlainText(html: string): string {
-  const document = parseDocument(html);
-  for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) {
-    for (const el of document.querySelectorAll(selector)) {
-      el.remove();
-    }
-  }
-  const text = document.body?.textContent ?? document.documentElement?.textContent ?? "";
-  return normalizeMarkdown(text.replace(/\s+/g, " "));
-}
-
-export function htmlToMarkdown(html: string): string {
-  if (!html || !html.trim()) return "";
-
-  const extracted = extractFromHtml(html);
-  if (!extracted) {
-    return fallbackPlainText(html);
-  }
-
-  let markdown = extracted.html ? convertHtmlToMarkdown(extracted.html) : "";
-  if (!markdown.trim()) {
-    markdown = extracted.textContent;
-  }
-
-  return normalizeMarkdown(markdown);
 }
 
 function escapeYamlValue(value: string): string {
@@ -836,6 +81,7 @@ export function formatMetadataYaml(meta: PageMetadata): string {
   if (meta.description) lines.push(`description: "${escapeYamlValue(meta.description)}"`);
   if (meta.author) lines.push(`author: "${escapeYamlValue(meta.author)}"`);
   if (meta.published) lines.push(`published: "${escapeYamlValue(meta.published)}"`);
+  if (meta.coverImage) lines.push(`coverImage: "${escapeYamlValue(meta.coverImage)}"`);
   lines.push(`captured_at: "${escapeYamlValue(meta.captured_at)}"`);
   lines.push("---");
   return lines.join("\n");

skills/baoyu-url-to-markdown/scripts/main.ts

@@ -4,7 +4,8 @@ import path from "node:path";
 import process from "node:process";
 
 import { CdpConnection, getFreePort, launchChrome, waitForChromeDebugPort, waitForNetworkIdle, waitForPageLoad, autoScroll, evaluateScript, killChrome } from "./cdp.js";
-import { cleanupAndExtractScript, htmlToMarkdown, createMarkdownDocument, type PageMetadata, type ConversionResult } from "./html-to-markdown.js";
+import { absolutizeUrlsScript, extractContent, createMarkdownDocument, type ConversionResult } from "./html-to-markdown.js";
+import { localizeMarkdownMedia, countRemoteMedia } from "./media-localizer.js";
 import { resolveUrlToMarkdownDataDir } from "./paths.js";
 import { DEFAULT_TIMEOUT_MS, CDP_CONNECT_TIMEOUT_MS, NETWORK_IDLE_TIMEOUT_MS, POST_LOAD_DELAY_MS, SCROLL_STEP_WAIT_MS, SCROLL_MAX_STEPS } from "./constants.js";
 
@@ -26,10 +27,11 @@ interface Args {
   output?: string;
   wait: boolean;
   timeout: number;
+  downloadMedia: boolean;
 }
 
 function parseArgs(argv: string[]): Args {
-  const args: Args = { url: "", wait: false, timeout: DEFAULT_TIMEOUT_MS };
+  const args: Args = { url: "", wait: false, timeout: DEFAULT_TIMEOUT_MS, downloadMedia: false };
   for (let i = 2; i < argv.length; i++) {
     const arg = argv[i];
     if (arg === "--wait" || arg === "-w") {
@@ -38,6 +40,8 @@ function parseArgs(argv: string[]): Args {
       args.output = argv[++i];
     } else if (arg === "--timeout" || arg === "-t") {
       args.timeout = parseInt(argv[++i], 10) || DEFAULT_TIMEOUT_MS;
+    } else if (arg === "--download-media") {
+      args.downloadMedia = true;
     } else if (!arg.startsWith("-") && !args.url) {
       args.url = arg;
     }
@@ -117,21 +121,11 @@ async function captureUrl(args: Args): Promise<ConversionResult> {
     }
 
     console.log("Capturing page content...");
-    const extracted = await evaluateScript<{ title: string; description?: string; author?: string; published?: string; html: string }>(
+    const { html } = await evaluateScript<{ html: string }>(
-      cdp, sessionId, cleanupAndExtractScript, args.timeout
+      cdp, sessionId, absolutizeUrlsScript, args.timeout
     );
 
-    const metadata: PageMetadata = {
+    return await extractContent(html, args.url);
-      url: args.url,
-      title: extracted.title || "",
-      description: extracted.description,
-      author: extracted.author,
-      published: extracted.published,
-      captured_at: new Date().toISOString()
-    };
-
-    const markdown = htmlToMarkdown(extracted.html);
-    return { metadata, markdown };
   } finally {
     if (cdp) {
       try { await cdp.send("Browser.close", {}, { timeoutMs: 5_000 }); } catch {}
@@ -163,7 +157,24 @@ async function main(): Promise<void> {
   const outputDir = path.dirname(outputPath);
   await mkdir(outputDir, { recursive: true });
 
-  const document = createMarkdownDocument(result);
+  let document = createMarkdownDocument(result);
+
+  if (args.downloadMedia) {
+    const mediaResult = await localizeMarkdownMedia(document, {
+      markdownPath: outputPath,
+      log: console.log,
+    });
+    document = mediaResult.markdown;
+    if (mediaResult.downloadedImages > 0 || mediaResult.downloadedVideos > 0) {
+      console.log(`Downloaded: ${mediaResult.downloadedImages} images, ${mediaResult.downloadedVideos} videos`);
+    }
+  } else {
+    const { images, videos } = countRemoteMedia(document);
+    if (images > 0 || videos > 0) {
+      console.log(`Remote media found: ${images} images, ${videos} videos`);
+    }
+  }
+
   await writeFile(outputPath, document, "utf-8");
 
   console.log(`Saved: ${outputPath}`);

skills/baoyu-url-to-markdown/scripts/media-localizer.ts

@@ -0,0 +1,317 @@
+import path from "node:path";
+import { mkdir, writeFile } from "node:fs/promises";
+
+type MediaKind = "image" | "video";
+type MediaHint = "image" | "unknown";
+
+type MarkdownLinkCandidate = {
+  url: string;
+  hint: MediaHint;
+};
+
+export type LocalizeMarkdownMediaOptions = {
+  markdownPath: string;
+  log?: (message: string) => void;
+};
+
+export type LocalizeMarkdownMediaResult = {
+  markdown: string;
+  downloadedImages: number;
+  downloadedVideos: number;
+  imageDir: string | null;
+  videoDir: string | null;
+};
+
+const MARKDOWN_LINK_RE = /(!?\[[^\]\n]*\])\((<)?(https?:\/\/[^)\s>]+)(>)?\)/g;
+const FRONTMATTER_COVER_RE = /^(coverImage:\s*")(https?:\/\/[^"]+)(")/m;
+
+const IMAGE_EXTENSIONS = new Set([
+  "jpg",
+  "jpeg",
+  "png",
+  "webp",
+  "gif",
+  "bmp",
+  "avif",
+  "heic",
+  "heif",
+  "svg",
+]);
+
+const VIDEO_EXTENSIONS = new Set(["mp4", "m4v", "mov", "webm", "mkv"]);
+
+const MIME_EXTENSION_MAP: Record<string, string> = {
+  "image/jpeg": "jpg",
+  "image/jpg": "jpg",
+  "image/png": "png",
+  "image/webp": "webp",
+  "image/gif": "gif",
+  "image/bmp": "bmp",
+  "image/avif": "avif",
+  "image/heic": "heic",
+  "image/heif": "heif",
+  "image/svg+xml": "svg",
+  "video/mp4": "mp4",
+  "video/webm": "webm",
+  "video/quicktime": "mov",
+  "video/x-m4v": "m4v",
+};
+
+const DOWNLOAD_USER_AGENT =
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
+
+function normalizeContentType(raw: string | null): string {
+  return raw?.split(";")[0]?.trim().toLowerCase() ?? "";
+}
+
+function normalizeExtension(raw: string | undefined | null): string | undefined {
+  if (!raw) return undefined;
+  const trimmed = raw.replace(/^\./, "").trim().toLowerCase();
+  if (!trimmed) return undefined;
+  if (trimmed === "jpeg") return "jpg";
+  if (trimmed === "jpg") return "jpg";
+  return trimmed;
+}
+
+function resolveExtensionFromUrl(rawUrl: string): string | undefined {
+  try {
+    const parsed = new URL(rawUrl);
+    const extFromPath = normalizeExtension(path.posix.extname(parsed.pathname));
+    if (extFromPath) return extFromPath;
+    const extFromFormat = normalizeExtension(parsed.searchParams.get("format"));
+    if (extFromFormat) return extFromFormat;
+  } catch {
+    return undefined;
+  }
+  return undefined;
+}
+
+function resolveKindFromContentType(contentType: string): MediaKind | undefined {
+  if (!contentType) return undefined;
+  if (contentType.startsWith("image/")) return "image";
+  if (contentType.startsWith("video/")) return "video";
+  return undefined;
+}
+
+function resolveKindFromExtension(ext: string | undefined): MediaKind | undefined {
+  if (!ext) return undefined;
+  if (IMAGE_EXTENSIONS.has(ext)) return "image";
+  if (VIDEO_EXTENSIONS.has(ext)) return "video";
+  return undefined;
+}
+
+function resolveMediaKind(
+  rawUrl: string,
+  contentType: string,
+  extension: string | undefined,
+  hint: MediaHint
+): MediaKind | undefined {
+  const kindFromType = resolveKindFromContentType(contentType);
+  if (kindFromType) return kindFromType;
+
+  const kindFromExtension = resolveKindFromExtension(extension);
+  if (kindFromExtension) return kindFromExtension;
+
+  if (contentType && contentType !== "application/octet-stream") {
+    return undefined;
+  }
+
+  return hint === "image" ? "image" : undefined;
+}
+
+function resolveOutputExtension(
+  contentType: string,
+  extension: string | undefined,
+  kind: MediaKind
+): string {
+  const extFromMime = normalizeExtension(MIME_EXTENSION_MAP[contentType]);
+  if (extFromMime) return extFromMime;
+
+  const normalizedExt = normalizeExtension(extension);
+  if (normalizedExt) return normalizedExt;
+
+  return kind === "video" ? "mp4" : "jpg";
+}
+
+function safeDecodeURIComponent(value: string): string {
+  try {
+    return decodeURIComponent(value);
+  } catch {
+    return value;
+  }
+}
+
+function sanitizeFileSegment(input: string): string {
+  return input
+    .replace(/[^a-zA-Z0-9_-]+/g, "-")
+    .replace(/-+/g, "-")
+    .replace(/^[-_]+|[-_]+$/g, "")
+    .slice(0, 48);
+}
+
+function resolveFileStem(rawUrl: string, extension: string): string {
+  try {
+    const parsed = new URL(rawUrl);
+    const base = path.posix.basename(parsed.pathname);
+    if (!base) return "";
+    const decodedBase = safeDecodeURIComponent(base);
+    const normalizedExt = normalizeExtension(extension);
+    const stripExt = normalizedExt ? new RegExp(`\\.${normalizedExt}$`, "i") : null;
+    const rawStem = stripExt ? decodedBase.replace(stripExt, "") : decodedBase;
+    return sanitizeFileSegment(rawStem);
+  } catch {
+    return "";
+  }
+}
+
+function buildFileName(kind: MediaKind, index: number, sourceUrl: string, extension: string): string {
+  const stem = resolveFileStem(sourceUrl, extension);
+  const prefix = kind === "image" ? "img" : "video";
+  const serial = String(index).padStart(3, "0");
+  const suffix = stem ? `-${stem}` : "";
+  return `${prefix}-${serial}${suffix}.${extension}`;
+}
+
+function collectMarkdownLinkCandidates(markdown: string): MarkdownLinkCandidate[] {
+  const candidates: MarkdownLinkCandidate[] = [];
+  const seen = new Set<string>();
+
+  const fmMatch = markdown.match(/^---\n([\s\S]*?)\n---/);
+  if (fmMatch) {
+    const coverMatch = fmMatch[1]?.match(FRONTMATTER_COVER_RE);
+    if (coverMatch?.[2] && !seen.has(coverMatch[2])) {
+      seen.add(coverMatch[2]);
+      candidates.push({ url: coverMatch[2], hint: "image" });
+    }
+  }
+
+  MARKDOWN_LINK_RE.lastIndex = 0;
+  let match: RegExpExecArray | null;
+  while ((match = MARKDOWN_LINK_RE.exec(markdown))) {
+    const label = match[1] ?? "";
+    const rawUrl = match[3] ?? "";
+    if (!rawUrl || seen.has(rawUrl)) continue;
+    seen.add(rawUrl);
+    candidates.push({
+      url: rawUrl,
+      hint: label.startsWith("![") ? "image" : "unknown",
+    });
+  }
+
+  return candidates;
+}
+
+function rewriteMarkdownMediaLinks(markdown: string, replacements: Map<string, string>): string {
+  if (replacements.size === 0) return markdown;
+  MARKDOWN_LINK_RE.lastIndex = 0;
+
+  let result = markdown.replace(MARKDOWN_LINK_RE, (full, label, _openAngle, rawUrl) => {
+    const localPath = replacements.get(rawUrl);
+    if (!localPath) return full;
+    return `${label}(${localPath})`;
+  });
+
+  result = result.replace(FRONTMATTER_COVER_RE, (full, prefix, rawUrl, suffix) => {
+    const localPath = replacements.get(rawUrl);
+    if (!localPath) return full;
+    return `${prefix}${localPath}${suffix}`;
+  });
+
+  return result;
+}
+
+export async function localizeMarkdownMedia(
+  markdown: string,
+  options: LocalizeMarkdownMediaOptions
+): Promise<LocalizeMarkdownMediaResult> {
+  const log = options.log ?? (() => {});
+  const markdownDir = path.dirname(options.markdownPath);
+  const candidates = collectMarkdownLinkCandidates(markdown);
+
+  if (candidates.length === 0) {
+    return {
+      markdown,
+      downloadedImages: 0,
+      downloadedVideos: 0,
+      imageDir: null,
+      videoDir: null,
+    };
+  }
+
+  const replacements = new Map<string, string>();
+  let downloadedImages = 0;
+  let downloadedVideos = 0;
+
+  for (const candidate of candidates) {
+    try {
+      const response = await fetch(candidate.url, {
+        method: "GET",
+        redirect: "follow",
+        headers: {
+          "user-agent": DOWNLOAD_USER_AGENT,
+        },
+      });
+
+      if (!response.ok) {
+        log(`[url-to-markdown] Skip media (${response.status}): ${candidate.url}`);
+        continue;
+      }
+
+      const sourceUrl = response.url || candidate.url;
+      const contentType = normalizeContentType(response.headers.get("content-type"));
+      const extension = resolveExtensionFromUrl(sourceUrl) ?? resolveExtensionFromUrl(candidate.url);
+      const kind = resolveMediaKind(sourceUrl, contentType, extension, candidate.hint);
+      if (!kind) {
+        continue;
+      }
+
+      const outputExtension = resolveOutputExtension(contentType, extension, kind);
+      const nextIndex = kind === "image" ? downloadedImages + 1 : downloadedVideos + 1;
+      const dirName = kind === "image" ? "imgs" : "videos";
+      const targetDir = path.join(markdownDir, dirName);
+      await mkdir(targetDir, { recursive: true });
+
+      const fileName = buildFileName(kind, nextIndex, sourceUrl, outputExtension);
+      const absolutePath = path.join(targetDir, fileName);
+      const relativePath = path.posix.join(dirName, fileName);
+      const bytes = Buffer.from(await response.arrayBuffer());
+      await writeFile(absolutePath, bytes);
+      replacements.set(candidate.url, relativePath);
+
+      if (kind === "image") {
+        downloadedImages = nextIndex;
+      } else {
+        downloadedVideos = nextIndex;
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error ?? "");
+      log(`[url-to-markdown] Failed to download media ${candidate.url}: ${message}`);
+    }
+  }
+
+  return {
+    markdown: rewriteMarkdownMediaLinks(markdown, replacements),
+    downloadedImages,
+    downloadedVideos,
+    imageDir: downloadedImages > 0 ? path.join(markdownDir, "imgs") : null,
+    videoDir: downloadedVideos > 0 ? path.join(markdownDir, "videos") : null,
+  };
+}
+
+export function countRemoteMedia(markdown: string): { images: number; videos: number; hasCoverImage: boolean } {
+  const fmMatch = markdown.match(/^---\n([\s\S]*?)\n---/);
+  const hasCoverImage = !!(fmMatch?.[1]?.match(FRONTMATTER_COVER_RE)?.[2]);
+  const candidates = collectMarkdownLinkCandidates(markdown);
+  let images = 0;
+  let videos = 0;
+  for (const c of candidates) {
+    const ext = resolveExtensionFromUrl(c.url);
+    const kind = resolveKindFromExtension(ext);
+    if (kind === "video") {
+      videos++;
+    } else if (kind === "image" || c.hint === "image") {
+      images++;
+    }
+  }
+  return { images, videos, hasCoverImage };
+}