mirror of
https://github.com/JimLiu/baoyu-skills.git
synced 2026-07-12 22:09:48 +08:00
Compare commits
24 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 46af919634 | |||
| 5560db595a | |||
| 3b031c7768 | |||
| 39c7e86a8d | |||
| f00ca11d9a | |||
| 87c2509268 | |||
| 20bf6d9ad8 | |||
| 33bc0a6255 | |||
| c15a44b439 | |||
| 6e533f938f | |||
| c60eb85629 | |||
| 20d2a78f87 | |||
| bb63ee2a2e | |||
| 5fc697166d | |||
| 1ed204bd5a | |||
| be2cbecfb0 | |||
| bce96e411d | |||
| fe404c493d | |||
| 6dd754ccdd | |||
| 1f4415c8ff | |||
| b25eb87bb4 | |||
| dbf35503a3 | |||
| 36ccbd0300 | |||
| bc878e5157 |
@@ -6,7 +6,7 @@
|
||||
},
|
||||
"metadata": {
|
||||
"description": "Skills shared by Baoyu for improving daily work efficiency",
|
||||
"version": "1.48.0"
|
||||
"version": "1.53.0"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
@@ -18,6 +18,7 @@
|
||||
"./skills/baoyu-xhs-images",
|
||||
"./skills/baoyu-post-to-x",
|
||||
"./skills/baoyu-post-to-wechat",
|
||||
"./skills/baoyu-post-to-weibo",
|
||||
"./skills/baoyu-article-illustrator",
|
||||
"./skills/baoyu-cover-image",
|
||||
"./skills/baoyu-slide-deck",
|
||||
|
||||
@@ -50,16 +50,16 @@ Just run `/release-skills` - auto-detects your project configuration.
|
||||
|
||||
**Language Detection Rules**:
|
||||
|
||||
| Filename Pattern | Language |
|
||||
|------------------|----------|
|
||||
| `CHANGELOG.md` (no suffix) | en (default) |
|
||||
| `CHANGELOG.zh.md` / `CHANGELOG_CN.md` / `CHANGELOG.zh-CN.md` | zh |
|
||||
| `CHANGELOG.ja.md` / `CHANGELOG_JP.md` | ja |
|
||||
| `CHANGELOG.ko.md` / `CHANGELOG_KR.md` | ko |
|
||||
| `CHANGELOG.de.md` / `CHANGELOG_DE.md` | de |
|
||||
| `CHANGELOG.fr.md` / `CHANGELOG_FR.md` | fr |
|
||||
| `CHANGELOG.es.md` / `CHANGELOG_ES.md` | es |
|
||||
| `CHANGELOG.{lang}.md` | Corresponding language code |
|
||||
Changelog files follow the pattern `CHANGELOG_{LANG}.md` or `CHANGELOG.{lang}.md`, where `{lang}` / `{LANG}` is a language or region code.
|
||||
|
||||
| Pattern | Example | Language |
|
||||
|---------|---------|----------|
|
||||
| No suffix | `CHANGELOG.md` | en (default) |
|
||||
| `_{LANG}` (uppercase) | `CHANGELOG_CN.md`, `CHANGELOG_JP.md` | Corresponding language |
|
||||
| `.{lang}` (lowercase) | `CHANGELOG.zh.md`, `CHANGELOG.ja.md` | Corresponding language |
|
||||
| `.{lang-region}` | `CHANGELOG.zh-CN.md` | Corresponding region variant |
|
||||
|
||||
Common language codes: `zh` (Chinese), `ja` (Japanese), `ko` (Korean), `de` (German), `fr` (French), `es` (Spanish).
|
||||
|
||||
**Output Example**:
|
||||
```
|
||||
|
||||
@@ -153,6 +153,7 @@ infographic/
|
||||
illustrations/
|
||||
comic/
|
||||
translate/
|
||||
posts/
|
||||
### IntelliJ IDEA ###
|
||||
.idea
|
||||
*.iws
|
||||
|
||||
@@ -2,6 +2,75 @@
|
||||
|
||||
English | [中文](./CHANGELOG.zh.md)
|
||||
|
||||
## 1.53.0 - 2026-03-06
|
||||
|
||||
### Features
|
||||
- `baoyu-url-to-markdown`: save rendered HTML snapshot as `-captured.html` alongside markdown output
|
||||
- `baoyu-url-to-markdown`: Defuddle-first markdown conversion with automatic fallback to legacy Readability/selector extractor
|
||||
|
||||
## 1.52.0 - 2026-03-06
|
||||
|
||||
### Features
|
||||
- `baoyu-post-to-weibo`: add video upload support via `--video` flag (max 18 files total)
|
||||
- `baoyu-post-to-weibo`: switch from clipboard paste to `DOM.setFileInputFiles` for more reliable uploads
|
||||
|
||||
### Fixes
|
||||
- `baoyu-post-to-weibo`: add Chrome health check with auto-restart for unresponsive instances
|
||||
- `baoyu-post-to-weibo`: add navigation check to ensure Weibo home page before posting
|
||||
|
||||
## 1.51.2 - 2026-03-06
|
||||
|
||||
### Fixes
|
||||
- `release-skills`: replace explicit language filename patterns (e.g. `CHANGELOG.de.md`) with generic pattern to avoid Gen Agent Trust Hub URL scanner false positive
|
||||
- `baoyu-infographic`: add credential/secret stripping instructions to address Snyk W007 insecure credential handling audit
|
||||
|
||||
## 1.51.1 - 2026-03-06
|
||||
|
||||
### Refactor
|
||||
- Unify Chrome CDP profile path — all skills now share `baoyu-skills/chrome-profile` instead of per-skill directories
|
||||
- Fix `baoyu-post-to-weibo` incorrectly reusing `x-browser-profile` path
|
||||
|
||||
### Fixes
|
||||
- Remove `curl | bash` remote code execution pattern from all install instructions
|
||||
- Enforce HTTPS-only for remote image downloads in `md-to-html` scripts
|
||||
- Add redirect limit (max 5) to prevent infinite redirect loops
|
||||
- Add Security Guidelines section to CLAUDE.md
|
||||
|
||||
## 1.51.0 - 2026-03-06
|
||||
|
||||
### Features
|
||||
- `baoyu-post-to-weibo`: new skill for posting to Weibo — supports text posts with images and headline articles (头条文章) via Chrome CDP
|
||||
- `baoyu-format-markdown`: add title/summary multi-candidate selection — generates 3 candidates for user to pick, with `auto_select` EXTEND.md support
|
||||
|
||||
## 1.50.0 - 2026-03-06
|
||||
|
||||
### Features
|
||||
- `baoyu-translate`: expand translation style presets from 4 to 9 — add academic, business, humorous, conversational, and elegant styles
|
||||
- `baoyu-translate`: add `--style` CLI flag for per-invocation style override
|
||||
- `baoyu-translate`: integrate style instructions into subagent prompt template
|
||||
|
||||
## 1.49.0 - 2026-03-06
|
||||
|
||||
### Features
|
||||
- `baoyu-format-markdown`: add reader-perspective content analysis phase — analyzes highlights, structure, and formatting issues before applying formatting
|
||||
- `baoyu-format-markdown`: restructure workflow from 8 steps to 7 with explicit do/don't formatting principles and completion report
|
||||
- `baoyu-translate`: extract Step 2 workflow mechanics to separate reference file for cleaner SKILL.md
|
||||
- `baoyu-translate`: expand trigger keywords (改成中文, 快翻, 本地化, etc.) for better skill activation
|
||||
- `baoyu-translate`: add proactive warning for long content in quick mode
|
||||
- `baoyu-translate`: save frontmatter to `chunks/frontmatter.md` during chunking
|
||||
|
||||
## 1.48.2 - 2026-03-06
|
||||
|
||||
### Features
|
||||
- `baoyu-translate`: add figurative language & emotional fidelity review steps to refined workflow critique and revision stages
|
||||
- `baoyu-translate`: enhance quick mode to enforce meaning-first translation principles for figurative language
|
||||
|
||||
## 1.48.1 - 2026-03-05
|
||||
|
||||
### Features
|
||||
- `baoyu-translate`: add figurative language & metaphor mapping to analysis step — interprets metaphors, idioms, and implied meanings before translation instead of translating literally
|
||||
- `baoyu-translate`: add "meaning over words", "figurative language", and "emotional fidelity" translation principles to SKILL.md, refined workflow, and subagent prompt template
|
||||
|
||||
## 1.48.0 - 2026-03-05
|
||||
|
||||
### Features
|
||||
|
||||
@@ -2,6 +2,75 @@
|
||||
|
||||
[English](./CHANGELOG.md) | 中文
|
||||
|
||||
## 1.53.0 - 2026-03-06
|
||||
|
||||
### 新功能
|
||||
- `baoyu-url-to-markdown`:将渲染后的 HTML 快照保存为 `-captured.html`,与 Markdown 文件并列输出
|
||||
- `baoyu-url-to-markdown`:优先使用 Defuddle 转换,失败时自动回退到旧版 Readability/选择器提取器
|
||||
|
||||
## 1.52.0 - 2026-03-06
|
||||
|
||||
### 新功能
|
||||
- `baoyu-post-to-weibo`:新增 `--video` 视频上传支持(图片+视频最多 18 个文件)
|
||||
- `baoyu-post-to-weibo`:上传方式从剪贴板粘贴改为 `DOM.setFileInputFiles`,提升上传可靠性
|
||||
|
||||
### 修复
|
||||
- `baoyu-post-to-weibo`:新增 Chrome 健康检查,无响应时自动重启
|
||||
- `baoyu-post-to-weibo`:发布前检查页面是否在微博首页,避免在错误页面操作
|
||||
|
||||
## 1.51.2 - 2026-03-06
|
||||
|
||||
### 修复
|
||||
- `release-skills`:将显式语言文件名模式(如 `CHANGELOG.de.md`)替换为通用模式,避免 Gen Agent Trust Hub URL 扫描器误报
|
||||
- `baoyu-infographic`:新增凭证/密钥剥离指令,解决 Snyk W007 不安全凭证处理审计问题
|
||||
|
||||
## 1.51.1 - 2026-03-06
|
||||
|
||||
### 重构
|
||||
- 统一 Chrome CDP profile 路径——所有 skill 共享 `baoyu-skills/chrome-profile`,不再各自独立目录
|
||||
- 修复 `baoyu-post-to-weibo` 错误复用 `x-browser-profile` 路径的问题
|
||||
|
||||
### 修复
|
||||
- 移除所有安装说明中的 `curl | bash` 远程代码执行模式
|
||||
- `md-to-html` 脚本强制仅允许 HTTPS 下载远程图片
|
||||
- 添加重定向次数限制(最多 5 次),防止无限重定向
|
||||
- 在 CLAUDE.md 中新增安全准则章节
|
||||
|
||||
## 1.51.0 - 2026-03-06
|
||||
|
||||
### 新功能
|
||||
- `baoyu-post-to-weibo`:新增微博发布技能——支持带图文本发布和头条文章,通过 Chrome CDP 自动化操作
|
||||
- `baoyu-format-markdown`:新增标题/摘要多候选项选择——生成 3 个候选供用户选择,支持 EXTEND.md 中的 `auto_select` 配置
|
||||
|
||||
## 1.50.0 - 2026-03-06
|
||||
|
||||
### 新功能
|
||||
- `baoyu-translate`:翻译风格预设从 4 种扩展到 9 种——新增学术、商务、幽默、口语化和优雅风格
|
||||
- `baoyu-translate`:新增 `--style` 命令行参数,支持按次指定翻译风格
|
||||
- `baoyu-translate`:将风格指令集成到子代理提示词模板
|
||||
|
||||
## 1.49.0 - 2026-03-06
|
||||
|
||||
### 新功能
|
||||
- `baoyu-format-markdown`:新增读者视角内容分析阶段——在应用格式之前先分析要点、结构和格式问题
|
||||
- `baoyu-format-markdown`:重构工作流从 8 步精简为 7 步,新增明确的格式化原则和完成报告模板
|
||||
- `baoyu-translate`:将步骤 2 的工作流机制提取到独立参考文件,精简 SKILL.md
|
||||
- `baoyu-translate`:扩展触发关键词(改成中文、快翻、本地化等),提升技能激活准确度
|
||||
- `baoyu-translate`:快速翻译模式下对长内容主动提示切换建议
|
||||
- `baoyu-translate`:分块时将 frontmatter 保存到 `chunks/frontmatter.md`
|
||||
|
||||
## 1.48.2 - 2026-03-06
|
||||
|
||||
### 新功能
|
||||
- `baoyu-translate`:在精翻工作流的审查和修订阶段新增比喻语言与情感忠实度检查
|
||||
- `baoyu-translate`:增强快速翻译模式,强制执行比喻语言的意义优先翻译原则
|
||||
|
||||
## 1.48.1 - 2026-03-05
|
||||
|
||||
### 新功能
|
||||
- `baoyu-translate`:在分析阶段新增比喻语言与隐喻映射——翻译前先解读隐喻、习语和隐含意义,避免字面直译
|
||||
- `baoyu-translate`:新增"意义优先于字面"、"比喻语言解读"、"情感忠实度"三项翻译原则,同步更新 SKILL.md、精翻工作流和子代理提示词模板
|
||||
|
||||
## 1.48.0 - 2026-03-05
|
||||
|
||||
### 新功能
|
||||
|
||||
@@ -56,7 +56,7 @@ if command -v bun &>/dev/null; then
|
||||
elif command -v npx &>/dev/null; then
|
||||
BUN_X="npx -y bun"
|
||||
else
|
||||
echo "Error: Neither bun nor npx found. Install bun: curl -fsSL https://bun.sh/install | bash"
|
||||
echo "Error: Neither bun nor npx found. Install bun: brew install oven-sh/bun/bun (macOS) or npm install -g bun"
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
@@ -65,7 +65,7 @@ fi
|
||||
|----------|-----------|-------------------|-------|
|
||||
| 1 | `bun` installed | `bun` | Fastest, native execution |
|
||||
| 2 | `npx` available | `npx -y bun` | Downloads bun on first run via npm |
|
||||
| 3 | Neither found | Error + install guide | Suggest: `curl -fsSL https://bun.sh/install \| bash` |
|
||||
| 3 | Neither found | Error + install guide | `brew install oven-sh/bun/bun` (macOS) or `npm install -g bun` |
|
||||
|
||||
### Script Execution
|
||||
|
||||
@@ -91,6 +91,74 @@ ${BUN_X} skills/baoyu-danger-gemini-web/scripts/main.ts --promptfiles system.md
|
||||
- **Chrome**: Required for `baoyu-danger-gemini-web` auth and `baoyu-post-to-x` automation
|
||||
- **No npm packages**: Self-contained TypeScript, no external dependencies
|
||||
|
||||
## Chrome Profile (Unified)
|
||||
|
||||
All skills that use Chrome CDP share a **single** profile directory. Do NOT create per-skill profiles.
|
||||
|
||||
| Platform | Default Path |
|
||||
|----------|-------------|
|
||||
| macOS | `~/Library/Application Support/baoyu-skills/chrome-profile` |
|
||||
| Linux | `$XDG_DATA_HOME/baoyu-skills/chrome-profile` (fallback `~/.local/share/baoyu-skills/chrome-profile`) |
|
||||
| Windows | `%APPDATA%/baoyu-skills/chrome-profile` |
|
||||
| WSL | Windows home `/.local/share/baoyu-skills/chrome-profile` |
|
||||
|
||||
**Environment variable override**: `BAOYU_CHROME_PROFILE_DIR` (takes priority, all skills respect it).
|
||||
|
||||
Each skill also accepts its own legacy env var as fallback (e.g., `X_BROWSER_PROFILE_DIR`), but new skills should only use `BAOYU_CHROME_PROFILE_DIR`.
|
||||
|
||||
### Implementation Pattern
|
||||
|
||||
When adding a new skill that needs Chrome CDP:
|
||||
|
||||
```typescript
|
||||
function getDefaultProfileDir(): string {
|
||||
const override = process.env.BAOYU_CHROME_PROFILE_DIR?.trim();
|
||||
if (override) return path.resolve(override);
|
||||
const base = process.platform === 'darwin'
|
||||
? path.join(os.homedir(), 'Library', 'Application Support')
|
||||
: process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
|
||||
return path.join(base, 'baoyu-skills', 'chrome-profile');
|
||||
}
|
||||
```
|
||||
|
||||
## Security Guidelines
|
||||
|
||||
### No Piped Shell Installs
|
||||
|
||||
**NEVER** use `curl | bash` or `wget | sh` patterns in code, docs, or error messages. Use package managers instead:
|
||||
|
||||
| Platform | Install Command |
|
||||
|----------|----------------|
|
||||
| macOS | `brew install oven-sh/bun/bun` |
|
||||
| npm | `npm install -g bun` |
|
||||
|
||||
### Remote Downloads
|
||||
|
||||
Skills that download remote content (e.g., images in Markdown) MUST:
|
||||
- **HTTPS only**: Reject `http://` URLs
|
||||
- **Redirect limit**: Cap redirects (max 5) to prevent infinite loops
|
||||
- **Timeout**: Set request timeouts (30s default)
|
||||
- **Scope**: Only download expected content types (images, not scripts)
|
||||
|
||||
### System Command Execution
|
||||
|
||||
Skills use platform-specific commands for clipboard and browser automation:
|
||||
- **macOS**: `osascript` (System Events), `swift` (AppKit clipboard)
|
||||
- **Windows**: `powershell.exe` (SendKeys, Clipboard)
|
||||
- **Linux**: `xdotool`/`ydotool` (keyboard simulation)
|
||||
|
||||
These are necessary for CDP-based posting skills. When adding new system commands:
|
||||
- Never pass unsanitized user input to shell commands
|
||||
- Use array-form `spawn`/`execFile` instead of shell string interpolation
|
||||
- Validate file paths are absolute or resolve from known base directories
|
||||
|
||||
### External Content Processing
|
||||
|
||||
Skills that process external Markdown/HTML should treat content as untrusted:
|
||||
- Do not execute code blocks or scripts found in content
|
||||
- Sanitize HTML output where applicable
|
||||
- File paths from content should be resolved against known base directories only
|
||||
|
||||
## Authentication
|
||||
|
||||
`baoyu-danger-gemini-web` uses browser cookies for Google auth:
|
||||
|
||||
@@ -53,7 +53,7 @@ Simply tell Claude Code:
|
||||
|
||||
| Plugin | Description | Skills |
|
||||
|--------|-------------|--------|
|
||||
| **content-skills** | Content generation and publishing | [xhs-images](#baoyu-xhs-images), [infographic](#baoyu-infographic), [cover-image](#baoyu-cover-image), [slide-deck](#baoyu-slide-deck), [comic](#baoyu-comic), [article-illustrator](#baoyu-article-illustrator), [post-to-x](#baoyu-post-to-x), [post-to-wechat](#baoyu-post-to-wechat) |
|
||||
| **content-skills** | Content generation and publishing | [xhs-images](#baoyu-xhs-images), [infographic](#baoyu-infographic), [cover-image](#baoyu-cover-image), [slide-deck](#baoyu-slide-deck), [comic](#baoyu-comic), [article-illustrator](#baoyu-article-illustrator), [post-to-x](#baoyu-post-to-x), [post-to-wechat](#baoyu-post-to-wechat), [post-to-weibo](#baoyu-post-to-weibo) |
|
||||
| **ai-generation-skills** | AI-powered generation backends | [image-gen](#baoyu-image-gen), [danger-gemini-web](#baoyu-danger-gemini-web) |
|
||||
| **utility-skills** | Utility tools for content processing | [url-to-markdown](#baoyu-url-to-markdown), [danger-x-to-markdown](#baoyu-danger-x-to-markdown), [compress-image](#baoyu-compress-image), [format-markdown](#baoyu-format-markdown), [translate](#baoyu-translate) |
|
||||
|
||||
@@ -558,6 +558,42 @@ To obtain credentials:
|
||||
|
||||
**Browser Method** (no API setup needed): Requires Google Chrome. First run opens browser for QR code login (session preserved).
|
||||
|
||||
#### baoyu-post-to-weibo
|
||||
|
||||
Post content to Weibo (微博). Supports regular posts with text, images, and videos, and headline articles (头条文章) with Markdown input. Uses real Chrome with CDP to bypass anti-automation.
|
||||
|
||||
**Regular Posts** - Text + images/videos (max 18 files):
|
||||
|
||||
```bash
|
||||
# Post with text
|
||||
/baoyu-post-to-weibo "Hello Weibo!"
|
||||
|
||||
# Post with images
|
||||
/baoyu-post-to-weibo "Check this out" --image photo.png
|
||||
|
||||
# Post with video
|
||||
/baoyu-post-to-weibo "Watch this" --video clip.mp4
|
||||
```
|
||||
|
||||
**Headline Articles (头条文章)** - Long-form Markdown:
|
||||
|
||||
```bash
|
||||
# Publish article
|
||||
/baoyu-post-to-weibo --article article.md
|
||||
|
||||
# With cover image
|
||||
/baoyu-post-to-weibo --article article.md --cover cover.jpg
|
||||
```
|
||||
|
||||
**Article Options**:
|
||||
| Option | Description |
|
||||
|--------|-------------|
|
||||
| `--cover <path>` | Cover image |
|
||||
| `--title <text>` | Override title (max 32 chars) |
|
||||
| `--summary <text>` | Override summary (max 44 chars) |
|
||||
|
||||
**Note**: Scripts fill content into the browser. User reviews and publishes manually. First run requires manual Weibo login (session persists).
|
||||
|
||||
### AI Generation Skills
|
||||
|
||||
AI-powered generation backends.
|
||||
@@ -641,7 +677,7 @@ Utility tools for content processing.
|
||||
|
||||
#### baoyu-url-to-markdown
|
||||
|
||||
Fetch any URL via Chrome CDP and convert to clean markdown. Supports two capture modes for different scenarios.
|
||||
Fetch any URL via Chrome CDP and convert to clean markdown. Saves rendered HTML snapshot alongside the markdown, and automatically falls back to a legacy extractor when Defuddle fails.
|
||||
|
||||
```bash
|
||||
# Auto mode (default) - capture when page loads
|
||||
@@ -741,7 +777,7 @@ Format plain text or markdown files with proper frontmatter, titles, summaries,
|
||||
|
||||
#### baoyu-translate
|
||||
|
||||
Translate articles and documents between languages with three modes: quick (direct), normal (analysis-informed), and refined (full publication-quality workflow).
|
||||
Translate articles and documents between languages with three modes: quick (direct), normal (analysis-informed), and refined (full publication-quality workflow with review and polish).
|
||||
|
||||
```bash
|
||||
# Normal mode (default) - analyze then translate
|
||||
@@ -752,8 +788,31 @@ Translate articles and documents between languages with three modes: quick (dire
|
||||
|
||||
# Refined mode - full workflow with review and polish
|
||||
/translate article.md --mode refined --to zh-CN
|
||||
|
||||
# Translate a URL
|
||||
/translate https://example.com/article --to zh-CN
|
||||
|
||||
# Specify audience
|
||||
/translate article.md --to zh-CN --audience technical
|
||||
|
||||
# Specify style
|
||||
/translate article.md --to zh-CN --style humorous
|
||||
|
||||
# With additional glossary
|
||||
/translate article.md --to zh-CN --glossary my-terms.md
|
||||
```
|
||||
|
||||
**Options**:
|
||||
| Option | Description |
|
||||
|--------|-------------|
|
||||
| `<source>` | File path, URL, or inline text |
|
||||
| `--mode <mode>` | `quick`, `normal` (default), `refined` |
|
||||
| `--from <lang>` | Source language (auto-detect if omitted) |
|
||||
| `--to <lang>` | Target language (default: `zh-CN`) |
|
||||
| `--audience <type>` | Target reader profile (default: `general`) |
|
||||
| `--style <style>` | Translation style (default: `storytelling`) |
|
||||
| `--glossary <file>` | Additional glossary file |
|
||||
|
||||
**Modes**:
|
||||
| Mode | Steps | Use Case |
|
||||
|------|-------|----------|
|
||||
@@ -761,7 +820,40 @@ Translate articles and documents between languages with three modes: quick (dire
|
||||
| Normal | Analyze → Translate | Articles, blog posts |
|
||||
| Refined | Analyze → Translate → Review → Polish | Publication-quality documents |
|
||||
|
||||
**Features**: Custom glossaries via EXTEND.md, audience-aware translation (general/technical/academic/business), automatic chunking for long documents with parallel subagent translation.
|
||||
After normal mode completes, you can reply "继续润色" or "refine" to continue with review and polish steps.
|
||||
|
||||
**Audience Presets**:
|
||||
| Value | Description |
|
||||
|-------|-------------|
|
||||
| `general` | General readers (default) — plain language, more translator's notes |
|
||||
| `technical` | Developers / engineers — less annotation on common tech terms |
|
||||
| `academic` | Researchers / scholars — formal register, precise terminology |
|
||||
| `business` | Business professionals — business-friendly tone |
|
||||
|
||||
Custom audience descriptions are also accepted, e.g., `--audience "AI-interested general readers"`.
|
||||
|
||||
**Style Presets**:
|
||||
| Value | Description |
|
||||
|-------|-------------|
|
||||
| `storytelling` | Engaging narrative flow (default) — smooth transitions, vivid phrasing |
|
||||
| `formal` | Professional, structured — neutral tone, no colloquialisms |
|
||||
| `technical` | Precise, documentation-style — concise, terminology-heavy |
|
||||
| `literal` | Close to original structure — minimal restructuring |
|
||||
| `academic` | Scholarly, rigorous — formal register, complex clauses OK |
|
||||
| `business` | Concise, results-focused — action-oriented, executive-friendly |
|
||||
| `humorous` | Preserves and adapts humor — witty, recreates comedic effect |
|
||||
| `conversational` | Casual, spoken-like — friendly, as if explaining to a friend |
|
||||
| `elegant` | Literary, polished prose — aesthetically refined, carefully crafted |
|
||||
|
||||
Custom style descriptions are also accepted, e.g., `--style "poetic and lyrical"`.
|
||||
|
||||
**Features**:
|
||||
- Custom glossaries via EXTEND.md with built-in EN→ZH glossary
|
||||
- Audience-aware translation with adjustable annotation depth
|
||||
- Automatic chunking for long documents (4000+ words) with parallel subagent translation
|
||||
- Figurative language interpreted by meaning, not word-for-word
|
||||
- Translator's notes for cultural/domain-specific references
|
||||
- Output directory with all intermediate files preserved
|
||||
|
||||
## Environment Configuration
|
||||
|
||||
|
||||
+95
-3
@@ -53,7 +53,7 @@ npx skills add jimliu/baoyu-skills
|
||||
|
||||
| 插件 | 说明 | 包含技能 |
|
||||
|------|------|----------|
|
||||
| **content-skills** | 内容生成和发布 | [xhs-images](#baoyu-xhs-images), [infographic](#baoyu-infographic), [cover-image](#baoyu-cover-image), [slide-deck](#baoyu-slide-deck), [comic](#baoyu-comic), [article-illustrator](#baoyu-article-illustrator), [post-to-x](#baoyu-post-to-x), [post-to-wechat](#baoyu-post-to-wechat) |
|
||||
| **content-skills** | 内容生成和发布 | [xhs-images](#baoyu-xhs-images), [infographic](#baoyu-infographic), [cover-image](#baoyu-cover-image), [slide-deck](#baoyu-slide-deck), [comic](#baoyu-comic), [article-illustrator](#baoyu-article-illustrator), [post-to-x](#baoyu-post-to-x), [post-to-wechat](#baoyu-post-to-wechat), [post-to-weibo](#baoyu-post-to-weibo) |
|
||||
| **ai-generation-skills** | AI 生成后端 | [image-gen](#baoyu-image-gen), [danger-gemini-web](#baoyu-danger-gemini-web) |
|
||||
| **utility-skills** | 内容处理工具 | [url-to-markdown](#baoyu-url-to-markdown), [danger-x-to-markdown](#baoyu-danger-x-to-markdown), [compress-image](#baoyu-compress-image), [format-markdown](#baoyu-format-markdown), [translate](#baoyu-translate) |
|
||||
|
||||
@@ -558,6 +558,42 @@ WECHAT_APP_SECRET=你的AppSecret
|
||||
|
||||
**浏览器方式**(无需 API 配置):需已安装 Google Chrome,首次运行需扫码登录(登录状态会保存)
|
||||
|
||||
#### baoyu-post-to-weibo
|
||||
|
||||
发布内容到微博。支持文字、图片、视频发布和头条文章(长篇 Markdown)。使用真实 Chrome + CDP 绕过反自动化检测。
|
||||
|
||||
**普通微博** - 文字 + 图片/视频(最多 18 个文件):
|
||||
|
||||
```bash
|
||||
# 发布文字
|
||||
/baoyu-post-to-weibo "Hello Weibo!"
|
||||
|
||||
# 发布带图片
|
||||
/baoyu-post-to-weibo "看看这个" --image photo.png
|
||||
|
||||
# 发布带视频
|
||||
/baoyu-post-to-weibo "看这个" --video clip.mp4
|
||||
```
|
||||
|
||||
**头条文章** - 长篇 Markdown 文章:
|
||||
|
||||
```bash
|
||||
# 发布文章
|
||||
/baoyu-post-to-weibo --article article.md
|
||||
|
||||
# 带封面图
|
||||
/baoyu-post-to-weibo --article article.md --cover cover.jpg
|
||||
```
|
||||
|
||||
**文章选项**:
|
||||
| 选项 | 说明 |
|
||||
|------|------|
|
||||
| `--cover <path>` | 封面图 |
|
||||
| `--title <text>` | 覆盖标题(最多 32 字) |
|
||||
| `--summary <text>` | 覆盖摘要(最多 44 字) |
|
||||
|
||||
**说明**:脚本会将内容填入浏览器,用户需手动检查并发布。首次运行需手动登录微博(登录状态会保存)。
|
||||
|
||||
### AI 生成技能 (AI Generation Skills)
|
||||
|
||||
AI 驱动的生成后端。
|
||||
@@ -641,7 +677,7 @@ AI 驱动的生成后端。
|
||||
|
||||
#### baoyu-url-to-markdown
|
||||
|
||||
通过 Chrome CDP 抓取任意 URL 并转换为干净的 Markdown。支持两种抓取模式,适应不同场景。
|
||||
通过 Chrome CDP 抓取任意 URL 并转换为 Markdown。同时保存渲染后的 HTML 快照,Defuddle 失败时自动回退到旧版提取器。
|
||||
|
||||
```bash
|
||||
# 自动模式(默认)- 页面加载后立即抓取
|
||||
@@ -752,8 +788,31 @@ AI 驱动的生成后端。
|
||||
|
||||
# 精翻模式 - 完整工作流,含审校与润色
|
||||
/translate article.md --mode refined --to zh-CN
|
||||
|
||||
# 翻译 URL
|
||||
/translate https://example.com/article --to zh-CN
|
||||
|
||||
# 指定受众
|
||||
/translate article.md --to zh-CN --audience technical
|
||||
|
||||
# 指定风格
|
||||
/translate article.md --to zh-CN --style humorous
|
||||
|
||||
# 附加术语表
|
||||
/translate article.md --to zh-CN --glossary my-terms.md
|
||||
```
|
||||
|
||||
**选项**:
|
||||
| 选项 | 说明 |
|
||||
|------|------|
|
||||
| `<source>` | 文件路径、URL 或行内文本 |
|
||||
| `--mode <mode>` | `quick`、`normal`(默认)、`refined` |
|
||||
| `--from <lang>` | 源语言(省略则自动检测) |
|
||||
| `--to <lang>` | 目标语言(默认:`zh-CN`) |
|
||||
| `--audience <type>` | 目标读者(默认:`general`) |
|
||||
| `--style <style>` | 翻译风格(默认:`storytelling`) |
|
||||
| `--glossary <file>` | 附加术语表文件 |
|
||||
|
||||
**模式**:
|
||||
| 模式 | 步骤 | 适用场景 |
|
||||
|------|------|----------|
|
||||
@@ -761,7 +820,40 @@ AI 驱动的生成后端。
|
||||
| 标准 | 分析 → 翻译 | 文章、博客 |
|
||||
| 精翻 | 分析 → 翻译 → 审校 → 润色 | 出版级文档 |
|
||||
|
||||
**特性**:通过 EXTEND.md 自定义术语表、面向受众的翻译(通用/技术/学术/商务)、长文档自动分块并行翻译。
|
||||
标准模式完成后,可回复「继续润色」或「refine」继续审校润色步骤。
|
||||
|
||||
**受众预设**:
|
||||
| 值 | 说明 |
|
||||
|----|------|
|
||||
| `general` | 普通读者(默认)— 通俗语言,更多译注 |
|
||||
| `technical` | 开发者/工程师 — 常见技术术语少加注释 |
|
||||
| `academic` | 研究者/学者 — 正式语体,精确术语 |
|
||||
| `business` | 商务人士 — 商务友好语气 |
|
||||
|
||||
也支持自定义受众描述,如 `--audience "对 AI 感兴趣的普通读者"`。
|
||||
|
||||
**风格预设**:
|
||||
| 值 | 说明 |
|
||||
|----|------|
|
||||
| `storytelling` | 叙事流畅(默认)— 过渡自然,表达生动 |
|
||||
| `formal` | 正式、结构化 — 中性语气,无口语化表达 |
|
||||
| `technical` | 精确、文档风格 — 简洁,术语密集 |
|
||||
| `literal` | 贴近原文结构 — 最小化重构 |
|
||||
| `academic` | 学术、严谨 — 正式语体,复杂从句可接受 |
|
||||
| `business` | 简洁、结果导向 — 行动导向,高管友好 |
|
||||
| `humorous` | 保留幽默感 — 诙谐,在目标语言中重现喜剧效果 |
|
||||
| `conversational` | 口语化、亲切 — 友好,如同朋友间解释 |
|
||||
| `elegant` | 文学性、优雅 — 精心雕琢,注重韵律美感 |
|
||||
|
||||
也支持自定义风格描述,如 `--style "诗意而抒情"`。
|
||||
|
||||
**特性**:
|
||||
- 通过 EXTEND.md 自定义术语表,内置英中术语表
|
||||
- 面向受众的翻译,可调节注释深度
|
||||
- 长文档(4000+ 词)自动分块并行翻译
|
||||
- 比喻和修辞按意译而非逐字翻译
|
||||
- 为文化/专业术语添加译注
|
||||
- 输出目录保留所有中间文件
|
||||
|
||||
## 环境配置
|
||||
|
||||
|
||||
@@ -42,11 +42,11 @@ function getWslWindowsHome(): string | null {
|
||||
}
|
||||
|
||||
export function resolveGeminiWebChromeProfileDir(): string {
|
||||
const override = process.env.GEMINI_WEB_CHROME_PROFILE_DIR?.trim();
|
||||
const override = process.env.BAOYU_CHROME_PROFILE_DIR?.trim() || process.env.GEMINI_WEB_CHROME_PROFILE_DIR?.trim();
|
||||
if (override) return path.resolve(override);
|
||||
const wslHome = getWslWindowsHome();
|
||||
if (wslHome) return path.join(wslHome, '.local', 'share', APP_DATA_DIR, GEMINI_DATA_DIR, PROFILE_DIR_NAME);
|
||||
return path.join(resolveGeminiWebDataDir(), PROFILE_DIR_NAME);
|
||||
if (wslHome) return path.join(wslHome, '.local', 'share', APP_DATA_DIR, PROFILE_DIR_NAME);
|
||||
return path.join(resolveUserDataRoot(), APP_DATA_DIR, PROFILE_DIR_NAME);
|
||||
}
|
||||
|
||||
export function resolveGeminiWebSessionsDir(): string {
|
||||
|
||||
@@ -43,11 +43,11 @@ function getWslWindowsHome(): string | null {
|
||||
}
|
||||
|
||||
export function resolveXToMarkdownChromeProfileDir(): string {
|
||||
const override = process.env.X_CHROME_PROFILE_DIR?.trim();
|
||||
const override = process.env.BAOYU_CHROME_PROFILE_DIR?.trim() || process.env.X_CHROME_PROFILE_DIR?.trim();
|
||||
if (override) return path.resolve(override);
|
||||
const wslHome = getWslWindowsHome();
|
||||
if (wslHome) return path.join(wslHome, ".local", "share", APP_DATA_DIR, X_TO_MARKDOWN_DATA_DIR, PROFILE_DIR_NAME);
|
||||
return path.join(resolveXToMarkdownDataDir(), PROFILE_DIR_NAME);
|
||||
if (wslHome) return path.join(wslHome, ".local", "share", APP_DATA_DIR, PROFILE_DIR_NAME);
|
||||
return path.join(resolveUserDataRoot(), APP_DATA_DIR, PROFILE_DIR_NAME);
|
||||
}
|
||||
|
||||
export function resolveXToMarkdownConsentPath(): string {
|
||||
|
||||
@@ -5,7 +5,9 @@ description: Formats plain text or markdown files with frontmatter, titles, summ
|
||||
|
||||
# Markdown Formatter
|
||||
|
||||
Transforms plain text or markdown files into well-structured markdown with proper frontmatter, formatting, and typography.
|
||||
Transforms plain text or markdown into well-structured, reader-friendly markdown. The goal is to help readers quickly grasp key points, highlights, and structure — without changing any original content.
|
||||
|
||||
**Core principle**: Only adjust formatting and fix obvious typos. Never add, delete, or rewrite content.
|
||||
|
||||
## Script Directory
|
||||
|
||||
@@ -49,55 +51,45 @@ if (Test-Path "$HOME/.baoyu-skills/baoyu-format-markdown/EXTEND.md") { "user" }
|
||||
│ Not found │ Use defaults │
|
||||
└───────────┴───────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
**EXTEND.md Supports**: Default formatting options | Summary length preferences
|
||||
**EXTEND.md Supports**:
|
||||
|
||||
| Setting | Values | Default | Description |
|
||||
|---------|--------|---------|-------------|
|
||||
| `auto_select` | `true`/`false` | `false` | Skip both title and summary selection, auto-pick best |
|
||||
| `auto_select_title` | `true`/`false` | `false` | Skip title selection only |
|
||||
| `auto_select_summary` | `true`/`false` | `false` | Skip summary selection only |
|
||||
| Other | — | — | Default formatting options, typography preferences |
|
||||
|
||||
## Usage
|
||||
|
||||
Claude performs content analysis and formatting (Steps 1-6), then runs the script for typography fixes (Step 7).
|
||||
The workflow has two phases: **Analyze** (understand the content) then **Format** (apply formatting). Claude performs content analysis and formatting (Steps 1-5), then runs the script for typography fixes (Step 6).
|
||||
|
||||
## Workflow
|
||||
|
||||
### Step 1: Read Source File
|
||||
### Step 1: Read & Detect Content Type
|
||||
|
||||
Read the user-specified markdown or plain text file.
|
||||
|
||||
### Step 1.5: Detect Content Type & Confirm
|
||||
|
||||
**Content Type Detection:**
|
||||
Read the user-specified file, then detect content type:
|
||||
|
||||
| Indicator | Classification |
|
||||
|-----------|----------------|
|
||||
| Has `---` YAML frontmatter | Markdown |
|
||||
| Has `#`, `##`, `###` headings | Markdown |
|
||||
| Has `**bold**`, `*italic*` | Markdown |
|
||||
| Has `- ` or `1. ` lists | Markdown |
|
||||
| Has ``` code blocks | Markdown |
|
||||
| Has `> ` blockquotes | Markdown |
|
||||
| Has `**bold**`, `*italic*`, lists, code blocks, blockquotes | Markdown |
|
||||
| None of above | Plain text |
|
||||
|
||||
**Decision Flow:**
|
||||
|
||||
┌─────────────────┬────────────────────────────────────────────────┐
|
||||
│ Content Type │ Action │
|
||||
├─────────────────┼────────────────────────────────────────────────┤
|
||||
│ Plain text │ Proceed to Step 2 (format to markdown) │
|
||||
├─────────────────┼────────────────────────────────────────────────┤
|
||||
│ Markdown │ Use AskUserQuestion to confirm optimization │
|
||||
└─────────────────┴────────────────────────────────────────────────┘
|
||||
|
||||
**If Markdown detected, ask user:**
|
||||
|
||||
```
|
||||
Detected existing markdown formatting. What would you like to do?
|
||||
|
||||
1. Optimize formatting (Recommended)
|
||||
- Add/improve frontmatter, headings, bold, lists
|
||||
- Analyze content, improve headings, bold, lists for readability
|
||||
- Run typography script (spacing, emphasis fixes)
|
||||
- Output: {filename}-formatted.md
|
||||
|
||||
2. Keep original formatting
|
||||
- Preserve existing markdown structure
|
||||
- Run typography script (spacing, emphasis fixes)
|
||||
- Run typography script only
|
||||
- Output: {filename}-formatted.md
|
||||
|
||||
3. Typography fixes only
|
||||
@@ -106,112 +98,168 @@ Detected existing markdown formatting. What would you like to do?
|
||||
```
|
||||
|
||||
**Based on user choice:**
|
||||
- **Optimize**: Continue to Step 2-8 (full workflow)
|
||||
- **Keep original**: Skip Steps 2-5, copy file → Step 6-8 (run script on copy)
|
||||
- **Typography only**: Skip Steps 2-6, run Step 7 on original file directly
|
||||
- **Optimize**: Continue to Step 2 (full workflow)
|
||||
- **Keep original**: Skip to Step 5, copy file then run Step 6
|
||||
- **Typography only**: Skip to Step 6, run on original file directly
|
||||
|
||||
### Step 2: Analyze Content Structure
|
||||
### Step 2: Analyze Content (Reader's Perspective)
|
||||
|
||||
Identify:
|
||||
- Existing title (H1 `#`)
|
||||
- Paragraph separations
|
||||
- Keywords suitable for **bold**
|
||||
- Parallel content convertible to lists
|
||||
- Code snippets
|
||||
- Quotations
|
||||
Read the entire content carefully. Think from a reader's perspective: what would help them quickly understand and remember the key information?
|
||||
|
||||
### Step 3: Check/Create Frontmatter
|
||||
Produce an analysis covering these dimensions:
|
||||
|
||||
**2.1 Highlights & Key Insights**
|
||||
- Core arguments or conclusions the author makes
|
||||
- Surprising facts, data points, or counterintuitive claims
|
||||
- Memorable quotes or well-phrased sentences (golden quotes)
|
||||
|
||||
**2.2 Structure Assessment**
|
||||
- Does the content have a clear logical flow? What is it?
|
||||
- Are there natural section boundaries that lack headings?
|
||||
- Are there long walls of text that could benefit from visual breaks?
|
||||
|
||||
**2.3 Reader-Important Information**
|
||||
- Actionable advice or takeaways
|
||||
- Definitions, explanations of key concepts
|
||||
- Lists or enumerations buried in prose
|
||||
- Comparisons or contrasts that would be clearer as tables
|
||||
|
||||
**2.4 Formatting Issues**
|
||||
- Missing or inconsistent heading hierarchy
|
||||
- Paragraphs that mix multiple topics
|
||||
- Parallel items written as prose instead of lists
|
||||
- Code, commands, or technical terms not marked as code
|
||||
- Obvious typos or formatting errors
|
||||
|
||||
**Save analysis to file**: `{original-filename}-analysis.md`
|
||||
|
||||
The analysis file serves as the blueprint for Step 3. Use this format:
|
||||
|
||||
```markdown
|
||||
# Content Analysis: {filename}
|
||||
|
||||
## Highlights & Key Insights
|
||||
- [list findings]
|
||||
|
||||
## Structure Assessment
|
||||
- Current flow: [describe]
|
||||
- Suggested sections: [list heading candidates with brief rationale]
|
||||
|
||||
## Reader-Important Information
|
||||
- [list actionable items, key concepts, buried lists, potential tables]
|
||||
|
||||
## Formatting Issues
|
||||
- [list specific issues with location references]
|
||||
|
||||
## Typos Found
|
||||
- [list any obvious typos with corrections, or "None found"]
|
||||
```
|
||||
|
||||
### Step 3: Check/Create Frontmatter, Title & Summary
|
||||
|
||||
Check for YAML frontmatter (`---` block). Create if missing.
|
||||
|
||||
**Meta field handling:**
|
||||
|
||||
| Field | Processing |
|
||||
|-------|------------|
|
||||
| `title` | See Step 4 |
|
||||
| `slug` | Infer from file path (e.g., `posts/2026/01/10/vibe-coding/` → `vibe-coding`) or generate from title |
|
||||
| `summary` | Generate engaging summary (100-150 characters) |
|
||||
| `coverImage` | Check if `imgs/cover.png` exists in same directory; if so, use relative path (also accepted: `featureImage`) |
|
||||
| `title` | See **Title Generation** below |
|
||||
| `slug` | Infer from file path or generate from title |
|
||||
| `summary` | See **Summary Generation** below |
|
||||
| `coverImage` | Check if `imgs/cover.png` exists in same directory; if so, use relative path |
|
||||
|
||||
### Step 4: Title Handling
|
||||
**Title Generation:**
|
||||
|
||||
**Logic:**
|
||||
1. If frontmatter already has `title` → use it, no H1 in body
|
||||
2. If first line is H1 → extract to frontmatter `title`, remove H1 from body
|
||||
3. If neither exists → generate candidate titles
|
||||
|
||||
**Title generation flow:**
|
||||
|
||||
1. Generate 3 candidate titles based on content
|
||||
2. Use `AskUserQuestion` tool:
|
||||
Generate 3 candidate titles with different angles/styles. Present to user for selection:
|
||||
|
||||
```
|
||||
Select a title:
|
||||
Pick a title:
|
||||
|
||||
1. [Title A] (Recommended)
|
||||
2. [Title B]
|
||||
3. [Title C]
|
||||
1. [Title A] — [angle/style note]
|
||||
2. [Title B] — [angle/style note]
|
||||
3. [Title C] — [angle/style note]
|
||||
|
||||
Enter number, or type a custom title:
|
||||
```
|
||||
|
||||
3. If no selection within a few seconds, use recommended (option 1)
|
||||
|
||||
**Title principles:**
|
||||
- Concise, max 20 characters
|
||||
- Captures core message
|
||||
Title principles:
|
||||
- Engaging, sparks reading interest
|
||||
- Captures core message or most compelling angle
|
||||
- Accurate, avoids clickbait
|
||||
- Vary angles: e.g. story-driven, conclusion-driven, question-driven
|
||||
|
||||
**Important:** Once title is in frontmatter, body should NOT have H1 (avoid duplication)
|
||||
If frontmatter already has `title`, skip selection and use it. If first line is H1, extract to frontmatter as default title but still offer alternatives.
|
||||
|
||||
### Step 5: Format Processing
|
||||
**Summary Generation:**
|
||||
|
||||
**Formatting rules:**
|
||||
Generate 3 candidate summaries with different focuses. Present to user for selection:
|
||||
|
||||
| Element | Format |
|
||||
|---------|--------|
|
||||
| Titles | Use `#`, `##`, `###` hierarchy |
|
||||
| Key points | Use `**bold**` |
|
||||
| Parallel items | Convert to `-` unordered or `1.` ordered lists |
|
||||
| Code/commands | Use `` `inline` `` or ` ```block``` ` |
|
||||
| Quotes/sayings | Use `>` blockquote |
|
||||
| Separators | Use `---` where appropriate |
|
||||
```
|
||||
Pick a summary:
|
||||
|
||||
**Formatting principles:**
|
||||
- Preserve original content and viewpoints
|
||||
- Add formatting only, do not modify text
|
||||
- Formatting serves readability
|
||||
- Avoid over-formatting
|
||||
1. [Summary A] — [focus note]
|
||||
2. [Summary B] — [focus note]
|
||||
3. [Summary C] — [focus note]
|
||||
|
||||
### Step 6: Save Formatted File
|
||||
Enter number, or type a custom summary:
|
||||
```
|
||||
|
||||
Summary principles:
|
||||
- 80-150 characters, precise and information-rich
|
||||
- Convey article's core value, not just topic
|
||||
- Vary focuses: e.g. problem-driven, result-driven, insight-driven
|
||||
- Avoid generic descriptions like "本文介绍了..."
|
||||
|
||||
If frontmatter already has `summary`, skip selection and use it.
|
||||
|
||||
**EXTEND.md skip behavior:** If `auto_select: true` is set in EXTEND.md, skip title and summary selection — generate the best candidate directly without asking. User can also set `auto_select_title: true` or `auto_select_summary: true` independently.
|
||||
|
||||
Once title is in frontmatter, body should NOT have H1 (avoid duplication).
|
||||
|
||||
### Step 4: Format Content
|
||||
|
||||
Apply formatting guided by the Step 2 analysis. The goal is making the content scannable and the key points impossible to miss.
|
||||
|
||||
**Formatting toolkit:**
|
||||
|
||||
| Element | When to use | Format |
|
||||
|---------|-------------|--------|
|
||||
| Headings | Natural topic boundaries, section breaks | `##`, `###` hierarchy |
|
||||
| Bold | Key conclusions, important terms, core takeaways | `**bold**` |
|
||||
| Unordered lists | Parallel items, feature lists, examples | `- item` |
|
||||
| Ordered lists | Sequential steps, ranked items, procedures | `1. item` |
|
||||
| Tables | Comparisons, structured data, option matrices | Markdown table |
|
||||
| Code | Commands, file paths, technical terms, variable names | `` `inline` `` or fenced blocks |
|
||||
| Blockquotes | Notable quotes, important warnings, cited text | `> quote` |
|
||||
| Separators | Major topic transitions | `---` |
|
||||
|
||||
**Formatting principles — what NOT to do:**
|
||||
- Do NOT add sentences, explanations, or commentary
|
||||
- Do NOT delete or shorten any content
|
||||
- Do NOT rephrase or rewrite the author's words
|
||||
- Do NOT add headings that editorialize (e.g., "Amazing Discovery" — use neutral descriptive headings)
|
||||
- Do NOT over-format: not every sentence needs bold, not every paragraph needs a heading
|
||||
|
||||
**Formatting principles — what TO do:**
|
||||
- Preserve the author's voice, tone, and every word
|
||||
- Use bold sparingly for genuinely important points
|
||||
- Extract parallel items from prose into lists only when the structure is clearly there
|
||||
- Add headings where the topic genuinely shifts
|
||||
- Fix obvious typos (based on Step 2 findings)
|
||||
|
||||
### Step 5: Save Formatted File
|
||||
|
||||
Save as `{original-filename}-formatted.md`
|
||||
|
||||
Examples:
|
||||
- `final.md` → `final-formatted.md`
|
||||
- `draft-v1.md` → `draft-v1-formatted.md`
|
||||
|
||||
**If user chose "Keep original formatting" (from Step 1.5):**
|
||||
- Copy original file to `{filename}-formatted.md` without modifications
|
||||
- Proceed to Step 7 for typography fixes only
|
||||
|
||||
**Backup existing file:**
|
||||
|
||||
If `{filename}-formatted.md` already exists, backup before overwriting:
|
||||
|
||||
```bash
|
||||
# Check if formatted file exists
|
||||
if [ -f "{filename}-formatted.md" ]; then
|
||||
# Backup with timestamp
|
||||
mv "{filename}-formatted.md" "{filename}-formatted.backup-$(date +%Y%m%d-%H%M%S).md"
|
||||
fi
|
||||
```
|
||||
|
||||
Example:
|
||||
- `final-formatted.md` exists → backup to `final-formatted.backup-20260128-143052.md`
|
||||
### Step 6: Execute Typography Script
|
||||
|
||||
### Step 7: Execute Text Formatting Script
|
||||
|
||||
After saving, **must** run the formatting script:
|
||||
Run the formatting script on the output file:
|
||||
|
||||
```bash
|
||||
${BUN_X} ${SKILL_DIR}/scripts/main.ts {output-file-path} [options]
|
||||
@@ -227,7 +275,6 @@ ${BUN_X} ${SKILL_DIR}/scripts/main.ts {output-file-path} [options]
|
||||
| `--no-spacing` | | Do not add CJK/English spacing | |
|
||||
| `--emphasis` | `-e` | Fix CJK emphasis punctuation issues | true |
|
||||
| `--no-emphasis` | | Do not fix CJK emphasis issues | |
|
||||
| `--help` | `-h` | Show help message | |
|
||||
|
||||
**Examples:**
|
||||
|
||||
@@ -240,62 +287,54 @@ ${BUN_X} ${SKILL_DIR}/scripts/main.ts article.md --quotes
|
||||
|
||||
# Only fix emphasis issues, skip spacing
|
||||
${BUN_X} ${SKILL_DIR}/scripts/main.ts article.md --no-spacing
|
||||
|
||||
# Disable all processing except frontmatter formatting
|
||||
${BUN_X} ${SKILL_DIR}/scripts/main.ts article.md --no-spacing --no-emphasis
|
||||
```
|
||||
|
||||
**Script performs (based on options):**
|
||||
1. Fix CJK emphasis/bold punctuation issues (default: enabled)
|
||||
2. Add CJK/English mixed text spacing via autocorrect (default: enabled)
|
||||
3. Replace ASCII quotes `"..."` with fullwidth quotes `"..."` (default: disabled)
|
||||
3. Replace ASCII quotes with fullwidth quotes (default: disabled)
|
||||
4. Format frontmatter YAML (always enabled)
|
||||
|
||||
### Step 8: Display Results
|
||||
### Step 7: Completion Report
|
||||
|
||||
Display a report summarizing all changes made:
|
||||
|
||||
```
|
||||
**Formatting complete**
|
||||
**Formatting Complete**
|
||||
|
||||
File: posts/2026/01/09/example/final-formatted.md
|
||||
**Files:**
|
||||
- Analysis: {filename}-analysis.md
|
||||
- Formatted: {filename}-formatted.md
|
||||
|
||||
Changes:
|
||||
- Added title: [title content]
|
||||
- Added X bold markers
|
||||
- Added X lists
|
||||
- Added X code blocks
|
||||
**Content Analysis Summary:**
|
||||
- Highlights found: X key insights
|
||||
- Golden quotes: X memorable sentences
|
||||
- Formatting issues fixed: X items
|
||||
|
||||
**Changes Applied:**
|
||||
- Frontmatter: [added/updated] (title, slug, summary)
|
||||
- Headings added: X (##: N, ###: N)
|
||||
- Bold markers added: X
|
||||
- Lists created: X (from prose → list conversion)
|
||||
- Tables created: X
|
||||
- Code markers added: X
|
||||
- Blockquotes added: X
|
||||
- Typos fixed: X [list each: "原文" → "修正"]
|
||||
|
||||
**Typography Script:**
|
||||
- CJK spacing: [applied/skipped]
|
||||
- Emphasis fixes: [applied/skipped]
|
||||
- Quote replacement: [applied/skipped]
|
||||
```
|
||||
|
||||
## Formatting Example
|
||||
|
||||
**Before:**
|
||||
```
|
||||
This is plain text. First point is efficiency improvement. Second point is cost reduction. Third point is experience optimization. Use npm install to install dependencies.
|
||||
```
|
||||
|
||||
**After:**
|
||||
```markdown
|
||||
---
|
||||
title: Three Core Advantages
|
||||
slug: three-core-advantages
|
||||
summary: Discover the three key benefits that drive success in modern projects.
|
||||
---
|
||||
|
||||
This is plain text.
|
||||
|
||||
**Main advantages:**
|
||||
- Efficiency improvement
|
||||
- Cost reduction
|
||||
- Experience optimization
|
||||
|
||||
Use `npm install` to install dependencies.
|
||||
```
|
||||
Adjust the report to reflect actual changes — omit categories where no changes were made.
|
||||
|
||||
## Notes
|
||||
|
||||
- Preserve original writing style and tone
|
||||
- Specify correct language for code blocks (e.g., `python`, `javascript`)
|
||||
- Maintain CJK/English spacing standards
|
||||
- Do not add content not present in original
|
||||
- The analysis file is a working document — it helps maintain consistency between what was identified and what was formatted
|
||||
|
||||
## Extension Support
|
||||
|
||||
|
||||
@@ -128,7 +128,7 @@ Slug: 2-4 words kebab-case from topic. Conflict: append `-YYYYMMDD-HHMMSS`.
|
||||
|
||||
## Core Principles
|
||||
|
||||
- Preserve all source data **verbatim**—no summarization or rephrasing
|
||||
- Preserve source data faithfully—no summarization or rephrasing (but **strip any credentials, API keys, tokens, or secrets** before including in outputs)
|
||||
- Define learning objectives before structuring content
|
||||
- Structure for visual communication (headlines, labels, visual elements)
|
||||
|
||||
@@ -192,7 +192,7 @@ Transform content into infographic structure:
|
||||
3. Data points (all statistics/quotes copied exactly)
|
||||
4. Design instructions from user
|
||||
|
||||
**Rules**: Markdown only. No new information. All data verbatim.
|
||||
**Rules**: Markdown only. No new information. Preserve data faithfully. Strip any credentials or secrets from output.
|
||||
|
||||
See `references/structured-content-template.md` for detailed format.
|
||||
|
||||
|
||||
@@ -102,8 +102,8 @@ Checks: Chrome, profile isolation, Bun, Accessibility, clipboard, paste keystrok
|
||||
| Check | Fix |
|
||||
|-------|-----|
|
||||
| Chrome | Install Chrome or set `WECHAT_BROWSER_CHROME_PATH` env var |
|
||||
| Profile dir | Ensure `~/.local/share/wechat-browser-profile` is writable |
|
||||
| Bun runtime | `curl -fsSL https://bun.sh/install \| bash` |
|
||||
| Profile dir | Shared profile at `baoyu-skills/chrome-profile` (see CLAUDE.md Chrome Profile section) |
|
||||
| Bun runtime | `brew install oven-sh/bun/bun` (macOS) or `npm install -g bun` |
|
||||
| Accessibility (macOS) | System Settings → Privacy & Security → Accessibility → enable terminal app |
|
||||
| Clipboard copy | Ensure Swift/AppKit available (macOS Xcode CLI tools: `xcode-select --install`) |
|
||||
| Paste keystroke (macOS) | Same as Accessibility fix above |
|
||||
|
||||
@@ -61,8 +61,12 @@ export function findChromeExecutable(): string | undefined {
|
||||
}
|
||||
|
||||
export function getDefaultProfileDir(): string {
|
||||
const base = process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
|
||||
return path.join(base, 'wechat-browser-profile');
|
||||
const override = process.env.BAOYU_CHROME_PROFILE_DIR?.trim();
|
||||
if (override) return path.resolve(override);
|
||||
const base = process.platform === 'darwin'
|
||||
? path.join(os.homedir(), 'Library', 'Application Support')
|
||||
: process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
|
||||
return path.join(base, 'baoyu-skills', 'chrome-profile');
|
||||
}
|
||||
|
||||
async function fetchJson<T = unknown>(url: string): Promise<T> {
|
||||
|
||||
@@ -181,7 +181,7 @@ async function checkBun(): Promise<void> {
|
||||
if (result.status === 0) {
|
||||
log('Bun runtime', true, `v${result.stdout?.toString().trim()}`);
|
||||
} else {
|
||||
log('Bun runtime', false, 'Cannot run bun. Install: curl -fsSL https://bun.sh/install | bash');
|
||||
log('Bun runtime', false, 'Cannot run bun. Install: brew install oven-sh/bun/bun (macOS) or npm install -g bun');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -183,11 +183,14 @@ function getWslWindowsHome(): string | null {
|
||||
}
|
||||
|
||||
function getDefaultProfileDir(): string {
|
||||
const override = process.env.WECHAT_BROWSER_PROFILE_DIR?.trim();
|
||||
const override = process.env.BAOYU_CHROME_PROFILE_DIR?.trim() || process.env.WECHAT_BROWSER_PROFILE_DIR?.trim();
|
||||
if (override) return path.resolve(override);
|
||||
const home = getWslWindowsHome() ?? os.homedir();
|
||||
const base = process.env.XDG_DATA_HOME || path.join(home, '.local', 'share');
|
||||
return path.join(base, 'wechat-browser-profile');
|
||||
const wslHome = getWslWindowsHome();
|
||||
if (wslHome) return path.join(wslHome, '.local', 'share', 'baoyu-skills', 'chrome-profile');
|
||||
const base = process.platform === 'darwin'
|
||||
? path.join(os.homedir(), 'Library', 'Application Support')
|
||||
: process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
|
||||
return path.join(base, 'baoyu-skills', 'chrome-profile');
|
||||
}
|
||||
|
||||
async function fetchJson<T = unknown>(url: string): Promise<T> {
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
---
|
||||
name: baoyu-post-to-weibo
|
||||
description: Posts content to Weibo (微博). Supports regular posts with text, images, and videos, and headline articles (头条文章) with Markdown input via Chrome CDP. Use when user asks to "post to Weibo", "发微博", "发布微博", "publish to Weibo", "share on Weibo", "写微博", or "微博头条文章".
|
||||
---
|
||||
|
||||
# Post to Weibo
|
||||
|
||||
Posts text, images, videos, and long-form articles to Weibo via real Chrome browser (bypasses anti-bot detection).
|
||||
|
||||
## Script Directory
|
||||
|
||||
**Important**: All scripts are located in the `scripts/` subdirectory of this skill.
|
||||
|
||||
**Agent Execution Instructions**:
|
||||
1. Determine this SKILL.md file's directory path as `SKILL_DIR`
|
||||
2. Script path = `${SKILL_DIR}/scripts/<script-name>.ts`
|
||||
3. Replace all `${SKILL_DIR}` in this document with the actual path
|
||||
4. Resolve `${BUN_X}` runtime: if `bun` installed → `bun`; if `npx` available → `npx -y bun`; else suggest installing bun
|
||||
|
||||
**Script Reference**:
|
||||
| Script | Purpose |
|
||||
|--------|---------|
|
||||
| `scripts/weibo-post.ts` | Regular posts (text + images) |
|
||||
| `scripts/weibo-article.ts` | Headline article publishing (Markdown) |
|
||||
| `scripts/copy-to-clipboard.ts` | Copy content to clipboard |
|
||||
| `scripts/paste-from-clipboard.ts` | Send real paste keystroke |
|
||||
|
||||
## Preferences (EXTEND.md)
|
||||
|
||||
Check EXTEND.md existence (priority order):
|
||||
|
||||
```bash
|
||||
# macOS, Linux, WSL, Git Bash
|
||||
test -f .baoyu-skills/baoyu-post-to-weibo/EXTEND.md && echo "project"
|
||||
test -f "$HOME/.baoyu-skills/baoyu-post-to-weibo/EXTEND.md" && echo "user"
|
||||
```
|
||||
|
||||
```powershell
|
||||
# PowerShell (Windows)
|
||||
if (Test-Path .baoyu-skills/baoyu-post-to-weibo/EXTEND.md) { "project" }
|
||||
if (Test-Path "$HOME/.baoyu-skills/baoyu-post-to-weibo/EXTEND.md") { "user" }
|
||||
```
|
||||
|
||||
┌──────────────────────────────────────────────────┬───────────────────┐
|
||||
│ Path │ Location │
|
||||
├──────────────────────────────────────────────────┼───────────────────┤
|
||||
│ .baoyu-skills/baoyu-post-to-weibo/EXTEND.md │ Project directory │
|
||||
├──────────────────────────────────────────────────┼───────────────────┤
|
||||
│ $HOME/.baoyu-skills/baoyu-post-to-weibo/EXTEND.md│ User home │
|
||||
└──────────────────────────────────────────────────┴───────────────────┘
|
||||
|
||||
┌───────────┬───────────────────────────────────────────────────────────────────────────┐
|
||||
│ Result │ Action │
|
||||
├───────────┼───────────────────────────────────────────────────────────────────────────┤
|
||||
│ Found │ Read, parse, apply settings │
|
||||
├───────────┼───────────────────────────────────────────────────────────────────────────┤
|
||||
│ Not found │ Use defaults │
|
||||
└───────────┴───────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
**EXTEND.md Supports**: Default Chrome profile
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Google Chrome or Chromium
|
||||
- `bun` runtime
|
||||
- First run: log in to Weibo manually (session saved)
|
||||
|
||||
---
|
||||
|
||||
## Regular Posts
|
||||
|
||||
Text + images/videos (max 18 files total). Posted on Weibo homepage.
|
||||
|
||||
```bash
|
||||
${BUN_X} ${SKILL_DIR}/scripts/weibo-post.ts "Hello Weibo!" --image ./photo.png
|
||||
${BUN_X} ${SKILL_DIR}/scripts/weibo-post.ts "Watch this" --video ./clip.mp4
|
||||
```
|
||||
|
||||
**Parameters**:
|
||||
| Parameter | Description |
|
||||
|-----------|-------------|
|
||||
| `<text>` | Post content (positional) |
|
||||
| `--image <path>` | Image file (repeatable) |
|
||||
| `--video <path>` | Video file (repeatable) |
|
||||
| `--profile <dir>` | Custom Chrome profile |
|
||||
|
||||
**Note**: Script opens browser with content filled in. User reviews and publishes manually.
|
||||
|
||||
---
|
||||
|
||||
## Headline Articles (头条文章)
|
||||
|
||||
Long-form Markdown articles published at `https://card.weibo.com/article/v3/editor`.
|
||||
|
||||
```bash
|
||||
${BUN_X} ${SKILL_DIR}/scripts/weibo-article.ts article.md
|
||||
${BUN_X} ${SKILL_DIR}/scripts/weibo-article.ts article.md --cover ./cover.jpg
|
||||
```
|
||||
|
||||
**Parameters**:
|
||||
| Parameter | Description |
|
||||
|-----------|-------------|
|
||||
| `<markdown>` | Markdown file (positional) |
|
||||
| `--cover <path>` | Cover image |
|
||||
| `--title <text>` | Override title (max 32 chars, truncated if longer) |
|
||||
| `--summary <text>` | Override summary (max 44 chars, auto-regenerated if longer) |
|
||||
| `--profile <dir>` | Custom Chrome profile |
|
||||
|
||||
**Frontmatter**: `title`, `summary`, `cover_image` supported in YAML front matter.
|
||||
|
||||
**Character Limits**:
|
||||
- Title: 32 characters max (truncated with warning if longer)
|
||||
- Summary/导语: 44 characters max (auto-regenerated from content if longer)
|
||||
|
||||
**Article Workflow**:
|
||||
1. Opens `https://card.weibo.com/article/v3/editor`
|
||||
2. Clicks "写文章" button, waits for editor to become editable
|
||||
3. Fills title (validated for 32-char limit)
|
||||
4. Fills summary/导语 (validated for 44-char limit)
|
||||
5. Inserts HTML content into ProseMirror editor via paste
|
||||
6. Replaces image placeholders one by one (copy image → select placeholder → paste)
|
||||
|
||||
**Post-Composition Check**: The script automatically verifies after all images are inserted:
|
||||
- Remaining `WBIMGPH_` placeholders in editor content
|
||||
- Expected vs actual image count
|
||||
|
||||
If the check fails (warnings in output), alert the user with the specific issues before they publish.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Chrome debug port not ready
|
||||
|
||||
If a script fails with `Chrome debug port not ready` or `Unable to connect`, kill existing Chrome CDP instances first, then retry:
|
||||
|
||||
```bash
|
||||
pkill -f "Chrome.*remote-debugging-port" 2>/dev/null; pkill -f "Chromium.*remote-debugging-port" 2>/dev/null; sleep 2
|
||||
```
|
||||
|
||||
**Important**: This should be done automatically -- when encountering this error, kill Chrome CDP instances and retry the command without asking the user.
|
||||
|
||||
## Notes
|
||||
|
||||
- First run: manual login required (session persists)
|
||||
- All scripts only fill content into the browser, user must review and publish manually
|
||||
- Cross-platform: macOS, Linux, Windows
|
||||
|
||||
## Extension Support
|
||||
|
||||
Custom configurations via EXTEND.md. See **Preferences** section for paths and supported options.
|
||||
@@ -0,0 +1,380 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import process from 'node:process';
|
||||
|
||||
const SUPPORTED_IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp']);
|
||||
|
||||
function printUsage(exitCode = 0): never {
|
||||
console.log(`Copy image or HTML to system clipboard
|
||||
|
||||
Supports:
|
||||
- Image files (jpg, png, gif, webp) - copies as image data
|
||||
- HTML content - copies as rich text for paste
|
||||
|
||||
Usage:
|
||||
# Copy image to clipboard
|
||||
npx -y bun copy-to-clipboard.ts image /path/to/image.jpg
|
||||
|
||||
# Copy HTML to clipboard
|
||||
npx -y bun copy-to-clipboard.ts html "<p>Hello</p>"
|
||||
|
||||
# Copy HTML from file
|
||||
npx -y bun copy-to-clipboard.ts html --file /path/to/content.html
|
||||
`);
|
||||
process.exit(exitCode);
|
||||
}
|
||||
|
||||
function resolvePath(filePath: string): string {
|
||||
return path.isAbsolute(filePath) ? filePath : path.resolve(process.cwd(), filePath);
|
||||
}
|
||||
|
||||
function inferImageMimeType(imagePath: string): string {
|
||||
const ext = path.extname(imagePath).toLowerCase();
|
||||
switch (ext) {
|
||||
case '.jpg':
|
||||
case '.jpeg':
|
||||
return 'image/jpeg';
|
||||
case '.png':
|
||||
return 'image/png';
|
||||
case '.gif':
|
||||
return 'image/gif';
|
||||
case '.webp':
|
||||
return 'image/webp';
|
||||
default:
|
||||
return 'application/octet-stream';
|
||||
}
|
||||
}
|
||||
|
||||
type RunResult = { stdout: string; stderr: string; exitCode: number };
|
||||
|
||||
async function runCommand(
|
||||
command: string,
|
||||
args: string[],
|
||||
options?: { input?: string | Buffer; allowNonZeroExit?: boolean },
|
||||
): Promise<RunResult> {
|
||||
return await new Promise<RunResult>((resolve, reject) => {
|
||||
const child = spawn(command, args, { stdio: ['pipe', 'pipe', 'pipe'] });
|
||||
const stdoutChunks: Buffer[] = [];
|
||||
const stderrChunks: Buffer[] = [];
|
||||
|
||||
child.stdout.on('data', (chunk) => stdoutChunks.push(Buffer.from(chunk)));
|
||||
child.stderr.on('data', (chunk) => stderrChunks.push(Buffer.from(chunk)));
|
||||
child.on('error', reject);
|
||||
child.on('close', (code) => {
|
||||
resolve({
|
||||
stdout: Buffer.concat(stdoutChunks).toString('utf8'),
|
||||
stderr: Buffer.concat(stderrChunks).toString('utf8'),
|
||||
exitCode: code ?? 0,
|
||||
});
|
||||
});
|
||||
|
||||
if (options?.input != null) child.stdin.write(options.input);
|
||||
child.stdin.end();
|
||||
}).then((result) => {
|
||||
if (!options?.allowNonZeroExit && result.exitCode !== 0) {
|
||||
const details = result.stderr.trim() || result.stdout.trim();
|
||||
throw new Error(`Command failed (${command}): exit ${result.exitCode}${details ? `\n${details}` : ''}`);
|
||||
}
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
async function commandExists(command: string): Promise<boolean> {
|
||||
if (process.platform === 'win32') {
|
||||
const result = await runCommand('where', [command], { allowNonZeroExit: true });
|
||||
return result.exitCode === 0 && result.stdout.trim().length > 0;
|
||||
}
|
||||
const result = await runCommand('which', [command], { allowNonZeroExit: true });
|
||||
return result.exitCode === 0 && result.stdout.trim().length > 0;
|
||||
}
|
||||
|
||||
async function runCommandWithFileStdin(command: string, args: string[], filePath: string): Promise<void> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(command, args, { stdio: ['pipe', 'pipe', 'pipe'] });
|
||||
const stderrChunks: Buffer[] = [];
|
||||
const stdoutChunks: Buffer[] = [];
|
||||
|
||||
child.stdout.on('data', (chunk) => stdoutChunks.push(Buffer.from(chunk)));
|
||||
child.stderr.on('data', (chunk) => stderrChunks.push(Buffer.from(chunk)));
|
||||
child.on('error', reject);
|
||||
child.on('close', (code) => {
|
||||
const exitCode = code ?? 0;
|
||||
if (exitCode !== 0) {
|
||||
const details = Buffer.concat(stderrChunks).toString('utf8').trim() || Buffer.concat(stdoutChunks).toString('utf8').trim();
|
||||
reject(
|
||||
new Error(`Command failed (${command}): exit ${exitCode}${details ? `\n${details}` : ''}`),
|
||||
);
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
|
||||
fs.createReadStream(filePath).on('error', reject).pipe(child.stdin);
|
||||
});
|
||||
}
|
||||
|
||||
async function withTempDir<T>(prefix: string, fn: (tempDir: string) => Promise<T>): Promise<T> {
|
||||
const tempDir = await mkdtemp(path.join(os.tmpdir(), prefix));
|
||||
try {
|
||||
return await fn(tempDir);
|
||||
} finally {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function getMacSwiftClipboardSource(): string {
|
||||
return `import AppKit
|
||||
import Foundation
|
||||
|
||||
func die(_ message: String, _ code: Int32 = 1) -> Never {
|
||||
FileHandle.standardError.write(message.data(using: .utf8)!)
|
||||
exit(code)
|
||||
}
|
||||
|
||||
if CommandLine.arguments.count < 3 {
|
||||
die("Usage: clipboard.swift <image|html> <path>\\n")
|
||||
}
|
||||
|
||||
let mode = CommandLine.arguments[1]
|
||||
let inputPath = CommandLine.arguments[2]
|
||||
let pasteboard = NSPasteboard.general
|
||||
pasteboard.clearContents()
|
||||
|
||||
switch mode {
|
||||
case "image":
|
||||
guard let image = NSImage(contentsOfFile: inputPath) else {
|
||||
die("Failed to load image: \\(inputPath)\\n")
|
||||
}
|
||||
if !pasteboard.writeObjects([image]) {
|
||||
die("Failed to write image to clipboard\\n")
|
||||
}
|
||||
|
||||
case "html":
|
||||
let url = URL(fileURLWithPath: inputPath)
|
||||
let data: Data
|
||||
do {
|
||||
data = try Data(contentsOf: url)
|
||||
} catch {
|
||||
die("Failed to read HTML file: \\(inputPath)\\n")
|
||||
}
|
||||
|
||||
_ = pasteboard.setData(data, forType: .html)
|
||||
|
||||
let options: [NSAttributedString.DocumentReadingOptionKey: Any] = [
|
||||
.documentType: NSAttributedString.DocumentType.html,
|
||||
.characterEncoding: String.Encoding.utf8.rawValue
|
||||
]
|
||||
|
||||
if let attr = try? NSAttributedString(data: data, options: options, documentAttributes: nil) {
|
||||
pasteboard.setString(attr.string, forType: .string)
|
||||
if let rtf = try? attr.data(
|
||||
from: NSRange(location: 0, length: attr.length),
|
||||
documentAttributes: [.documentType: NSAttributedString.DocumentType.rtf]
|
||||
) {
|
||||
_ = pasteboard.setData(rtf, forType: .rtf)
|
||||
}
|
||||
} else if let html = String(data: data, encoding: .utf8) {
|
||||
pasteboard.setString(html, forType: .string)
|
||||
}
|
||||
|
||||
default:
|
||||
die("Unknown mode: \\(mode)\\n")
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
async function copyImageMac(imagePath: string): Promise<void> {
|
||||
await withTempDir('copy-to-clipboard-', async (tempDir) => {
|
||||
const swiftPath = path.join(tempDir, 'clipboard.swift');
|
||||
await writeFile(swiftPath, getMacSwiftClipboardSource(), 'utf8');
|
||||
await runCommand('swift', [swiftPath, 'image', imagePath]);
|
||||
});
|
||||
}
|
||||
|
||||
async function copyHtmlMac(htmlFilePath: string): Promise<void> {
|
||||
await withTempDir('copy-to-clipboard-', async (tempDir) => {
|
||||
const swiftPath = path.join(tempDir, 'clipboard.swift');
|
||||
await writeFile(swiftPath, getMacSwiftClipboardSource(), 'utf8');
|
||||
await runCommand('swift', [swiftPath, 'html', htmlFilePath]);
|
||||
});
|
||||
}
|
||||
|
||||
async function copyImageLinux(imagePath: string): Promise<void> {
|
||||
const mime = inferImageMimeType(imagePath);
|
||||
if (await commandExists('wl-copy')) {
|
||||
await runCommandWithFileStdin('wl-copy', ['--type', mime], imagePath);
|
||||
return;
|
||||
}
|
||||
if (await commandExists('xclip')) {
|
||||
await runCommand('xclip', ['-selection', 'clipboard', '-t', mime, '-i', imagePath]);
|
||||
return;
|
||||
}
|
||||
throw new Error('No clipboard tool found. Install `wl-clipboard` (wl-copy) or `xclip`.');
|
||||
}
|
||||
|
||||
async function copyHtmlLinux(htmlFilePath: string): Promise<void> {
|
||||
if (await commandExists('wl-copy')) {
|
||||
await runCommandWithFileStdin('wl-copy', ['--type', 'text/html'], htmlFilePath);
|
||||
return;
|
||||
}
|
||||
if (await commandExists('xclip')) {
|
||||
await runCommand('xclip', ['-selection', 'clipboard', '-t', 'text/html', '-i', htmlFilePath]);
|
||||
return;
|
||||
}
|
||||
throw new Error('No clipboard tool found. Install `wl-clipboard` (wl-copy) or `xclip`.');
|
||||
}
|
||||
|
||||
async function copyImageWindows(imagePath: string): Promise<void> {
|
||||
const escaped = imagePath.replace(/'/g, "''");
|
||||
const ps = [
|
||||
'Add-Type -AssemblyName System.Windows.Forms',
|
||||
'Add-Type -AssemblyName System.Drawing',
|
||||
`$img = [System.Drawing.Image]::FromFile('${escaped}')`,
|
||||
'[System.Windows.Forms.Clipboard]::SetImage($img)',
|
||||
'$img.Dispose()',
|
||||
].join('; ');
|
||||
await runCommand('powershell.exe', ['-NoProfile', '-Sta', '-Command', ps]);
|
||||
}
|
||||
|
||||
async function copyHtmlWindows(htmlFilePath: string): Promise<void> {
|
||||
const escaped = htmlFilePath.replace(/'/g, "''");
|
||||
const ps = [
|
||||
'Add-Type -AssemblyName System.Windows.Forms',
|
||||
`$html = Get-Content -Raw -LiteralPath '${escaped}'`,
|
||||
'[System.Windows.Forms.Clipboard]::SetText($html, [System.Windows.Forms.TextDataFormat]::Html)',
|
||||
].join('; ');
|
||||
await runCommand('powershell.exe', ['-NoProfile', '-Sta', '-Command', ps]);
|
||||
}
|
||||
|
||||
async function copyImageToClipboard(imagePathInput: string): Promise<void> {
|
||||
const imagePath = resolvePath(imagePathInput);
|
||||
const ext = path.extname(imagePath).toLowerCase();
|
||||
if (!SUPPORTED_IMAGE_EXTS.has(ext)) {
|
||||
throw new Error(
|
||||
`Unsupported image type: ${ext || '(none)'} (supported: ${Array.from(SUPPORTED_IMAGE_EXTS).join(', ')})`,
|
||||
);
|
||||
}
|
||||
if (!fs.existsSync(imagePath)) throw new Error(`File not found: ${imagePath}`);
|
||||
|
||||
switch (process.platform) {
|
||||
case 'darwin':
|
||||
await copyImageMac(imagePath);
|
||||
return;
|
||||
case 'linux':
|
||||
await copyImageLinux(imagePath);
|
||||
return;
|
||||
case 'win32':
|
||||
await copyImageWindows(imagePath);
|
||||
return;
|
||||
default:
|
||||
throw new Error(`Unsupported platform: ${process.platform}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function copyHtmlFileToClipboard(htmlFilePathInput: string): Promise<void> {
|
||||
const htmlFilePath = resolvePath(htmlFilePathInput);
|
||||
if (!fs.existsSync(htmlFilePath)) throw new Error(`File not found: ${htmlFilePath}`);
|
||||
|
||||
switch (process.platform) {
|
||||
case 'darwin':
|
||||
await copyHtmlMac(htmlFilePath);
|
||||
return;
|
||||
case 'linux':
|
||||
await copyHtmlLinux(htmlFilePath);
|
||||
return;
|
||||
case 'win32':
|
||||
await copyHtmlWindows(htmlFilePath);
|
||||
return;
|
||||
default:
|
||||
throw new Error(`Unsupported platform: ${process.platform}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function readStdinText(): Promise<string | null> {
|
||||
if (process.stdin.isTTY) return null;
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of process.stdin) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
const text = Buffer.concat(chunks).toString('utf8');
|
||||
return text.length > 0 ? text : null;
|
||||
}
|
||||
|
||||
async function copyHtmlToClipboard(args: string[]): Promise<void> {
|
||||
let htmlFile: string | undefined;
|
||||
const positional: string[] = [];
|
||||
|
||||
for (let i = 0; i < args.length; i += 1) {
|
||||
const arg = args[i] ?? '';
|
||||
if (arg === '--help' || arg === '-h') printUsage(0);
|
||||
if (arg === '--file') {
|
||||
htmlFile = args[i + 1];
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg.startsWith('--file=')) {
|
||||
htmlFile = arg.slice('--file='.length);
|
||||
continue;
|
||||
}
|
||||
if (arg === '--') {
|
||||
positional.push(...args.slice(i + 1));
|
||||
break;
|
||||
}
|
||||
if (arg.startsWith('-')) {
|
||||
throw new Error(`Unknown option: ${arg}`);
|
||||
}
|
||||
positional.push(arg);
|
||||
}
|
||||
|
||||
if (htmlFile && positional.length > 0) {
|
||||
throw new Error('Do not pass HTML text when using --file.');
|
||||
}
|
||||
|
||||
if (htmlFile) {
|
||||
await copyHtmlFileToClipboard(htmlFile);
|
||||
return;
|
||||
}
|
||||
|
||||
const htmlFromArgs = positional.join(' ').trim();
|
||||
const htmlFromStdin = (await readStdinText())?.trim() ?? '';
|
||||
const html = htmlFromArgs || htmlFromStdin;
|
||||
if (!html) throw new Error('Missing HTML input. Provide a string or use --file.');
|
||||
|
||||
await withTempDir('copy-to-clipboard-', async (tempDir) => {
|
||||
const htmlPath = path.join(tempDir, 'input.html');
|
||||
await writeFile(htmlPath, html, 'utf8');
|
||||
await copyHtmlFileToClipboard(htmlPath);
|
||||
});
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const argv = process.argv.slice(2);
|
||||
if (argv.length === 0) printUsage(1);
|
||||
|
||||
const command = argv[0];
|
||||
if (command === '--help' || command === '-h') printUsage(0);
|
||||
|
||||
if (command === 'image') {
|
||||
const imagePath = argv[1];
|
||||
if (!imagePath) throw new Error('Missing image path.');
|
||||
await copyImageToClipboard(imagePath);
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === 'html') {
|
||||
await copyHtmlToClipboard(argv.slice(1));
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error(`Unknown command: ${command}`);
|
||||
}
|
||||
|
||||
await main().catch((err) => {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error(`Error: ${message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,452 @@
|
||||
import fs from 'node:fs';
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import https from 'node:https';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import process from 'node:process';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
import frontMatter from 'front-matter';
|
||||
import hljs from 'highlight.js/lib/common';
|
||||
import { Lexer, Marked, type RendererObject, type Tokens } from 'marked';
|
||||
|
||||
interface ImageInfo {
|
||||
placeholder: string;
|
||||
localPath: string;
|
||||
originalPath: string;
|
||||
alt: string;
|
||||
blockIndex: number;
|
||||
}
|
||||
|
||||
interface ParsedMarkdown {
|
||||
title: string;
|
||||
summary: string;
|
||||
shortSummary: string;
|
||||
coverImage: string | null;
|
||||
contentImages: ImageInfo[];
|
||||
html: string;
|
||||
totalBlocks: number;
|
||||
}
|
||||
|
||||
type FrontmatterFields = Record<string, unknown>;
|
||||
|
||||
function parseFrontmatter(content: string): { frontmatter: FrontmatterFields; body: string } {
|
||||
try {
|
||||
const parsed = frontMatter<FrontmatterFields>(content);
|
||||
return {
|
||||
frontmatter: parsed.attributes ?? {},
|
||||
body: parsed.body,
|
||||
};
|
||||
} catch {
|
||||
return { frontmatter: {}, body: content };
|
||||
}
|
||||
}
|
||||
|
||||
function stripWrappingQuotes(value: string): string {
|
||||
if (!value) return value;
|
||||
const doubleQuoted = value.startsWith('"') && value.endsWith('"');
|
||||
const singleQuoted = value.startsWith("'") && value.endsWith("'");
|
||||
const cjkDoubleQuoted = value.startsWith('\u201c') && value.endsWith('\u201d');
|
||||
const cjkSingleQuoted = value.startsWith('\u2018') && value.endsWith('\u2019');
|
||||
if (doubleQuoted || singleQuoted || cjkDoubleQuoted || cjkSingleQuoted) {
|
||||
return value.slice(1, -1).trim();
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
function toFrontmatterString(value: unknown): string | undefined {
|
||||
if (typeof value === 'string') {
|
||||
return stripWrappingQuotes(value);
|
||||
}
|
||||
if (typeof value === 'number' || typeof value === 'boolean') {
|
||||
return String(value);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function pickFirstString(frontmatter: FrontmatterFields, keys: string[]): string | undefined {
|
||||
for (const key of keys) {
|
||||
const value = toFrontmatterString(frontmatter[key]);
|
||||
if (value) return value;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function extractTitleFromMarkdown(markdown: string): string {
|
||||
const tokens = Lexer.lex(markdown, { gfm: true, breaks: true });
|
||||
for (const token of tokens) {
|
||||
if (token.type === 'heading' && token.depth === 1) {
|
||||
return stripWrappingQuotes(token.text);
|
||||
}
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function extractSummaryFromBody(body: string, maxLen: number): string {
|
||||
const lines = body.split('\n').filter(l => l.trim() && !l.startsWith('#') && !l.startsWith('!') && !l.startsWith('```'));
|
||||
const firstParagraph = lines[0]?.replace(/[*_`\[\]()]/g, '').trim() || '';
|
||||
if (firstParagraph.length <= maxLen) return firstParagraph;
|
||||
return firstParagraph.slice(0, maxLen - 1) + '\u2026';
|
||||
}
|
||||
|
||||
function downloadFile(url: string, destPath: string, maxRedirects = 5): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!url.startsWith('https://')) {
|
||||
reject(new Error(`Refusing non-HTTPS download: ${url}`));
|
||||
return;
|
||||
}
|
||||
if (maxRedirects <= 0) {
|
||||
reject(new Error('Too many redirects'));
|
||||
return;
|
||||
}
|
||||
const file = fs.createWriteStream(destPath);
|
||||
|
||||
const request = https.get(url, { headers: { 'User-Agent': 'Mozilla/5.0' } }, (response) => {
|
||||
if (response.statusCode === 301 || response.statusCode === 302) {
|
||||
const redirectUrl = response.headers.location;
|
||||
if (redirectUrl) {
|
||||
file.close();
|
||||
fs.unlinkSync(destPath);
|
||||
downloadFile(redirectUrl, destPath, maxRedirects - 1).then(resolve).catch(reject);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (response.statusCode !== 200) {
|
||||
file.close();
|
||||
fs.unlinkSync(destPath);
|
||||
reject(new Error(`Failed to download: ${response.statusCode}`));
|
||||
return;
|
||||
}
|
||||
|
||||
response.pipe(file);
|
||||
file.on('finish', () => {
|
||||
file.close();
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
request.on('error', (err) => {
|
||||
file.close();
|
||||
fs.unlink(destPath, () => {});
|
||||
reject(err);
|
||||
});
|
||||
|
||||
request.setTimeout(30000, () => {
|
||||
request.destroy();
|
||||
reject(new Error('Download timeout'));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function getImageExtension(urlOrPath: string): string {
|
||||
const match = urlOrPath.match(/\.(jpg|jpeg|png|gif|webp)(\?|$)/i);
|
||||
return match ? match[1]!.toLowerCase() : 'png';
|
||||
}
|
||||
|
||||
function resolveLocalWithFallback(resolved: string): string {
|
||||
if (fs.existsSync(resolved)) return resolved;
|
||||
const ext = path.extname(resolved);
|
||||
const base = resolved.slice(0, -ext.length);
|
||||
const alternatives = [
|
||||
base + '.webp',
|
||||
base + '.jpg',
|
||||
base + '.jpeg',
|
||||
base + '.png',
|
||||
base + '.gif',
|
||||
base + '_original.png',
|
||||
base + '_original.jpg',
|
||||
].filter((p) => p !== resolved);
|
||||
for (const alt of alternatives) {
|
||||
if (fs.existsSync(alt)) {
|
||||
console.error(`[md-to-html] Image fallback: ${path.basename(resolved)} → ${path.basename(alt)}`);
|
||||
return alt;
|
||||
}
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
async function resolveImagePath(imagePath: string, baseDir: string, tempDir: string): Promise<string> {
|
||||
if (imagePath.startsWith('http://')) {
|
||||
console.error(`[md-to-html] Skipping non-HTTPS image: ${imagePath}`);
|
||||
return '';
|
||||
}
|
||||
if (imagePath.startsWith('https://')) {
|
||||
const hash = createHash('md5').update(imagePath).digest('hex').slice(0, 8);
|
||||
const ext = getImageExtension(imagePath);
|
||||
const localPath = path.join(tempDir, `remote_${hash}.${ext}`);
|
||||
|
||||
if (!fs.existsSync(localPath)) {
|
||||
console.error(`[md-to-html] Downloading: ${imagePath}`);
|
||||
await downloadFile(imagePath, localPath);
|
||||
}
|
||||
return localPath;
|
||||
}
|
||||
|
||||
const resolved = path.isAbsolute(imagePath) ? imagePath : path.resolve(baseDir, imagePath);
|
||||
return resolveLocalWithFallback(resolved);
|
||||
}
|
||||
|
||||
function escapeHtml(text: string): string {
|
||||
return text
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
}
|
||||
|
||||
function highlightCode(code: string, lang: string): string {
|
||||
try {
|
||||
if (lang && hljs.getLanguage(lang)) {
|
||||
return hljs.highlight(code, { language: lang, ignoreIllegals: true }).value;
|
||||
}
|
||||
return hljs.highlightAuto(code).value;
|
||||
} catch {
|
||||
return escapeHtml(code);
|
||||
}
|
||||
}
|
||||
|
||||
const EMPTY_PARAGRAPH = '<p></p>';
|
||||
|
||||
function convertMarkdownToHtml(markdown: string, imageCallback: (src: string, alt: string) => string): { html: string; totalBlocks: number } {
|
||||
const blockTokens = Lexer.lex(markdown, { gfm: true, breaks: true });
|
||||
|
||||
const renderer: RendererObject = {
|
||||
heading({ depth, tokens }: Tokens.Heading): string {
|
||||
if (depth === 1) {
|
||||
return '';
|
||||
}
|
||||
return `<h2>${this.parser.parseInline(tokens)}</h2>`;
|
||||
},
|
||||
|
||||
paragraph({ tokens }: Tokens.Paragraph): string {
|
||||
const text = this.parser.parseInline(tokens).trim();
|
||||
if (!text) return '';
|
||||
return `<p>${text}</p>`;
|
||||
},
|
||||
|
||||
blockquote({ tokens }: Tokens.Blockquote): string {
|
||||
return `<blockquote>${this.parser.parse(tokens)}</blockquote>`;
|
||||
},
|
||||
|
||||
code({ text, lang = '' }: Tokens.Code): string {
|
||||
const language = lang.split(/\s+/)[0]!.toLowerCase();
|
||||
const source = text.replace(/\n$/, '');
|
||||
const highlighted = highlightCode(source, language).replace(/\n/g, '<br>');
|
||||
const label = language ? `<strong>[${escapeHtml(language)}]</strong><br>` : '';
|
||||
return `<blockquote>${label}${highlighted}</blockquote>`;
|
||||
},
|
||||
|
||||
image({ href, text }: Tokens.Image): string {
|
||||
if (!href) return '';
|
||||
return imageCallback(href, text ?? '');
|
||||
},
|
||||
|
||||
link({ href, title, tokens, text }: Tokens.Link): string {
|
||||
const label = tokens?.length ? this.parser.parseInline(tokens) : escapeHtml(text || href || '');
|
||||
if (!href) return label;
|
||||
|
||||
const titleAttr = title ? ` title="${escapeHtml(title)}"` : '';
|
||||
return `<a href="${escapeHtml(href)}"${titleAttr} rel="noopener noreferrer nofollow">${label}</a>`;
|
||||
},
|
||||
};
|
||||
|
||||
const parser = new Marked({
|
||||
gfm: true,
|
||||
breaks: true,
|
||||
});
|
||||
parser.use({ renderer });
|
||||
|
||||
const rendered = parser.parse(markdown);
|
||||
if (typeof rendered !== 'string') {
|
||||
throw new Error('Unexpected async markdown parse result');
|
||||
}
|
||||
|
||||
const totalBlocks = blockTokens.filter((token) => {
|
||||
if (token.type === 'space') return false;
|
||||
if (token.type === 'heading' && token.depth === 1) return false;
|
||||
return true;
|
||||
}).length;
|
||||
|
||||
const blocks = rendered
|
||||
.replace(/(<\/(?:p|h[1-6]|blockquote|ol|ul|hr|pre)>)/gi, '$1\n')
|
||||
.split('\n')
|
||||
.filter((l) => l.trim());
|
||||
|
||||
const spaced: string[] = [];
|
||||
const nestTags = ['ol', 'ul', 'blockquote'];
|
||||
let depth = 0;
|
||||
for (let i = 0; i < blocks.length; i++) {
|
||||
const block = blocks[i]!;
|
||||
const opens = (block.match(new RegExp(`<(${nestTags.join('|')})[\\s>]`, 'gi')) || []).length;
|
||||
const closes = (block.match(new RegExp(`</(${nestTags.join('|')})>`, 'gi')) || []).length;
|
||||
|
||||
spaced.push(block);
|
||||
depth += opens - closes;
|
||||
|
||||
if (depth <= 0 && i < blocks.length - 1) {
|
||||
const lastIsEmpty = spaced.length > 0 && spaced[spaced.length - 1] === EMPTY_PARAGRAPH;
|
||||
if (!lastIsEmpty) {
|
||||
spaced.push(EMPTY_PARAGRAPH);
|
||||
}
|
||||
}
|
||||
if (depth < 0) depth = 0;
|
||||
}
|
||||
|
||||
return {
|
||||
html: spaced.join('\n'),
|
||||
totalBlocks,
|
||||
};
|
||||
}
|
||||
|
||||
export async function parseMarkdown(
|
||||
markdownPath: string,
|
||||
options?: { coverImage?: string; title?: string; tempDir?: string },
|
||||
): Promise<ParsedMarkdown> {
|
||||
const content = fs.readFileSync(markdownPath, 'utf-8');
|
||||
const baseDir = path.dirname(markdownPath);
|
||||
const tempDir = options?.tempDir ?? path.join(os.tmpdir(), 'weibo-article-images');
|
||||
|
||||
await mkdir(tempDir, { recursive: true });
|
||||
|
||||
const { frontmatter, body } = parseFrontmatter(content);
|
||||
|
||||
let title = stripWrappingQuotes(options?.title ?? '') || pickFirstString(frontmatter, ['title']) || '';
|
||||
if (!title) {
|
||||
title = extractTitleFromMarkdown(body);
|
||||
}
|
||||
if (!title) {
|
||||
title = path.basename(markdownPath, path.extname(markdownPath));
|
||||
}
|
||||
|
||||
let summary = pickFirstString(frontmatter, ['summary', 'description', 'excerpt']) || '';
|
||||
if (!summary) summary = extractSummaryFromBody(body, 44);
|
||||
const shortSummary = extractSummaryFromBody(body, 44);
|
||||
|
||||
let coverImagePath = stripWrappingQuotes(options?.coverImage ?? '') || pickFirstString(frontmatter, [
|
||||
'featureImage', 'cover_image', 'coverImage', 'cover', 'image',
|
||||
]) || null;
|
||||
|
||||
const images: Array<{ src: string; alt: string; blockIndex: number }> = [];
|
||||
let imageCounter = 0;
|
||||
|
||||
const { html, totalBlocks } = convertMarkdownToHtml(body, (src, alt) => {
|
||||
const placeholder = `WBIMGPH_${++imageCounter}`;
|
||||
images.push({ src, alt, blockIndex: -1 });
|
||||
return placeholder;
|
||||
});
|
||||
|
||||
const htmlLines = html.split('\n');
|
||||
for (let i = 0; i < images.length; i++) {
|
||||
const placeholder = `WBIMGPH_${i + 1}`;
|
||||
for (let lineIndex = 0; lineIndex < htmlLines.length; lineIndex++) {
|
||||
const regex = new RegExp(`\\b${placeholder}\\b`);
|
||||
if (regex.test(htmlLines[lineIndex]!)) {
|
||||
images[i]!.blockIndex = lineIndex;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const contentImages: ImageInfo[] = [];
|
||||
|
||||
for (let i = 0; i < images.length; i++) {
|
||||
const img = images[i]!;
|
||||
const localPath = await resolveImagePath(img.src, baseDir, tempDir);
|
||||
|
||||
contentImages.push({
|
||||
placeholder: `WBIMGPH_${i + 1}`,
|
||||
localPath,
|
||||
originalPath: img.src,
|
||||
alt: img.alt,
|
||||
blockIndex: img.blockIndex,
|
||||
});
|
||||
}
|
||||
|
||||
const finalHtml = html.replace(/\n{3,}/g, '\n\n').trim();
|
||||
|
||||
let resolvedCoverImage: string | null = null;
|
||||
if (coverImagePath) {
|
||||
resolvedCoverImage = await resolveImagePath(coverImagePath, baseDir, tempDir);
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
summary,
|
||||
shortSummary,
|
||||
coverImage: resolvedCoverImage,
|
||||
contentImages,
|
||||
html: finalHtml,
|
||||
totalBlocks,
|
||||
};
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = process.argv.slice(2);
|
||||
if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
|
||||
console.log(`Convert Markdown to HTML for Weibo article publishing
|
||||
|
||||
Usage:
|
||||
npx -y bun md-to-html.ts <markdown_file> [options]
|
||||
|
||||
Options:
|
||||
--title <title> Override title
|
||||
--cover <image> Override cover image
|
||||
--output <json|html> Output format (default: json)
|
||||
--html-only Output only the HTML content
|
||||
--save-html <path> Save HTML to file
|
||||
--help Show this help
|
||||
`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
let markdownPath: string | undefined;
|
||||
let title: string | undefined;
|
||||
let coverImage: string | undefined;
|
||||
let outputFormat: 'json' | 'html' = 'json';
|
||||
let htmlOnly = false;
|
||||
let saveHtmlPath: string | undefined;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i]!;
|
||||
if (arg === '--title' && args[i + 1]) {
|
||||
title = args[++i];
|
||||
} else if (arg === '--cover' && args[i + 1]) {
|
||||
coverImage = args[++i];
|
||||
} else if (arg === '--output' && args[i + 1]) {
|
||||
outputFormat = args[++i] as 'json' | 'html';
|
||||
} else if (arg === '--html-only') {
|
||||
htmlOnly = true;
|
||||
} else if (arg === '--save-html' && args[i + 1]) {
|
||||
saveHtmlPath = args[++i];
|
||||
} else if (!arg.startsWith('-')) {
|
||||
markdownPath = arg;
|
||||
}
|
||||
}
|
||||
|
||||
if (!markdownPath || !fs.existsSync(markdownPath)) {
|
||||
console.error('Error: Valid markdown file path required');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const result = await parseMarkdown(markdownPath, { title, coverImage });
|
||||
|
||||
if (saveHtmlPath) {
|
||||
await writeFile(saveHtmlPath, result.html, 'utf-8');
|
||||
console.error(`[md-to-html] HTML saved to: ${saveHtmlPath}`);
|
||||
}
|
||||
|
||||
if (htmlOnly) {
|
||||
console.log(result.html);
|
||||
} else if (outputFormat === 'html') {
|
||||
console.log(result.html);
|
||||
} else {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
await main().catch((err) => {
|
||||
console.error(`Error: ${err instanceof Error ? err.message : String(err)}`);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,194 @@
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import process from 'node:process';
|
||||
|
||||
function printUsage(exitCode = 0): never {
|
||||
console.log(`Send real paste keystroke (Cmd+V / Ctrl+V) to the frontmost application
|
||||
|
||||
This bypasses CDP's synthetic events which websites can detect and ignore.
|
||||
|
||||
Usage:
|
||||
npx -y bun paste-from-clipboard.ts [options]
|
||||
|
||||
Options:
|
||||
--retries <n> Number of retry attempts (default: 3)
|
||||
--delay <ms> Delay between retries in ms (default: 500)
|
||||
--app <name> Target application to activate first (macOS only)
|
||||
--help Show this help
|
||||
|
||||
Examples:
|
||||
# Simple paste
|
||||
npx -y bun paste-from-clipboard.ts
|
||||
|
||||
# Paste to Chrome with retries
|
||||
npx -y bun paste-from-clipboard.ts --app "Google Chrome" --retries 5
|
||||
|
||||
# Quick paste with shorter delay
|
||||
npx -y bun paste-from-clipboard.ts --delay 200
|
||||
`);
|
||||
process.exit(exitCode);
|
||||
}
|
||||
|
||||
function sleepSync(ms: number): void {
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
||||
}
|
||||
|
||||
function activateApp(appName: string): boolean {
|
||||
if (process.platform !== 'darwin') return false;
|
||||
|
||||
// Activate and wait for app to be frontmost
|
||||
const script = `
|
||||
tell application "${appName}"
|
||||
activate
|
||||
delay 0.5
|
||||
end tell
|
||||
|
||||
-- Verify app is frontmost
|
||||
tell application "System Events"
|
||||
set frontApp to name of first application process whose frontmost is true
|
||||
if frontApp is not "${appName}" then
|
||||
tell application "${appName}" to activate
|
||||
delay 0.3
|
||||
end if
|
||||
end tell
|
||||
`;
|
||||
const result = spawnSync('osascript', ['-e', script], { stdio: 'pipe' });
|
||||
return result.status === 0;
|
||||
}
|
||||
|
||||
function pasteMac(retries: number, delayMs: number, targetApp?: string): boolean {
|
||||
for (let i = 0; i < retries; i++) {
|
||||
// Build script that activates app (if specified) and sends keystroke in one atomic operation
|
||||
const script = targetApp
|
||||
? `
|
||||
tell application "${targetApp}"
|
||||
activate
|
||||
end tell
|
||||
delay 0.3
|
||||
tell application "System Events"
|
||||
keystroke "v" using command down
|
||||
end tell
|
||||
`
|
||||
: `
|
||||
tell application "System Events"
|
||||
keystroke "v" using command down
|
||||
end tell
|
||||
`;
|
||||
|
||||
const result = spawnSync('osascript', ['-e', script], { stdio: 'pipe' });
|
||||
if (result.status === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const stderr = result.stderr?.toString().trim();
|
||||
if (stderr) {
|
||||
console.error(`[paste] osascript error: ${stderr}`);
|
||||
}
|
||||
|
||||
if (i < retries - 1) {
|
||||
console.error(`[paste] Attempt ${i + 1}/${retries} failed, retrying in ${delayMs}ms...`);
|
||||
sleepSync(delayMs);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function pasteLinux(retries: number, delayMs: number): boolean {
|
||||
// Try xdotool first (X11), then ydotool (Wayland)
|
||||
const tools = [
|
||||
{ cmd: 'xdotool', args: ['key', 'ctrl+v'] },
|
||||
{ cmd: 'ydotool', args: ['key', '29:1', '47:1', '47:0', '29:0'] }, // Ctrl down, V down, V up, Ctrl up
|
||||
];
|
||||
|
||||
for (const tool of tools) {
|
||||
const which = spawnSync('which', [tool.cmd], { stdio: 'pipe' });
|
||||
if (which.status !== 0) continue;
|
||||
|
||||
for (let i = 0; i < retries; i++) {
|
||||
const result = spawnSync(tool.cmd, tool.args, { stdio: 'pipe' });
|
||||
if (result.status === 0) {
|
||||
return true;
|
||||
}
|
||||
if (i < retries - 1) {
|
||||
console.error(`[paste] Attempt ${i + 1}/${retries} failed, retrying in ${delayMs}ms...`);
|
||||
sleepSync(delayMs);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
console.error('[paste] No supported tool found. Install xdotool (X11) or ydotool (Wayland).');
|
||||
return false;
|
||||
}
|
||||
|
||||
function pasteWindows(retries: number, delayMs: number): boolean {
|
||||
const ps = `
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
[System.Windows.Forms.SendKeys]::SendWait("^v")
|
||||
`;
|
||||
|
||||
for (let i = 0; i < retries; i++) {
|
||||
const result = spawnSync('powershell.exe', ['-NoProfile', '-Command', ps], { stdio: 'pipe' });
|
||||
if (result.status === 0) {
|
||||
return true;
|
||||
}
|
||||
if (i < retries - 1) {
|
||||
console.error(`[paste] Attempt ${i + 1}/${retries} failed, retrying in ${delayMs}ms...`);
|
||||
sleepSync(delayMs);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function paste(retries: number, delayMs: number, targetApp?: string): boolean {
|
||||
switch (process.platform) {
|
||||
case 'darwin':
|
||||
return pasteMac(retries, delayMs, targetApp);
|
||||
case 'linux':
|
||||
return pasteLinux(retries, delayMs);
|
||||
case 'win32':
|
||||
return pasteWindows(retries, delayMs);
|
||||
default:
|
||||
console.error(`[paste] Unsupported platform: ${process.platform}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
let retries = 3;
|
||||
let delayMs = 500;
|
||||
let targetApp: string | undefined;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i] ?? '';
|
||||
if (arg === '--help' || arg === '-h') {
|
||||
printUsage(0);
|
||||
}
|
||||
if (arg === '--retries' && args[i + 1]) {
|
||||
retries = parseInt(args[++i]!, 10) || 3;
|
||||
} else if (arg === '--delay' && args[i + 1]) {
|
||||
delayMs = parseInt(args[++i]!, 10) || 500;
|
||||
} else if (arg === '--app' && args[i + 1]) {
|
||||
targetApp = args[++i];
|
||||
} else if (arg.startsWith('-')) {
|
||||
console.error(`Unknown option: ${arg}`);
|
||||
printUsage(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (targetApp) {
|
||||
console.log(`[paste] Target app: ${targetApp}`);
|
||||
}
|
||||
console.log(`[paste] Sending paste keystroke (retries=${retries}, delay=${delayMs}ms)...`);
|
||||
const success = paste(retries, delayMs, targetApp);
|
||||
|
||||
if (success) {
|
||||
console.log('[paste] Paste keystroke sent successfully');
|
||||
} else {
|
||||
console.error('[paste] Failed to send paste keystroke');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
await main();
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,294 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import process from 'node:process';
|
||||
import {
|
||||
CdpConnection,
|
||||
findChromeExecutable,
|
||||
findExistingChromeDebugPort,
|
||||
getDefaultProfileDir,
|
||||
getFreePort,
|
||||
killChromeByProfile,
|
||||
sleep,
|
||||
waitForChromeDebugPort,
|
||||
} from './weibo-utils.js';
|
||||
|
||||
const WEIBO_HOME_URL = 'https://weibo.com/';
|
||||
|
||||
const MAX_FILES = 18;
|
||||
|
||||
interface WeiboPostOptions {
|
||||
text?: string;
|
||||
images?: string[];
|
||||
videos?: string[];
|
||||
timeoutMs?: number;
|
||||
profileDir?: string;
|
||||
chromePath?: string;
|
||||
}
|
||||
|
||||
export async function postToWeibo(options: WeiboPostOptions): Promise<void> {
|
||||
const { text, images = [], videos = [], timeoutMs = 120_000, profileDir = getDefaultProfileDir() } = options;
|
||||
|
||||
const allFiles = [...images, ...videos];
|
||||
if (allFiles.length > MAX_FILES) {
|
||||
throw new Error(`Too many files: ${allFiles.length} (max ${MAX_FILES})`);
|
||||
}
|
||||
|
||||
await mkdir(profileDir, { recursive: true });
|
||||
|
||||
const chromePath = options.chromePath ?? findChromeExecutable();
|
||||
if (!chromePath) throw new Error('Chrome not found. Set WEIBO_BROWSER_CHROME_PATH env var.');
|
||||
|
||||
const launchChrome = async (): Promise<number> => {
|
||||
const port = await getFreePort();
|
||||
console.log(`[weibo-post] Launching Chrome (profile: ${profileDir})`);
|
||||
const chromeArgs = [
|
||||
`--remote-debugging-port=${port}`,
|
||||
`--user-data-dir=${profileDir}`,
|
||||
'--no-first-run',
|
||||
'--no-default-browser-check',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--start-maximized',
|
||||
WEIBO_HOME_URL,
|
||||
];
|
||||
if (process.platform === 'darwin') {
|
||||
const appPath = chromePath.replace(/\/Contents\/MacOS\/Google Chrome$/, '');
|
||||
spawn('open', ['-na', appPath, '--args', ...chromeArgs], { stdio: 'ignore' });
|
||||
} else {
|
||||
spawn(chromePath, chromeArgs, { stdio: 'ignore' });
|
||||
}
|
||||
return port;
|
||||
};
|
||||
|
||||
let port: number;
|
||||
const existingPort = findExistingChromeDebugPort(profileDir);
|
||||
|
||||
if (existingPort) {
|
||||
console.log(`[weibo-post] Found existing Chrome on port ${existingPort}, checking health...`);
|
||||
try {
|
||||
const wsUrl = await waitForChromeDebugPort(existingPort, 5_000);
|
||||
const testCdp = await CdpConnection.connect(wsUrl, 5_000, { defaultTimeoutMs: 5_000 });
|
||||
await testCdp.send('Target.getTargets');
|
||||
testCdp.close();
|
||||
console.log('[weibo-post] Existing Chrome is responsive, reusing.');
|
||||
port = existingPort;
|
||||
} catch {
|
||||
console.log('[weibo-post] Existing Chrome unresponsive, restarting...');
|
||||
killChromeByProfile(profileDir);
|
||||
await sleep(2000);
|
||||
port = await launchChrome();
|
||||
}
|
||||
} else {
|
||||
port = await launchChrome();
|
||||
}
|
||||
|
||||
let cdp: CdpConnection | null = null;
|
||||
|
||||
try {
|
||||
const wsUrl = await waitForChromeDebugPort(port, 30_000);
|
||||
cdp = await CdpConnection.connect(wsUrl, 30_000, { defaultTimeoutMs: 15_000 });
|
||||
|
||||
const targets = await cdp.send<{ targetInfos: Array<{ targetId: string; url: string; type: string }> }>('Target.getTargets');
|
||||
let pageTarget = targets.targetInfos.find((t) => t.type === 'page' && t.url.includes('weibo.com'));
|
||||
|
||||
if (!pageTarget) {
|
||||
const { targetId } = await cdp.send<{ targetId: string }>('Target.createTarget', { url: WEIBO_HOME_URL });
|
||||
pageTarget = { targetId, url: WEIBO_HOME_URL, type: 'page' };
|
||||
}
|
||||
|
||||
const { sessionId } = await cdp.send<{ sessionId: string }>('Target.attachToTarget', { targetId: pageTarget.targetId, flatten: true });
|
||||
|
||||
await cdp.send('Target.activateTarget', { targetId: pageTarget.targetId });
|
||||
|
||||
await cdp.send('Page.enable', {}, { sessionId });
|
||||
await cdp.send('Runtime.enable', {}, { sessionId });
|
||||
await cdp.send('Input.setIgnoreInputEvents', { ignore: false }, { sessionId });
|
||||
|
||||
const currentUrl = await cdp.send<{ result: { value: string } }>('Runtime.evaluate', {
|
||||
expression: `window.location.href`,
|
||||
returnByValue: true,
|
||||
}, { sessionId });
|
||||
|
||||
if (!currentUrl.result.value.includes('weibo.com/') || currentUrl.result.value.includes('card.weibo.com')) {
|
||||
console.log('[weibo-post] Navigating to Weibo home...');
|
||||
await cdp.send('Page.navigate', { url: WEIBO_HOME_URL }, { sessionId });
|
||||
await sleep(3000);
|
||||
}
|
||||
|
||||
console.log('[weibo-post] Waiting for Weibo editor...');
|
||||
await sleep(3000);
|
||||
|
||||
const waitForEditor = async (): Promise<boolean> => {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
const result = await cdp!.send<{ result: { value: boolean } }>('Runtime.evaluate', {
|
||||
expression: `!!document.querySelector('#homeWrap textarea')`,
|
||||
returnByValue: true,
|
||||
}, { sessionId });
|
||||
if (result.result.value) return true;
|
||||
await sleep(1000);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const editorFound = await waitForEditor();
|
||||
if (!editorFound) {
|
||||
console.log('[weibo-post] Editor not found. Please log in to Weibo in the browser window.');
|
||||
console.log('[weibo-post] Waiting for login...');
|
||||
const loggedIn = await waitForEditor();
|
||||
if (!loggedIn) throw new Error('Timed out waiting for Weibo editor. Please log in first.');
|
||||
}
|
||||
|
||||
if (text) {
|
||||
console.log('[weibo-post] Typing text...');
|
||||
|
||||
// Focus and use Input.insertText via CDP
|
||||
await cdp.send('Runtime.evaluate', {
|
||||
expression: `(() => {
|
||||
const editor = document.querySelector('#homeWrap textarea');
|
||||
if (editor) { editor.focus(); editor.value = ''; }
|
||||
})()`,
|
||||
}, { sessionId });
|
||||
await sleep(200);
|
||||
|
||||
await cdp.send('Input.insertText', { text }, { sessionId });
|
||||
await sleep(500);
|
||||
|
||||
// Verify text was entered
|
||||
const textCheck = await cdp.send<{ result: { value: string } }>('Runtime.evaluate', {
|
||||
expression: `document.querySelector('#homeWrap textarea')?.value || ''`,
|
||||
returnByValue: true,
|
||||
}, { sessionId });
|
||||
|
||||
if (textCheck.result.value.length > 0) {
|
||||
console.log(`[weibo-post] Text verified (${textCheck.result.value.length} chars)`);
|
||||
} else {
|
||||
console.warn('[weibo-post] Text input appears empty, trying execCommand fallback...');
|
||||
await cdp.send('Runtime.evaluate', {
|
||||
expression: `(() => {
|
||||
const editor = document.querySelector('#homeWrap textarea');
|
||||
if (editor) { editor.focus(); document.execCommand('insertText', false, ${JSON.stringify(text)}); }
|
||||
})()`,
|
||||
}, { sessionId });
|
||||
await sleep(300);
|
||||
|
||||
const textRecheck = await cdp.send<{ result: { value: string } }>('Runtime.evaluate', {
|
||||
expression: `document.querySelector('#homeWrap textarea')?.value || ''`,
|
||||
returnByValue: true,
|
||||
}, { sessionId });
|
||||
console.log(`[weibo-post] Text after fallback: ${textRecheck.result.value.length} chars`);
|
||||
}
|
||||
}
|
||||
|
||||
if (allFiles.length > 0) {
|
||||
const missing = allFiles.filter((f) => !fs.existsSync(f));
|
||||
if (missing.length > 0) {
|
||||
throw new Error(`Files not found: ${missing.join(', ')}`);
|
||||
}
|
||||
|
||||
const absolutePaths = allFiles.map((f) => path.resolve(f));
|
||||
console.log(`[weibo-post] Uploading ${absolutePaths.length} file(s) via file input...`);
|
||||
|
||||
await cdp.send('DOM.enable', {}, { sessionId });
|
||||
|
||||
const { root } = await cdp.send<{ root: { nodeId: number } }>('DOM.getDocument', {}, { sessionId });
|
||||
|
||||
const { nodeId } = await cdp.send<{ nodeId: number }>('DOM.querySelector', {
|
||||
nodeId: root.nodeId,
|
||||
selector: '#homeWrap input[type="file"]',
|
||||
}, { sessionId });
|
||||
|
||||
if (!nodeId || nodeId === 0) {
|
||||
throw new Error('File input not found. Make sure the Weibo compose area is visible.');
|
||||
}
|
||||
|
||||
await cdp.send('DOM.setFileInputFiles', {
|
||||
nodeId,
|
||||
files: absolutePaths,
|
||||
}, { sessionId });
|
||||
|
||||
console.log('[weibo-post] Files set on input. Waiting for upload...');
|
||||
await sleep(2000);
|
||||
|
||||
const uploadCheck = await cdp.send<{ result: { value: number } }>('Runtime.evaluate', {
|
||||
expression: `document.querySelectorAll('#homeWrap img[src^="blob:"], #homeWrap img[src^="data:"], #homeWrap video').length`,
|
||||
returnByValue: true,
|
||||
}, { sessionId });
|
||||
|
||||
if (uploadCheck.result.value > 0) {
|
||||
console.log(`[weibo-post] Upload verified (${uploadCheck.result.value} media item(s) detected)`);
|
||||
} else {
|
||||
console.warn('[weibo-post] Upload may still be in progress. Please verify in browser.');
|
||||
}
|
||||
}
|
||||
|
||||
console.log('[weibo-post] Post composed. Please review and click the publish button in the browser.');
|
||||
console.log('[weibo-post] Browser remains open for manual review.');
|
||||
|
||||
} finally {
|
||||
if (cdp) {
|
||||
cdp.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function printUsage(): never {
|
||||
console.log(`Post to Weibo using real Chrome browser
|
||||
|
||||
Usage:
|
||||
npx -y bun weibo-post.ts [options] [text]
|
||||
|
||||
Options:
|
||||
--image <path> Add image (can be repeated)
|
||||
--video <path> Add video (can be repeated)
|
||||
--profile <dir> Chrome profile directory
|
||||
--help Show this help
|
||||
|
||||
Max ${MAX_FILES} files total (images + videos combined).
|
||||
|
||||
Examples:
|
||||
npx -y bun weibo-post.ts "Hello from CLI!"
|
||||
npx -y bun weibo-post.ts "Check this out" --image ./screenshot.png
|
||||
npx -y bun weibo-post.ts "Post it!" --image a.png --image b.png
|
||||
npx -y bun weibo-post.ts "Watch this" --video ./clip.mp4
|
||||
`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = process.argv.slice(2);
|
||||
if (args.includes('--help') || args.includes('-h')) printUsage();
|
||||
|
||||
const images: string[] = [];
|
||||
const videos: string[] = [];
|
||||
let profileDir: string | undefined;
|
||||
const textParts: string[] = [];
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i]!;
|
||||
if (arg === '--image' && args[i + 1]) {
|
||||
images.push(args[++i]!);
|
||||
} else if (arg === '--video' && args[i + 1]) {
|
||||
videos.push(args[++i]!);
|
||||
} else if (arg === '--profile' && args[i + 1]) {
|
||||
profileDir = args[++i];
|
||||
} else if (!arg.startsWith('-')) {
|
||||
textParts.push(arg);
|
||||
}
|
||||
}
|
||||
|
||||
const text = textParts.join(' ').trim() || undefined;
|
||||
|
||||
if (!text && images.length === 0 && videos.length === 0) {
|
||||
console.error('Error: Provide text or at least one image/video.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
await postToWeibo({ text, images, videos, profileDir });
|
||||
}
|
||||
|
||||
await main().catch((err) => {
|
||||
console.error(`Error: ${err instanceof Error ? err.message : String(err)}`);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,229 @@
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import net from 'node:net';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import process from 'node:process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
export const CHROME_CANDIDATES = {
|
||||
darwin: [
|
||||
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
||||
'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
|
||||
'/Applications/Chromium.app/Contents/MacOS/Chromium',
|
||||
],
|
||||
win32: [
|
||||
'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
|
||||
'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
|
||||
],
|
||||
default: [
|
||||
'/usr/bin/google-chrome',
|
||||
'/usr/bin/chromium',
|
||||
'/usr/bin/chromium-browser',
|
||||
],
|
||||
};
|
||||
|
||||
export function findChromeExecutable(): string | undefined {
|
||||
const override = process.env.WEIBO_BROWSER_CHROME_PATH?.trim();
|
||||
if (override && fs.existsSync(override)) return override;
|
||||
|
||||
const candidates = process.platform === 'darwin'
|
||||
? CHROME_CANDIDATES.darwin
|
||||
: process.platform === 'win32'
|
||||
? CHROME_CANDIDATES.win32
|
||||
: CHROME_CANDIDATES.default;
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (fs.existsSync(candidate)) return candidate;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function findExistingChromeDebugPort(profileDir: string): number | null {
|
||||
try {
|
||||
const result = spawnSync('ps', ['aux'], { encoding: 'utf-8', timeout: 5000 });
|
||||
if (result.status !== 0 || !result.stdout) return null;
|
||||
const lines = result.stdout.split('\n');
|
||||
for (const line of lines) {
|
||||
if (!line.includes('--remote-debugging-port=') || !line.includes(profileDir)) continue;
|
||||
const portMatch = line.match(/--remote-debugging-port=(\d+)/);
|
||||
if (portMatch) return Number(portMatch[1]);
|
||||
}
|
||||
} catch {}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function killChromeByProfile(profileDir: string): void {
|
||||
try {
|
||||
const result = spawnSync('ps', ['aux'], { encoding: 'utf-8', timeout: 5000 });
|
||||
if (result.status !== 0 || !result.stdout) return;
|
||||
for (const line of result.stdout.split('\n')) {
|
||||
if (!line.includes(profileDir) || !line.includes('--remote-debugging-port=')) continue;
|
||||
const pidMatch = line.trim().split(/\s+/)[1];
|
||||
if (pidMatch) {
|
||||
try { process.kill(Number(pidMatch), 'SIGTERM'); } catch {}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
export function getDefaultProfileDir(): string {
|
||||
const override = process.env.BAOYU_CHROME_PROFILE_DIR?.trim() || process.env.WEIBO_BROWSER_PROFILE_DIR?.trim();
|
||||
if (override) return path.resolve(override);
|
||||
const base = process.platform === 'darwin'
|
||||
? path.join(os.homedir(), 'Library', 'Application Support')
|
||||
: process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
|
||||
return path.join(base, 'baoyu-skills', 'chrome-profile');
|
||||
}
|
||||
|
||||
export function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export async function getFreePort(): Promise<number> {
|
||||
const fixed = parseInt(process.env.WEIBO_BROWSER_DEBUG_PORT || '', 10);
|
||||
if (fixed > 0) return fixed;
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = net.createServer();
|
||||
server.unref();
|
||||
server.on('error', reject);
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
const address = server.address();
|
||||
if (!address || typeof address === 'string') {
|
||||
server.close(() => reject(new Error('Unable to allocate a free TCP port.')));
|
||||
return;
|
||||
}
|
||||
const port = address.port;
|
||||
server.close((err) => {
|
||||
if (err) reject(err);
|
||||
else resolve(port);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchJson<T = unknown>(url: string): Promise<T> {
|
||||
const res = await fetch(url, { redirect: 'follow' });
|
||||
if (!res.ok) throw new Error(`Request failed: ${res.status} ${res.statusText}`);
|
||||
return (await res.json()) as T;
|
||||
}
|
||||
|
||||
export async function waitForChromeDebugPort(port: number, timeoutMs: number): Promise<string> {
|
||||
const start = Date.now();
|
||||
let lastError: unknown = null;
|
||||
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
try {
|
||||
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(`http://127.0.0.1:${port}/json/version`);
|
||||
if (version.webSocketDebuggerUrl) return version.webSocketDebuggerUrl;
|
||||
lastError = new Error('Missing webSocketDebuggerUrl');
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
await sleep(200);
|
||||
}
|
||||
|
||||
throw new Error(`Chrome debug port not ready: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
|
||||
}
|
||||
|
||||
type PendingRequest = {
|
||||
resolve: (value: unknown) => void;
|
||||
reject: (error: Error) => void;
|
||||
timer: ReturnType<typeof setTimeout> | null;
|
||||
};
|
||||
|
||||
export class CdpConnection {
|
||||
private ws: WebSocket;
|
||||
private nextId = 0;
|
||||
private pending = new Map<number, PendingRequest>();
|
||||
private defaultTimeoutMs: number;
|
||||
|
||||
private constructor(ws: WebSocket, options?: { defaultTimeoutMs?: number }) {
|
||||
this.ws = ws;
|
||||
this.defaultTimeoutMs = options?.defaultTimeoutMs ?? 15_000;
|
||||
|
||||
this.ws.addEventListener('message', (event) => {
|
||||
try {
|
||||
const data = typeof event.data === 'string' ? event.data : new TextDecoder().decode(event.data as ArrayBuffer);
|
||||
const msg = JSON.parse(data) as { id?: number; method?: string; params?: unknown; result?: unknown; error?: { message?: string } };
|
||||
|
||||
if (msg.id) {
|
||||
const pending = this.pending.get(msg.id);
|
||||
if (pending) {
|
||||
this.pending.delete(msg.id);
|
||||
if (pending.timer) clearTimeout(pending.timer);
|
||||
if (msg.error?.message) pending.reject(new Error(msg.error.message));
|
||||
else pending.resolve(msg.result);
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
});
|
||||
|
||||
this.ws.addEventListener('close', () => {
|
||||
for (const [id, pending] of this.pending.entries()) {
|
||||
this.pending.delete(id);
|
||||
if (pending.timer) clearTimeout(pending.timer);
|
||||
pending.reject(new Error('CDP connection closed.'));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
static async connect(url: string, timeoutMs: number, options?: { defaultTimeoutMs?: number }): Promise<CdpConnection> {
|
||||
const ws = new WebSocket(url);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error('CDP connection timeout.')), timeoutMs);
|
||||
ws.addEventListener('open', () => { clearTimeout(timer); resolve(); });
|
||||
ws.addEventListener('error', () => { clearTimeout(timer); reject(new Error('CDP connection failed.')); });
|
||||
});
|
||||
return new CdpConnection(ws, options);
|
||||
}
|
||||
|
||||
async send<T = unknown>(method: string, params?: Record<string, unknown>, options?: { sessionId?: string; timeoutMs?: number }): Promise<T> {
|
||||
const id = ++this.nextId;
|
||||
const message: Record<string, unknown> = { id, method };
|
||||
if (params) message.params = params;
|
||||
if (options?.sessionId) message.sessionId = options.sessionId;
|
||||
|
||||
const timeoutMs = options?.timeoutMs ?? this.defaultTimeoutMs;
|
||||
|
||||
const result = await new Promise<unknown>((resolve, reject) => {
|
||||
const timer = timeoutMs > 0
|
||||
? setTimeout(() => { this.pending.delete(id); reject(new Error(`CDP timeout: ${method}`)); }, timeoutMs)
|
||||
: null;
|
||||
this.pending.set(id, { resolve, reject, timer });
|
||||
this.ws.send(JSON.stringify(message));
|
||||
});
|
||||
|
||||
return result as T;
|
||||
}
|
||||
|
||||
close(): void {
|
||||
try { this.ws.close(); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
export function getScriptDir(): string {
|
||||
return path.dirname(fileURLToPath(import.meta.url));
|
||||
}
|
||||
|
||||
function runBunScript(scriptPath: string, args: string[]): boolean {
|
||||
const result = spawnSync('npx', ['-y', 'bun', scriptPath, ...args], { stdio: 'inherit' });
|
||||
return result.status === 0;
|
||||
}
|
||||
|
||||
export function copyImageToClipboard(imagePath: string): boolean {
|
||||
const copyScript = path.join(getScriptDir(), 'copy-to-clipboard.ts');
|
||||
return runBunScript(copyScript, ['image', imagePath]);
|
||||
}
|
||||
|
||||
export function copyHtmlToClipboard(htmlPath: string): boolean {
|
||||
const copyScript = path.join(getScriptDir(), 'copy-to-clipboard.ts');
|
||||
return runBunScript(copyScript, ['html', '--file', htmlPath]);
|
||||
}
|
||||
|
||||
export function pasteFromClipboard(targetApp?: string, retries = 3, delayMs = 500): boolean {
|
||||
const pasteScript = path.join(getScriptDir(), 'paste-from-clipboard.ts');
|
||||
const args = ['--retries', String(retries), '--delay', String(delayMs)];
|
||||
if (targetApp) args.push('--app', targetApp);
|
||||
return runBunScript(pasteScript, args);
|
||||
}
|
||||
@@ -84,8 +84,8 @@ Checks: Chrome, profile isolation, Bun, Accessibility, clipboard, paste keystrok
|
||||
| Check | Fix |
|
||||
|-------|-----|
|
||||
| Chrome | Install Chrome or set `X_BROWSER_CHROME_PATH` env var |
|
||||
| Profile dir | Ensure `~/.local/share/x-browser-profile` is writable |
|
||||
| Bun runtime | `curl -fsSL https://bun.sh/install \| bash` |
|
||||
| Profile dir | Shared profile at `baoyu-skills/chrome-profile` (see CLAUDE.md Chrome Profile section) |
|
||||
| Bun runtime | `brew install oven-sh/bun/bun` (macOS) or `npm install -g bun` |
|
||||
| Accessibility (macOS) | System Settings → Privacy & Security → Accessibility → enable terminal app |
|
||||
| Clipboard copy | Ensure Swift/AppKit available (macOS Xcode CLI tools: `xcode-select --install`) |
|
||||
| Paste keystroke (macOS) | Same as Accessibility fix above |
|
||||
|
||||
@@ -182,7 +182,7 @@ async function checkBun(): Promise<void> {
|
||||
if (result.status === 0) {
|
||||
log('Bun runtime', true, `v${result.stdout?.toString().trim()}`);
|
||||
} else {
|
||||
log('Bun runtime', false, 'Cannot run bun. Install: curl -fsSL https://bun.sh/install | bash');
|
||||
log('Bun runtime', false, 'Cannot run bun. Install: brew install oven-sh/bun/bun (macOS) or npm install -g bun');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import fs from 'node:fs';
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import https from 'node:https';
|
||||
import http from 'node:http';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import process from 'node:process';
|
||||
@@ -106,18 +105,25 @@ function extractTitleFromMarkdown(markdown: string): string {
|
||||
return '';
|
||||
}
|
||||
|
||||
function downloadFile(url: string, destPath: string): Promise<void> {
|
||||
function downloadFile(url: string, destPath: string, maxRedirects = 5): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const protocol = url.startsWith('https') ? https : http;
|
||||
if (!url.startsWith('https://')) {
|
||||
reject(new Error(`Refusing non-HTTPS download: ${url}`));
|
||||
return;
|
||||
}
|
||||
if (maxRedirects <= 0) {
|
||||
reject(new Error('Too many redirects'));
|
||||
return;
|
||||
}
|
||||
const file = fs.createWriteStream(destPath);
|
||||
|
||||
const request = protocol.get(url, { headers: { 'User-Agent': 'Mozilla/5.0' } }, (response) => {
|
||||
const request = https.get(url, { headers: { 'User-Agent': 'Mozilla/5.0' } }, (response) => {
|
||||
if (response.statusCode === 301 || response.statusCode === 302) {
|
||||
const redirectUrl = response.headers.location;
|
||||
if (redirectUrl) {
|
||||
file.close();
|
||||
fs.unlinkSync(destPath);
|
||||
downloadFile(redirectUrl, destPath).then(resolve).catch(reject);
|
||||
downloadFile(redirectUrl, destPath, maxRedirects - 1).then(resolve).catch(reject);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -155,7 +161,11 @@ function getImageExtension(urlOrPath: string): string {
|
||||
}
|
||||
|
||||
async function resolveImagePath(imagePath: string, baseDir: string, tempDir: string): Promise<string> {
|
||||
if (imagePath.startsWith('http://') || imagePath.startsWith('https://')) {
|
||||
if (imagePath.startsWith('http://')) {
|
||||
console.error(`[md-to-html] Skipping non-HTTPS image: ${imagePath}`);
|
||||
return '';
|
||||
}
|
||||
if (imagePath.startsWith('https://')) {
|
||||
const hash = createHash('md5').update(imagePath).digest('hex').slice(0, 8);
|
||||
const ext = getImageExtension(imagePath);
|
||||
const localPath = path.join(tempDir, `remote_${hash}.${ext}`);
|
||||
|
||||
@@ -81,11 +81,14 @@ function getWslWindowsHome(): string | null {
|
||||
}
|
||||
|
||||
export function getDefaultProfileDir(): string {
|
||||
const override = process.env.X_BROWSER_PROFILE_DIR?.trim();
|
||||
const override = process.env.BAOYU_CHROME_PROFILE_DIR?.trim() || process.env.X_BROWSER_PROFILE_DIR?.trim();
|
||||
if (override) return path.resolve(override);
|
||||
const home = getWslWindowsHome() ?? os.homedir();
|
||||
const base = process.env.XDG_DATA_HOME || path.join(home, '.local', 'share');
|
||||
return path.join(base, 'x-browser-profile');
|
||||
const wslHome = getWslWindowsHome();
|
||||
if (wslHome) return path.join(wslHome, '.local', 'share', 'baoyu-skills', 'chrome-profile');
|
||||
const base = process.platform === 'darwin'
|
||||
? path.join(os.homedir(), 'Library', 'Application Support')
|
||||
: process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
|
||||
return path.join(base, 'baoyu-skills', 'chrome-profile');
|
||||
}
|
||||
|
||||
export function sleep(ms: number): Promise<void> {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: baoyu-translate
|
||||
description: Translates articles and documents between languages with three modes - quick (direct), normal (analyze then translate), and refined (analyze, translate, review, polish). Supports custom glossaries and terminology consistency via EXTEND.md. Use when user asks to "translate", "翻译", "精翻", "translate article", "translate to Chinese/English", or needs any document translation. Also triggers for "refined translation", "精细翻译", "proofread translation", or "快速翻译".
|
||||
description: Translates articles and documents between languages with three modes - quick (direct), normal (analyze then translate), and refined (analyze, translate, review, polish). Supports custom glossaries and terminology consistency via EXTEND.md. Use when user asks to "translate", "翻译", "精翻", "translate article", "translate to Chinese/English", "改成中文", "改成英文", "convert to Chinese", "localize", "本地化", or needs any document translation. Also triggers for "refined translation", "精细翻译", "proofread translation", "快速翻译", "快翻", "这篇文章翻译一下", or when a URL or file is provided with translation intent.
|
||||
---
|
||||
|
||||
# Translator
|
||||
@@ -62,7 +62,7 @@ All configurable values in one place. EXTEND.md overrides these; CLI flags overr
|
||||
| Target language | `zh-CN` | `target_language` | `--to` | Translation target language |
|
||||
| Mode | `normal` | `default_mode` | `--mode` | Translation mode |
|
||||
| Audience | `general` | `audience` | `--audience` | Target reader profile |
|
||||
| Style | `storytelling` | `style` | — | Translation style preference |
|
||||
| Style | `storytelling` | `style` | `--style` | Translation style preference |
|
||||
| Chunk threshold | `4000` | `chunk_threshold` | — | Word count to trigger chunked translation |
|
||||
| Chunk max words | `5000` | `chunk_max_words` | — | Max words per chunk |
|
||||
|
||||
@@ -76,6 +76,22 @@ All configurable values in one place. EXTEND.md overrides these; CLI flags overr
|
||||
|
||||
**Default mode**: Normal (can be overridden in EXTEND.md `default_mode` setting).
|
||||
|
||||
**Style presets** — control the voice and tone of the translation (independent of audience):
|
||||
|
||||
| Value | Description | Effect |
|
||||
|-------|-------------|--------|
|
||||
| `storytelling` | Engaging narrative flow (default) | Draws readers in, smooth transitions, vivid phrasing |
|
||||
| `formal` | Professional, structured | Neutral tone, clear organization, no colloquialisms |
|
||||
| `technical` | Precise, documentation-style | Concise, terminology-heavy, minimal embellishment |
|
||||
| `literal` | Close to original structure | Minimal restructuring, preserves source sentence patterns |
|
||||
| `academic` | Scholarly, rigorous | Formal register, complex clauses OK, citation-aware |
|
||||
| `business` | Concise, results-focused | Action-oriented, executive-friendly, bullet-point mindset |
|
||||
| `humorous` | Preserves and adapts humor | Witty, playful, recreates comedic effect in target language |
|
||||
| `conversational` | Casual, spoken-like | Friendly, approachable, as if explaining to a friend |
|
||||
| `elegant` | Literary, polished prose | Aesthetically refined, rhythmic, carefully crafted word choices |
|
||||
|
||||
Custom style descriptions are also accepted, e.g., `--style "poetic and lyrical"`.
|
||||
|
||||
**Auto-detection**:
|
||||
- "快翻", "quick", "直接翻译" → quick mode
|
||||
- "精翻", "refined", "publication quality", "proofread" → refined mode
|
||||
@@ -84,18 +100,19 @@ All configurable values in one place. EXTEND.md overrides these; CLI flags overr
|
||||
**Upgrade prompt**: After normal mode completes, display:
|
||||
> Translation saved. To further review and polish, reply "继续润色" or "refine".
|
||||
|
||||
If user responds, continue with review → polish steps (same as refined mode Steps 3-4) on the existing output.
|
||||
If user responds, continue with review → polish steps (same as refined mode Steps 4-6 in refined-workflow.md) on the existing output.
|
||||
|
||||
## Usage
|
||||
|
||||
```
|
||||
/translate [--mode quick|normal|refined] [--from <lang>] [--to <lang>] [--audience <audience>] [--glossary <file>] <source>
|
||||
/translate [--mode quick|normal|refined] [--from <lang>] [--to <lang>] [--audience <audience>] [--style <style>] [--glossary <file>] <source>
|
||||
```
|
||||
|
||||
- `<source>`: File path, URL, or inline text
|
||||
- `--from`: Source language (auto-detect if omitted)
|
||||
- `--to`: Target language (from EXTEND.md or default `zh-CN`)
|
||||
- `--audience`: Target reader profile (from EXTEND.md or default `general`)
|
||||
- `--style`: Translation style (from EXTEND.md or default `storytelling`)
|
||||
- `--glossary`: Additional glossary file to merge with EXTEND.md glossary
|
||||
|
||||
**Audience presets**:
|
||||
@@ -122,49 +139,25 @@ Custom audience descriptions are also accepted, e.g., `--audience "AI感兴趣
|
||||
|
||||
### Step 2: Materialize Source & Create Output Directory
|
||||
|
||||
If the input is not a file, save it as one first. Then create an output subdirectory next to the source file.
|
||||
Materialize source (file as-is, inline text/URL → save to `translate/{slug}.md`), then create output directory: `{source-dir}/{source-basename}-{target-lang}/`. Detect source language if `--from` not specified.
|
||||
|
||||
**2.1 Materialize source**
|
||||
Full details: [references/workflow-mechanics.md](references/workflow-mechanics.md)
|
||||
|
||||
| Input Type | Action |
|
||||
|------------|--------|
|
||||
| File | Use as-is (no copy needed) |
|
||||
| Inline text | Save to `translate/{slug}.md` |
|
||||
| URL | Fetch content, save to `translate/{slug}.md` |
|
||||
|
||||
`{slug}`: 2-4 word kebab-case slug derived from content topic.
|
||||
|
||||
**2.2 Create output directory**
|
||||
|
||||
Create a subdirectory next to the source file: `{source-dir}/{source-basename}-{target-lang}/`
|
||||
|
||||
Examples:
|
||||
- `posts/article.md` → `posts/article-zh/`
|
||||
- `translate/ai-future.md` → `translate/ai-future-zh/`
|
||||
|
||||
**Conflict resolution**: If the output directory already exists, rename the existing one to `{name}.backup-YYYYMMDD-HHMMSS/` before creating the new one. Never overwrite existing results.
|
||||
|
||||
**2.3 Output directory contents**
|
||||
|
||||
All intermediate and final files go into this directory:
|
||||
**Output directory contents** (all intermediate and final files go here):
|
||||
|
||||
| File | Mode | Description |
|
||||
|------|------|-------------|
|
||||
| `translation.md` | All | Final translation (always this name) |
|
||||
| `01-analysis.md` | Normal, Refined | Content analysis (domain, tone, terminology) |
|
||||
| `02-prompt.md` | Normal, Refined | Assembled translation prompt (used by subagent or inline) |
|
||||
| `02-prompt.md` | Normal, Refined | Assembled translation prompt |
|
||||
| `03-draft.md` | Refined | Initial draft before review |
|
||||
| `04-critique.md` | Refined | Critical review findings (diagnosis only, no rewriting) |
|
||||
| `04-critique.md` | Refined | Critical review findings (diagnosis only) |
|
||||
| `05-revision.md` | Refined | Revised translation based on critique |
|
||||
| `chunks/` | Chunked | Source chunks + translated chunks |
|
||||
| `chunks/chunk-01.md` | Chunked | Source chunk |
|
||||
| `chunks/chunk-01-draft.md` | Chunked | Translated chunk |
|
||||
|
||||
Detect source language if `--from` not specified.
|
||||
|
||||
### Step 3: Assess Content Length
|
||||
|
||||
Quick mode does not chunk — translate directly regardless of length. If content exceeds model context limits, suggest the user switch to normal or refined mode.
|
||||
Quick mode does not chunk — translate directly regardless of length. Before translating, estimate word count. If content exceeds chunk threshold (default 4000 words), proactively warn: "This article is ~{N} words. Quick mode translates in one pass without chunking — for long content, `--mode normal` produces better results with terminology consistency." Then proceed if user doesn't switch.
|
||||
|
||||
For normal and refined modes:
|
||||
|
||||
@@ -192,7 +185,7 @@ Before translating chunks:
|
||||
- Terminology consistency is guaranteed by the shared `02-prompt.md` (glossary + comprehension challenges from analysis)
|
||||
- If no chunks (content under threshold): spawn one subagent for the entire source file
|
||||
- If Agent tool is unavailable, translate chunks sequentially inline using `02-prompt.md`
|
||||
6. **Merge**: Once all subagents complete, combine translated chunks in order, prepend frontmatter if present → save as `03-draft.md` (refined) or `translation.md` (normal)
|
||||
6. **Merge**: Once all subagents complete, combine translated chunks in order. If `chunks/frontmatter.md` exists, prepend it. Save as `03-draft.md` (refined) or `translation.md` (normal)
|
||||
7. All intermediate files (source chunks + translated chunks) are preserved in `chunks/`
|
||||
|
||||
**After chunked draft is merged**, return control to main agent for critical review, revision, and polish (Step 4).
|
||||
@@ -202,19 +195,22 @@ Before translating chunks:
|
||||
**Translation principles** (apply to all modes):
|
||||
|
||||
- **Accuracy first**: Facts, data, and logic must match the original exactly
|
||||
- **Natural flow**: Use idiomatic target language word order; break long sentences into shorter ones
|
||||
- **Meaning over words**: Translate what the author means, not just what the words say. When a literal translation sounds unnatural or fails to convey the intended effect, restructure freely to express the same meaning in idiomatic target language
|
||||
- **Figurative language**: Interpret metaphors, idioms, and figurative expressions by their intended meaning rather than translating them word-for-word. When a source-language image does not carry the same connotation in the target language, replace it with a natural expression that conveys the same idea and emotional effect
|
||||
- **Emotional fidelity**: Preserve the emotional connotations of word choices, not just their dictionary meanings. Words that carry subjective feelings (e.g., "alarming", "haunting") should be rendered to evoke the same response in target-language readers
|
||||
- **Natural flow**: Use idiomatic target language word order and sentence patterns; break or restructure sentences freely when the source structure doesn't work naturally in the target language
|
||||
- **Terminology**: Use standard translations; annotate with original term in parentheses on first occurrence
|
||||
- **Preserve format**: Keep all markdown formatting (headings, bold, italic, images, links, code blocks)
|
||||
- **Respect original**: Maintain original structure and meaning; do not add, remove, or editorialize
|
||||
- **Respect original**: Maintain original meaning and intent; do not add, remove, or editorialize — but sentence structure and imagery may be adapted freely to serve the meaning
|
||||
- **Translator's notes**: For terms, concepts, or cultural references that target readers may not understand — due to jargon, cultural gaps, or domain-specific knowledge — add a concise explanatory note in parentheses immediately after the term. The note should explain *what it means* in plain language, not just provide the English original. Format: `译文(English original,通俗解释)`. Calibrate annotation depth to the target audience: general readers need more notes than technical readers. Only add notes where genuinely needed; do not over-annotate obvious terms.
|
||||
|
||||
#### Quick Mode
|
||||
|
||||
Translate directly → save to `translation.md`.
|
||||
Translate directly → save to `translation.md`. No analysis file, but still apply all translation principles above — especially: interpret figurative language by meaning (not word-for-word), preserve emotional connotations, and restructure sentences for natural target-language flow.
|
||||
|
||||
#### Normal Mode
|
||||
|
||||
1. **Analyze** → `01-analysis.md` (domain, tone, audience, terminology, reader comprehension challenges)
|
||||
1. **Analyze** → `01-analysis.md` (domain, tone, audience, terminology, reader comprehension challenges, figurative language & metaphor mapping)
|
||||
2. **Assemble prompt** → `02-prompt.md` (translation instructions with inlined context)
|
||||
3. **Translate** (following `02-prompt.md`) → `translation.md`
|
||||
|
||||
@@ -229,7 +225,7 @@ Full workflow for publication quality. See [references/refined-workflow.md](refe
|
||||
The subagent (if used in Step 3.1) only handles the initial draft. All subsequent steps (critical review, revision, polish) are handled by the main agent, which may delegate to subagents at its discretion.
|
||||
|
||||
Steps and saved files (all in output directory):
|
||||
1. **Analyze** → `01-analysis.md` (domain, tone, terminology, reader comprehension challenges)
|
||||
1. **Analyze** → `01-analysis.md` (domain, tone, terminology, reader comprehension challenges, figurative language & metaphor mapping)
|
||||
2. **Assemble prompt** → `02-prompt.md` (translation instructions with inlined context)
|
||||
3. **Draft** → `03-draft.md` (initial translation with translator's notes; from subagent if chunked)
|
||||
4. **Critical review** → `04-critique.md` (diagnosis only: accuracy, Europeanized language, strategy execution, expression issues)
|
||||
|
||||
@@ -15,7 +15,7 @@ default_mode: normal # quick | normal | refined
|
||||
audience: general # general | technical | academic | business | or custom string
|
||||
|
||||
# Translation style preference
|
||||
style: storytelling # storytelling | formal | technical | literal
|
||||
style: storytelling # storytelling | formal | technical | literal | academic | business | humorous | conversational | elegant | or custom string
|
||||
|
||||
# Word count threshold to trigger chunked translation
|
||||
chunk_threshold: 4000
|
||||
@@ -49,7 +49,7 @@ glossaries:
|
||||
| `target_language` | string | `zh-CN` | Default target language code |
|
||||
| `default_mode` | string | `normal` | Default translation mode (`quick` / `normal` / `refined`) |
|
||||
| `audience` | string | `general` | Target reader profile (`general` / `technical` / `academic` / `business` / custom) |
|
||||
| `style` | string | `storytelling` | Translation style preference |
|
||||
| `style` | string | `storytelling` | Translation style (`storytelling` / `formal` / `technical` / `literal` / `academic` / `business` / `humorous` / `conversational` / `elegant` / custom) |
|
||||
| `chunk_threshold` | number | `4000` | Word count threshold to trigger chunked translation |
|
||||
| `chunk_max_words` | number | `5000` | Max words per chunk |
|
||||
| `glossary` | array | `[]` | Universal glossary entries |
|
||||
|
||||
@@ -99,15 +99,27 @@ header: "Style"
|
||||
question: "Translation style?"
|
||||
options:
|
||||
- label: "Storytelling (Recommended)"
|
||||
description: "Engaging, narrative-like flow"
|
||||
description: "Engaging narrative flow, smooth transitions"
|
||||
- label: "Formal"
|
||||
description: "Professional, structured"
|
||||
description: "Professional, structured, neutral tone"
|
||||
- label: "Technical"
|
||||
description: "Precise, documentation-style"
|
||||
description: "Precise, documentation-style, concise"
|
||||
- label: "Literal"
|
||||
description: "Close to original structure"
|
||||
- label: "Academic"
|
||||
description: "Scholarly, rigorous, formal register"
|
||||
- label: "Business"
|
||||
description: "Concise, results-focused, action-oriented"
|
||||
- label: "Humorous"
|
||||
description: "Preserves humor, witty, playful"
|
||||
- label: "Conversational"
|
||||
description: "Casual, friendly, spoken-like"
|
||||
- label: "Elegant"
|
||||
description: "Literary, polished, aesthetically refined"
|
||||
```
|
||||
|
||||
Note: User may type a custom style description.
|
||||
|
||||
### Question 5: Save Location
|
||||
|
||||
```yaml
|
||||
@@ -141,7 +153,7 @@ options:
|
||||
target_language: [zh-CN/zh-TW/en/ja/...]
|
||||
default_mode: [quick/normal/refined]
|
||||
audience: [general/technical/academic/business/custom]
|
||||
style: [storytelling/formal/technical/literal]
|
||||
style: [storytelling/formal/technical/literal/academic/business/humorous/conversational/elegant]
|
||||
|
||||
# Custom glossary (optional) — add your own term translations here
|
||||
# glossary:
|
||||
|
||||
@@ -63,10 +63,26 @@ For each identified challenge, note:
|
||||
2. Why it may confuse target readers
|
||||
3. A concise plain-language explanation to use as a translator's note
|
||||
|
||||
### 1.7 Translation Challenges
|
||||
### 1.7 Figurative Language & Metaphor Mapping
|
||||
|
||||
Identify all metaphors, similes, idioms, and figurative expressions in the source. For each:
|
||||
|
||||
1. **Original expression**: The exact phrase
|
||||
2. **Intended meaning**: What the author is actually communicating (the idea behind the image)
|
||||
3. **Literal translation risk**: Would a word-for-word translation sound unnatural, lose the connotation, or confuse target readers?
|
||||
4. **Target-language approach**: One of:
|
||||
- **Interpret**: Discard the source image entirely, express the intended meaning directly in natural target language
|
||||
- **Substitute**: Replace with a target-language idiom or image that conveys the same idea and emotional effect
|
||||
- **Retain**: Keep the original image if it works equally well in the target language
|
||||
|
||||
Also flag:
|
||||
- **Emotional connotations carried by word choice**: Words like "alarming" that convey subjective feeling, not just objective description — note the emotional effect to preserve
|
||||
- **Implied meanings**: Sentences where the surface meaning is simple but the implication is richer — note what the author really means so the translator can convey the full intent
|
||||
|
||||
### 1.8 Structural & Creative Challenges
|
||||
|
||||
- Complex sentence patterns (long subordinate clauses, nested modifiers, participial phrases) that need restructuring for natural target-language flow
|
||||
- Structural challenges (wordplay, ambiguity, puns that don't translate)
|
||||
- Passages where literal translation would lose meaning
|
||||
- Content where the author's voice or humor requires creative adaptation
|
||||
|
||||
**Save `01-analysis.md`** with:
|
||||
@@ -95,8 +111,12 @@ Implicit assumptions: [unstated premises]
|
||||
- [term/passage] → [why confusing] → [proposed note]
|
||||
- ...
|
||||
|
||||
## Translation Challenges
|
||||
[structural issues, creative adaptation needs]
|
||||
## Figurative Language & Metaphor Mapping
|
||||
- [original expression] → [intended meaning] → [approach: interpret/substitute/retain] → [suggested rendering]
|
||||
- ...
|
||||
|
||||
## Structural & Creative Challenges
|
||||
[sentence restructuring needs, wordplay, creative adaptation needs]
|
||||
```
|
||||
|
||||
## Step 2: Assemble Translation Prompt
|
||||
@@ -111,15 +131,12 @@ Save to `03-draft.md` in the output directory.
|
||||
|
||||
For chunked content, the subagent produces this draft (merged from chunk translations). For non-chunked content, the main agent produces it directly.
|
||||
|
||||
Translate the full content following `02-prompt.md`:
|
||||
Translate the full content following `02-prompt.md`. Apply all **Translation principles** from SKILL.md Step 4, plus these step-specific guidelines:
|
||||
|
||||
- Use the terminology decisions from Step 1 consistently
|
||||
- Match the identified tone and register
|
||||
- Break complex sentences into natural target-language patterns
|
||||
- Preserve all formatting and structure
|
||||
- First occurrence of technical terms: translated term (original in parentheses)
|
||||
- Add translator's notes for comprehension challenges identified in Step 1: use parentheses with a plain-language explanation, e.g., `译文(English original,通俗解释)`
|
||||
- Only annotate where genuinely needed — do not over-explain obvious terms
|
||||
- Follow the metaphor mapping from Step 1 for figurative language handling
|
||||
- Add translator's notes for comprehension challenges identified in Step 1
|
||||
|
||||
## Step 4: Critical Review
|
||||
|
||||
@@ -140,25 +157,31 @@ The main agent critically reviews the draft against the source. Save review find
|
||||
- **Over-nominalization**: Abstract nouns where verbs or adjectives would be more natural (e.g., "进行了讨论" → "讨论了")
|
||||
- **Awkward pronouns**: Overuse of 他/她/它/我们/你 where they can be omitted
|
||||
|
||||
### 4.3 Strategy Execution
|
||||
### 4.3 Figurative Language & Emotional Fidelity
|
||||
- Cross-check against the metaphor mapping in `01-analysis.md`: were all flagged metaphors/idioms handled per the recommended approach (interpret/substitute/retain)?
|
||||
- Flag any metaphors or figurative expressions that were translated literally and sound unnatural or lose the intended meaning in the target language
|
||||
- Check emotional connotations: do words that carry subjective feelings in the source (e.g., "alarming", "haunting", "striking") evoke the same response in the translation, or were they flattened into neutral/objective descriptions?
|
||||
- Flag implied meanings that were lost: sentences where the author's deeper intent was not conveyed because the translator stayed too close to the surface meaning
|
||||
|
||||
### 4.4 Strategy Execution
|
||||
- Were the translation strategies from `02-prompt.md` actually followed?
|
||||
- Did the translator apply the tone and register identified in analysis?
|
||||
- Were comprehension challenges from `01-analysis.md` addressed with appropriate notes?
|
||||
- Were glossary terms used consistently?
|
||||
|
||||
### 4.4 Expression & Logic
|
||||
### 4.5 Expression & Logic
|
||||
- Flag sentences that read like "translationese" — unnatural word order, calques, stiff phrasing
|
||||
- Check logical flow between sentences and paragraphs
|
||||
- Identify where sentence restructuring would improve readability
|
||||
- Note where the target language idiom was missed (e.g., English metaphor translated literally instead of finding a target-language equivalent)
|
||||
- Note where the target language idiom was missed
|
||||
|
||||
### 4.5 Translator's Notes Quality
|
||||
### 4.6 Translator's Notes Quality
|
||||
- Are notes accurate, concise, and genuinely helpful?
|
||||
- Identify missed comprehension challenges that need notes
|
||||
- Flag over-annotations on terms obvious to the target audience
|
||||
- Check that cultural references are explained where needed
|
||||
|
||||
### 4.6 Cultural Adaptation
|
||||
### 4.7 Cultural Adaptation
|
||||
- Do metaphors and idioms work in the target language?
|
||||
- Are any references potentially confusing or offensive in the target culture?
|
||||
- Could any passage be misinterpreted due to cultural context differences?
|
||||
@@ -173,6 +196,11 @@ The main agent critically reviews the draft against the source. Save review find
|
||||
- [issue type]: [example from draft] → [suggested fix]
|
||||
- ...
|
||||
|
||||
## Figurative Language & Emotional Fidelity
|
||||
- [literal metaphor]: [original] → [draft rendering] → [suggested interpretation]
|
||||
- [flattened emotion]: [original word/phrase] → [draft rendering] → [how to restore emotional effect]
|
||||
- ...
|
||||
|
||||
## Strategy Execution
|
||||
- [strategy]: [followed/missed] — [details]
|
||||
- ...
|
||||
@@ -201,6 +229,8 @@ The revision reads `03-draft.md` (the original draft) and `04-critique.md` (the
|
||||
|
||||
- Fix all accuracy issues identified in the critique
|
||||
- Rewrite Europeanized expressions into natural target-language patterns
|
||||
- Re-interpret literally translated metaphors and figurative expressions per the metaphor mapping; replace with natural target-language renderings that convey the intended meaning and emotional effect
|
||||
- Restore flattened emotional connotations: ensure words carrying subjective feelings evoke the same response as the source
|
||||
- Apply missed translation strategies
|
||||
- Restructure stiff or awkward sentences for fluency
|
||||
- Add, remove, or revise translator's notes per critique recommendations
|
||||
@@ -216,7 +246,8 @@ Final pass on `05-revision.md` for publication quality:
|
||||
- Read the entire translation as a standalone piece — does it flow as native content?
|
||||
- Smooth any remaining rough transitions between paragraphs
|
||||
- Ensure the narrative voice is consistent throughout
|
||||
- For storytelling-style content: ensure the narrative flow draws readers in
|
||||
- Apply the selected translation style consistently: storytelling should flow like a narrative, formal should maintain neutral professionalism, humorous should land jokes naturally in the target language, etc.
|
||||
- Final scan for surviving literal metaphors or flattened emotions: any figurative expression that still reads as "translated" rather than "written" should be recast into natural target-language expression
|
||||
- Final consistency check on terminology across the full text
|
||||
- Verify formatting is preserved correctly (headings, bold, links, code blocks)
|
||||
- Remove any remaining traces of translationese
|
||||
|
||||
@@ -19,9 +19,15 @@ You are a professional translator. Your task is to translate markdown content fr
|
||||
|
||||
{audience description}
|
||||
|
||||
## Translation Style
|
||||
|
||||
{style description — e.g., "storytelling: engaging narrative flow, smooth transitions, vivid phrasing" or custom style from user}
|
||||
|
||||
Apply this style consistently: it determines the voice, tone, and sentence-level choices throughout the translation. Style is independent of audience — a technical audience can still get a storytelling-style translation, or a general audience can get a formal one.
|
||||
|
||||
## Content Background
|
||||
|
||||
{Inlined from 01-analysis.md if analysis exists: quick summary, core argument, author background, writing context, tone assessment.}
|
||||
{Inlined from 01-analysis.md if analysis exists: quick summary, core argument, author background, writing context, tone assessment, figurative language & metaphor mapping.}
|
||||
|
||||
## Glossary
|
||||
|
||||
@@ -38,10 +44,13 @@ The following terms or references may confuse target readers. Add translator's n
|
||||
## Translation Principles
|
||||
|
||||
- **Accuracy first**: Facts, data, and logic must match the original exactly
|
||||
- **Natural flow**: Use idiomatic {target_lang} word order; break long sentences into shorter ones
|
||||
- **Meaning over words**: Translate what the author means, not just what the words say. When a literal translation sounds unnatural or fails to convey the intended effect, restructure freely to express the same meaning in idiomatic {target_lang}
|
||||
- **Figurative language**: Interpret metaphors, idioms, and figurative expressions by their intended meaning. When a source-language image does not carry the same connotation in {target_lang}, replace it with a natural expression that conveys the same idea and emotional effect. Refer to the Figurative Language section in Content Background (if provided) for pre-analyzed metaphor mappings
|
||||
- **Emotional fidelity**: Preserve the emotional connotations of word choices, not just their dictionary meanings
|
||||
- **Natural flow**: Use idiomatic {target_lang} word order and sentence patterns; break or restructure sentences freely when the source structure doesn't work naturally
|
||||
- **Terminology**: Use glossary translations consistently; annotate with original term in parentheses on first occurrence
|
||||
- **Preserve format**: Keep all markdown formatting (headings, bold, italic, images, links, code blocks)
|
||||
- **Respect original**: Maintain original structure and meaning; do not add, remove, or editorialize
|
||||
- **Respect original**: Maintain original meaning and intent; do not add, remove, or editorialize — but sentence structure and imagery may be adapted freely to serve the meaning
|
||||
- **Translator's notes**: For terms or cultural references listed in Comprehension Challenges above, add a concise explanatory note in parentheses. Only annotate where genuinely needed for the target audience.
|
||||
```
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
# Workflow Mechanics
|
||||
|
||||
Details for source materialization, output directory creation, and conflict resolution.
|
||||
|
||||
## Materialize Source
|
||||
|
||||
| Input Type | Action |
|
||||
|------------|--------|
|
||||
| File | Use as-is (no copy needed) |
|
||||
| Inline text | Save to `translate/{slug}.md` |
|
||||
| URL | Fetch content, save to `translate/{slug}.md` |
|
||||
|
||||
`{slug}`: 2-4 word kebab-case slug derived from content topic.
|
||||
|
||||
## Create Output Directory
|
||||
|
||||
Create a subdirectory next to the source file: `{source-dir}/{source-basename}-{target-lang}/`
|
||||
|
||||
Examples:
|
||||
- `posts/article.md` → `posts/article-zh/`
|
||||
- `translate/ai-future.md` → `translate/ai-future-zh/`
|
||||
|
||||
## Conflict Resolution
|
||||
|
||||
If the output directory already exists, rename the existing one to `{name}.backup-YYYYMMDD-HHMMSS/` before creating the new one. Never overwrite existing results.
|
||||
@@ -118,6 +118,10 @@ if (cur.length > 0) chunks.push({ blocks: cur, words: curWords })
|
||||
const dir = outputDir ? join(outputDir, "chunks") : join(dirname(file), "chunks")
|
||||
mkdirSync(dir, { recursive: true })
|
||||
|
||||
if (frontmatter) {
|
||||
writeFileSync(join(dir, "frontmatter.md"), frontmatter)
|
||||
}
|
||||
|
||||
chunks.forEach((chunk, i) => {
|
||||
const num = String(i + 1).padStart(2, "0")
|
||||
const out = join(dir, `chunk-${num}.md`)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
---
|
||||
name: baoyu-url-to-markdown
|
||||
description: Fetch any URL and convert to markdown using Chrome CDP. Supports two modes - auto-capture on page load, or wait for user signal (for pages requiring login). Use when user wants to save a webpage as markdown.
|
||||
description: Fetch any URL and convert to markdown using Chrome CDP. Saves the rendered HTML snapshot alongside the markdown, and automatically falls back to the pre-Defuddle HTML-to-Markdown pipeline when Defuddle fails. Supports two modes - auto-capture on page load, or wait for user signal (for pages requiring login). Use when user wants to save a webpage as markdown.
|
||||
---
|
||||
|
||||
# URL to Markdown
|
||||
|
||||
Fetches any URL via Chrome CDP and converts HTML to clean markdown.
|
||||
Fetches any URL via Chrome CDP, saves the rendered HTML snapshot, and converts it to clean markdown.
|
||||
|
||||
## Script Directory
|
||||
|
||||
@@ -21,6 +21,7 @@ Fetches any URL via Chrome CDP and converts HTML to clean markdown.
|
||||
| Script | Purpose |
|
||||
|--------|---------|
|
||||
| `scripts/main.ts` | CLI entry point for URL fetching |
|
||||
| `scripts/html-to-markdown.ts` | Defuddle-first conversion with automatic legacy fallback |
|
||||
|
||||
## Preferences (EXTEND.md)
|
||||
|
||||
@@ -101,7 +102,9 @@ Full reference: [references/config/first-time-setup.md](references/config/first-
|
||||
|
||||
- Chrome CDP for full JavaScript rendering
|
||||
- Two capture modes: auto or wait-for-user
|
||||
- Save rendered HTML as a sibling `-captured.html` file
|
||||
- Clean markdown output with metadata
|
||||
- Defuddle-first markdown conversion with automatic fallback to the pre-Defuddle extractor from git history
|
||||
- Handles login-required pages via wait mode
|
||||
- Download images and videos to local directories
|
||||
|
||||
@@ -149,13 +152,23 @@ ${BUN_X} ${SKILL_DIR}/scripts/main.ts <url> --download-media
|
||||
|
||||
## Output Format
|
||||
|
||||
YAML front matter with `url`, `title`, `description`, `author`, `published`, `captured_at` fields, followed by converted markdown content.
|
||||
Each run saves two files side by side:
|
||||
|
||||
- Markdown: YAML front matter with `url`, `title`, `description`, `author`, `published`, optional `coverImage`, and `captured_at`, followed by converted markdown content
|
||||
- HTML snapshot: `*-captured.html`, containing the rendered page HTML captured from Chrome
|
||||
|
||||
The HTML snapshot is saved before any markdown media localization, so it stays a faithful capture of the page DOM used for conversion.
|
||||
|
||||
## Output Directory
|
||||
|
||||
Default: `url-to-markdown/<domain>/<slug>.md`
|
||||
With `--output-dir ./posts/`: `./posts/<domain>/<slug>.md`
|
||||
|
||||
HTML snapshot path uses the same basename:
|
||||
|
||||
- `url-to-markdown/<domain>/<slug>-captured.html`
|
||||
- `./posts/<domain>/<slug>-captured.html`
|
||||
|
||||
- `<slug>`: From page title or URL path (kebab-case, 2-6 words)
|
||||
- Conflict resolution: Append timestamp `<slug>-YYYYMMDD-HHMMSS.md`
|
||||
|
||||
@@ -164,6 +177,19 @@ When `--download-media` is enabled:
|
||||
- Videos are saved to `videos/` next to the markdown file
|
||||
- Markdown media links are rewritten to local relative paths
|
||||
|
||||
## Conversion Fallback
|
||||
|
||||
Conversion order:
|
||||
|
||||
1. Try Defuddle first
|
||||
2. If Defuddle throws, cannot load, returns obviously incomplete markdown, or captures lower-quality content than the legacy pipeline, automatically fall back to the pre-Defuddle extractor
|
||||
3. The fallback path uses the older Readability/selector/Next.js-data based HTML-to-Markdown implementation recovered from git history
|
||||
|
||||
CLI output will show:
|
||||
|
||||
- `Converter: defuddle` when Defuddle succeeds
|
||||
- `Converter: legacy:...` plus `Fallback used: ...` when fallback was needed
|
||||
|
||||
## Media Download Workflow
|
||||
|
||||
Based on `download_media` setting in EXTEND.md:
|
||||
@@ -193,7 +219,7 @@ Based on `download_media` setting in EXTEND.md:
|
||||
| `URL_DATA_DIR` | Custom data directory |
|
||||
| `URL_CHROME_PROFILE_DIR` | Custom Chrome profile directory |
|
||||
|
||||
**Troubleshooting**: Chrome not found → set `URL_CHROME_PATH`. Timeout → increase `--timeout`. Complex pages → try `--wait` mode.
|
||||
**Troubleshooting**: Chrome not found → set `URL_CHROME_PATH`. Timeout → increase `--timeout`. Complex pages → try `--wait` mode. If markdown quality is poor, inspect the saved `-captured.html` and check whether the run logged a legacy fallback.
|
||||
|
||||
## Extension Support
|
||||
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
{
|
||||
"lockfileVersion": 1,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "baoyu-url-to-markdown-scripts",
|
||||
"dependencies": {
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"defuddle": "^0.10.0",
|
||||
"jsdom": "^24.1.3",
|
||||
"linkedom": "^0.18.12",
|
||||
"turndown": "^7.2.2",
|
||||
"turndown-plugin-gfm": "^1.0.2",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@asamuzakjp/css-color": ["@asamuzakjp/css-color@3.2.0", "", { "dependencies": { "@csstools/css-calc": "^2.1.3", "@csstools/css-color-parser": "^3.0.9", "@csstools/css-parser-algorithms": "^3.0.4", "@csstools/css-tokenizer": "^3.0.3", "lru-cache": "^10.4.3" } }, "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw=="],
|
||||
|
||||
"@csstools/color-helpers": ["@csstools/color-helpers@5.1.0", "", {}, "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA=="],
|
||||
|
||||
"@csstools/css-calc": ["@csstools/css-calc@2.1.4", "", { "peerDependencies": { "@csstools/css-parser-algorithms": "^3.0.5", "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ=="],
|
||||
|
||||
"@csstools/css-color-parser": ["@csstools/css-color-parser@3.1.0", "", { "dependencies": { "@csstools/color-helpers": "^5.1.0", "@csstools/css-calc": "^2.1.4" }, "peerDependencies": { "@csstools/css-parser-algorithms": "^3.0.5", "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA=="],
|
||||
|
||||
"@csstools/css-parser-algorithms": ["@csstools/css-parser-algorithms@3.0.5", "", { "peerDependencies": { "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ=="],
|
||||
|
||||
"@csstools/css-tokenizer": ["@csstools/css-tokenizer@3.0.4", "", {}, "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw=="],
|
||||
|
||||
"@mixmark-io/domino": ["@mixmark-io/domino@2.2.0", "", {}, "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw=="],
|
||||
|
||||
"@mozilla/readability": ["@mozilla/readability@0.6.0", "", {}, "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ=="],
|
||||
|
||||
"@xmldom/xmldom": ["@xmldom/xmldom@0.8.11", "", {}, "sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw=="],
|
||||
|
||||
"agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="],
|
||||
|
||||
"asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="],
|
||||
|
||||
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
||||
|
||||
"call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="],
|
||||
|
||||
"combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="],
|
||||
|
||||
"commander": ["commander@12.1.0", "", {}, "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA=="],
|
||||
|
||||
"css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
||||
|
||||
"css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="],
|
||||
|
||||
"cssom": ["cssom@0.5.0", "", {}, "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="],
|
||||
|
||||
"cssstyle": ["cssstyle@4.6.0", "", { "dependencies": { "@asamuzakjp/css-color": "^3.2.0", "rrweb-cssom": "^0.8.0" } }, "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg=="],
|
||||
|
||||
"data-urls": ["data-urls@5.0.0", "", { "dependencies": { "whatwg-mimetype": "^4.0.0", "whatwg-url": "^14.0.0" } }, "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg=="],
|
||||
|
||||
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
|
||||
|
||||
"decimal.js": ["decimal.js@10.6.0", "", {}, "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg=="],
|
||||
|
||||
"defuddle": ["defuddle@0.10.0", "", { "dependencies": { "commander": "^12.1.0" }, "optionalDependencies": { "mathml-to-latex": "^1.5.0", "temml": "^0.13.1", "turndown": "^7.2.0" }, "peerDependencies": { "jsdom": "^24.0.0" }, "bin": { "defuddle": "dist/cli.js" } }, "sha512-a43juTtHv6Vs4+sxvahVLM5NxoyDsarO1Ag3UxLORI4Fo/nsNFwzDxuQBvosKVGTIRxCwN/mfnWAzNXmQfieqw=="],
|
||||
|
||||
"delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="],
|
||||
|
||||
"dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="],
|
||||
|
||||
"domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="],
|
||||
|
||||
"domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="],
|
||||
|
||||
"domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
|
||||
|
||||
"dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
|
||||
|
||||
"entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
|
||||
|
||||
"es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="],
|
||||
|
||||
"es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="],
|
||||
|
||||
"es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="],
|
||||
|
||||
"es-set-tostringtag": ["es-set-tostringtag@2.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA=="],
|
||||
|
||||
"form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="],
|
||||
|
||||
"function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="],
|
||||
|
||||
"get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="],
|
||||
|
||||
"get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="],
|
||||
|
||||
"gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="],
|
||||
|
||||
"has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="],
|
||||
|
||||
"has-tostringtag": ["has-tostringtag@1.0.2", "", { "dependencies": { "has-symbols": "^1.0.3" } }, "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw=="],
|
||||
|
||||
"hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
|
||||
|
||||
"html-encoding-sniffer": ["html-encoding-sniffer@4.0.0", "", { "dependencies": { "whatwg-encoding": "^3.1.1" } }, "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ=="],
|
||||
|
||||
"html-escaper": ["html-escaper@3.0.3", "", {}, "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="],
|
||||
|
||||
"htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="],
|
||||
|
||||
"http-proxy-agent": ["http-proxy-agent@7.0.2", "", { "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" } }, "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig=="],
|
||||
|
||||
"https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="],
|
||||
|
||||
"iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="],
|
||||
|
||||
"is-potential-custom-element-name": ["is-potential-custom-element-name@1.0.1", "", {}, "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ=="],
|
||||
|
||||
"jsdom": ["jsdom@24.1.3", "", { "dependencies": { "cssstyle": "^4.0.1", "data-urls": "^5.0.0", "decimal.js": "^10.4.3", "form-data": "^4.0.0", "html-encoding-sniffer": "^4.0.0", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.5", "is-potential-custom-element-name": "^1.0.1", "nwsapi": "^2.2.12", "parse5": "^7.1.2", "rrweb-cssom": "^0.7.1", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^4.1.4", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^7.0.0", "whatwg-encoding": "^3.1.1", "whatwg-mimetype": "^4.0.0", "whatwg-url": "^14.0.0", "ws": "^8.18.0", "xml-name-validator": "^5.0.0" }, "peerDependencies": { "canvas": "^2.11.2" }, "optionalPeers": ["canvas"] }, "sha512-MyL55p3Ut3cXbeBEG7Hcv0mVM8pp8PBNWxRqchZnSfAiES1v1mRnMeFfaHWIPULpwsYfvO+ZmMZz5tGCnjzDUQ=="],
|
||||
|
||||
"linkedom": ["linkedom@0.18.12", "", { "dependencies": { "css-select": "^5.1.0", "cssom": "^0.5.0", "html-escaper": "^3.0.3", "htmlparser2": "^10.0.0", "uhyphen": "^0.2.0" }, "peerDependencies": { "canvas": ">= 2" }, "optionalPeers": ["canvas"] }, "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q=="],
|
||||
|
||||
"lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="],
|
||||
|
||||
"math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
|
||||
|
||||
"mathml-to-latex": ["mathml-to-latex@1.5.0", "", { "dependencies": { "@xmldom/xmldom": "^0.8.10" } }, "sha512-rrWn0eEvcEcdMM4xfHcSGIy+i01DX9byOdXTLWg+w1iJ6O6ohP5UXY1dVzNUZLhzfl3EGcRekWLhY7JT5Omaew=="],
|
||||
|
||||
"mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="],
|
||||
|
||||
"mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="],
|
||||
|
||||
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
|
||||
|
||||
"nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
|
||||
|
||||
"nwsapi": ["nwsapi@2.2.23", "", {}, "sha512-7wfH4sLbt4M0gCDzGE6vzQBo0bfTKjU7Sfpqy/7gs1qBfYz2vEJH6vXcBKpO3+6Yu1telwd0t9HpyOoLEQQbIQ=="],
|
||||
|
||||
"parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="],
|
||||
|
||||
"psl": ["psl@1.15.0", "", { "dependencies": { "punycode": "^2.3.1" } }, "sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w=="],
|
||||
|
||||
"punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="],
|
||||
|
||||
"querystringify": ["querystringify@2.2.0", "", {}, "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ=="],
|
||||
|
||||
"requires-port": ["requires-port@1.0.0", "", {}, "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ=="],
|
||||
|
||||
"rrweb-cssom": ["rrweb-cssom@0.7.1", "", {}, "sha512-TrEMa7JGdVm0UThDJSx7ddw5nVm3UJS9o9CCIZ72B1vSyEZoziDqBYP3XIoi/12lKrJR8rE3jeFHMok2F/Mnsg=="],
|
||||
|
||||
"safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
|
||||
|
||||
"saxes": ["saxes@6.0.0", "", { "dependencies": { "xmlchars": "^2.2.0" } }, "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA=="],
|
||||
|
||||
"symbol-tree": ["symbol-tree@3.2.4", "", {}, "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="],
|
||||
|
||||
"temml": ["temml@0.13.1", "", {}, "sha512-/fL1utq8QUD9YpcLeZHPRnp9Cbzbexq5hZl5uSBhf8mNYiKkcS4eYbLidDB+/nF8C+RHAcBQbKw2bKoS83mz1Q=="],
|
||||
|
||||
"tough-cookie": ["tough-cookie@4.1.4", "", { "dependencies": { "psl": "^1.1.33", "punycode": "^2.1.1", "universalify": "^0.2.0", "url-parse": "^1.5.3" } }, "sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag=="],
|
||||
|
||||
"tr46": ["tr46@5.1.1", "", { "dependencies": { "punycode": "^2.3.1" } }, "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw=="],
|
||||
|
||||
"turndown": ["turndown@7.2.2", "", { "dependencies": { "@mixmark-io/domino": "^2.2.0" } }, "sha512-1F7db8BiExOKxjSMU2b7if62D/XOyQyZbPKq/nUwopfgnHlqXHqQ0lvfUTeUIr1lZJzOPFn43dODyMSIfvWRKQ=="],
|
||||
|
||||
"turndown-plugin-gfm": ["turndown-plugin-gfm@1.0.2", "", {}, "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg=="],
|
||||
|
||||
"uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],
|
||||
|
||||
"universalify": ["universalify@0.2.0", "", {}, "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg=="],
|
||||
|
||||
"url-parse": ["url-parse@1.5.10", "", { "dependencies": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" } }, "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ=="],
|
||||
|
||||
"w3c-xmlserializer": ["w3c-xmlserializer@5.0.0", "", { "dependencies": { "xml-name-validator": "^5.0.0" } }, "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA=="],
|
||||
|
||||
"webidl-conversions": ["webidl-conversions@7.0.0", "", {}, "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="],
|
||||
|
||||
"whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="],
|
||||
|
||||
"whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="],
|
||||
|
||||
"whatwg-url": ["whatwg-url@14.2.0", "", { "dependencies": { "tr46": "^5.1.0", "webidl-conversions": "^7.0.0" } }, "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw=="],
|
||||
|
||||
"ws": ["ws@8.19.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg=="],
|
||||
|
||||
"xml-name-validator": ["xml-name-validator@5.0.0", "", {}, "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg=="],
|
||||
|
||||
"xmlchars": ["xmlchars@2.2.0", "", {}, "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="],
|
||||
|
||||
"cssstyle/rrweb-cssom": ["rrweb-cssom@0.8.0", "", {}, "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw=="],
|
||||
|
||||
"dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
|
||||
|
||||
"htmlparser2/entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="],
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
import { JSDOM } from "jsdom";
|
||||
import { Defuddle } from "defuddle/node";
|
||||
import { parseHTML } from "linkedom";
|
||||
import { Readability } from "@mozilla/readability";
|
||||
import TurndownService from "turndown";
|
||||
import { gfm } from "turndown-plugin-gfm";
|
||||
|
||||
export interface PageMetadata {
|
||||
url: string;
|
||||
@@ -14,8 +16,103 @@ export interface PageMetadata {
|
||||
export interface ConversionResult {
|
||||
metadata: PageMetadata;
|
||||
markdown: string;
|
||||
rawHtml: string;
|
||||
conversionMethod: string;
|
||||
fallbackReason?: string;
|
||||
}
|
||||
|
||||
interface ExtractionCandidate {
|
||||
title: string | null;
|
||||
byline: string | null;
|
||||
excerpt: string | null;
|
||||
published: string | null;
|
||||
html: string | null;
|
||||
textContent: string;
|
||||
method: string;
|
||||
}
|
||||
|
||||
type AnyRecord = Record<string, unknown>;
|
||||
|
||||
const MIN_CONTENT_LENGTH = 120;
|
||||
const GOOD_CONTENT_LENGTH = 900;
|
||||
|
||||
const CONTENT_SELECTORS = [
|
||||
"article",
|
||||
"main article",
|
||||
"[role='main'] article",
|
||||
"[itemprop='articleBody']",
|
||||
".article-content",
|
||||
".article-body",
|
||||
".post-content",
|
||||
".entry-content",
|
||||
".story-body",
|
||||
"main",
|
||||
"[role='main']",
|
||||
"#content",
|
||||
".content",
|
||||
];
|
||||
|
||||
const REMOVE_SELECTORS = [
|
||||
"script",
|
||||
"style",
|
||||
"noscript",
|
||||
"template",
|
||||
"iframe",
|
||||
"svg",
|
||||
"path",
|
||||
"nav",
|
||||
"aside",
|
||||
"footer",
|
||||
"header",
|
||||
"form",
|
||||
".advertisement",
|
||||
".ads",
|
||||
".social-share",
|
||||
".related-articles",
|
||||
".comments",
|
||||
".newsletter",
|
||||
".cookie-banner",
|
||||
".cookie-consent",
|
||||
"[role='navigation']",
|
||||
"[aria-label*='cookie' i]",
|
||||
];
|
||||
|
||||
const PUBLISHED_TIME_SELECTORS = [
|
||||
"meta[property='article:published_time']",
|
||||
"meta[name='pubdate']",
|
||||
"meta[name='publishdate']",
|
||||
"meta[name='date']",
|
||||
"time[datetime]",
|
||||
];
|
||||
|
||||
const ARTICLE_TYPES = new Set([
|
||||
"Article",
|
||||
"NewsArticle",
|
||||
"BlogPosting",
|
||||
"WebPage",
|
||||
"ReportageNewsArticle",
|
||||
]);
|
||||
|
||||
const NEXT_DATA_CONTENT_PATHS = [
|
||||
"props.pageProps.content.body",
|
||||
"props.pageProps.article.body",
|
||||
"props.pageProps.article.content",
|
||||
"props.pageProps.post.body",
|
||||
"props.pageProps.post.content",
|
||||
"props.pageProps.data.body",
|
||||
"props.pageProps.story.body.content",
|
||||
];
|
||||
|
||||
const LOW_QUALITY_MARKERS = [
|
||||
/Join The Conversation/i,
|
||||
/One Community\. Many Voices/i,
|
||||
/Read our community guidelines/i,
|
||||
/Create a free account to share your thoughts/i,
|
||||
/Become a Forbes Member/i,
|
||||
/Subscribe to trusted journalism/i,
|
||||
/\bComments\b/i,
|
||||
];
|
||||
|
||||
export const absolutizeUrlsScript = String.raw`
|
||||
(function() {
|
||||
const baseUrl = document.baseURI || location.href;
|
||||
@@ -53,21 +150,791 @@ export const absolutizeUrlsScript = String.raw`
|
||||
})()
|
||||
`;
|
||||
|
||||
export async function extractContent(html: string, url: string): Promise<ConversionResult> {
|
||||
const dom = new JSDOM(html, { url });
|
||||
const result = await Defuddle(dom, url, { markdown: true });
|
||||
function pickString(...values: unknown[]): string | null {
|
||||
for (const value of values) {
|
||||
if (typeof value === "string") {
|
||||
const trimmed = value.trim();
|
||||
if (trimmed) return trimmed;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const metadata: PageMetadata = {
|
||||
function normalizeMarkdown(markdown: string): string {
|
||||
return markdown
|
||||
.replace(/\r\n/g, "\n")
|
||||
.replace(/[ \t]+\n/g, "\n")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function parseDocument(html: string): Document {
|
||||
const normalized = /<\s*html[\s>]/i.test(html)
|
||||
? html
|
||||
: `<!doctype html><html><body>${html}</body></html>`;
|
||||
return parseHTML(normalized).document as unknown as Document;
|
||||
}
|
||||
|
||||
function sanitizeHtml(html: string): string {
|
||||
const { document } = parseHTML(`<div id="__root">${html}</div>`);
|
||||
const root = document.querySelector("#__root");
|
||||
if (!root) return html;
|
||||
|
||||
for (const selector of ["script", "style", "iframe", "noscript", "template", "svg", "path"]) {
|
||||
for (const el of root.querySelectorAll(selector)) {
|
||||
el.remove();
|
||||
}
|
||||
}
|
||||
|
||||
return root.innerHTML;
|
||||
}
|
||||
|
||||
function extractTextFromHtml(html: string): string {
|
||||
const { document } = parseHTML(`<!doctype html><html><body>${html}</body></html>`);
|
||||
for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) {
|
||||
for (const el of document.querySelectorAll(selector)) {
|
||||
el.remove();
|
||||
}
|
||||
}
|
||||
return document.body?.textContent?.replace(/\s+/g, " ").trim() ?? "";
|
||||
}
|
||||
|
||||
function getMetaContent(document: Document, names: string[]): string | null {
|
||||
for (const name of names) {
|
||||
const element =
|
||||
document.querySelector(`meta[name="${name}"]`) ??
|
||||
document.querySelector(`meta[property="${name}"]`);
|
||||
const content = element?.getAttribute("content");
|
||||
if (content && content.trim()) return content.trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function flattenJsonLdItems(data: unknown): AnyRecord[] {
|
||||
if (!data || typeof data !== "object") return [];
|
||||
if (Array.isArray(data)) return data.flatMap(flattenJsonLdItems);
|
||||
|
||||
const item = data as AnyRecord;
|
||||
if (Array.isArray(item["@graph"])) {
|
||||
return (item["@graph"] as unknown[]).flatMap(flattenJsonLdItems);
|
||||
}
|
||||
|
||||
return [item];
|
||||
}
|
||||
|
||||
function parseJsonLdScripts(document: Document): AnyRecord[] {
|
||||
const results: AnyRecord[] = [];
|
||||
const scripts = document.querySelectorAll("script[type='application/ld+json']");
|
||||
|
||||
for (const script of scripts) {
|
||||
try {
|
||||
const data = JSON.parse(script.textContent ?? "");
|
||||
results.push(...flattenJsonLdItems(data));
|
||||
} catch {
|
||||
// Ignore malformed blocks.
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
function isArticleType(item: AnyRecord): boolean {
|
||||
const value = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"];
|
||||
return typeof value === "string" && ARTICLE_TYPES.has(value);
|
||||
}
|
||||
|
||||
function extractAuthorFromJsonLd(authorData: unknown): string | null {
|
||||
if (typeof authorData === "string") return authorData;
|
||||
if (!authorData || typeof authorData !== "object") return null;
|
||||
|
||||
if (Array.isArray(authorData)) {
|
||||
const names = authorData
|
||||
.map((author) => extractAuthorFromJsonLd(author))
|
||||
.filter((name): name is string => Boolean(name));
|
||||
return names.length > 0 ? names.join(", ") : null;
|
||||
}
|
||||
|
||||
const author = authorData as AnyRecord;
|
||||
return typeof author.name === "string" ? author.name : null;
|
||||
}
|
||||
|
||||
function extractPrimaryJsonLdMeta(document: Document): Partial<PageMetadata> {
|
||||
for (const item of parseJsonLdScripts(document)) {
|
||||
if (!isArticleType(item)) continue;
|
||||
|
||||
return {
|
||||
title: pickString(item.headline, item.name) ?? undefined,
|
||||
description: pickString(item.description) ?? undefined,
|
||||
author: extractAuthorFromJsonLd(item.author) ?? undefined,
|
||||
published: pickString(item.datePublished, item.dateCreated) ?? undefined,
|
||||
coverImage:
|
||||
pickString(
|
||||
item.image,
|
||||
(item.image as AnyRecord | undefined)?.url,
|
||||
(Array.isArray(item.image) ? item.image[0] : undefined) as unknown
|
||||
) ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
function extractPublishedTime(document: Document): string | null {
|
||||
for (const selector of PUBLISHED_TIME_SELECTORS) {
|
||||
const el = document.querySelector(selector);
|
||||
if (!el) continue;
|
||||
const value = el.getAttribute("content") ?? el.getAttribute("datetime");
|
||||
if (value && value.trim()) return value.trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractTitle(document: Document): string | null {
|
||||
const ogTitle = document.querySelector("meta[property='og:title']")?.getAttribute("content");
|
||||
if (ogTitle && ogTitle.trim()) return ogTitle.trim();
|
||||
|
||||
const twitterTitle = document.querySelector("meta[name='twitter:title']")?.getAttribute("content");
|
||||
if (twitterTitle && twitterTitle.trim()) return twitterTitle.trim();
|
||||
|
||||
const title = document.querySelector("title")?.textContent?.trim();
|
||||
if (title) {
|
||||
const cleaned = title.split(/\s*[-|–—]\s*/)[0]?.trim();
|
||||
if (cleaned) return cleaned;
|
||||
}
|
||||
|
||||
const h1 = document.querySelector("h1")?.textContent?.trim();
|
||||
return h1 || null;
|
||||
}
|
||||
|
||||
function extractMetadataFromHtml(html: string, url: string, capturedAt: string): PageMetadata {
|
||||
const document = parseDocument(html);
|
||||
const jsonLd = extractPrimaryJsonLdMeta(document);
|
||||
const timeEl = document.querySelector("time[datetime]");
|
||||
|
||||
return {
|
||||
url,
|
||||
title: result.title || "",
|
||||
description: result.description || undefined,
|
||||
author: result.author || undefined,
|
||||
published: result.published || undefined,
|
||||
coverImage: result.image || undefined,
|
||||
captured_at: new Date().toISOString(),
|
||||
title:
|
||||
pickString(
|
||||
getMetaContent(document, ["og:title", "twitter:title"]),
|
||||
jsonLd.title,
|
||||
document.querySelector("h1")?.textContent,
|
||||
document.title
|
||||
) ?? "",
|
||||
description:
|
||||
pickString(
|
||||
getMetaContent(document, ["description", "og:description", "twitter:description"]),
|
||||
jsonLd.description
|
||||
) ?? undefined,
|
||||
author:
|
||||
pickString(
|
||||
getMetaContent(document, ["author", "article:author", "twitter:creator"]),
|
||||
jsonLd.author
|
||||
) ?? undefined,
|
||||
published:
|
||||
pickString(
|
||||
timeEl?.getAttribute("datetime"),
|
||||
getMetaContent(document, ["article:published_time", "datePublished", "publishdate", "date"]),
|
||||
jsonLd.published,
|
||||
extractPublishedTime(document)
|
||||
) ?? undefined,
|
||||
coverImage:
|
||||
pickString(
|
||||
getMetaContent(document, ["og:image", "twitter:image", "twitter:image:src"]),
|
||||
jsonLd.coverImage
|
||||
) ?? undefined,
|
||||
captured_at: capturedAt,
|
||||
};
|
||||
}
|
||||
|
||||
return { metadata, markdown: result.content || "" };
|
||||
function generateExcerpt(excerpt: string | null, textContent: string | null): string | null {
|
||||
if (excerpt) return excerpt;
|
||||
if (!textContent) return null;
|
||||
const trimmed = textContent.trim();
|
||||
if (!trimmed) return null;
|
||||
return trimmed.length > 200 ? `${trimmed.slice(0, 200)}...` : trimmed;
|
||||
}
|
||||
|
||||
function parseJsonLdItem(item: AnyRecord): ExtractionCandidate | null {
|
||||
if (!isArticleType(item)) return null;
|
||||
|
||||
const rawContent =
|
||||
(typeof item.articleBody === "string" && item.articleBody) ||
|
||||
(typeof item.text === "string" && item.text) ||
|
||||
(typeof item.description === "string" && item.description) ||
|
||||
null;
|
||||
|
||||
if (!rawContent) return null;
|
||||
|
||||
const content = rawContent.trim();
|
||||
const htmlLike = /<\/?[a-z][\s\S]*>/i.test(content);
|
||||
const textContent = htmlLike ? extractTextFromHtml(content) : content;
|
||||
|
||||
if (textContent.length < MIN_CONTENT_LENGTH) return null;
|
||||
|
||||
return {
|
||||
title: pickString(item.headline, item.name),
|
||||
byline: extractAuthorFromJsonLd(item.author),
|
||||
excerpt: pickString(item.description),
|
||||
published: pickString(item.datePublished, item.dateCreated),
|
||||
html: htmlLike ? content : null,
|
||||
textContent,
|
||||
method: "json-ld",
|
||||
};
|
||||
}
|
||||
|
||||
function tryJsonLdExtraction(document: Document): ExtractionCandidate | null {
|
||||
for (const item of parseJsonLdScripts(document)) {
|
||||
const extracted = parseJsonLdItem(item);
|
||||
if (extracted) return extracted;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function getByPath(value: unknown, path: string): unknown {
|
||||
let current = value;
|
||||
for (const part of path.split(".")) {
|
||||
if (!current || typeof current !== "object") return undefined;
|
||||
current = (current as AnyRecord)[part];
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function isContentBlockArray(value: unknown): value is AnyRecord[] {
|
||||
if (!Array.isArray(value) || value.length === 0) return false;
|
||||
return value.slice(0, 5).some((item) => {
|
||||
if (!item || typeof item !== "object") return false;
|
||||
const obj = item as AnyRecord;
|
||||
return "type" in obj || "text" in obj || "textHtml" in obj || "content" in obj;
|
||||
});
|
||||
}
|
||||
|
||||
function extractTextFromContentBlocks(blocks: AnyRecord[]): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
function pushParagraph(text: string): void {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return;
|
||||
parts.push(trimmed, "\n\n");
|
||||
}
|
||||
|
||||
function walk(node: unknown): void {
|
||||
if (!node || typeof node !== "object") return;
|
||||
const block = node as AnyRecord;
|
||||
|
||||
if (typeof block.text === "string") {
|
||||
pushParagraph(block.text);
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof block.textHtml === "string") {
|
||||
pushParagraph(extractTextFromHtml(block.textHtml));
|
||||
return;
|
||||
}
|
||||
|
||||
if (Array.isArray(block.items)) {
|
||||
for (const item of block.items) {
|
||||
if (item && typeof item === "object") {
|
||||
const text = pickString((item as AnyRecord).text);
|
||||
if (text) parts.push(`- ${text}\n`);
|
||||
}
|
||||
}
|
||||
parts.push("\n");
|
||||
}
|
||||
|
||||
if (Array.isArray(block.components)) {
|
||||
for (const component of block.components) {
|
||||
walk(component);
|
||||
}
|
||||
}
|
||||
|
||||
if (Array.isArray(block.content)) {
|
||||
for (const child of block.content) {
|
||||
walk(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const block of blocks) {
|
||||
walk(block);
|
||||
}
|
||||
|
||||
return parts.join("").replace(/\n{3,}/g, "\n\n").trim();
|
||||
}
|
||||
|
||||
function tryStringBodyExtraction(
|
||||
content: string,
|
||||
meta: AnyRecord,
|
||||
document: Document,
|
||||
method: string
|
||||
): ExtractionCandidate | null {
|
||||
if (!content || content.length < MIN_CONTENT_LENGTH) return null;
|
||||
|
||||
const isHtml = /<\/?[a-z][\s\S]*>/i.test(content);
|
||||
const html = isHtml ? sanitizeHtml(content) : null;
|
||||
const textContent = isHtml ? extractTextFromHtml(html) : content.trim();
|
||||
|
||||
if (textContent.length < MIN_CONTENT_LENGTH) return null;
|
||||
|
||||
return {
|
||||
title: pickString(meta.headline, meta.title, extractTitle(document)),
|
||||
byline: pickString(meta.byline, meta.author),
|
||||
excerpt: pickString(meta.description, meta.excerpt, generateExcerpt(null, textContent)),
|
||||
published: pickString(meta.datePublished, meta.publishedAt, extractPublishedTime(document)),
|
||||
html,
|
||||
textContent,
|
||||
method,
|
||||
};
|
||||
}
|
||||
|
||||
function tryNextDataExtraction(document: Document): ExtractionCandidate | null {
|
||||
try {
|
||||
const script = document.querySelector("script#__NEXT_DATA__");
|
||||
if (!script?.textContent) return null;
|
||||
|
||||
const data = JSON.parse(script.textContent) as AnyRecord;
|
||||
const pageProps = (getByPath(data, "props.pageProps") ?? {}) as AnyRecord;
|
||||
|
||||
for (const path of NEXT_DATA_CONTENT_PATHS) {
|
||||
const value = getByPath(data, path);
|
||||
|
||||
if (typeof value === "string") {
|
||||
const parentPath = path.split(".").slice(0, -1).join(".");
|
||||
const parent = (getByPath(data, parentPath) ?? {}) as AnyRecord;
|
||||
const meta = {
|
||||
...pageProps,
|
||||
...parent,
|
||||
title: parent.title ?? (pageProps.title as string | undefined),
|
||||
};
|
||||
|
||||
const candidate = tryStringBodyExtraction(value, meta, document, "next-data");
|
||||
if (candidate) return candidate;
|
||||
}
|
||||
|
||||
if (isContentBlockArray(value)) {
|
||||
const textContent = extractTextFromContentBlocks(value);
|
||||
if (textContent.length < MIN_CONTENT_LENGTH) continue;
|
||||
|
||||
return {
|
||||
title: pickString(
|
||||
getByPath(data, "props.pageProps.content.headline"),
|
||||
getByPath(data, "props.pageProps.article.headline"),
|
||||
getByPath(data, "props.pageProps.article.title"),
|
||||
getByPath(data, "props.pageProps.post.title"),
|
||||
pageProps.title,
|
||||
extractTitle(document)
|
||||
),
|
||||
byline: pickString(
|
||||
getByPath(data, "props.pageProps.author.name"),
|
||||
getByPath(data, "props.pageProps.article.author.name")
|
||||
),
|
||||
excerpt: pickString(
|
||||
getByPath(data, "props.pageProps.content.description"),
|
||||
getByPath(data, "props.pageProps.article.description"),
|
||||
pageProps.description,
|
||||
generateExcerpt(null, textContent)
|
||||
),
|
||||
published: pickString(
|
||||
getByPath(data, "props.pageProps.content.datePublished"),
|
||||
getByPath(data, "props.pageProps.article.datePublished"),
|
||||
getByPath(data, "props.pageProps.publishedAt"),
|
||||
extractPublishedTime(document)
|
||||
),
|
||||
html: null,
|
||||
textContent,
|
||||
method: "next-data",
|
||||
};
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function buildReadabilityCandidate(
|
||||
article: ReturnType<Readability["parse"]>,
|
||||
document: Document,
|
||||
method: string
|
||||
): ExtractionCandidate | null {
|
||||
const textContent = article?.textContent?.trim() ?? "";
|
||||
if (textContent.length < MIN_CONTENT_LENGTH) return null;
|
||||
|
||||
return {
|
||||
title: pickString(article?.title, extractTitle(document)),
|
||||
byline: pickString((article as { byline?: string } | null)?.byline),
|
||||
excerpt: pickString(article?.excerpt, generateExcerpt(null, textContent)),
|
||||
published: pickString((article as { publishedTime?: string } | null)?.publishedTime, extractPublishedTime(document)),
|
||||
html: article?.content ? sanitizeHtml(article.content) : null,
|
||||
textContent,
|
||||
method,
|
||||
};
|
||||
}
|
||||
|
||||
function tryReadability(document: Document): ExtractionCandidate | null {
|
||||
try {
|
||||
const strictClone = document.cloneNode(true) as Document;
|
||||
const strictResult = buildReadabilityCandidate(
|
||||
new Readability(strictClone).parse(),
|
||||
document,
|
||||
"readability"
|
||||
);
|
||||
if (strictResult) return strictResult;
|
||||
|
||||
const relaxedClone = document.cloneNode(true) as Document;
|
||||
return buildReadabilityCandidate(
|
||||
new Readability(relaxedClone, { charThreshold: 120 }).parse(),
|
||||
document,
|
||||
"readability-relaxed"
|
||||
);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function trySelectorExtraction(document: Document): ExtractionCandidate | null {
|
||||
for (const selector of CONTENT_SELECTORS) {
|
||||
const element = document.querySelector(selector);
|
||||
if (!element) continue;
|
||||
|
||||
const clone = element.cloneNode(true) as Element;
|
||||
for (const removeSelector of REMOVE_SELECTORS) {
|
||||
for (const node of clone.querySelectorAll(removeSelector)) {
|
||||
node.remove();
|
||||
}
|
||||
}
|
||||
|
||||
const html = sanitizeHtml(clone.innerHTML);
|
||||
const textContent = extractTextFromHtml(html);
|
||||
if (textContent.length < MIN_CONTENT_LENGTH) continue;
|
||||
|
||||
return {
|
||||
title: extractTitle(document),
|
||||
byline: null,
|
||||
excerpt: generateExcerpt(null, textContent),
|
||||
published: extractPublishedTime(document),
|
||||
html,
|
||||
textContent,
|
||||
method: `selector:${selector}`,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function tryBodyExtraction(document: Document): ExtractionCandidate | null {
|
||||
const body = document.body;
|
||||
if (!body) return null;
|
||||
|
||||
const clone = body.cloneNode(true) as Element;
|
||||
for (const removeSelector of REMOVE_SELECTORS) {
|
||||
for (const node of clone.querySelectorAll(removeSelector)) {
|
||||
node.remove();
|
||||
}
|
||||
}
|
||||
|
||||
const html = sanitizeHtml(clone.innerHTML);
|
||||
const textContent = extractTextFromHtml(html);
|
||||
if (!textContent) return null;
|
||||
|
||||
return {
|
||||
title: extractTitle(document),
|
||||
byline: null,
|
||||
excerpt: generateExcerpt(null, textContent),
|
||||
published: extractPublishedTime(document),
|
||||
html,
|
||||
textContent,
|
||||
method: "body-fallback",
|
||||
};
|
||||
}
|
||||
|
||||
function pickBestCandidate(candidates: ExtractionCandidate[]): ExtractionCandidate | null {
|
||||
if (candidates.length === 0) return null;
|
||||
|
||||
const methodOrder = [
|
||||
"readability",
|
||||
"readability-relaxed",
|
||||
"next-data",
|
||||
"json-ld",
|
||||
"selector:",
|
||||
"body-fallback",
|
||||
];
|
||||
|
||||
function methodRank(method: string): number {
|
||||
const idx = methodOrder.findIndex((entry) =>
|
||||
entry.endsWith(":") ? method.startsWith(entry) : method === entry
|
||||
);
|
||||
return idx === -1 ? methodOrder.length : idx;
|
||||
}
|
||||
|
||||
const ranked = [...candidates].sort((a, b) => {
|
||||
const rankA = methodRank(a.method);
|
||||
const rankB = methodRank(b.method);
|
||||
if (rankA !== rankB) return rankA - rankB;
|
||||
return (b.textContent.length ?? 0) - (a.textContent.length ?? 0);
|
||||
});
|
||||
|
||||
for (const candidate of ranked) {
|
||||
if (candidate.textContent.length >= GOOD_CONTENT_LENGTH) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
for (const candidate of ranked) {
|
||||
if (candidate.textContent.length >= MIN_CONTENT_LENGTH) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return ranked[0];
|
||||
}
|
||||
|
||||
function extractFromHtml(html: string): ExtractionCandidate | null {
|
||||
const document = parseDocument(html);
|
||||
|
||||
const readabilityCandidate = tryReadability(document);
|
||||
const nextDataCandidate = tryNextDataExtraction(document);
|
||||
const jsonLdCandidate = tryJsonLdExtraction(document);
|
||||
const selectorCandidate = trySelectorExtraction(document);
|
||||
const bodyCandidate = tryBodyExtraction(document);
|
||||
|
||||
const candidates = [
|
||||
readabilityCandidate,
|
||||
nextDataCandidate,
|
||||
jsonLdCandidate,
|
||||
selectorCandidate,
|
||||
bodyCandidate,
|
||||
].filter((candidate): candidate is ExtractionCandidate => Boolean(candidate));
|
||||
|
||||
const winner = pickBestCandidate(candidates);
|
||||
if (!winner) return null;
|
||||
|
||||
return {
|
||||
...winner,
|
||||
title: winner.title ?? extractTitle(document),
|
||||
published: winner.published ?? extractPublishedTime(document),
|
||||
excerpt: winner.excerpt ?? generateExcerpt(null, winner.textContent),
|
||||
};
|
||||
}
|
||||
|
||||
const turndown = new TurndownService({
|
||||
headingStyle: "atx",
|
||||
hr: "---",
|
||||
bulletListMarker: "-",
|
||||
codeBlockStyle: "fenced",
|
||||
emDelimiter: "*",
|
||||
strongDelimiter: "**",
|
||||
linkStyle: "inlined",
|
||||
});
|
||||
|
||||
turndown.use(gfm);
|
||||
turndown.remove(["script", "style", "iframe", "noscript", "template", "svg", "path"]);
|
||||
|
||||
turndown.addRule("collapseFigure", {
|
||||
filter: "figure",
|
||||
replacement(content) {
|
||||
return `\n\n${content.trim()}\n\n`;
|
||||
},
|
||||
});
|
||||
|
||||
turndown.addRule("dropInvisibleAnchors", {
|
||||
filter(node) {
|
||||
return node.nodeName === "A" && !(node as Element).textContent?.trim();
|
||||
},
|
||||
replacement() {
|
||||
return "";
|
||||
},
|
||||
});
|
||||
|
||||
function convertHtmlToMarkdown(html: string): string {
|
||||
if (!html || !html.trim()) return "";
|
||||
|
||||
try {
|
||||
const sanitized = sanitizeHtml(html);
|
||||
return turndown.turndown(sanitized);
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function fallbackPlainText(html: string): string {
|
||||
const document = parseDocument(html);
|
||||
for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) {
|
||||
for (const el of document.querySelectorAll(selector)) {
|
||||
el.remove();
|
||||
}
|
||||
}
|
||||
const text = document.body?.textContent ?? document.documentElement?.textContent ?? "";
|
||||
return normalizeMarkdown(text.replace(/\s+/g, " "));
|
||||
}
|
||||
|
||||
function countBylines(markdown: string): number {
|
||||
return (markdown.match(/(^|\n)By\s+/g) || []).length;
|
||||
}
|
||||
|
||||
function countUsefulParagraphs(markdown: string): number {
|
||||
const paragraphs = normalizeMarkdown(markdown).split(/\n{2,}/);
|
||||
let count = 0;
|
||||
|
||||
for (const paragraph of paragraphs) {
|
||||
const trimmed = paragraph.trim();
|
||||
if (!trimmed) continue;
|
||||
if (/^!?\[[^\]]*\]\([^)]+\)$/.test(trimmed)) continue;
|
||||
if (/^#{1,6}\s+/.test(trimmed)) continue;
|
||||
if ((trimmed.match(/\b[\p{L}\p{N}']+\b/gu) || []).length < 8) continue;
|
||||
count++;
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
function countMarkerHits(markdown: string, markers: RegExp[]): number {
|
||||
let hits = 0;
|
||||
for (const marker of markers) {
|
||||
if (marker.test(markdown)) hits++;
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
function scoreMarkdownQuality(markdown: string): number {
|
||||
const normalized = normalizeMarkdown(markdown);
|
||||
const wordCount = (normalized.match(/\b[\p{L}\p{N}']+\b/gu) || []).length;
|
||||
const usefulParagraphs = countUsefulParagraphs(normalized);
|
||||
const headingCount = (normalized.match(/^#{1,6}\s+/gm) || []).length;
|
||||
const markerHits = countMarkerHits(normalized, LOW_QUALITY_MARKERS);
|
||||
const bylineCount = countBylines(normalized);
|
||||
const staffCount = (normalized.match(/\bForbes Staff\b/gi) || []).length;
|
||||
|
||||
return (
|
||||
Math.min(wordCount, 4000) +
|
||||
usefulParagraphs * 40 +
|
||||
headingCount * 10 -
|
||||
markerHits * 180 -
|
||||
Math.max(0, bylineCount - 1) * 120 -
|
||||
Math.max(0, staffCount - 1) * 80
|
||||
);
|
||||
}
|
||||
|
||||
function shouldCompareWithLegacy(markdown: string): boolean {
|
||||
const normalized = normalizeMarkdown(markdown);
|
||||
return (
|
||||
countMarkerHits(normalized, LOW_QUALITY_MARKERS) > 0 ||
|
||||
countBylines(normalized) > 1 ||
|
||||
countUsefulParagraphs(normalized) < 6
|
||||
);
|
||||
}
|
||||
|
||||
function isMarkdownUsable(markdown: string, html: string): boolean {
|
||||
const normalized = normalizeMarkdown(markdown);
|
||||
if (!normalized) return false;
|
||||
|
||||
const htmlTextLength = extractTextFromHtml(html).length;
|
||||
if (htmlTextLength < MIN_CONTENT_LENGTH) return true;
|
||||
|
||||
if (normalized.length >= 80) return true;
|
||||
return normalized.length >= Math.min(200, Math.floor(htmlTextLength * 0.2));
|
||||
}
|
||||
|
||||
async function tryDefuddleConversion(
|
||||
html: string,
|
||||
url: string,
|
||||
baseMetadata: PageMetadata
|
||||
): Promise<{ ok: true; result: ConversionResult } | { ok: false; reason: string }> {
|
||||
try {
|
||||
const [{ JSDOM, VirtualConsole }, { Defuddle }] = await Promise.all([
|
||||
import("jsdom"),
|
||||
import("defuddle/node"),
|
||||
]);
|
||||
|
||||
const virtualConsole = new VirtualConsole();
|
||||
virtualConsole.on("jsdomError", (error: Error & { type?: string }) => {
|
||||
if (error.type === "css parsing" || /Could not parse CSS stylesheet/i.test(error.message)) {
|
||||
return;
|
||||
}
|
||||
console.warn(`[url-to-markdown] jsdom: ${error.message}`);
|
||||
});
|
||||
|
||||
const dom = new JSDOM(html, { url, virtualConsole });
|
||||
const result = await Defuddle(dom, url, { markdown: true });
|
||||
const markdown = normalizeMarkdown(result.content || "");
|
||||
|
||||
if (!isMarkdownUsable(markdown, html)) {
|
||||
return { ok: false, reason: "Defuddle returned empty or incomplete markdown" };
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
result: {
|
||||
metadata: {
|
||||
...baseMetadata,
|
||||
title: pickString(result.title, baseMetadata.title) ?? "",
|
||||
description: pickString(result.description, baseMetadata.description) ?? undefined,
|
||||
author: pickString(result.author, baseMetadata.author) ?? undefined,
|
||||
published: pickString(result.published, baseMetadata.published) ?? undefined,
|
||||
coverImage: pickString(result.image, baseMetadata.coverImage) ?? undefined,
|
||||
},
|
||||
markdown,
|
||||
rawHtml: html,
|
||||
conversionMethod: "defuddle",
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function convertWithLegacyExtractor(html: string, baseMetadata: PageMetadata): ConversionResult {
|
||||
const extracted = extractFromHtml(html);
|
||||
|
||||
let markdown = extracted?.html ? convertHtmlToMarkdown(extracted.html) : "";
|
||||
if (!markdown.trim()) {
|
||||
markdown = extracted?.textContent?.trim() || fallbackPlainText(html);
|
||||
}
|
||||
|
||||
return {
|
||||
metadata: {
|
||||
...baseMetadata,
|
||||
title: pickString(extracted?.title, baseMetadata.title) ?? "",
|
||||
description: pickString(extracted?.excerpt, baseMetadata.description) ?? undefined,
|
||||
author: pickString(extracted?.byline, baseMetadata.author) ?? undefined,
|
||||
published: pickString(extracted?.published, baseMetadata.published) ?? undefined,
|
||||
},
|
||||
markdown: normalizeMarkdown(markdown),
|
||||
rawHtml: html,
|
||||
conversionMethod: extracted ? `legacy:${extracted.method}` : "legacy:plain-text",
|
||||
};
|
||||
}
|
||||
|
||||
export async function extractContent(html: string, url: string): Promise<ConversionResult> {
|
||||
const capturedAt = new Date().toISOString();
|
||||
const baseMetadata = extractMetadataFromHtml(html, url, capturedAt);
|
||||
|
||||
const defuddleResult = await tryDefuddleConversion(html, url, baseMetadata);
|
||||
if (defuddleResult.ok) {
|
||||
if (shouldCompareWithLegacy(defuddleResult.result.markdown)) {
|
||||
const legacyResult = convertWithLegacyExtractor(html, baseMetadata);
|
||||
const legacyScore = scoreMarkdownQuality(legacyResult.markdown);
|
||||
const defuddleScore = scoreMarkdownQuality(defuddleResult.result.markdown);
|
||||
|
||||
if (legacyScore > defuddleScore + 120) {
|
||||
return {
|
||||
...legacyResult,
|
||||
fallbackReason: "Legacy extractor produced higher-quality markdown than Defuddle",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return defuddleResult.result;
|
||||
}
|
||||
|
||||
const fallbackResult = convertWithLegacyExtractor(html, baseMetadata);
|
||||
return {
|
||||
...fallbackResult,
|
||||
fallbackReason: defuddleResult.reason,
|
||||
};
|
||||
}
|
||||
|
||||
function escapeYamlValue(value: string): string {
|
||||
|
||||
@@ -69,6 +69,12 @@ function formatTimestamp(): string {
|
||||
return `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
|
||||
}
|
||||
|
||||
function deriveHtmlSnapshotPath(markdownPath: string): string {
|
||||
const parsed = path.parse(markdownPath);
|
||||
const basename = parsed.ext ? parsed.name : parsed.base;
|
||||
return path.join(parsed.dir, `${basename}-captured.html`);
|
||||
}
|
||||
|
||||
async function generateOutputPath(url: string, title: string, outputDir?: string): Promise<string> {
|
||||
const domain = new URL(url).hostname.replace(/^www\./, "");
|
||||
const slug = generateSlug(title, url);
|
||||
@@ -141,7 +147,7 @@ async function captureUrl(args: Args): Promise<ConversionResult> {
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs(process.argv);
|
||||
if (!args.url) {
|
||||
console.error("Usage: bun main.ts <url> [-o output.md] [--wait] [--timeout ms]");
|
||||
console.error("Usage: bun main.ts <url> [-o output.md] [--output-dir dir] [--wait] [--timeout ms] [--download-media]");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -166,7 +172,9 @@ async function main(): Promise<void> {
|
||||
const result = await captureUrl(args);
|
||||
const outputPath = args.output || await generateOutputPath(args.url, result.metadata.title, args.outputDir);
|
||||
const outputDir = path.dirname(outputPath);
|
||||
const htmlSnapshotPath = deriveHtmlSnapshotPath(outputPath);
|
||||
await mkdir(outputDir, { recursive: true });
|
||||
await writeFile(htmlSnapshotPath, result.rawHtml, "utf-8");
|
||||
|
||||
let document = createMarkdownDocument(result);
|
||||
|
||||
@@ -189,7 +197,12 @@ async function main(): Promise<void> {
|
||||
await writeFile(outputPath, document, "utf-8");
|
||||
|
||||
console.log(`Saved: ${outputPath}`);
|
||||
console.log(`Saved HTML: ${htmlSnapshotPath}`);
|
||||
console.log(`Title: ${result.metadata.title || "(no title)"}`);
|
||||
console.log(`Converter: ${result.conversionMethod}`);
|
||||
if (result.fallbackReason) {
|
||||
console.warn(`Fallback used: ${result.fallbackReason}`);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "baoyu-url-to-markdown-scripts",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"defuddle": "^0.10.0",
|
||||
"jsdom": "^24.1.3",
|
||||
"linkedom": "^0.18.12",
|
||||
"turndown": "^7.2.2",
|
||||
"turndown-plugin-gfm": "^1.0.2"
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,7 @@ export function resolveUrlToMarkdownDataDir(): string {
|
||||
}
|
||||
|
||||
export function resolveUrlToMarkdownChromeProfileDir(): string {
|
||||
const override = process.env.URL_CHROME_PROFILE_DIR?.trim();
|
||||
const override = process.env.BAOYU_CHROME_PROFILE_DIR?.trim() || process.env.URL_CHROME_PROFILE_DIR?.trim();
|
||||
if (override) return path.resolve(override);
|
||||
return path.join(resolveUserDataRoot(), APP_DATA_DIR, URL_TO_MARKDOWN_DATA_DIR, PROFILE_DIR_NAME);
|
||||
return path.join(resolveUserDataRoot(), APP_DATA_DIR, PROFILE_DIR_NAME);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user