mirror of
https://github.com/JimLiu/baoyu-skills.git
synced 2026-08-08 01:43:03 +08:00
Compare commits
50 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 12e207dc3f | |||
| 00e74ab071 | |||
| 1653b8544b | |||
| dad8f3a800 | |||
| 35298d7c9d | |||
| f22374ab62 | |||
| d4e80b1bc3 | |||
| a5761dc71a | |||
| a5189dff37 | |||
| 39fe872bf3 | |||
| 52813504f8 | |||
| a4d4108cd1 | |||
| d7e763f1f5 | |||
| 097c09c59b | |||
| e4cd8bfefc | |||
| 3dc5f2e06f | |||
| e5d6c8ec68 | |||
| 6a4b312146 | |||
| 2d6fe533eb | |||
| f53af25e65 | |||
| c7e32b4590 | |||
| 8d973f2bc5 | |||
| ba20cf89f2 | |||
| 1827be9234 | |||
| 93c98dfc3c | |||
| fbd9f9b622 | |||
| b6e293d059 | |||
| bb78aab095 | |||
| 5071a1d0d0 | |||
| e413ade164 | |||
| e52f92b193 | |||
| 603cabaef4 | |||
| 7d12526e90 | |||
| e7f9764a49 | |||
| e55f91b0ea | |||
| fe3b3d9125 | |||
| 105339cf3f | |||
| dcfd9033ae | |||
| eb416d174c | |||
| 83afacb00e | |||
| 0e6bfbcabd | |||
| 7b67ff44b8 | |||
| 79c289ca92 | |||
| fc5ad4b762 | |||
| 747977416d | |||
| e43eec260a | |||
| 96ef6e2251 | |||
| efb7a1917a | |||
| 1af984a64f | |||
| e79a42fd94 |
@@ -6,48 +6,33 @@
|
||||
},
|
||||
"metadata": {
|
||||
"description": "Skills shared by Baoyu for improving daily work efficiency",
|
||||
"version": "1.73.1"
|
||||
"version": "1.80.0"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "content-skills",
|
||||
"description": "Content generation and publishing skills",
|
||||
"name": "baoyu-skills",
|
||||
"description": "Content generation, AI backends, and utility tools for daily work efficiency",
|
||||
"source": "./",
|
||||
"strict": true,
|
||||
"skills": [
|
||||
"./skills/baoyu-xhs-images",
|
||||
"./skills/baoyu-post-to-x",
|
||||
"./skills/baoyu-post-to-wechat",
|
||||
"./skills/baoyu-post-to-weibo",
|
||||
"./skills/baoyu-article-illustrator",
|
||||
"./skills/baoyu-cover-image",
|
||||
"./skills/baoyu-slide-deck",
|
||||
"./skills/baoyu-comic",
|
||||
"./skills/baoyu-infographic"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "ai-generation-skills",
|
||||
"description": "AI-powered generation backends",
|
||||
"source": "./",
|
||||
"strict": true,
|
||||
"skills": [
|
||||
"./skills/baoyu-danger-gemini-web",
|
||||
"./skills/baoyu-image-gen"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "utility-skills",
|
||||
"description": "Utility tools for content processing",
|
||||
"source": "./",
|
||||
"strict": true,
|
||||
"skills": [
|
||||
"./skills/baoyu-danger-x-to-markdown",
|
||||
"./skills/baoyu-compress-image",
|
||||
"./skills/baoyu-url-to-markdown",
|
||||
"./skills/baoyu-cover-image",
|
||||
"./skills/baoyu-danger-gemini-web",
|
||||
"./skills/baoyu-danger-x-to-markdown",
|
||||
"./skills/baoyu-format-markdown",
|
||||
"./skills/baoyu-image-gen",
|
||||
"./skills/baoyu-infographic",
|
||||
"./skills/baoyu-markdown-to-html",
|
||||
"./skills/baoyu-translate"
|
||||
"./skills/baoyu-post-to-weibo",
|
||||
"./skills/baoyu-post-to-wechat",
|
||||
"./skills/baoyu-post-to-x",
|
||||
"./skills/baoyu-slide-deck",
|
||||
"./skills/baoyu-translate",
|
||||
"./skills/baoyu-url-to-markdown",
|
||||
"./skills/baoyu-xhs-images",
|
||||
"./skills/baoyu-youtube-transcript"
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -166,3 +166,4 @@ posts/
|
||||
.clawdhub/
|
||||
.release-artifacts/
|
||||
.worktrees/
|
||||
youtube-transcript/
|
||||
|
||||
@@ -2,6 +2,99 @@
|
||||
|
||||
English | [中文](./CHANGELOG.zh.md)
|
||||
|
||||
## 1.80.0 - 2026-03-24
|
||||
|
||||
### Features
|
||||
- `baoyu-image-gen`: add Azure OpenAI as independent image generation provider with flexible endpoint parsing, deployment-name resolution, quality mapping, and reference image validation
|
||||
|
||||
## 1.79.2 - 2026-03-23
|
||||
|
||||
### Fixes
|
||||
- `baoyu-cover-image`: simplify reference image handling — use `--ref` when model supports it, only create description files for models without reference image support
|
||||
- `baoyu-post-to-weibo`: add no-theme rule for article markdown-to-HTML conversion
|
||||
|
||||
### Tests
|
||||
- Fix Node-compatible parser tests and add parser test dependencies
|
||||
|
||||
## 1.79.1 - 2026-03-23
|
||||
|
||||
### Fixes
|
||||
- Consolidate to single plugin to prevent duplicate skill registration (by @TyrealQ)
|
||||
- `baoyu-article-illustrator`: remove opacity parameter from watermark prompt
|
||||
- `baoyu-comic`: fix Doraemon naming spacing and remove opacity from watermark prompt
|
||||
- `baoyu-xhs-images`: remove opacity from watermark prompt and fix CJK spacing
|
||||
|
||||
### Documentation
|
||||
- Update project documentation to reflect single-plugin architecture
|
||||
|
||||
## 1.79.0 - 2026-03-22
|
||||
|
||||
### Features
|
||||
- `baoyu-post-to-wechat`: improve credential loading with multi-source resolution, priority ordering, and diagnostics for skipped incomplete sources
|
||||
|
||||
## 1.78.0 - 2026-03-22
|
||||
|
||||
### Features
|
||||
- `baoyu-url-to-markdown`: add URL-specific parser layer for X/Twitter and archive.ph sites
|
||||
- `baoyu-url-to-markdown`: improved slug generation with stop words removal and subdirectory output structure
|
||||
|
||||
### Fixes
|
||||
- `baoyu-url-to-markdown`: preserve anchor elements containing media in legacy converter
|
||||
- `baoyu-url-to-markdown`: smarter title deduplication to avoid redundant headings
|
||||
|
||||
## 1.77.0 - 2026-03-22
|
||||
|
||||
### Features
|
||||
- `baoyu-youtube-transcript`: add end times to chapter data (by @jzOcb)
|
||||
|
||||
### Fixes
|
||||
- `sync-clawhub`: skip failed skills instead of aborting
|
||||
|
||||
## 1.76.1 - 2026-03-21
|
||||
|
||||
### Documentation
|
||||
- `baoyu-youtube-transcript`: fix zsh glob issue — always single-quote YouTube URLs when running the script
|
||||
|
||||
## 1.76.0 - 2026-03-21
|
||||
|
||||
### Features
|
||||
- `baoyu-youtube-transcript`: add title heading, description summary, and cover image to markdown output
|
||||
|
||||
### Fixes
|
||||
- `baoyu-markdown-to-html`: use process.execPath and tsx import in test runner
|
||||
|
||||
## 1.75.0 - 2026-03-21
|
||||
|
||||
### Features
|
||||
- `baoyu-youtube-transcript`: new skill — download YouTube video transcripts/subtitles and cover images with multi-language, chapters, and speaker identification support
|
||||
|
||||
## 1.74.1 - 2026-03-21
|
||||
|
||||
### Fixes
|
||||
- `baoyu-image-gen`: align OpenRouter image generation with current API, harden image support, and narrow Gemini aspect ratios (by @cwandev)
|
||||
- `baoyu-image-gen`: broaden OpenRouter model detection and aspect ratio validation
|
||||
|
||||
## 1.74.0 - 2026-03-20
|
||||
|
||||
### Features
|
||||
- `baoyu-markdown-to-html`: CLI now supports all rendering options — color, font-family, font-size, code-theme, mac-code-block, line-number, count, legend
|
||||
|
||||
### Fixes
|
||||
- `baoyu-markdown-to-html`: fix CSS custom property regex to handle quoted values; grace/simple themes now layer default CSS
|
||||
|
||||
## 1.73.3 - 2026-03-20
|
||||
|
||||
### Fixes
|
||||
- `baoyu-post-to-wechat`: fix placeholder replacement to avoid shorter placeholders matching longer numbered variants
|
||||
|
||||
## 1.73.2 - 2026-03-20
|
||||
|
||||
### Fixes
|
||||
- `baoyu-post-to-wechat`: fix body image upload to correctly use media/uploadimg API with format and size validation (by @AICreator-Wind)
|
||||
|
||||
### Refactor
|
||||
- `baoyu-post-to-wechat`: extract image processor module for local format conversion (WebP/BMP/GIF → JPEG/PNG) instead of material API fallback
|
||||
|
||||
## 1.73.1 - 2026-03-18
|
||||
|
||||
### Refactor
|
||||
|
||||
@@ -2,6 +2,99 @@
|
||||
|
||||
[English](./CHANGELOG.md) | 中文
|
||||
|
||||
## 1.80.0 - 2026-03-24
|
||||
|
||||
### 新功能
|
||||
- `baoyu-image-gen`:新增 Azure OpenAI 作为独立图像生成服务商,支持灵活的端点解析、部署名称推断、质量映射及参考图片格式校验
|
||||
|
||||
## 1.79.2 - 2026-03-23
|
||||
|
||||
### 修复
|
||||
- `baoyu-cover-image`:简化参考图片处理流程 — 模型支持 `--ref` 时直接传递,仅在模型不支持参考图时创建描述文件
|
||||
- `baoyu-post-to-weibo`:文章 Markdown 转 HTML 时不传递 --theme 参数
|
||||
|
||||
### 测试
|
||||
- 修复 Node 兼容的解析器测试,添加解析器测试依赖
|
||||
|
||||
## 1.79.1 - 2026-03-23
|
||||
|
||||
### 修复
|
||||
- 合并为单一插件,防止 skill 重复注册 (by @TyrealQ)
|
||||
- `baoyu-article-illustrator`:移除水印提示词中的不透明度参数
|
||||
- `baoyu-comic`:修正哆啦 A 梦命名间距,移除水印不透明度参数
|
||||
- `baoyu-xhs-images`:移除水印不透明度参数,修正中英文间距
|
||||
|
||||
### 文档
|
||||
- 更新项目文档以反映单一插件架构
|
||||
|
||||
## 1.79.0 - 2026-03-22
|
||||
|
||||
### 新功能
|
||||
- `baoyu-post-to-wechat`:改进凭据加载机制,支持多来源优先级解析,并提供不完整凭据来源的诊断信息
|
||||
|
||||
## 1.78.0 - 2026-03-22
|
||||
|
||||
### 新功能
|
||||
- `baoyu-url-to-markdown`:新增 URL 专用解析层,支持 X/Twitter 和 archive.ph 站点的定制化 HTML 提取
|
||||
- `baoyu-url-to-markdown`:改进 slug 生成算法,去除停用词并采用子目录输出结构
|
||||
|
||||
### 修复
|
||||
- `baoyu-url-to-markdown`:旧版转换器保留包含媒体元素的锚标签
|
||||
- `baoyu-url-to-markdown`:更智能的标题去重,避免重复添加标题
|
||||
|
||||
## 1.77.0 - 2026-03-22
|
||||
|
||||
### 新功能
|
||||
- `baoyu-youtube-transcript`:为章节数据添加结束时间 (by @jzOcb)
|
||||
|
||||
### 修复
|
||||
- `sync-clawhub`:跳过失败的技能而不是中止同步
|
||||
|
||||
## 1.76.1 - 2026-03-21
|
||||
|
||||
### 文档
|
||||
- `baoyu-youtube-transcript`:修复 zsh glob 问题 — 运行脚本时始终对 YouTube URL 使用单引号
|
||||
|
||||
## 1.76.0 - 2026-03-21
|
||||
|
||||
### 新功能
|
||||
- `baoyu-youtube-transcript`:Markdown 输出中新增标题、描述摘要和封面图片
|
||||
|
||||
### 修复
|
||||
- `baoyu-markdown-to-html`:测试运行器改用 process.execPath 和 tsx import
|
||||
|
||||
## 1.75.0 - 2026-03-21
|
||||
|
||||
### 新功能
|
||||
- `baoyu-youtube-transcript`:新技能 — 下载 YouTube 视频字幕/转录文本和封面图片,支持多语言、章节分段和说话人识别
|
||||
|
||||
## 1.74.1 - 2026-03-21
|
||||
|
||||
### 修复
|
||||
- `baoyu-image-gen`:对齐 OpenRouter 图像生成与当前 API,增强图像支持,收窄 Gemini 宽高比范围 (by @cwandev)
|
||||
- `baoyu-image-gen`:扩展 OpenRouter 模型检测和宽高比验证
|
||||
|
||||
## 1.74.0 - 2026-03-20
|
||||
|
||||
### 新功能
|
||||
- `baoyu-markdown-to-html`:CLI 支持全部渲染选项 — color、font-family、font-size、code-theme、mac-code-block、line-number、count、legend
|
||||
|
||||
### 修复
|
||||
- `baoyu-markdown-to-html`:修复 CSS 自定义属性正则无法处理带引号值的问题;grace/simple 主题现在会叠加 default 主题 CSS
|
||||
|
||||
## 1.73.3 - 2026-03-20
|
||||
|
||||
### 修复
|
||||
- `baoyu-post-to-wechat`:修复占位符替换时短占位符错误匹配更长编号变体的问题
|
||||
|
||||
## 1.73.2 - 2026-03-20
|
||||
|
||||
### 修复
|
||||
- `baoyu-post-to-wechat`:修复正文图片上传,正确使用 media/uploadimg 接口并处理格式和大小限制 (by @AICreator-Wind)
|
||||
|
||||
### 重构
|
||||
- `baoyu-post-to-wechat`:提取图片处理模块,本地转换不支持的格式(WebP/BMP/GIF → JPEG/PNG)而非回退到 material 接口
|
||||
|
||||
## 1.73.1 - 2026-03-18
|
||||
|
||||
### 重构
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
# CLAUDE.md
|
||||
|
||||
Claude Code marketplace plugin providing AI-powered content generation skills. Version: **1.73.1**.
|
||||
Claude Code marketplace plugin providing AI-powered content generation skills. Version: **1.80.0**.
|
||||
|
||||
## Architecture
|
||||
|
||||
Skills organized into three categories in `.claude-plugin/marketplace.json` (defines plugin metadata, version, and skill paths):
|
||||
Skills are exposed through the single `baoyu-skills` plugin in `.claude-plugin/marketplace.json` (which defines plugin metadata, version, and skill paths). The repo docs still group them into three logical areas:
|
||||
|
||||
| Category | Description |
|
||||
|----------|-------------|
|
||||
| `content-skills` | Generate or publish content (images, slides, comics, posts) |
|
||||
| `ai-generation-skills` | AI generation backends |
|
||||
| `utility-skills` | Content processing (conversion, compression, translation) |
|
||||
| Group | Description |
|
||||
|-------|-------------|
|
||||
| Content Skills | Generate or publish content (images, slides, comics, posts) |
|
||||
| AI Generation Skills | AI generation backends |
|
||||
| Utility Skills | Content processing (conversion, compression, translation) |
|
||||
|
||||
Each skill contains `SKILL.md` (YAML front matter + docs), optional `scripts/`, `references/`, `prompts/`.
|
||||
|
||||
@@ -31,7 +31,7 @@ Execute: `${BUN_X} skills/<skill>/scripts/main.ts [options]`
|
||||
|
||||
- **Bun**: TypeScript runtime (`bun` preferred, fallback `npx -y bun`)
|
||||
- **Chrome**: Required for CDP-based skills (gemini-web, post-to-x/wechat/weibo, url-to-markdown). All CDP skills share a single profile, override via `BAOYU_CHROME_PROFILE_DIR` env var. Platform paths: [docs/chrome-profile.md](docs/chrome-profile.md)
|
||||
- **Image generation APIs**: `baoyu-image-gen` requires API key (OpenAI, Google, OpenRouter, DashScope, or Replicate) configured in EXTEND.md
|
||||
- **Image generation APIs**: `baoyu-image-gen` requires API key (OpenAI, Azure OpenAI, Google, OpenRouter, DashScope, or Replicate) configured in EXTEND.md
|
||||
- **Gemini Web auth**: Browser cookies (first run opens Chrome for login, `--login` to refresh)
|
||||
|
||||
## Security
|
||||
|
||||
@@ -52,16 +52,14 @@ Run the following command in Claude Code:
|
||||
|
||||
1. Select **Browse and install plugins**
|
||||
2. Select **baoyu-skills**
|
||||
3. Select the plugin(s) you want to install
|
||||
3. Select the **baoyu-skills** plugin
|
||||
4. Select **Install now**
|
||||
|
||||
**Option 2: Direct Install**
|
||||
|
||||
```bash
|
||||
# Install specific plugin
|
||||
/plugin install content-skills@baoyu-skills
|
||||
/plugin install ai-generation-skills@baoyu-skills
|
||||
/plugin install utility-skills@baoyu-skills
|
||||
# Install the marketplace's single plugin
|
||||
/plugin install baoyu-skills@baoyu-skills
|
||||
```
|
||||
|
||||
**Option 3: Ask the Agent**
|
||||
@@ -70,13 +68,13 @@ Simply tell Claude Code:
|
||||
|
||||
> Please install Skills from github.com/JimLiu/baoyu-skills
|
||||
|
||||
### Available Plugins
|
||||
### Available Plugin
|
||||
|
||||
| Plugin | Description | Skills |
|
||||
|--------|-------------|--------|
|
||||
| **content-skills** | Content generation and publishing | [xhs-images](#baoyu-xhs-images), [infographic](#baoyu-infographic), [cover-image](#baoyu-cover-image), [slide-deck](#baoyu-slide-deck), [comic](#baoyu-comic), [article-illustrator](#baoyu-article-illustrator), [post-to-x](#baoyu-post-to-x), [post-to-wechat](#baoyu-post-to-wechat), [post-to-weibo](#baoyu-post-to-weibo) |
|
||||
| **ai-generation-skills** | AI-powered generation backends | [image-gen](#baoyu-image-gen), [danger-gemini-web](#baoyu-danger-gemini-web) |
|
||||
| **utility-skills** | Utility tools for content processing | [url-to-markdown](#baoyu-url-to-markdown), [danger-x-to-markdown](#baoyu-danger-x-to-markdown), [compress-image](#baoyu-compress-image), [format-markdown](#baoyu-format-markdown), [markdown-to-html](#baoyu-markdown-to-html), [translate](#baoyu-translate) |
|
||||
The marketplace now exposes a single plugin so each skill is registered exactly once.
|
||||
|
||||
| Plugin | Description | Includes |
|
||||
|--------|-------------|----------|
|
||||
| **baoyu-skills** | Content generation, AI backends, and utility tools for daily work efficiency | All skills in this repository, organized below as Content Skills, AI Generation Skills, and Utility Skills |
|
||||
|
||||
## Update Skills
|
||||
|
||||
@@ -665,7 +663,7 @@ AI-powered generation backends.
|
||||
|
||||
#### baoyu-image-gen
|
||||
|
||||
AI SDK-based image generation using OpenAI, Google, OpenRouter, DashScope (Aliyun Tongyi Wanxiang), Jimeng (即梦), Seedream (豆包), and Replicate APIs. Supports text-to-image, reference images, aspect ratios, and quality presets.
|
||||
AI SDK-based image generation using OpenAI, Azure OpenAI, Google, OpenRouter, DashScope (Aliyun Tongyi Wanxiang), Jimeng (即梦), Seedream (豆包), and Replicate APIs. Supports text-to-image, reference images, aspect ratios, and quality presets.
|
||||
|
||||
```bash
|
||||
# Basic generation (auto-detect provider)
|
||||
@@ -680,6 +678,9 @@ AI SDK-based image generation using OpenAI, Google, OpenRouter, DashScope (Aliyu
|
||||
# Specific provider
|
||||
/baoyu-image-gen --prompt "A cat" --image cat.png --provider openai
|
||||
|
||||
# Azure OpenAI (model = deployment name)
|
||||
/baoyu-image-gen --prompt "A cat" --image cat.png --provider azure --model gpt-image-1.5
|
||||
|
||||
# OpenRouter
|
||||
/baoyu-image-gen --prompt "A cat" --image cat.png --provider openrouter
|
||||
|
||||
@@ -695,7 +696,7 @@ AI SDK-based image generation using OpenAI, Google, OpenRouter, DashScope (Aliyu
|
||||
# Seedream (豆包)
|
||||
/baoyu-image-gen --prompt "一只可爱的猫" --image cat.png --provider seedream
|
||||
|
||||
# With reference images (Google, OpenAI, OpenRouter, Replicate, or Seedream 5.0/4.5/4.0)
|
||||
# With reference images (Google, OpenAI, Azure OpenAI, OpenRouter, Replicate, or Seedream 5.0/4.5/4.0)
|
||||
/baoyu-image-gen --prompt "Make it blue" --image out.png --ref source.png
|
||||
```
|
||||
|
||||
@@ -766,6 +767,40 @@ Interacts with Gemini Web to generate text and images.
|
||||
|
||||
Utility tools for content processing.
|
||||
|
||||
#### baoyu-youtube-transcript
|
||||
|
||||
Download YouTube video transcripts/subtitles and cover images. Supports multiple languages, translation, chapters, and speaker identification. Caches raw data for fast re-formatting.
|
||||
|
||||
```bash
|
||||
# Default: markdown with timestamps
|
||||
/baoyu-youtube-transcript https://www.youtube.com/watch?v=VIDEO_ID
|
||||
|
||||
# Specify languages (priority order)
|
||||
/baoyu-youtube-transcript https://youtu.be/VIDEO_ID --languages zh,en,ja
|
||||
|
||||
# With chapters and speaker identification
|
||||
/baoyu-youtube-transcript https://youtu.be/VIDEO_ID --chapters --speakers
|
||||
|
||||
# SRT subtitle format
|
||||
/baoyu-youtube-transcript https://youtu.be/VIDEO_ID --format srt
|
||||
|
||||
# List available transcripts
|
||||
/baoyu-youtube-transcript https://youtu.be/VIDEO_ID --list
|
||||
```
|
||||
|
||||
**Options**:
|
||||
| Option | Description | Default |
|
||||
|--------|-------------|---------|
|
||||
| `<url-or-id>` | YouTube URL or video ID | Required |
|
||||
| `--languages <codes>` | Language codes, comma-separated | `en` |
|
||||
| `--format <fmt>` | Output format: `text`, `srt` | `text` |
|
||||
| `--translate <code>` | Translate to specified language | |
|
||||
| `--chapters` | Chapter segmentation from video description | |
|
||||
| `--speakers` | Speaker identification (requires AI post-processing) | |
|
||||
| `--no-timestamps` | Disable timestamps | |
|
||||
| `--list` | List available transcripts | |
|
||||
| `--refresh` | Force re-fetch, ignore cache | |
|
||||
|
||||
#### baoyu-url-to-markdown
|
||||
|
||||
Fetch any URL via Chrome CDP and convert to clean markdown. Saves rendered HTML snapshot alongside the markdown, and automatically falls back to a legacy extractor when Defuddle fails.
|
||||
|
||||
+46
-11
@@ -52,16 +52,14 @@ clawhub install baoyu-markdown-to-html
|
||||
|
||||
1. 选择 **Browse and install plugins**
|
||||
2. 选择 **baoyu-skills**
|
||||
3. 选择要安装的插件
|
||||
3. 选择 **baoyu-skills** 插件
|
||||
4. 选择 **Install now**
|
||||
|
||||
**方式二:直接安装**
|
||||
|
||||
```bash
|
||||
# 安装指定插件
|
||||
/plugin install content-skills@baoyu-skills
|
||||
/plugin install ai-generation-skills@baoyu-skills
|
||||
/plugin install utility-skills@baoyu-skills
|
||||
# 安装 marketplace 中唯一的插件
|
||||
/plugin install baoyu-skills@baoyu-skills
|
||||
```
|
||||
|
||||
**方式三:告诉 Agent**
|
||||
@@ -72,11 +70,11 @@ clawhub install baoyu-markdown-to-html
|
||||
|
||||
### 可用插件
|
||||
|
||||
| 插件 | 说明 | 包含技能 |
|
||||
现在 marketplace 只暴露一个插件,这样每个 skill 只会注册一次。
|
||||
|
||||
| 插件 | 说明 | 包含内容 |
|
||||
|------|------|----------|
|
||||
| **content-skills** | 内容生成和发布 | [xhs-images](#baoyu-xhs-images), [infographic](#baoyu-infographic), [cover-image](#baoyu-cover-image), [slide-deck](#baoyu-slide-deck), [comic](#baoyu-comic), [article-illustrator](#baoyu-article-illustrator), [post-to-x](#baoyu-post-to-x), [post-to-wechat](#baoyu-post-to-wechat), [post-to-weibo](#baoyu-post-to-weibo) |
|
||||
| **ai-generation-skills** | AI 生成后端 | [image-gen](#baoyu-image-gen), [danger-gemini-web](#baoyu-danger-gemini-web) |
|
||||
| **utility-skills** | 内容处理工具 | [url-to-markdown](#baoyu-url-to-markdown), [danger-x-to-markdown](#baoyu-danger-x-to-markdown), [compress-image](#baoyu-compress-image), [format-markdown](#baoyu-format-markdown), [markdown-to-html](#baoyu-markdown-to-html), [translate](#baoyu-translate) |
|
||||
| **baoyu-skills** | 提供内容生成、AI 后端和日常效率工具技能 | 仓库中的全部 skills,仍按下方的内容技能、AI 生成技能、工具技能三个分类展示 |
|
||||
|
||||
## 更新技能
|
||||
|
||||
@@ -665,7 +663,7 @@ AI 驱动的生成后端。
|
||||
|
||||
#### baoyu-image-gen
|
||||
|
||||
基于 AI SDK 的图像生成,支持 OpenAI、Google、OpenRouter、DashScope(阿里通义万相)、即梦(Jimeng)、豆包(Seedream)和 Replicate API。支持文生图、参考图、宽高比和质量预设。
|
||||
基于 AI SDK 的图像生成,支持 OpenAI、Azure OpenAI、Google、OpenRouter、DashScope(阿里通义万相)、即梦(Jimeng)、豆包(Seedream)和 Replicate API。支持文生图、参考图、宽高比和质量预设。
|
||||
|
||||
```bash
|
||||
# 基础生成(自动检测服务商)
|
||||
@@ -680,6 +678,9 @@ AI 驱动的生成后端。
|
||||
# 指定服务商
|
||||
/baoyu-image-gen --prompt "一只猫" --image cat.png --provider openai
|
||||
|
||||
# Azure OpenAI(model 为部署名称)
|
||||
/baoyu-image-gen --prompt "一只猫" --image cat.png --provider azure --model gpt-image-1.5
|
||||
|
||||
# OpenRouter
|
||||
/baoyu-image-gen --prompt "一只猫" --image cat.png --provider openrouter
|
||||
|
||||
@@ -695,7 +696,7 @@ AI 驱动的生成后端。
|
||||
# 豆包(Seedream)
|
||||
/baoyu-image-gen --prompt "一只可爱的猫" --image cat.png --provider seedream
|
||||
|
||||
# 带参考图(Google、OpenAI、OpenRouter、Replicate 或 Seedream 5.0/4.5/4.0)
|
||||
# 带参考图(Google、OpenAI、Azure OpenAI、OpenRouter、Replicate 或 Seedream 5.0/4.5/4.0)
|
||||
/baoyu-image-gen --prompt "把它变成蓝色" --image out.png --ref source.png
|
||||
```
|
||||
|
||||
@@ -766,6 +767,40 @@ AI 驱动的生成后端。
|
||||
|
||||
内容处理工具。
|
||||
|
||||
#### baoyu-youtube-transcript
|
||||
|
||||
下载 YouTube 视频字幕/转录文本和封面图片。支持多语言、翻译、章节分段和说话人识别。缓存原始数据以便快速重新格式化。
|
||||
|
||||
```bash
|
||||
# 默认:带时间戳的 Markdown
|
||||
/baoyu-youtube-transcript https://www.youtube.com/watch?v=VIDEO_ID
|
||||
|
||||
# 指定语言(按优先级排列)
|
||||
/baoyu-youtube-transcript https://youtu.be/VIDEO_ID --languages zh,en,ja
|
||||
|
||||
# 章节分段 + 说话人识别
|
||||
/baoyu-youtube-transcript https://youtu.be/VIDEO_ID --chapters --speakers
|
||||
|
||||
# SRT 字幕格式
|
||||
/baoyu-youtube-transcript https://youtu.be/VIDEO_ID --format srt
|
||||
|
||||
# 列出可用字幕
|
||||
/baoyu-youtube-transcript https://youtu.be/VIDEO_ID --list
|
||||
```
|
||||
|
||||
**选项**:
|
||||
| 选项 | 说明 | 默认值 |
|
||||
|------|------|--------|
|
||||
| `<url-or-id>` | YouTube URL 或视频 ID | 必填 |
|
||||
| `--languages <codes>` | 语言代码,逗号分隔 | `en` |
|
||||
| `--format <fmt>` | 输出格式:`text`、`srt` | `text` |
|
||||
| `--translate <code>` | 翻译为指定语言 | |
|
||||
| `--chapters` | 根据视频描述进行章节分段 | |
|
||||
| `--speakers` | 说话人识别(需 AI 后处理) | |
|
||||
| `--no-timestamps` | 禁用时间戳 | |
|
||||
| `--list` | 列出可用字幕 | |
|
||||
| `--refresh` | 强制重新获取,忽略缓存 | |
|
||||
|
||||
#### baoyu-url-to-markdown
|
||||
|
||||
通过 Chrome CDP 抓取任意 URL 并转换为 Markdown。同时保存渲染后的 HTML 快照,Defuddle 失败时自动回退到旧版提取器。
|
||||
|
||||
+11
-9
@@ -34,20 +34,22 @@ metadata:
|
||||
1. Create `skills/baoyu-<name>/SKILL.md` with YAML front matter
|
||||
2. Add TypeScript in `skills/baoyu-<name>/scripts/` (if applicable)
|
||||
3. Add prompt templates in `skills/baoyu-<name>/prompts/` if needed
|
||||
4. Register in `marketplace.json` under appropriate category
|
||||
4. Register the skill in `.claude-plugin/marketplace.json` under the `baoyu-skills` plugin entry
|
||||
5. Add Script Directory section to SKILL.md if skill has scripts
|
||||
6. Add openclaw metadata to frontmatter
|
||||
|
||||
## Category Selection
|
||||
## Skill Grouping
|
||||
|
||||
| If your skill... | Use category |
|
||||
|------------------|--------------|
|
||||
| Generates visual content (images, slides, comics) | `content-skills` |
|
||||
| Publishes to platforms (X, WeChat, Weibo) | `content-skills` |
|
||||
| Provides AI generation backend | `ai-generation-skills` |
|
||||
| Converts or processes content | `utility-skills` |
|
||||
All skills are registered under the single `baoyu-skills` plugin. Use these logical groups when deciding where the skill should appear in the docs:
|
||||
|
||||
New category: add plugin object to `marketplace.json` with `name`, `description`, `skills[]`.
|
||||
| If your skill... | Use group |
|
||||
|------------------|-----------|
|
||||
| Generates visual content (images, slides, comics) | Content Skills |
|
||||
| Publishes to platforms (X, WeChat, Weibo) | Content Skills |
|
||||
| Provides AI generation backend | AI Generation Skills |
|
||||
| Converts or processes content | Utility Skills |
|
||||
|
||||
If you add a new logical group, update the docs that present grouped skills, but keep the skill registered under the single `baoyu-skills` plugin entry.
|
||||
|
||||
## Writing Descriptions
|
||||
|
||||
|
||||
Generated
+105
-1
@@ -9,7 +9,11 @@
|
||||
"packages/*"
|
||||
],
|
||||
"devDependencies": {
|
||||
"tsx": "^4.20.5"
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"linkedom": "^0.18.12",
|
||||
"tsx": "^4.20.5",
|
||||
"turndown": "^7.2.2",
|
||||
"turndown-plugin-gfm": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/aix-ppc64": {
|
||||
@@ -454,6 +458,23 @@
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@mixmark-io/domino": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",
|
||||
"integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",
|
||||
"dev": true,
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/@mozilla/readability": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.6.0.tgz",
|
||||
"integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/debug": {
|
||||
"version": "4.1.12",
|
||||
"resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz",
|
||||
@@ -615,6 +636,13 @@
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
},
|
||||
"node_modules/cssom": {
|
||||
"version": "0.5.0",
|
||||
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
|
||||
"integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.4.3",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
||||
@@ -896,6 +924,13 @@
|
||||
"node": ">=12.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/html-escaper": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz",
|
||||
"integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/htmlparser2": {
|
||||
"version": "9.1.0",
|
||||
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz",
|
||||
@@ -984,6 +1019,51 @@
|
||||
"node": ">=18.17"
|
||||
}
|
||||
},
|
||||
"node_modules/linkedom": {
|
||||
"version": "0.18.12",
|
||||
"resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.12.tgz",
|
||||
"integrity": "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==",
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"css-select": "^5.1.0",
|
||||
"cssom": "^0.5.0",
|
||||
"html-escaper": "^3.0.3",
|
||||
"htmlparser2": "^10.0.0",
|
||||
"uhyphen": "^0.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"canvas": ">= 2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"canvas": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/linkedom/node_modules/htmlparser2": {
|
||||
"version": "10.1.0",
|
||||
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz",
|
||||
"integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
"https://github.com/fb55/htmlparser2?sponsor=1",
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"domelementtype": "^2.3.0",
|
||||
"domhandler": "^5.0.3",
|
||||
"domutils": "^3.2.2",
|
||||
"entities": "^7.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/longest-streak": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz",
|
||||
@@ -1768,6 +1848,30 @@
|
||||
"fsevents": "~2.3.3"
|
||||
}
|
||||
},
|
||||
"node_modules/turndown": {
|
||||
"version": "7.2.2",
|
||||
"resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.2.tgz",
|
||||
"integrity": "sha512-1F7db8BiExOKxjSMU2b7if62D/XOyQyZbPKq/nUwopfgnHlqXHqQ0lvfUTeUIr1lZJzOPFn43dODyMSIfvWRKQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@mixmark-io/domino": "^2.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/turndown-plugin-gfm": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.2.tgz",
|
||||
"integrity": "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/uhyphen": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz",
|
||||
"integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==",
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/undici": {
|
||||
"version": "6.24.0",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-6.24.0.tgz",
|
||||
|
||||
@@ -10,6 +10,10 @@
|
||||
"test:coverage": "node --import tsx --experimental-test-coverage --test"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"linkedom": "^0.18.12",
|
||||
"turndown": "^7.2.2",
|
||||
"turndown-plugin-gfm": "^1.0.2",
|
||||
"tsx": "^4.20.5"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,12 +9,26 @@ import { COLOR_PRESETS, FONT_FAMILY_MAP } from "./constants.ts";
|
||||
import {
|
||||
buildMarkdownDocumentMeta,
|
||||
formatTimestamp,
|
||||
renderMarkdownDocument,
|
||||
resolveColorToken,
|
||||
resolveFontFamilyToken,
|
||||
resolveMarkdownStyle,
|
||||
resolveRenderOptions,
|
||||
} from "./document.ts";
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, `\\$&`);
|
||||
}
|
||||
|
||||
function findInlineStyle(html: string, tagName: string, text: string): string {
|
||||
const pattern = new RegExp(
|
||||
`<${tagName}[^>]*style="([^"]*)"[^>]*>${escapeRegExp(text)}</${tagName}>`,
|
||||
);
|
||||
const match = html.match(pattern);
|
||||
assert.ok(match, `Expected inline style for <${tagName}>${text}</${tagName}>`);
|
||||
return match![1]!;
|
||||
}
|
||||
|
||||
function useCwd(t: TestContext, cwd: string): void {
|
||||
const previous = process.cwd();
|
||||
process.chdir(cwd);
|
||||
@@ -138,3 +152,23 @@ keep_title: true
|
||||
assert.equal(explicit.fontSize, "18px");
|
||||
assert.equal(explicit.keepTitle, false);
|
||||
});
|
||||
|
||||
test("renderMarkdownDocument layers default rules into grace theme before CSS inlining", async () => {
|
||||
const { html } = await renderMarkdownDocument(
|
||||
`## Section\n\nParagraph with **bold** text.`,
|
||||
{ keepTitle: true, theme: "grace" },
|
||||
);
|
||||
|
||||
const h2Style = findInlineStyle(html, "h2", "Section");
|
||||
assert.match(h2Style, /background: #92617E/);
|
||||
assert.match(h2Style, /box-shadow: 0 4px 6px rgba\(0, 0, 0, 0\.1\)/);
|
||||
|
||||
const pMatch = html.match(/<p[^>]*style="([^"]*)"[^>]*>/);
|
||||
assert.ok(pMatch, "Expected inline style on <p> tag");
|
||||
assert.match(pMatch![1]!, /color:/);
|
||||
|
||||
const strongPattern = /<strong[^>]*style="([^"]*)"[^>]*>bold<\/strong>/;
|
||||
const strongMatch = html.match(strongPattern);
|
||||
assert.ok(strongMatch, "Expected inline style for <strong>bold</strong>");
|
||||
assert.match(strongMatch![1]!, /font-weight:/);
|
||||
});
|
||||
|
||||
@@ -59,6 +59,17 @@ test("normalizeCssText and normalizeInlineCss replace variables and strip declar
|
||||
assert.doesNotMatch(normalizedHtml, /var\(--md-primary-color\)/);
|
||||
});
|
||||
|
||||
test("normalizeInlineCss removes quoted custom property values without leaving fragments behind", () => {
|
||||
const normalizedHtml = normalizeInlineCss(
|
||||
`<html style="--md-font-family: Menlo, Monaco, 'Courier New', monospace; color: var(--md-primary-color)"></html>`,
|
||||
DEFAULT_STYLE,
|
||||
);
|
||||
|
||||
assert.match(normalizedHtml, /style=" color: #0F4C81"/);
|
||||
assert.doesNotMatch(normalizedHtml, /Courier New/);
|
||||
assert.doesNotMatch(normalizedHtml, /--md-font-family/);
|
||||
});
|
||||
|
||||
test("HTML structure helpers hoist nested lists and remove the first heading", () => {
|
||||
const nestedList = `<ul><li>Parent<ul><li>Child</li></ul></li></ul>`;
|
||||
assert.equal(
|
||||
|
||||
@@ -100,13 +100,13 @@ export function normalizeCssText(cssText: string, style: StyleConfig = DEFAULT_S
|
||||
.replace(/var\(--md-accent-color\)/g, style.accentColor)
|
||||
.replace(/var\(--md-container-bg\)/g, style.containerBg)
|
||||
.replace(/hsl\(var\(--foreground\)\)/g, "#3f3f3f")
|
||||
.replace(/--md-primary-color:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-font-family:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-font-size:\s*[^;"']+;?/g, "")
|
||||
.replace(/--blockquote-background:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-accent-color:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-container-bg:\s*[^;"']+;?/g, "")
|
||||
.replace(/--foreground:\s*[^;"']+;?/g, "");
|
||||
.replace(/--md-primary-color:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-font-family:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-font-size:\s*[^;]+;?/g, "")
|
||||
.replace(/--blockquote-background:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-accent-color:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-container-bg:\s*[^;]+;?/g, "")
|
||||
.replace(/--foreground:\s*[^;]+;?/g, "");
|
||||
}
|
||||
|
||||
export function normalizeInlineCss(html: string, style: StyleConfig = DEFAULT_STYLE): string {
|
||||
|
||||
@@ -6,6 +6,7 @@ import type { ThemeName } from "./types.js";
|
||||
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
export const THEME_DIR = path.resolve(SCRIPT_DIR, "themes");
|
||||
const FALLBACK_THEMES: ThemeName[] = ["default", "grace", "simple"];
|
||||
const THEMES_EXTENDING_DEFAULT = new Set<ThemeName>(["grace", "simple"]);
|
||||
|
||||
function stripOutputScope(cssContent: string): string {
|
||||
let css = cssContent;
|
||||
@@ -41,6 +42,7 @@ export function loadThemeCss(theme: ThemeName): {
|
||||
themeCss: string;
|
||||
} {
|
||||
const basePath = path.join(THEME_DIR, "base.css");
|
||||
const defaultThemePath = path.join(THEME_DIR, "default.css");
|
||||
const themePath = path.join(THEME_DIR, `${theme}.css`);
|
||||
|
||||
if (!fs.existsSync(basePath)) {
|
||||
@@ -51,9 +53,18 @@ export function loadThemeCss(theme: ThemeName): {
|
||||
throw new Error(`Missing theme CSS for "${theme}": ${themePath}`);
|
||||
}
|
||||
|
||||
const layeredThemeCss: string[] = [];
|
||||
if (theme !== "default" && THEMES_EXTENDING_DEFAULT.has(theme)) {
|
||||
if (!fs.existsSync(defaultThemePath)) {
|
||||
throw new Error(`Missing default theme CSS: ${defaultThemePath}`);
|
||||
}
|
||||
layeredThemeCss.push(fs.readFileSync(defaultThemePath, "utf-8"));
|
||||
}
|
||||
layeredThemeCss.push(fs.readFileSync(themePath, "utf-8"));
|
||||
|
||||
return {
|
||||
baseCss: fs.readFileSync(basePath, "utf-8"),
|
||||
themeCss: fs.readFileSync(themePath, "utf-8"),
|
||||
themeCss: layeredThemeCss.join("\n"),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -151,6 +151,9 @@ async function main() {
|
||||
.map((tag) => tag.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
let succeeded = 0;
|
||||
const failed = [];
|
||||
|
||||
for (const candidate of actionable) {
|
||||
const version =
|
||||
candidate.status === "new"
|
||||
@@ -158,6 +161,7 @@ async function main() {
|
||||
: bumpSemver(candidate.latestVersion, options.bump);
|
||||
|
||||
console.log(`Publishing ${candidate.slug}@${version}`);
|
||||
try {
|
||||
const files = await listTextFiles(candidate.folder);
|
||||
await publishSkill({
|
||||
registry,
|
||||
@@ -168,10 +172,19 @@ async function main() {
|
||||
changelog: options.changelog,
|
||||
tags,
|
||||
});
|
||||
succeeded++;
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
console.error(`SKIPPED ${candidate.slug}: ${msg}`);
|
||||
failed.push(candidate.slug);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("");
|
||||
console.log(`Uploaded ${actionable.length} skill(s).`);
|
||||
console.log(`Uploaded ${succeeded}/${actionable.length} skill(s).`);
|
||||
if (failed.length > 0) {
|
||||
console.log(`Failed (${failed.length}): ${failed.join(", ")}`);
|
||||
}
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
|
||||
@@ -280,5 +280,5 @@ TEXTURE: Halftone transitions between sides
|
||||
If watermark enabled in preferences, append:
|
||||
|
||||
```
|
||||
Include a subtle watermark "[content]" positioned at [position] with approximately [opacity*100]% visibility.
|
||||
Include a subtle watermark "[content]" positioned at [position].
|
||||
```
|
||||
|
||||
@@ -359,8 +359,7 @@ Art: [art style] | Tone: [tone] | Layout: [layout type]
|
||||
**Watermark Application** (if enabled in preferences):
|
||||
Add to each prompt:
|
||||
```
|
||||
Include a subtle watermark "[content]" positioned at [position]
|
||||
with approximately [opacity*100]% visibility. The watermark should
|
||||
Include a subtle watermark "[content]" positioned at [position]. The watermark should
|
||||
be legible but not distracting from the comic panels and storytelling.
|
||||
Ensure watermark does not overlap speech bubbles or key action.
|
||||
```
|
||||
@@ -452,8 +451,8 @@ When skill does NOT support reference images, create combined prompt files:
|
||||
|
||||
## Character Reference (maintain consistency)
|
||||
[Copy relevant sections from characters/characters.md here]
|
||||
- 大雄: Japanese boy, round glasses, yellow shirt, navy shorts...
|
||||
- 哆啦A梦: Round blue robot cat, white belly, red nose, golden bell...
|
||||
- 大雄:Japanese boy, round glasses, yellow shirt, navy shorts...
|
||||
- 哆啦 A 梦:Round blue robot cat, white belly, red nose, golden bell...
|
||||
|
||||
## Page Content
|
||||
[Original page prompt here]
|
||||
|
||||
@@ -162,15 +162,14 @@ if (Test-Path "$HOME/.baoyu-skills/baoyu-cover-image/EXTEND.md") { "user" }
|
||||
5. **Detect language**: Compare source, user input, EXTEND.md preference
|
||||
6. **Determine output directory**: Per File Structure rules
|
||||
|
||||
**⚠️ People in Reference Images — MUST follow all 3 rules:**
|
||||
**⚠️ People in Reference Images:**
|
||||
|
||||
If reference images contain **people** who should appear in the cover:
|
||||
|
||||
1. **`usage: direct`** — MUST set in refs description file. NEVER use `style` or `palette` when people need to appear
|
||||
2. **Per-character description** — MUST describe each person's distinctive features (hair, glasses, skin tone, clothing) in `refs/ref-NN-{slug}.md`. Vague descriptions like "a man" will fail
|
||||
3. **`--ref` flag** — MUST pass reference image via `--ref` in Step 4 so the model sees actual faces
|
||||
- **Model supports `--ref`** (default): Copy image to `refs/`, pass via `--ref` at generation. No description file needed — the model sees the face directly.
|
||||
- **Model does NOT support `--ref`** (Jimeng, Seedream 3.0): Create `refs/ref-NN-{slug}.md` with per-character description (hair, glasses, skin tone, clothing). Embed as MUST/REQUIRED instructions in prompt text.
|
||||
|
||||
See [reference-images.md § Character Analysis](references/workflow/reference-images.md) for description format.
|
||||
See [reference-images.md](references/workflow/reference-images.md) for full decision table.
|
||||
|
||||
### Step 2: Confirm Options ⚠️
|
||||
|
||||
|
||||
@@ -16,17 +16,24 @@ Guide for processing user-provided reference images in cover generation.
|
||||
|
||||
**If user provides file path**:
|
||||
1. Copy to `refs/ref-NN-{slug}.{ext}` (NN = 01, 02, ...)
|
||||
2. Create description: `refs/ref-NN-{slug}.md`
|
||||
3. Verify files exist before proceeding
|
||||
2. **Only** create description file `refs/ref-NN-{slug}.md` when model does NOT support `--ref` (see below)
|
||||
3. Verify image file exists before proceeding
|
||||
|
||||
**Description File Format**:
|
||||
**When to create description file**:
|
||||
|
||||
| Situation | Action |
|
||||
|-----------|--------|
|
||||
| Model supports `--ref` (Google, OpenAI, OpenRouter, Replicate, Seedream 4.0+) | Copy image only. **No description file needed.** Pass via `--ref` at generation. |
|
||||
| Model does NOT support `--ref` (Jimeng, Seedream 3.0) | Copy image + create description file. Embed description in prompt text. |
|
||||
|
||||
**Description File Format** (only when needed):
|
||||
```yaml
|
||||
---
|
||||
ref_id: NN
|
||||
filename: ref-NN-{slug}.{ext}
|
||||
usage: direct | style | palette
|
||||
---
|
||||
[User's description or auto-generated description]
|
||||
[Character or style description to embed in prompt]
|
||||
```
|
||||
|
||||
| Usage | When to Use |
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: baoyu-image-gen
|
||||
description: AI image generation with OpenAI, Google, OpenRouter, DashScope, Jimeng, Seedream and Replicate APIs. Supports text-to-image, reference images, aspect ratios, and batch generation from saved prompt files. Sequential by default; use batch parallel generation when the user already has multiple prompts or wants stable multi-image throughput. Use when user asks to generate, create, or draw images.
|
||||
description: AI image generation with OpenAI, Azure OpenAI, Google, OpenRouter, DashScope, Jimeng, Seedream and Replicate APIs. Supports text-to-image, reference images, aspect ratios, and batch generation from saved prompt files. Sequential by default; use batch parallel generation when the user already has multiple prompts or wants stable multi-image throughput. Use when user asks to generate, create, or draw images.
|
||||
version: 1.56.3
|
||||
metadata:
|
||||
openclaw:
|
||||
@@ -13,7 +13,7 @@ metadata:
|
||||
|
||||
# Image Generation (AI SDK)
|
||||
|
||||
Official API-based image generation. Supports OpenAI, Google, OpenRouter, DashScope (阿里通义万象), Jimeng (即梦), Seedream (豆包) and Replicate providers.
|
||||
Official API-based image generation. Supports OpenAI, Azure OpenAI, Google, OpenRouter, DashScope (阿里通义万象), Jimeng (即梦), Seedream (豆包) and Replicate providers.
|
||||
|
||||
## Script Directory
|
||||
|
||||
@@ -74,12 +74,15 @@ ${BUN_X} {baseDir}/scripts/main.ts --prompt "A cat" --image out.png --quality 2k
|
||||
# From prompt files
|
||||
${BUN_X} {baseDir}/scripts/main.ts --promptfiles system.md content.md --image out.png
|
||||
|
||||
# With reference images (Google, OpenAI, OpenRouter, Replicate, or Seedream 4.0/4.5/5.0)
|
||||
# With reference images (Google, OpenAI, Azure OpenAI, OpenRouter, Replicate, or Seedream 4.0/4.5/5.0)
|
||||
${BUN_X} {baseDir}/scripts/main.ts --prompt "Make blue" --image out.png --ref source.png
|
||||
|
||||
# With reference images (explicit provider/model)
|
||||
${BUN_X} {baseDir}/scripts/main.ts --prompt "Make blue" --image out.png --provider google --model gemini-3-pro-image-preview --ref source.png
|
||||
|
||||
# Azure OpenAI (model means deployment name)
|
||||
${BUN_X} {baseDir}/scripts/main.ts --prompt "A cat" --image out.png --provider azure --model gpt-image-1.5
|
||||
|
||||
# OpenRouter (recommended default model)
|
||||
${BUN_X} {baseDir}/scripts/main.ts --prompt "A cat" --image out.png --provider openrouter
|
||||
|
||||
@@ -147,13 +150,13 @@ Paths in `promptFiles`, `image`, and `ref` are resolved relative to the batch fi
|
||||
| `--image <path>` | Output image path (required in single-image mode) |
|
||||
| `--batchfile <path>` | JSON batch file for multi-image generation |
|
||||
| `--jobs <count>` | Worker count for batch mode (default: auto, max from config, built-in default 10) |
|
||||
| `--provider google\|openai\|openrouter\|dashscope\|jimeng\|seedream\|replicate` | Force provider (default: auto-detect) |
|
||||
| `--model <id>`, `-m` | Model ID (Google: `gemini-3-pro-image-preview`; OpenAI: `gpt-image-1.5`; OpenRouter: `google/gemini-3.1-flash-image-preview`; DashScope: `qwen-image-2.0-pro`) |
|
||||
| `--provider google\|openai\|azure\|openrouter\|dashscope\|jimeng\|seedream\|replicate` | Force provider (default: auto-detect) |
|
||||
| `--model <id>`, `-m` | Model ID (Google: `gemini-3-pro-image-preview`; OpenAI: `gpt-image-1.5`; Azure: deployment name such as `gpt-image-1.5` or `image-prod`; OpenRouter: `google/gemini-3.1-flash-image-preview`; DashScope: `qwen-image-2.0-pro`) |
|
||||
| `--ar <ratio>` | Aspect ratio (e.g., `16:9`, `1:1`, `4:3`) |
|
||||
| `--size <WxH>` | Size (e.g., `1024x1024`) |
|
||||
| `--quality normal\|2k` | Quality preset (default: `2k`) |
|
||||
| `--imageSize 1K\|2K\|4K` | Image size for Google/OpenRouter (default: from quality) |
|
||||
| `--ref <files...>` | Reference images. Supported by Google multimodal, OpenAI GPT Image edits, OpenRouter multimodal models, Replicate, and Seedream 5.0/4.5/4.0. Not supported by Jimeng, Seedream 3.0, or removed SeedEdit 3.0 |
|
||||
| `--ref <files...>` | Reference images. Supported by Google multimodal, OpenAI GPT Image edits, Azure OpenAI edits (PNG/JPG only), OpenRouter multimodal models, Replicate, and Seedream 5.0/4.5/4.0. Not supported by Jimeng, Seedream 3.0, or removed SeedEdit 3.0 |
|
||||
| `--n <count>` | Number of images |
|
||||
| `--json` | JSON output |
|
||||
|
||||
@@ -162,6 +165,7 @@ Paths in `promptFiles`, `image`, and `ref` are resolved relative to the batch fi
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `OPENAI_API_KEY` | OpenAI API key |
|
||||
| `AZURE_OPENAI_API_KEY` | Azure OpenAI API key |
|
||||
| `OPENROUTER_API_KEY` | OpenRouter API key |
|
||||
| `GOOGLE_API_KEY` | Google API key |
|
||||
| `DASHSCOPE_API_KEY` | DashScope API key (阿里云) |
|
||||
@@ -170,6 +174,8 @@ Paths in `promptFiles`, `image`, and `ref` are resolved relative to the batch fi
|
||||
| `JIMENG_SECRET_ACCESS_KEY` | Jimeng (即梦) Volcengine secret key |
|
||||
| `ARK_API_KEY` | Seedream (豆包) Volcengine ARK API key |
|
||||
| `OPENAI_IMAGE_MODEL` | OpenAI model override |
|
||||
| `AZURE_OPENAI_DEPLOYMENT` | Azure default deployment name |
|
||||
| `AZURE_OPENAI_IMAGE_MODEL` | Backward-compatible alias for Azure default deployment/model name |
|
||||
| `OPENROUTER_IMAGE_MODEL` | OpenRouter model override (default: `google/gemini-3.1-flash-image-preview`) |
|
||||
| `GOOGLE_IMAGE_MODEL` | Google model override |
|
||||
| `DASHSCOPE_IMAGE_MODEL` | DashScope model override (default: `qwen-image-2.0-pro`) |
|
||||
@@ -177,6 +183,8 @@ Paths in `promptFiles`, `image`, and `ref` are resolved relative to the batch fi
|
||||
| `JIMENG_IMAGE_MODEL` | Jimeng model override (default: jimeng_t2i_v40) |
|
||||
| `SEEDREAM_IMAGE_MODEL` | Seedream model override (default: doubao-seedream-5-0-260128) |
|
||||
| `OPENAI_BASE_URL` | Custom OpenAI endpoint |
|
||||
| `AZURE_OPENAI_BASE_URL` | Azure resource endpoint or deployment endpoint |
|
||||
| `AZURE_API_VERSION` | Azure image API version (default: `2025-04-01-preview`) |
|
||||
| `OPENROUTER_BASE_URL` | Custom OpenRouter endpoint (default: `https://openrouter.ai/api/v1`) |
|
||||
| `OPENROUTER_HTTP_REFERER` | Optional app/site URL for OpenRouter attribution |
|
||||
| `OPENROUTER_TITLE` | Optional app name for OpenRouter attribution |
|
||||
@@ -201,6 +209,8 @@ Model priority (highest → lowest), applies to all providers:
|
||||
3. Env var: `<PROVIDER>_IMAGE_MODEL` (e.g., `GOOGLE_IMAGE_MODEL`)
|
||||
4. Built-in default
|
||||
|
||||
For Azure, `--model` / `default_model.azure` should be the Azure deployment name. `AZURE_OPENAI_DEPLOYMENT` is the preferred env var, and `AZURE_OPENAI_IMAGE_MODEL` remains as a backward-compatible alias.
|
||||
|
||||
**EXTEND.md overrides env vars**. If both EXTEND.md `default_model.google: "gemini-3-pro-image-preview"` and env var `GOOGLE_IMAGE_MODEL=gemini-3.1-flash-image-preview` exist, EXTEND.md wins.
|
||||
|
||||
**Agent MUST display model info** before each generation:
|
||||
|
||||
@@ -47,6 +47,8 @@ options:
|
||||
description: "Gemini multimodal - high quality, reference images, flexible sizes"
|
||||
- label: "OpenAI"
|
||||
description: "GPT Image - consistent quality, reliable output"
|
||||
- label: "Azure OpenAI"
|
||||
description: "Azure-hosted GPT Image deployments with resource-specific routing"
|
||||
- label: "OpenRouter"
|
||||
description: "Router for Gemini/FLUX/OpenAI-compatible image models"
|
||||
- label: "DashScope"
|
||||
@@ -87,6 +89,20 @@ options:
|
||||
description: "Strong text-to-image quality through OpenRouter"
|
||||
```
|
||||
|
||||
### Question 2c: Default Azure Deployment
|
||||
|
||||
Only show if user selected Azure OpenAI.
|
||||
|
||||
```yaml
|
||||
header: "Azure Deploy"
|
||||
question: "Default Azure image deployment name?"
|
||||
options:
|
||||
- label: "gpt-image-1.5 (Recommended)"
|
||||
description: "Best default if your Azure deployment uses the same name"
|
||||
- label: "gpt-image-1"
|
||||
description: "Previous GPT Image deployment name"
|
||||
```
|
||||
|
||||
### Question 3: Default Quality
|
||||
|
||||
```yaml
|
||||
@@ -130,6 +146,7 @@ default_image_size: null
|
||||
default_model:
|
||||
google: [selected google model or null]
|
||||
openai: null
|
||||
azure: [selected azure deployment or null]
|
||||
openrouter: [selected openrouter model or null]
|
||||
dashscope: null
|
||||
replicate: null
|
||||
@@ -166,6 +183,23 @@ options:
|
||||
description: "Previous generation GPT Image model"
|
||||
```
|
||||
|
||||
### Azure Deployment Selection
|
||||
|
||||
```yaml
|
||||
header: "Azure Deploy"
|
||||
question: "Choose a default Azure image deployment name?"
|
||||
options:
|
||||
- label: "gpt-image-1.5 (Recommended)"
|
||||
description: "Use when your Azure deployment name matches the GPT-image-1.5 model"
|
||||
- label: "gpt-image-1"
|
||||
description: "Use when your Azure deployment name matches GPT-image-1"
|
||||
```
|
||||
|
||||
Notes for Azure setup:
|
||||
|
||||
- In `baoyu-image-gen`, Azure `--model` / `default_model.azure` should be the Azure deployment name, not just the underlying model family.
|
||||
- If the deployment name is custom, save that exact deployment name in `default_model.azure`.
|
||||
|
||||
### OpenRouter Model Selection
|
||||
|
||||
```yaml
|
||||
@@ -230,6 +264,7 @@ After user selects a model:
|
||||
default_model:
|
||||
google: [value or null]
|
||||
openai: [value or null]
|
||||
azure: [value or null]
|
||||
openrouter: [value or null]
|
||||
dashscope: [value or null]
|
||||
replicate: [value or null]
|
||||
|
||||
@@ -11,7 +11,7 @@ description: EXTEND.md YAML schema for baoyu-image-gen user preferences
|
||||
---
|
||||
version: 1
|
||||
|
||||
default_provider: null # google|openai|openrouter|dashscope|replicate|null (null = auto-detect)
|
||||
default_provider: null # google|openai|azure|openrouter|dashscope|replicate|null (null = auto-detect)
|
||||
|
||||
default_quality: null # normal|2k|null (null = use default: 2k)
|
||||
|
||||
@@ -22,6 +22,7 @@ default_image_size: null # 1K|2K|4K|null (Google/OpenRouter, overrides qualit
|
||||
default_model:
|
||||
google: null # e.g., "gemini-3-pro-image-preview", "gemini-3.1-flash-image-preview"
|
||||
openai: null # e.g., "gpt-image-1.5", "gpt-image-1"
|
||||
azure: null # Azure deployment name, e.g., "gpt-image-1.5" or "image-prod"
|
||||
openrouter: null # e.g., "google/gemini-3.1-flash-image-preview"
|
||||
dashscope: null # e.g., "qwen-image-2.0-pro"
|
||||
replicate: null # e.g., "google/nano-banana-pro"
|
||||
@@ -38,6 +39,9 @@ batch:
|
||||
openai:
|
||||
concurrency: 3
|
||||
start_interval_ms: 1100
|
||||
azure:
|
||||
concurrency: 3
|
||||
start_interval_ms: 1100
|
||||
openrouter:
|
||||
concurrency: 3
|
||||
start_interval_ms: 1100
|
||||
@@ -58,6 +62,7 @@ batch:
|
||||
| `default_image_size` | string\|null | null | Google/OpenRouter image size (overrides quality) |
|
||||
| `default_model.google` | string\|null | null | Google default model |
|
||||
| `default_model.openai` | string\|null | null | OpenAI default model |
|
||||
| `default_model.azure` | string\|null | null | Azure default deployment name |
|
||||
| `default_model.openrouter` | string\|null | null | OpenRouter default model |
|
||||
| `default_model.dashscope` | string\|null | null | DashScope default model |
|
||||
| `default_model.replicate` | string\|null | null | Replicate default model |
|
||||
@@ -87,6 +92,7 @@ default_image_size: 2K
|
||||
default_model:
|
||||
google: "gemini-3-pro-image-preview"
|
||||
openai: "gpt-image-1.5"
|
||||
azure: "gpt-image-1.5"
|
||||
openrouter: "google/gemini-3.1-flash-image-preview"
|
||||
dashscope: "qwen-image-2.0-pro"
|
||||
replicate: "google/nano-banana-pro"
|
||||
@@ -96,6 +102,9 @@ batch:
|
||||
replicate:
|
||||
concurrency: 5
|
||||
start_interval_ms: 700
|
||||
azure:
|
||||
concurrency: 3
|
||||
start_interval_ms: 1100
|
||||
openrouter:
|
||||
concurrency: 3
|
||||
start_interval_ms: 1100
|
||||
|
||||
@@ -123,6 +123,7 @@ default_image_size: 2K
|
||||
default_model:
|
||||
google: gemini-3-pro-image-preview
|
||||
openai: gpt-image-1.5
|
||||
azure: image-prod
|
||||
batch:
|
||||
max_workers: 8
|
||||
provider_limits:
|
||||
@@ -131,6 +132,9 @@ batch:
|
||||
start_interval_ms: 900
|
||||
openai:
|
||||
concurrency: 4
|
||||
azure:
|
||||
concurrency: 1
|
||||
start_interval_ms: 1500
|
||||
`;
|
||||
|
||||
const config = parseSimpleYaml(yaml);
|
||||
@@ -142,6 +146,7 @@ batch:
|
||||
assert.equal(config.default_image_size, "2K");
|
||||
assert.equal(config.default_model?.google, "gemini-3-pro-image-preview");
|
||||
assert.equal(config.default_model?.openai, "gpt-image-1.5");
|
||||
assert.equal(config.default_model?.azure, "image-prod");
|
||||
assert.equal(config.batch?.max_workers, 8);
|
||||
assert.deepEqual(config.batch?.provider_limits?.google, {
|
||||
concurrency: 2,
|
||||
@@ -150,6 +155,10 @@ batch:
|
||||
assert.deepEqual(config.batch?.provider_limits?.openai, {
|
||||
concurrency: 4,
|
||||
});
|
||||
assert.deepEqual(config.batch?.provider_limits?.azure, {
|
||||
concurrency: 1,
|
||||
start_interval_ms: 1500,
|
||||
});
|
||||
});
|
||||
|
||||
test("mergeConfig only fills values missing from CLI args", () => {
|
||||
@@ -203,6 +212,8 @@ test("detectProvider selects an available ref-capable provider for reference-ima
|
||||
useEnv(t, {
|
||||
GOOGLE_API_KEY: null,
|
||||
OPENAI_API_KEY: "openai-key",
|
||||
AZURE_OPENAI_API_KEY: null,
|
||||
AZURE_OPENAI_BASE_URL: null,
|
||||
OPENROUTER_API_KEY: null,
|
||||
DASHSCOPE_API_KEY: null,
|
||||
REPLICATE_API_TOKEN: null,
|
||||
@@ -216,6 +227,27 @@ test("detectProvider selects an available ref-capable provider for reference-ima
|
||||
);
|
||||
});
|
||||
|
||||
test("detectProvider selects Azure when only Azure credentials are configured", (t) => {
|
||||
useEnv(t, {
|
||||
GOOGLE_API_KEY: null,
|
||||
OPENAI_API_KEY: null,
|
||||
AZURE_OPENAI_API_KEY: "azure-key",
|
||||
AZURE_OPENAI_BASE_URL: "https://example.openai.azure.com",
|
||||
OPENROUTER_API_KEY: null,
|
||||
DASHSCOPE_API_KEY: null,
|
||||
REPLICATE_API_TOKEN: null,
|
||||
JIMENG_ACCESS_KEY_ID: null,
|
||||
JIMENG_SECRET_ACCESS_KEY: null,
|
||||
ARK_API_KEY: null,
|
||||
});
|
||||
|
||||
assert.equal(detectProvider(makeArgs()), "azure");
|
||||
assert.equal(
|
||||
detectProvider(makeArgs({ referenceImages: ["ref.png"] })),
|
||||
"azure",
|
||||
);
|
||||
});
|
||||
|
||||
test("detectProvider infers Seedream from model id and allows Seedream reference-image workflows", (t) => {
|
||||
useEnv(t, {
|
||||
GOOGLE_API_KEY: null,
|
||||
|
||||
@@ -60,6 +60,7 @@ const DEFAULT_PROVIDER_RATE_LIMITS: Record<Provider, ProviderRateLimit> = {
|
||||
dashscope: { concurrency: 3, startIntervalMs: 1100 },
|
||||
jimeng: { concurrency: 3, startIntervalMs: 1100 },
|
||||
seedream: { concurrency: 3, startIntervalMs: 1100 },
|
||||
azure: { concurrency: 3, startIntervalMs: 1100 },
|
||||
};
|
||||
|
||||
function printUsage(): void {
|
||||
@@ -74,13 +75,13 @@ Options:
|
||||
--image <path> Output image path (required in single-image mode)
|
||||
--batchfile <path> JSON batch file for multi-image generation
|
||||
--jobs <count> Worker count for batch mode (default: auto, max from config, built-in default 10)
|
||||
--provider google|openai|openrouter|dashscope|replicate|jimeng|seedream Force provider (auto-detect by default)
|
||||
--provider google|openai|openrouter|dashscope|replicate|jimeng|seedream|azure Force provider (auto-detect by default)
|
||||
-m, --model <id> Model ID
|
||||
--ar <ratio> Aspect ratio (e.g., 16:9, 1:1, 4:3)
|
||||
--size <WxH> Size (e.g., 1024x1024)
|
||||
--quality normal|2k Quality preset (default: 2k)
|
||||
--imageSize 1K|2K|4K Image size for Google/OpenRouter (default: from quality)
|
||||
--ref <files...> Reference images (Google, OpenAI, OpenRouter, Replicate, or Seedream 4.0/4.5/5.0)
|
||||
--ref <files...> Reference images (Google, OpenAI, Azure, OpenRouter, Replicate, or Seedream 4.0/4.5/5.0)
|
||||
--n <count> Number of images for the current task (default: 1)
|
||||
--json JSON output
|
||||
-h, --help Show help
|
||||
@@ -131,6 +132,11 @@ Environment variables:
|
||||
DASHSCOPE_BASE_URL Custom DashScope endpoint
|
||||
REPLICATE_BASE_URL Custom Replicate endpoint
|
||||
JIMENG_BASE_URL Custom Jimeng endpoint
|
||||
AZURE_OPENAI_API_KEY Azure OpenAI API key
|
||||
AZURE_OPENAI_BASE_URL Azure OpenAI resource or deployment endpoint
|
||||
AZURE_OPENAI_DEPLOYMENT Default Azure deployment name
|
||||
AZURE_API_VERSION Azure API version (default: 2025-04-01-preview)
|
||||
AZURE_OPENAI_IMAGE_MODEL Backward-compatible Azure deployment/model alias (defaults to gpt-image-1.5)
|
||||
SEEDREAM_BASE_URL Custom Seedream endpoint
|
||||
BAOYU_IMAGE_GEN_MAX_WORKERS Override batch worker cap
|
||||
BAOYU_IMAGE_GEN_<PROVIDER>_CONCURRENCY Override provider concurrency
|
||||
@@ -231,7 +237,8 @@ export function parseArgs(argv: string[]): CliArgs {
|
||||
v !== "dashscope" &&
|
||||
v !== "replicate" &&
|
||||
v !== "jimeng" &&
|
||||
v !== "seedream"
|
||||
v !== "seedream" &&
|
||||
v !== "azure"
|
||||
) {
|
||||
throw new Error(`Invalid provider: ${v}`);
|
||||
}
|
||||
@@ -386,6 +393,7 @@ export function parseSimpleYaml(yaml: string): Partial<ExtendConfig> {
|
||||
replicate: null,
|
||||
jimeng: null,
|
||||
seedream: null,
|
||||
azure: null,
|
||||
};
|
||||
currentKey = "default_model";
|
||||
currentProvider = null;
|
||||
@@ -411,7 +419,8 @@ export function parseSimpleYaml(yaml: string): Partial<ExtendConfig> {
|
||||
key === "dashscope" ||
|
||||
key === "replicate" ||
|
||||
key === "jimeng" ||
|
||||
key === "seedream"
|
||||
key === "seedream" ||
|
||||
key === "azure"
|
||||
)
|
||||
) {
|
||||
config.batch ??= {};
|
||||
@@ -427,7 +436,8 @@ export function parseSimpleYaml(yaml: string): Partial<ExtendConfig> {
|
||||
key === "dashscope" ||
|
||||
key === "replicate" ||
|
||||
key === "jimeng" ||
|
||||
key === "seedream"
|
||||
key === "seedream" ||
|
||||
key === "azure"
|
||||
)
|
||||
) {
|
||||
const cleaned = value.replace(/['"]/g, "");
|
||||
@@ -520,9 +530,10 @@ export function getConfiguredProviderRateLimits(
|
||||
dashscope: { ...DEFAULT_PROVIDER_RATE_LIMITS.dashscope },
|
||||
jimeng: { ...DEFAULT_PROVIDER_RATE_LIMITS.jimeng },
|
||||
seedream: { ...DEFAULT_PROVIDER_RATE_LIMITS.seedream },
|
||||
azure: { ...DEFAULT_PROVIDER_RATE_LIMITS.azure },
|
||||
};
|
||||
|
||||
for (const provider of ["replicate", "google", "openai", "openrouter", "dashscope", "jimeng", "seedream"] as Provider[]) {
|
||||
for (const provider of ["replicate", "google", "openai", "openrouter", "dashscope", "jimeng", "seedream", "azure"] as Provider[]) {
|
||||
const envPrefix = `BAOYU_IMAGE_GEN_${provider.toUpperCase()}`;
|
||||
const extendLimit = extendConfig.batch?.provider_limits?.[provider];
|
||||
configured[provider] = {
|
||||
@@ -581,18 +592,20 @@ export function detectProvider(args: CliArgs): Provider {
|
||||
args.provider &&
|
||||
args.provider !== "google" &&
|
||||
args.provider !== "openai" &&
|
||||
args.provider !== "azure" &&
|
||||
args.provider !== "openrouter" &&
|
||||
args.provider !== "replicate" &&
|
||||
args.provider !== "seedream"
|
||||
) {
|
||||
throw new Error(
|
||||
"Reference images require a ref-capable provider. Use --provider google (Gemini multimodal), --provider openai (GPT Image edits), --provider openrouter (OpenRouter multimodal), --provider replicate, or --provider seedream for supported Seedream models."
|
||||
"Reference images require a ref-capable provider. Use --provider google (Gemini multimodal), --provider openai (GPT Image edits), --provider azure (Azure OpenAI), --provider openrouter (OpenRouter multimodal), --provider replicate, or --provider seedream for supported Seedream models."
|
||||
);
|
||||
}
|
||||
|
||||
if (args.provider) return args.provider;
|
||||
|
||||
const hasGoogle = !!(process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY);
|
||||
const hasAzure = !!(process.env.AZURE_OPENAI_API_KEY && process.env.AZURE_OPENAI_BASE_URL);
|
||||
const hasOpenai = !!process.env.OPENAI_API_KEY;
|
||||
const hasOpenrouter = !!process.env.OPENROUTER_API_KEY;
|
||||
const hasDashscope = !!process.env.DASHSCOPE_API_KEY;
|
||||
@@ -611,17 +624,19 @@ export function detectProvider(args: CliArgs): Provider {
|
||||
if (args.referenceImages.length > 0) {
|
||||
if (hasGoogle) return "google";
|
||||
if (hasOpenai) return "openai";
|
||||
if (hasAzure) return "azure";
|
||||
if (hasOpenrouter) return "openrouter";
|
||||
if (hasReplicate) return "replicate";
|
||||
if (hasSeedream) return "seedream";
|
||||
throw new Error(
|
||||
"Reference images require Google, OpenAI, OpenRouter, Replicate, or supported Seedream models. Set GOOGLE_API_KEY/GEMINI_API_KEY, OPENAI_API_KEY, OPENROUTER_API_KEY, REPLICATE_API_TOKEN, or ARK_API_KEY, or remove --ref."
|
||||
"Reference images require Google, OpenAI, Azure, OpenRouter, Replicate, or supported Seedream models. Set GOOGLE_API_KEY/GEMINI_API_KEY, OPENAI_API_KEY, AZURE_OPENAI_API_KEY+AZURE_OPENAI_BASE_URL, OPENROUTER_API_KEY, REPLICATE_API_TOKEN, or ARK_API_KEY, or remove --ref."
|
||||
);
|
||||
}
|
||||
|
||||
const available = [
|
||||
hasGoogle && "google",
|
||||
hasOpenai && "openai",
|
||||
hasAzure && "azure",
|
||||
hasOpenrouter && "openrouter",
|
||||
hasDashscope && "dashscope",
|
||||
hasReplicate && "replicate",
|
||||
@@ -633,7 +648,7 @@ export function detectProvider(args: CliArgs): Provider {
|
||||
if (available.length > 1) return available[0]!;
|
||||
|
||||
throw new Error(
|
||||
"No API key found. Set GOOGLE_API_KEY, GEMINI_API_KEY, OPENAI_API_KEY, OPENROUTER_API_KEY, DASHSCOPE_API_KEY, REPLICATE_API_TOKEN, JIMENG keys, or ARK_API_KEY.\n" +
|
||||
"No API key found. Set GOOGLE_API_KEY, GEMINI_API_KEY, OPENAI_API_KEY, AZURE_OPENAI_API_KEY+AZURE_OPENAI_BASE_URL, OPENROUTER_API_KEY, DASHSCOPE_API_KEY, REPLICATE_API_TOKEN, JIMENG keys, or ARK_API_KEY.\n" +
|
||||
"Create ~/.baoyu-skills/.env or <cwd>/.baoyu-skills/.env with your keys."
|
||||
);
|
||||
}
|
||||
@@ -676,6 +691,7 @@ async function loadProviderModule(provider: Provider): Promise<ProviderModule> {
|
||||
if (provider === "openrouter") return (await import("./providers/openrouter")) as ProviderModule;
|
||||
if (provider === "jimeng") return (await import("./providers/jimeng")) as ProviderModule;
|
||||
if (provider === "seedream") return (await import("./providers/seedream")) as ProviderModule;
|
||||
if (provider === "azure") return (await import("./providers/azure")) as ProviderModule;
|
||||
return (await import("./providers/openai")) as ProviderModule;
|
||||
}
|
||||
|
||||
@@ -704,6 +720,7 @@ function getModelForProvider(
|
||||
if (provider === "replicate" && extendConfig.default_model.replicate) return extendConfig.default_model.replicate;
|
||||
if (provider === "jimeng" && extendConfig.default_model.jimeng) return extendConfig.default_model.jimeng;
|
||||
if (provider === "seedream" && extendConfig.default_model.seedream) return extendConfig.default_model.seedream;
|
||||
if (provider === "azure" && extendConfig.default_model.azure) return extendConfig.default_model.azure;
|
||||
}
|
||||
return providerModule.getDefaultModel();
|
||||
}
|
||||
@@ -923,7 +940,7 @@ async function runBatchTasks(
|
||||
const acquireProvider = createProviderGate(providerRateLimits);
|
||||
const workerCount = getWorkerCount(tasks.length, jobs, maxWorkers);
|
||||
console.error(`Batch mode: ${tasks.length} tasks, ${workerCount} workers, parallel mode enabled.`);
|
||||
for (const provider of ["replicate", "google", "openai", "openrouter", "dashscope", "jimeng", "seedream"] as Provider[]) {
|
||||
for (const provider of ["replicate", "google", "openai", "openrouter", "dashscope", "jimeng", "seedream", "azure"] as Provider[]) {
|
||||
const limit = providerRateLimits[provider];
|
||||
console.error(`- ${provider}: concurrency=${limit.concurrency}, startIntervalMs=${limit.startIntervalMs}`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test, { type TestContext } from "node:test";
|
||||
|
||||
import type { CliArgs } from "../types.ts";
|
||||
import {
|
||||
generateImage,
|
||||
getDefaultModel,
|
||||
parseAzureBaseURL,
|
||||
validateArgs,
|
||||
} from "./azure.ts";
|
||||
|
||||
function useEnv(
|
||||
t: TestContext,
|
||||
values: Record<string, string | null>,
|
||||
): void {
|
||||
const previous = new Map<string, string | undefined>();
|
||||
for (const [key, value] of Object.entries(values)) {
|
||||
previous.set(key, process.env[key]);
|
||||
if (value == null) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
t.after(() => {
|
||||
for (const [key, value] of previous.entries()) {
|
||||
if (value == null) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = value;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function makeArgs(overrides: Partial<CliArgs> = {}): CliArgs {
|
||||
return {
|
||||
prompt: null,
|
||||
promptFiles: [],
|
||||
imagePath: null,
|
||||
provider: null,
|
||||
model: null,
|
||||
aspectRatio: null,
|
||||
size: null,
|
||||
quality: null,
|
||||
imageSize: null,
|
||||
referenceImages: [],
|
||||
n: 1,
|
||||
batchFile: null,
|
||||
jobs: null,
|
||||
json: false,
|
||||
help: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function makeTempDir(prefix: string): Promise<string> {
|
||||
return fs.mkdtemp(path.join(os.tmpdir(), prefix));
|
||||
}
|
||||
|
||||
test("Azure endpoint parsing and default deployment selection follow env precedence", (t) => {
|
||||
assert.deepEqual(parseAzureBaseURL("https://example.openai.azure.com"), {
|
||||
resourceBaseURL: "https://example.openai.azure.com/openai",
|
||||
deployment: null,
|
||||
});
|
||||
assert.deepEqual(
|
||||
parseAzureBaseURL("https://example.openai.azure.com/openai/deployments/from-url"),
|
||||
{
|
||||
resourceBaseURL: "https://example.openai.azure.com/openai",
|
||||
deployment: "from-url",
|
||||
},
|
||||
);
|
||||
|
||||
useEnv(t, {
|
||||
AZURE_OPENAI_BASE_URL: "https://example.openai.azure.com/openai/deployments/from-url",
|
||||
AZURE_OPENAI_DEPLOYMENT: "explicit-deploy",
|
||||
AZURE_OPENAI_IMAGE_MODEL: "env-fallback",
|
||||
});
|
||||
assert.equal(getDefaultModel(), "explicit-deploy");
|
||||
});
|
||||
|
||||
test("Azure validateArgs rejects unsupported edit input formats before the API call", () => {
|
||||
assert.doesNotThrow(() =>
|
||||
validateArgs("demo-deployment", makeArgs({ referenceImages: ["hero.png", "photo.jpeg"] })),
|
||||
);
|
||||
assert.throws(
|
||||
() => validateArgs("demo-deployment", makeArgs({ referenceImages: ["hero.webp"] })),
|
||||
/PNG or JPG\/JPEG/,
|
||||
);
|
||||
});
|
||||
|
||||
test("Azure image generation routes model to deployment and sends mapped quality", async (t) => {
|
||||
useEnv(t, {
|
||||
AZURE_OPENAI_API_KEY: "azure-key",
|
||||
AZURE_OPENAI_BASE_URL: "https://example.openai.azure.com/openai/deployments/default-deploy",
|
||||
AZURE_API_VERSION: null,
|
||||
AZURE_OPENAI_DEPLOYMENT: null,
|
||||
AZURE_OPENAI_IMAGE_MODEL: null,
|
||||
});
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
t.after(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
const calls: Array<{ url: string; body: string }> = [];
|
||||
globalThis.fetch = async (input, init) => {
|
||||
calls.push({
|
||||
url: String(input),
|
||||
body: String(init?.body ?? ""),
|
||||
});
|
||||
return Response.json({
|
||||
data: [{ b64_json: Buffer.from("azure-image").toString("base64") }],
|
||||
});
|
||||
};
|
||||
|
||||
const bytes = await generateImage(
|
||||
"A calm lake at sunset",
|
||||
"custom-deploy",
|
||||
makeArgs({ quality: "normal" }),
|
||||
);
|
||||
|
||||
assert.equal(Buffer.from(bytes).toString("utf8"), "azure-image");
|
||||
assert.equal(
|
||||
calls[0]?.url,
|
||||
"https://example.openai.azure.com/openai/deployments/custom-deploy/images/generations?api-version=2025-04-01-preview",
|
||||
);
|
||||
|
||||
const body = JSON.parse(calls[0]!.body) as Record<string, string>;
|
||||
assert.equal(body.quality, "medium");
|
||||
assert.equal(body.size, "1024x1024");
|
||||
});
|
||||
|
||||
test("Azure image edits include quality in multipart requests", async (t) => {
|
||||
const root = await makeTempDir("baoyu-image-gen-azure-");
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
|
||||
const pngPath = path.join(root, "ref.png");
|
||||
const jpgPath = path.join(root, "ref.jpg");
|
||||
await fs.writeFile(pngPath, "png-bytes");
|
||||
await fs.writeFile(jpgPath, "jpg-bytes");
|
||||
|
||||
useEnv(t, {
|
||||
AZURE_OPENAI_API_KEY: "azure-key",
|
||||
AZURE_OPENAI_BASE_URL: "https://example.openai.azure.com",
|
||||
AZURE_API_VERSION: "2025-04-01-preview",
|
||||
AZURE_OPENAI_DEPLOYMENT: null,
|
||||
AZURE_OPENAI_IMAGE_MODEL: null,
|
||||
});
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
t.after(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
const calls: Array<{ url: string; form: FormData }> = [];
|
||||
globalThis.fetch = async (input, init) => {
|
||||
calls.push({
|
||||
url: String(input),
|
||||
form: init?.body as FormData,
|
||||
});
|
||||
return Response.json({
|
||||
data: [{ b64_json: Buffer.from("edited-image").toString("base64") }],
|
||||
});
|
||||
};
|
||||
|
||||
const bytes = await generateImage(
|
||||
"Add warm lighting",
|
||||
"edit-deploy",
|
||||
makeArgs({
|
||||
quality: "2k",
|
||||
referenceImages: [pngPath, jpgPath],
|
||||
}),
|
||||
);
|
||||
|
||||
assert.equal(Buffer.from(bytes).toString("utf8"), "edited-image");
|
||||
assert.equal(
|
||||
calls[0]?.url,
|
||||
"https://example.openai.azure.com/openai/deployments/edit-deploy/images/edits?api-version=2025-04-01-preview",
|
||||
);
|
||||
assert.equal(calls[0]?.form.get("quality"), "high");
|
||||
assert.equal(calls[0]?.form.get("size"), "1024x1024");
|
||||
assert.equal(calls[0]?.form.getAll("image[]").length, 2);
|
||||
});
|
||||
@@ -0,0 +1,192 @@
|
||||
import path from "node:path";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import type { CliArgs } from "../types";
|
||||
import { getOpenAISize, extractImageFromResponse } from "./openai.ts";
|
||||
|
||||
type OpenAIImageResponse = { data: Array<{ url?: string; b64_json?: string }> };
|
||||
type AzureEndpoint = {
|
||||
resourceBaseURL: string;
|
||||
deployment: string | null;
|
||||
};
|
||||
|
||||
const DEFAULT_AZURE_API_VERSION = "2025-04-01-preview";
|
||||
const AZURE_EDIT_IMAGE_EXTENSIONS = new Set([".png", ".jpg", ".jpeg"]);
|
||||
|
||||
export function parseAzureBaseURL(url: string): AzureEndpoint {
|
||||
const parsed = new URL(url);
|
||||
const trimmedPath = parsed.pathname.replace(/\/+$/, "");
|
||||
const deploymentMatch = trimmedPath.match(/^(.*?)(?:\/openai)?\/deployments\/([^/]+)$/);
|
||||
|
||||
if (deploymentMatch) {
|
||||
parsed.pathname = `${deploymentMatch[1] || ""}/openai`;
|
||||
return {
|
||||
resourceBaseURL: parsed.toString().replace(/\/+$/, ""),
|
||||
deployment: decodeURIComponent(deploymentMatch[2]!),
|
||||
};
|
||||
}
|
||||
|
||||
parsed.pathname = trimmedPath.endsWith("/openai") ? trimmedPath : `${trimmedPath}/openai`;
|
||||
return {
|
||||
resourceBaseURL: parsed.toString().replace(/\/+$/, ""),
|
||||
deployment: null,
|
||||
};
|
||||
}
|
||||
|
||||
export function getDefaultModel(): string {
|
||||
const explicitDeployment = process.env.AZURE_OPENAI_DEPLOYMENT?.trim();
|
||||
if (explicitDeployment) return explicitDeployment;
|
||||
|
||||
const baseURL = process.env.AZURE_OPENAI_BASE_URL;
|
||||
if (baseURL) {
|
||||
try {
|
||||
const { deployment } = parseAzureBaseURL(baseURL);
|
||||
if (deployment) return deployment;
|
||||
} catch {
|
||||
// Ignore invalid URLs here so the required-env check can raise the user-facing error later.
|
||||
}
|
||||
}
|
||||
|
||||
return process.env.AZURE_OPENAI_IMAGE_MODEL || "gpt-image-1.5";
|
||||
}
|
||||
|
||||
function getEndpoint(): AzureEndpoint {
|
||||
const url = process.env.AZURE_OPENAI_BASE_URL;
|
||||
if (!url) {
|
||||
throw new Error(
|
||||
"AZURE_OPENAI_BASE_URL is required. Set it to your Azure resource or deployment endpoint, e.g.: https://your-resource.openai.azure.com or https://your-resource.openai.azure.com/openai/deployments/your-deployment"
|
||||
);
|
||||
}
|
||||
return parseAzureBaseURL(url);
|
||||
}
|
||||
|
||||
function getApiKey(): string {
|
||||
const key = process.env.AZURE_OPENAI_API_KEY;
|
||||
if (!key) {
|
||||
throw new Error(
|
||||
"AZURE_OPENAI_API_KEY is required. Get it from Azure Portal → your OpenAI resource → Keys and Endpoint."
|
||||
);
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
function getApiVersion(): string {
|
||||
return process.env.AZURE_API_VERSION || DEFAULT_AZURE_API_VERSION;
|
||||
}
|
||||
|
||||
function getDeployment(model: string): string {
|
||||
const deployment = model.trim();
|
||||
if (!deployment) {
|
||||
throw new Error(
|
||||
"Azure deployment name is required. Use --model <deployment>, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_IMAGE_MODEL, or embed the deployment in AZURE_OPENAI_BASE_URL."
|
||||
);
|
||||
}
|
||||
return deployment;
|
||||
}
|
||||
|
||||
function buildURL(deployment: string, pathSuffix: string): string {
|
||||
const { resourceBaseURL } = getEndpoint();
|
||||
return `${resourceBaseURL}/deployments/${encodeURIComponent(deployment)}${pathSuffix}?api-version=${getApiVersion()}`;
|
||||
}
|
||||
|
||||
function authHeaders(): Record<string, string> {
|
||||
return { "api-key": getApiKey() };
|
||||
}
|
||||
|
||||
function getAzureQuality(quality: CliArgs["quality"]): "medium" | "high" {
|
||||
return quality === "2k" ? "high" : "medium";
|
||||
}
|
||||
|
||||
export function validateArgs(_model: string, args: CliArgs): void {
|
||||
for (const refPath of args.referenceImages) {
|
||||
const ext = path.extname(refPath).toLowerCase();
|
||||
if (!AZURE_EDIT_IMAGE_EXTENSIONS.has(ext)) {
|
||||
throw new Error(
|
||||
`Azure OpenAI reference images must be PNG or JPG/JPEG. Unsupported file: ${refPath}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function generateImage(
|
||||
prompt: string,
|
||||
model: string,
|
||||
args: CliArgs
|
||||
): Promise<Uint8Array> {
|
||||
const deployment = getDeployment(model);
|
||||
const size = args.size || getOpenAISize(model, args.aspectRatio, args.quality);
|
||||
|
||||
if (args.referenceImages.length > 0) {
|
||||
return generateWithAzureEdits(prompt, deployment, size, args.referenceImages, args.quality);
|
||||
}
|
||||
|
||||
return generateWithAzureGenerations(prompt, deployment, size, args.quality);
|
||||
}
|
||||
|
||||
async function generateWithAzureGenerations(
|
||||
prompt: string,
|
||||
deployment: string,
|
||||
size: string,
|
||||
quality: CliArgs["quality"]
|
||||
): Promise<Uint8Array> {
|
||||
const body: Record<string, any> = {
|
||||
prompt,
|
||||
size,
|
||||
n: 1,
|
||||
quality: getAzureQuality(quality),
|
||||
};
|
||||
|
||||
const res = await fetch(buildURL(deployment, "/images/generations"), {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...authHeaders(),
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const err = await res.text();
|
||||
throw new Error(`Azure OpenAI API error: ${err}`);
|
||||
}
|
||||
|
||||
const result = (await res.json()) as OpenAIImageResponse;
|
||||
return extractImageFromResponse(result);
|
||||
}
|
||||
|
||||
async function generateWithAzureEdits(
|
||||
prompt: string,
|
||||
deployment: string,
|
||||
size: string,
|
||||
referenceImages: string[],
|
||||
quality: CliArgs["quality"]
|
||||
): Promise<Uint8Array> {
|
||||
const form = new FormData();
|
||||
form.append("prompt", prompt);
|
||||
form.append("size", size);
|
||||
form.append("n", "1");
|
||||
form.append("quality", getAzureQuality(quality));
|
||||
|
||||
for (const refPath of referenceImages) {
|
||||
const bytes = await readFile(refPath);
|
||||
const filename = path.basename(refPath);
|
||||
const mimeType = path.extname(filename).toLowerCase() === ".png" ? "image/png" : "image/jpeg";
|
||||
const blob = new Blob([bytes], { type: mimeType });
|
||||
form.append("image[]", blob, filename);
|
||||
}
|
||||
|
||||
const res = await fetch(buildURL(deployment, "/images/edits"), {
|
||||
method: "POST",
|
||||
headers: {
|
||||
...authHeaders(),
|
||||
},
|
||||
body: form,
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const err = await res.text();
|
||||
throw new Error(`Azure OpenAI edits API error: ${err}`);
|
||||
}
|
||||
|
||||
const result = (await res.json()) as OpenAIImageResponse;
|
||||
return extractImageFromResponse(result);
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import type { CliArgs } from "../types.ts";
|
||||
import {
|
||||
buildContent,
|
||||
buildRequestBody,
|
||||
extractImageFromResponse,
|
||||
getAspectRatio,
|
||||
getImageSize,
|
||||
validateArgs,
|
||||
} from "./openrouter.ts";
|
||||
|
||||
const GEMINI_MODEL = "google/gemini-3.1-flash-image-preview";
|
||||
const GEMINI_25_MODEL = "google/gemini-2.5-flash-image";
|
||||
const GPT_5_IMAGE_MODEL = "openai/gpt-5-image";
|
||||
const OPENROUTER_AUTO_MODEL = "openrouter/auto";
|
||||
const FLUX_MODEL = "black-forest-labs/flux.2-pro";
|
||||
|
||||
function makeArgs(overrides: Partial<CliArgs> = {}): CliArgs {
|
||||
return {
|
||||
prompt: null,
|
||||
promptFiles: [],
|
||||
imagePath: null,
|
||||
provider: null,
|
||||
model: null,
|
||||
aspectRatio: null,
|
||||
size: null,
|
||||
quality: null,
|
||||
imageSize: null,
|
||||
referenceImages: [],
|
||||
n: 1,
|
||||
batchFile: null,
|
||||
jobs: null,
|
||||
json: false,
|
||||
help: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test("OpenRouter request body uses image_config and string content for text-only prompts", () => {
|
||||
const args = makeArgs({ aspectRatio: "16:9", quality: "2k" });
|
||||
const body = buildRequestBody("hello", GEMINI_MODEL, args, []);
|
||||
|
||||
assert.deepEqual(body.image_config, {
|
||||
image_size: "2K",
|
||||
aspect_ratio: "16:9",
|
||||
});
|
||||
assert.deepEqual(body.provider, {
|
||||
require_parameters: true,
|
||||
});
|
||||
assert.deepEqual(body.modalities, ["image", "text"]);
|
||||
assert.equal(body.stream, false);
|
||||
assert.equal(body.messages[0].content, "hello");
|
||||
});
|
||||
|
||||
test("OpenRouter request body keeps text+image modalities for current text+image models", () => {
|
||||
for (const model of [GEMINI_MODEL, GEMINI_25_MODEL, GPT_5_IMAGE_MODEL, OPENROUTER_AUTO_MODEL]) {
|
||||
const body = buildRequestBody("hello", model, makeArgs({ quality: "2k" }), []);
|
||||
|
||||
assert.deepEqual(body.image_config, {
|
||||
image_size: "2K",
|
||||
});
|
||||
assert.deepEqual(body.provider, {
|
||||
require_parameters: true,
|
||||
});
|
||||
assert.deepEqual(body.modalities, ["image", "text"]);
|
||||
assert.equal(body.messages[0].content, "hello");
|
||||
}
|
||||
});
|
||||
|
||||
test("OpenRouter request body uses image-only modalities for image-only models under CLI defaults", () => {
|
||||
const body = buildRequestBody("hello", FLUX_MODEL, makeArgs({ quality: "2k" }), []);
|
||||
|
||||
assert.deepEqual(body.image_config, {
|
||||
image_size: "2K",
|
||||
});
|
||||
assert.deepEqual(body.provider, {
|
||||
require_parameters: true,
|
||||
});
|
||||
assert.deepEqual(body.modalities, ["image"]);
|
||||
assert.equal(body.stream, false);
|
||||
assert.equal(body.messages[0].content, "hello");
|
||||
});
|
||||
|
||||
test("OpenRouter helper omits image_config when no size or quality is passed", () => {
|
||||
const body = buildRequestBody("hello", FLUX_MODEL, makeArgs(), []);
|
||||
|
||||
assert.equal(body.image_config, undefined);
|
||||
assert.equal(body.provider, undefined);
|
||||
assert.deepEqual(body.modalities, ["image"]);
|
||||
assert.equal(body.stream, false);
|
||||
assert.equal(body.messages[0].content, "hello");
|
||||
});
|
||||
|
||||
test("OpenRouter request body keeps multimodal array content when references are provided", () => {
|
||||
const content = buildContent("hello", ["data:image/png;base64,abc"]);
|
||||
assert.ok(Array.isArray(content));
|
||||
assert.deepEqual(content[0], { type: "text", text: "hello" });
|
||||
assert.deepEqual(content[1], {
|
||||
type: "image_url",
|
||||
image_url: { url: "data:image/png;base64,abc" },
|
||||
});
|
||||
});
|
||||
|
||||
test("OpenRouter size and aspect helpers infer supported values", () => {
|
||||
assert.equal(getImageSize(makeArgs()), null);
|
||||
assert.equal(getImageSize(makeArgs({ quality: "normal" })), "1K");
|
||||
assert.equal(getImageSize(makeArgs({ size: "2048x1024" })), "2K");
|
||||
assert.equal(getAspectRatio(GEMINI_MODEL, makeArgs({ size: "1600x900" })), "16:9");
|
||||
assert.equal(getAspectRatio(GEMINI_MODEL, makeArgs({ size: "1024x4096" })), "1:4");
|
||||
assert.equal(getAspectRatio(GEMINI_25_MODEL, makeArgs({ size: "1600x900" })), "16:9");
|
||||
assert.equal(getAspectRatio(FLUX_MODEL, makeArgs({ size: "1024x4096" })), null);
|
||||
});
|
||||
|
||||
test("OpenRouter validates explicit aspect ratios and inferred size ratios against model support", () => {
|
||||
assert.doesNotThrow(() =>
|
||||
validateArgs(GEMINI_MODEL, makeArgs({ aspectRatio: "1:4" })),
|
||||
);
|
||||
assert.doesNotThrow(() =>
|
||||
validateArgs(GEMINI_MODEL, makeArgs({ size: "1024x4096" })),
|
||||
);
|
||||
assert.throws(
|
||||
() => validateArgs(GEMINI_25_MODEL, makeArgs({ aspectRatio: "1:4" })),
|
||||
/does not support aspect ratio 1:4/,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateArgs(FLUX_MODEL, makeArgs({ aspectRatio: "1:4" })),
|
||||
/does not support aspect ratio 1:4/,
|
||||
);
|
||||
assert.throws(
|
||||
() => validateArgs(GEMINI_MODEL, makeArgs({ size: "2048x1024" })),
|
||||
/does not support size 2048x1024 \(aspect ratio 2:1\)/,
|
||||
);
|
||||
});
|
||||
|
||||
test("OpenRouter response extraction supports inline image data and finish_reason errors", async () => {
|
||||
const bytes = await extractImageFromResponse({
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
images: [
|
||||
{
|
||||
image_url: {
|
||||
url: `data:image/png;base64,${Buffer.from("hello").toString("base64")}`,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.equal(Buffer.from(bytes).toString("utf8"), "hello");
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
extractImageFromResponse({
|
||||
choices: [
|
||||
{
|
||||
finish_reason: "error",
|
||||
native_finish_reason: "MALFORMED_FUNCTION_CALL",
|
||||
message: { content: null },
|
||||
},
|
||||
],
|
||||
}),
|
||||
/finish_reason=MALFORMED_FUNCTION_CALL/,
|
||||
);
|
||||
});
|
||||
@@ -3,6 +3,19 @@ import { readFile } from "node:fs/promises";
|
||||
import type { CliArgs } from "../types";
|
||||
|
||||
const DEFAULT_MODEL = "google/gemini-3.1-flash-image-preview";
|
||||
const COMMON_ASPECT_RATIOS = [
|
||||
"1:1",
|
||||
"2:3",
|
||||
"3:2",
|
||||
"3:4",
|
||||
"4:3",
|
||||
"4:5",
|
||||
"5:4",
|
||||
"9:16",
|
||||
"16:9",
|
||||
"21:9",
|
||||
];
|
||||
const GEMINI_EXTENDED_ASPECT_RATIOS = ["1:4", "4:1", "1:8", "8:1"];
|
||||
|
||||
type OpenRouterImageEntry = {
|
||||
image_url?: string | { url?: string | null } | null;
|
||||
@@ -18,9 +31,11 @@ type OpenRouterMessagePart = {
|
||||
|
||||
type OpenRouterResponse = {
|
||||
choices?: Array<{
|
||||
finish_reason?: string | null;
|
||||
native_finish_reason?: string | null;
|
||||
message?: {
|
||||
images?: OpenRouterImageEntry[];
|
||||
content?: string | OpenRouterMessagePart[];
|
||||
content?: string | OpenRouterMessagePart[] | null;
|
||||
};
|
||||
}>;
|
||||
};
|
||||
@@ -29,6 +44,36 @@ export function getDefaultModel(): string {
|
||||
return process.env.OPENROUTER_IMAGE_MODEL || DEFAULT_MODEL;
|
||||
}
|
||||
|
||||
function normalizeModelId(model: string): string {
|
||||
return model.trim().toLowerCase().split(":")[0]!;
|
||||
}
|
||||
|
||||
function isTextAndImageModel(model: string): boolean {
|
||||
const normalized = normalizeModelId(model);
|
||||
if (normalized === "openrouter/auto") {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (normalized.startsWith("google/gemini-") && normalized.includes("image")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (normalized.startsWith("openai/gpt-") && normalized.includes("image")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function getSupportedAspectRatios(model: string): Set<string> {
|
||||
const normalized = normalizeModelId(model);
|
||||
if (normalized !== "google/gemini-3.1-flash-image-preview") {
|
||||
return new Set(COMMON_ASPECT_RATIOS);
|
||||
}
|
||||
|
||||
return new Set([...COMMON_ASPECT_RATIOS, ...GEMINI_EXTENDED_ASPECT_RATIOS]);
|
||||
}
|
||||
|
||||
function getApiKey(): string | null {
|
||||
return process.env.OPENROUTER_API_KEY || null;
|
||||
}
|
||||
@@ -103,17 +148,50 @@ function inferImageSize(size: string | null): "1K" | "2K" | "4K" | null {
|
||||
return "4K";
|
||||
}
|
||||
|
||||
function getImageSize(args: CliArgs): "1K" | "2K" | "4K" {
|
||||
export function getImageSize(args: CliArgs): "1K" | "2K" | "4K" | null {
|
||||
if (args.imageSize) return args.imageSize as "1K" | "2K" | "4K";
|
||||
|
||||
const inferredFromSize = inferImageSize(args.size);
|
||||
if (inferredFromSize) return inferredFromSize;
|
||||
|
||||
return args.quality === "normal" ? "1K" : "2K";
|
||||
if (args.quality === "normal") return "1K";
|
||||
if (args.quality === "2k") return "2K";
|
||||
return null;
|
||||
}
|
||||
|
||||
function getAspectRatio(args: CliArgs): string | null {
|
||||
return args.aspectRatio || inferAspectRatio(args.size);
|
||||
export function getAspectRatio(model: string, args: CliArgs): string | null {
|
||||
if (args.aspectRatio) return args.aspectRatio;
|
||||
|
||||
const inferred = inferAspectRatio(args.size);
|
||||
if (!inferred || !getSupportedAspectRatios(model).has(inferred)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return inferred;
|
||||
}
|
||||
|
||||
function getModalities(model: string): string[] {
|
||||
return isTextAndImageModel(model) ? ["image", "text"] : ["image"];
|
||||
}
|
||||
|
||||
export function validateArgs(model: string, args: CliArgs): void {
|
||||
const requestedAspectRatio = args.aspectRatio || inferAspectRatio(args.size);
|
||||
if (!requestedAspectRatio) {
|
||||
return;
|
||||
}
|
||||
|
||||
const supported = getSupportedAspectRatios(model);
|
||||
if (supported.has(requestedAspectRatio)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const requestedValue = args.aspectRatio
|
||||
? `aspect ratio ${requestedAspectRatio}`
|
||||
: `size ${args.size} (aspect ratio ${requestedAspectRatio})`;
|
||||
|
||||
throw new Error(
|
||||
`OpenRouter model ${model} does not support ${requestedValue}. Supported values: ${Array.from(supported).join(", ")}`
|
||||
);
|
||||
}
|
||||
|
||||
function getMimeType(filename: string): string {
|
||||
@@ -129,7 +207,14 @@ async function readImageAsDataUrl(filePath: string): Promise<string> {
|
||||
return `data:${getMimeType(filePath)};base64,${bytes.toString("base64")}`;
|
||||
}
|
||||
|
||||
function buildContent(prompt: string, referenceImages: string[]): Array<Record<string, unknown>> {
|
||||
export function buildContent(
|
||||
prompt: string,
|
||||
referenceImages: string[],
|
||||
): string | Array<Record<string, unknown>> {
|
||||
if (referenceImages.length === 0) {
|
||||
return prompt;
|
||||
}
|
||||
|
||||
const content: Array<Record<string, unknown>> = [{ type: "text", text: prompt }];
|
||||
|
||||
for (const imageUrl of referenceImages) {
|
||||
@@ -171,8 +256,9 @@ async function downloadImage(value: string): Promise<Uint8Array> {
|
||||
return Uint8Array.from(Buffer.from(value, "base64"));
|
||||
}
|
||||
|
||||
async function extractImageFromResponse(result: OpenRouterResponse): Promise<Uint8Array> {
|
||||
const message = result.choices?.[0]?.message;
|
||||
export async function extractImageFromResponse(result: OpenRouterResponse): Promise<Uint8Array> {
|
||||
const choice = result.choices?.[0];
|
||||
const message = choice?.message;
|
||||
|
||||
for (const image of message?.images ?? []) {
|
||||
const imageUrl = extractImageUrl(image);
|
||||
@@ -194,7 +280,52 @@ async function extractImageFromResponse(result: OpenRouterResponse): Promise<Uin
|
||||
if (inline) return inline;
|
||||
}
|
||||
|
||||
throw new Error("No image in OpenRouter response");
|
||||
const finishReason =
|
||||
choice?.native_finish_reason || choice?.finish_reason || "unknown";
|
||||
throw new Error(
|
||||
`No image in OpenRouter response (finish_reason=${finishReason})`,
|
||||
);
|
||||
}
|
||||
|
||||
export function buildRequestBody(
|
||||
prompt: string,
|
||||
model: string,
|
||||
args: CliArgs,
|
||||
referenceImages: string[],
|
||||
): Record<string, unknown> {
|
||||
validateArgs(model, args);
|
||||
|
||||
const imageConfig: Record<string, string> = {};
|
||||
|
||||
const imageSize = getImageSize(args);
|
||||
if (imageSize) {
|
||||
imageConfig.image_size = imageSize;
|
||||
}
|
||||
|
||||
const aspectRatio = getAspectRatio(model, args);
|
||||
if (aspectRatio) {
|
||||
imageConfig.aspect_ratio = aspectRatio;
|
||||
}
|
||||
|
||||
const body: Record<string, unknown> = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: buildContent(prompt, referenceImages),
|
||||
},
|
||||
],
|
||||
modalities: getModalities(model),
|
||||
stream: false,
|
||||
};
|
||||
|
||||
if (Object.keys(imageConfig).length > 0) {
|
||||
body.image_config = imageConfig;
|
||||
body.provider = {
|
||||
require_parameters: true,
|
||||
};
|
||||
}
|
||||
|
||||
return body;
|
||||
}
|
||||
|
||||
export async function generateImage(
|
||||
@@ -212,32 +343,15 @@ export async function generateImage(
|
||||
referenceImages.push(await readImageAsDataUrl(refPath));
|
||||
}
|
||||
|
||||
const imageGenerationOptions: Record<string, string> = {
|
||||
size: getImageSize(args),
|
||||
};
|
||||
|
||||
const aspectRatio = getAspectRatio(args);
|
||||
if (aspectRatio) {
|
||||
imageGenerationOptions.aspect_ratio = aspectRatio;
|
||||
}
|
||||
|
||||
const body = {
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: buildContent(prompt, referenceImages),
|
||||
},
|
||||
],
|
||||
modalities: ["image", "text"],
|
||||
max_tokens: 256,
|
||||
imageGenerationOptions,
|
||||
providerPreferences: {
|
||||
require_parameters: true,
|
||||
},
|
||||
...buildRequestBody(prompt, model, args, referenceImages),
|
||||
};
|
||||
|
||||
console.log(`Generating image with OpenRouter (${model})...`, imageGenerationOptions);
|
||||
console.log(
|
||||
`Generating image with OpenRouter (${model})...`,
|
||||
(body.image_config as Record<string, string>),
|
||||
);
|
||||
|
||||
const response = await fetch(`${getBaseUrl()}/chat/completions`, {
|
||||
method: "POST",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export type Provider = "google" | "openai" | "openrouter" | "dashscope" | "replicate" | "jimeng" | "seedream";
|
||||
export type Provider = "google" | "openai" | "openrouter" | "dashscope" | "replicate" | "jimeng" | "seedream" | "azure";
|
||||
export type Quality = "normal" | "2k";
|
||||
|
||||
export type CliArgs = {
|
||||
@@ -55,6 +55,7 @@ export type ExtendConfig = {
|
||||
replicate: string | null;
|
||||
jimeng: string | null;
|
||||
seedream: string | null;
|
||||
azure: string | null;
|
||||
};
|
||||
batch?: {
|
||||
max_workers?: number | null;
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
const SCRIPT_PATH = path.join(SCRIPT_DIR, "main.ts");
|
||||
|
||||
async function makeTempDir(prefix: string): Promise<string> {
|
||||
return fs.mkdtemp(path.join(os.tmpdir(), prefix));
|
||||
}
|
||||
|
||||
test("CLI forwards wrapper title and vendor render options", async () => {
|
||||
const root = await makeTempDir("baoyu-markdown-to-html-cli-");
|
||||
const markdownPath = path.join(root, "article.md");
|
||||
await fs.writeFile(markdownPath, "## Section\n\nParagraph with **bold** text.\n", "utf-8");
|
||||
|
||||
const { stdout } = await execFileAsync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
SCRIPT_PATH,
|
||||
markdownPath,
|
||||
"--theme", "grace",
|
||||
"--color", "red",
|
||||
"--font-family", "mono",
|
||||
"--font-size", "18",
|
||||
"--keep-title",
|
||||
"--title", "Overridden",
|
||||
],
|
||||
{ cwd: SCRIPT_DIR },
|
||||
);
|
||||
|
||||
const result = JSON.parse(stdout.trim()) as {
|
||||
htmlPath: string;
|
||||
title: string;
|
||||
};
|
||||
|
||||
assert.equal(result.title, "Overridden");
|
||||
|
||||
const html = await fs.readFile(result.htmlPath, "utf-8");
|
||||
assert.match(html, /<title>Overridden<\/title>/);
|
||||
assert.match(html, /<h2[^>]*style="[^"]*background: #A93226/);
|
||||
assert.match(html, /<strong[^>]*style="[^"]*color: #A93226/);
|
||||
assert.match(
|
||||
html,
|
||||
/<body[^>]*style="[^"]*font-family: Menlo, Monaco, 'Courier New', monospace;[^"]*font-size: 18px/,
|
||||
);
|
||||
});
|
||||
@@ -4,16 +4,22 @@ import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
import {
|
||||
COLOR_PRESETS,
|
||||
FONT_FAMILY_MAP,
|
||||
FONT_SIZE_OPTIONS,
|
||||
THEME_NAMES,
|
||||
extractSummaryFromBody,
|
||||
extractTitleFromMarkdown,
|
||||
formatTimestamp,
|
||||
parseArgs,
|
||||
parseFrontmatter,
|
||||
renderMarkdownDocument,
|
||||
replaceMarkdownImagesWithPlaceholders,
|
||||
resolveContentImages,
|
||||
serializeFrontmatter,
|
||||
stripWrappingQuotes,
|
||||
} from "baoyu-md";
|
||||
} from "./vendor/baoyu-md/src/index.ts";
|
||||
import type { CliOptions } from "./vendor/baoyu-md/src/types.ts";
|
||||
|
||||
interface ImageInfo {
|
||||
placeholder: string;
|
||||
@@ -30,9 +36,13 @@ interface ParsedResult {
|
||||
contentImages: ImageInfo[];
|
||||
}
|
||||
|
||||
type ConvertMarkdownOptions = Partial<Omit<CliOptions, "inputPath">> & {
|
||||
title?: string;
|
||||
};
|
||||
|
||||
export async function convertMarkdown(
|
||||
markdownPath: string,
|
||||
options?: { title?: string; theme?: string; keepTitle?: boolean; citeStatus?: boolean },
|
||||
options?: ConvertMarkdownOptions,
|
||||
): Promise<ParsedResult> {
|
||||
const baseDir = path.dirname(markdownPath);
|
||||
const content = fs.readFileSync(markdownPath, "utf-8");
|
||||
@@ -56,20 +66,32 @@ export async function convertMarkdown(
|
||||
summary = extractSummaryFromBody(body, 120);
|
||||
}
|
||||
|
||||
const effectiveFrontmatter = options?.title
|
||||
? { ...frontmatter, title }
|
||||
: frontmatter;
|
||||
|
||||
const { images, markdown: rewrittenBody } = replaceMarkdownImagesWithPlaceholders(
|
||||
body,
|
||||
"MDTOHTMLIMGPH_",
|
||||
);
|
||||
const rewrittenMarkdown = `${serializeFrontmatter(frontmatter)}${rewrittenBody}`;
|
||||
const rewrittenMarkdown = `${serializeFrontmatter(effectiveFrontmatter)}${rewrittenBody}`;
|
||||
|
||||
console.error(
|
||||
`[markdown-to-html] Rendering with theme: ${theme ?? "default"}, keepTitle: ${keepTitle}, citeStatus: ${citeStatus}`,
|
||||
);
|
||||
|
||||
const { html } = await renderMarkdownDocument(rewrittenMarkdown, {
|
||||
codeTheme: options?.codeTheme,
|
||||
countStatus: options?.countStatus,
|
||||
citeStatus,
|
||||
defaultTitle: title,
|
||||
fontFamily: options?.fontFamily,
|
||||
fontSize: options?.fontSize,
|
||||
isMacCodeBlock: options?.isMacCodeBlock,
|
||||
isShowLineNumber: options?.isShowLineNumber,
|
||||
keepTitle,
|
||||
legend: options?.legend,
|
||||
primaryColor: options?.primaryColor,
|
||||
theme,
|
||||
});
|
||||
|
||||
@@ -111,7 +133,10 @@ export async function convertMarkdown(
|
||||
};
|
||||
}
|
||||
|
||||
function printUsage(): never {
|
||||
function printUsage(exitCode = 0): never {
|
||||
const colorNames = Object.keys(COLOR_PRESETS).join(", ");
|
||||
const fontFamilyNames = Object.keys(FONT_FAMILY_MAP).join(", ");
|
||||
|
||||
console.log(`Convert Markdown to styled HTML
|
||||
|
||||
Usage:
|
||||
@@ -119,8 +144,17 @@ Usage:
|
||||
|
||||
Options:
|
||||
--title <title> Override title
|
||||
--theme <name> Theme name (default, grace, simple). Default: default
|
||||
--theme <name> Theme name (${THEME_NAMES.join(", ")}). Default: default
|
||||
--color <name|hex> Primary color: ${colorNames}
|
||||
--font-family <name> Font: ${fontFamilyNames}, or CSS value
|
||||
--font-size <N> Font size: ${FONT_SIZE_OPTIONS.join(", ")} (default: 16px)
|
||||
--code-theme <name> Code highlight theme (default: github)
|
||||
--mac-code-block Show Mac-style code block header
|
||||
--no-mac-code-block Hide Mac-style code block header
|
||||
--line-number Show line numbers in code blocks
|
||||
--cite Convert ordinary external links to bottom citations. Default: off
|
||||
--count Show reading time / word count
|
||||
--legend <value> Image caption: title-alt, alt-title, title, alt, none
|
||||
--keep-title Keep the first heading in content. Default: false (removed)
|
||||
--help Show this help
|
||||
|
||||
@@ -142,40 +176,60 @@ Output JSON format:
|
||||
Example:
|
||||
npx -y bun main.ts article.md
|
||||
npx -y bun main.ts article.md --theme grace
|
||||
npx -y bun main.ts article.md --theme modern --color red
|
||||
npx -y bun main.ts article.md --cite
|
||||
`);
|
||||
process.exit(0);
|
||||
process.exit(exitCode);
|
||||
}
|
||||
|
||||
function parseArgValue(argv: string[], i: number, flag: string): string | null {
|
||||
const arg = argv[i]!;
|
||||
if (arg.includes("=")) {
|
||||
return arg.slice(flag.length + 1);
|
||||
}
|
||||
const next = argv[i + 1];
|
||||
return next ?? null;
|
||||
}
|
||||
|
||||
function extractTitleArg(argv: string[]): { renderArgs: string[]; title?: string } {
|
||||
let title: string | undefined;
|
||||
const renderArgs: string[] = [];
|
||||
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const arg = argv[i]!;
|
||||
if (arg === "--title" || arg.startsWith("--title=")) {
|
||||
const value = parseArgValue(argv, i, "--title");
|
||||
if (!value) {
|
||||
console.error("Missing value for --title");
|
||||
printUsage(1);
|
||||
}
|
||||
title = value;
|
||||
if (!arg.includes("=")) {
|
||||
i += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
renderArgs.push(arg);
|
||||
}
|
||||
|
||||
return { renderArgs, title };
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = process.argv.slice(2);
|
||||
if (args.length === 0 || args.includes("--help") || args.includes("-h")) {
|
||||
printUsage();
|
||||
printUsage(0);
|
||||
}
|
||||
|
||||
let markdownPath: string | undefined;
|
||||
let title: string | undefined;
|
||||
let theme: string | undefined;
|
||||
let citeStatus = false;
|
||||
let keepTitle = false;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i]!;
|
||||
if (arg === "--title" && args[i + 1]) {
|
||||
title = args[++i];
|
||||
} else if (arg === "--theme" && args[i + 1]) {
|
||||
theme = args[++i];
|
||||
} else if (arg === "--cite") {
|
||||
citeStatus = true;
|
||||
} else if (arg === "--keep-title") {
|
||||
keepTitle = true;
|
||||
} else if (!arg.startsWith("-")) {
|
||||
markdownPath = arg;
|
||||
}
|
||||
const { renderArgs, title } = extractTitleArg(args);
|
||||
const options = parseArgs(renderArgs);
|
||||
if (!options) {
|
||||
printUsage(1);
|
||||
}
|
||||
|
||||
if (!markdownPath) {
|
||||
console.error("Error: Markdown file path is required");
|
||||
const markdownPath = path.resolve(process.cwd(), options.inputPath);
|
||||
if (!markdownPath.toLowerCase().endsWith(".md")) {
|
||||
console.error("Input file must end with .md");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -184,7 +238,7 @@ async function main(): Promise<void> {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const result = await convertMarkdown(markdownPath, { title, theme, keepTitle, citeStatus });
|
||||
const result = await convertMarkdown(markdownPath, { ...options, title });
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
|
||||
@@ -9,12 +9,26 @@ import { COLOR_PRESETS, FONT_FAMILY_MAP } from "./constants.ts";
|
||||
import {
|
||||
buildMarkdownDocumentMeta,
|
||||
formatTimestamp,
|
||||
renderMarkdownDocument,
|
||||
resolveColorToken,
|
||||
resolveFontFamilyToken,
|
||||
resolveMarkdownStyle,
|
||||
resolveRenderOptions,
|
||||
} from "./document.ts";
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, `\\$&`);
|
||||
}
|
||||
|
||||
function findInlineStyle(html: string, tagName: string, text: string): string {
|
||||
const pattern = new RegExp(
|
||||
`<${tagName}[^>]*style="([^"]*)"[^>]*>${escapeRegExp(text)}</${tagName}>`,
|
||||
);
|
||||
const match = html.match(pattern);
|
||||
assert.ok(match, `Expected inline style for <${tagName}>${text}</${tagName}>`);
|
||||
return match![1]!;
|
||||
}
|
||||
|
||||
function useCwd(t: TestContext, cwd: string): void {
|
||||
const previous = process.cwd();
|
||||
process.chdir(cwd);
|
||||
@@ -138,3 +152,23 @@ keep_title: true
|
||||
assert.equal(explicit.fontSize, "18px");
|
||||
assert.equal(explicit.keepTitle, false);
|
||||
});
|
||||
|
||||
test("renderMarkdownDocument layers default rules into grace theme before CSS inlining", async () => {
|
||||
const { html } = await renderMarkdownDocument(
|
||||
`## Section\n\nParagraph with **bold** text.`,
|
||||
{ keepTitle: true, theme: "grace" },
|
||||
);
|
||||
|
||||
const h2Style = findInlineStyle(html, "h2", "Section");
|
||||
assert.match(h2Style, /background: #92617E/);
|
||||
assert.match(h2Style, /box-shadow: 0 4px 6px rgba\(0, 0, 0, 0\.1\)/);
|
||||
|
||||
const pMatch = html.match(/<p[^>]*style="([^"]*)"[^>]*>/);
|
||||
assert.ok(pMatch, "Expected inline style on <p> tag");
|
||||
assert.match(pMatch![1]!, /color:/);
|
||||
|
||||
const strongPattern = /<strong[^>]*style="([^"]*)"[^>]*>bold<\/strong>/;
|
||||
const strongMatch = html.match(strongPattern);
|
||||
assert.ok(strongMatch, "Expected inline style for <strong>bold</strong>");
|
||||
assert.match(strongMatch![1]!, /font-weight:/);
|
||||
});
|
||||
|
||||
+11
@@ -59,6 +59,17 @@ test("normalizeCssText and normalizeInlineCss replace variables and strip declar
|
||||
assert.doesNotMatch(normalizedHtml, /var\(--md-primary-color\)/);
|
||||
});
|
||||
|
||||
test("normalizeInlineCss removes quoted custom property values without leaving fragments behind", () => {
|
||||
const normalizedHtml = normalizeInlineCss(
|
||||
`<html style="--md-font-family: Menlo, Monaco, 'Courier New', monospace; color: var(--md-primary-color)"></html>`,
|
||||
DEFAULT_STYLE,
|
||||
);
|
||||
|
||||
assert.match(normalizedHtml, /style=" color: #0F4C81"/);
|
||||
assert.doesNotMatch(normalizedHtml, /Courier New/);
|
||||
assert.doesNotMatch(normalizedHtml, /--md-font-family/);
|
||||
});
|
||||
|
||||
test("HTML structure helpers hoist nested lists and remove the first heading", () => {
|
||||
const nestedList = `<ul><li>Parent<ul><li>Child</li></ul></li></ul>`;
|
||||
assert.equal(
|
||||
|
||||
@@ -100,13 +100,13 @@ export function normalizeCssText(cssText: string, style: StyleConfig = DEFAULT_S
|
||||
.replace(/var\(--md-accent-color\)/g, style.accentColor)
|
||||
.replace(/var\(--md-container-bg\)/g, style.containerBg)
|
||||
.replace(/hsl\(var\(--foreground\)\)/g, "#3f3f3f")
|
||||
.replace(/--md-primary-color:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-font-family:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-font-size:\s*[^;"']+;?/g, "")
|
||||
.replace(/--blockquote-background:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-accent-color:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-container-bg:\s*[^;"']+;?/g, "")
|
||||
.replace(/--foreground:\s*[^;"']+;?/g, "");
|
||||
.replace(/--md-primary-color:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-font-family:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-font-size:\s*[^;]+;?/g, "")
|
||||
.replace(/--blockquote-background:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-accent-color:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-container-bg:\s*[^;]+;?/g, "")
|
||||
.replace(/--foreground:\s*[^;]+;?/g, "");
|
||||
}
|
||||
|
||||
export function normalizeInlineCss(html: string, style: StyleConfig = DEFAULT_STYLE): string {
|
||||
|
||||
@@ -6,6 +6,7 @@ import type { ThemeName } from "./types.js";
|
||||
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
export const THEME_DIR = path.resolve(SCRIPT_DIR, "themes");
|
||||
const FALLBACK_THEMES: ThemeName[] = ["default", "grace", "simple"];
|
||||
const THEMES_EXTENDING_DEFAULT = new Set<ThemeName>(["grace", "simple"]);
|
||||
|
||||
function stripOutputScope(cssContent: string): string {
|
||||
let css = cssContent;
|
||||
@@ -41,6 +42,7 @@ export function loadThemeCss(theme: ThemeName): {
|
||||
themeCss: string;
|
||||
} {
|
||||
const basePath = path.join(THEME_DIR, "base.css");
|
||||
const defaultThemePath = path.join(THEME_DIR, "default.css");
|
||||
const themePath = path.join(THEME_DIR, `${theme}.css`);
|
||||
|
||||
if (!fs.existsSync(basePath)) {
|
||||
@@ -51,9 +53,18 @@ export function loadThemeCss(theme: ThemeName): {
|
||||
throw new Error(`Missing theme CSS for "${theme}": ${themePath}`);
|
||||
}
|
||||
|
||||
const layeredThemeCss: string[] = [];
|
||||
if (theme !== "default" && THEMES_EXTENDING_DEFAULT.has(theme)) {
|
||||
if (!fs.existsSync(defaultThemePath)) {
|
||||
throw new Error(`Missing default theme CSS: ${defaultThemePath}`);
|
||||
}
|
||||
layeredThemeCss.push(fs.readFileSync(defaultThemePath, "utf-8"));
|
||||
}
|
||||
layeredThemeCss.push(fs.readFileSync(themePath, "utf-8"));
|
||||
|
||||
return {
|
||||
baseCss: fs.readFileSync(basePath, "utf-8"),
|
||||
themeCss: fs.readFileSync(themePath, "utf-8"),
|
||||
themeCss: layeredThemeCss.join("\n"),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -4,32 +4,108 @@
|
||||
"": {
|
||||
"name": "baoyu-post-to-wechat-scripts",
|
||||
"dependencies": {
|
||||
"@jsquash/webp": "^1.5.0",
|
||||
"baoyu-chrome-cdp": "file:./vendor/baoyu-chrome-cdp",
|
||||
"baoyu-md": "file:./vendor/baoyu-md",
|
||||
"jimp": "^1.6.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@jimp/core": ["@jimp/core@1.6.0", "", { "dependencies": { "@jimp/file-ops": "1.6.0", "@jimp/types": "1.6.0", "@jimp/utils": "1.6.0", "await-to-js": "^3.0.0", "exif-parser": "^0.1.12", "file-type": "^16.0.0", "mime": "3" } }, "sha512-EQQlKU3s9QfdJqiSrZWNTxBs3rKXgO2W+GxNXDtwchF3a4IqxDheFX1ti+Env9hdJXDiYLp2jTRjlxhPthsk8w=="],
|
||||
|
||||
"@jimp/diff": ["@jimp/diff@1.6.0", "", { "dependencies": { "@jimp/plugin-resize": "1.6.0", "@jimp/types": "1.6.0", "@jimp/utils": "1.6.0", "pixelmatch": "^5.3.0" } }, "sha512-+yUAQ5gvRC5D1WHYxjBHZI7JBRusGGSLf8AmPRPCenTzh4PA+wZ1xv2+cYqQwTfQHU5tXYOhA0xDytfHUf1Zyw=="],
|
||||
|
||||
"@jimp/file-ops": ["@jimp/file-ops@1.6.0", "", {}, "sha512-Dx/bVDmgnRe1AlniRpCKrGRm5YvGmUwbDzt+MAkgmLGf+jvBT75hmMEZ003n9HQI/aPnm/YKnXjg/hOpzNCpHQ=="],
|
||||
|
||||
"@jimp/js-bmp": ["@jimp/js-bmp@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/types": "1.6.0", "@jimp/utils": "1.6.0", "bmp-ts": "^1.0.9" } }, "sha512-FU6Q5PC/e3yzLyBDXupR3SnL3htU7S3KEs4e6rjDP6gNEOXRFsWs6YD3hXuXd50jd8ummy+q2WSwuGkr8wi+Gw=="],
|
||||
|
||||
"@jimp/js-gif": ["@jimp/js-gif@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/types": "1.6.0", "gifwrap": "^0.10.1", "omggif": "^1.0.10" } }, "sha512-N9CZPHOrJTsAUoWkWZstLPpwT5AwJ0wge+47+ix3++SdSL/H2QzyMqxbcDYNFe4MoI5MIhATfb0/dl/wmX221g=="],
|
||||
|
||||
"@jimp/js-jpeg": ["@jimp/js-jpeg@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/types": "1.6.0", "jpeg-js": "^0.4.4" } }, "sha512-6vgFDqeusblf5Pok6B2DUiMXplH8RhIKAryj1yn+007SIAQ0khM1Uptxmpku/0MfbClx2r7pnJv9gWpAEJdMVA=="],
|
||||
|
||||
"@jimp/js-png": ["@jimp/js-png@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/types": "1.6.0", "pngjs": "^7.0.0" } }, "sha512-AbQHScy3hDDgMRNfG0tPjL88AV6qKAILGReIa3ATpW5QFjBKpisvUaOqhzJ7Reic1oawx3Riyv152gaPfqsBVg=="],
|
||||
|
||||
"@jimp/js-tiff": ["@jimp/js-tiff@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/types": "1.6.0", "utif2": "^4.1.0" } }, "sha512-zhReR8/7KO+adijj3h0ZQUOiun3mXUv79zYEAKvE0O+rP7EhgtKvWJOZfRzdZSNv0Pu1rKtgM72qgtwe2tFvyw=="],
|
||||
|
||||
"@jimp/plugin-blit": ["@jimp/plugin-blit@1.6.0", "", { "dependencies": { "@jimp/types": "1.6.0", "@jimp/utils": "1.6.0", "zod": "^3.23.8" } }, "sha512-M+uRWl1csi7qilnSK8uxK4RJMSuVeBiO1AY0+7APnfUbQNZm6hCe0CCFv1Iyw1D/Dhb8ph8fQgm5mwM0eSxgVA=="],
|
||||
|
||||
"@jimp/plugin-blur": ["@jimp/plugin-blur@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/utils": "1.6.0" } }, "sha512-zrM7iic1OTwUCb0g/rN5y+UnmdEsT3IfuCXCJJNs8SZzP0MkZ1eTvuwK9ZidCuMo4+J3xkzCidRwYXB5CyGZTw=="],
|
||||
|
||||
"@jimp/plugin-circle": ["@jimp/plugin-circle@1.6.0", "", { "dependencies": { "@jimp/types": "1.6.0", "zod": "^3.23.8" } }, "sha512-xt1Gp+LtdMKAXfDp3HNaG30SPZW6AQ7dtAtTnoRKorRi+5yCJjKqXRgkewS5bvj8DEh87Ko1ydJfzqS3P2tdWw=="],
|
||||
|
||||
"@jimp/plugin-color": ["@jimp/plugin-color@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/types": "1.6.0", "@jimp/utils": "1.6.0", "tinycolor2": "^1.6.0", "zod": "^3.23.8" } }, "sha512-J5q8IVCpkBsxIXM+45XOXTrsyfblyMZg3a9eAo0P7VPH4+CrvyNQwaYatbAIamSIN1YzxmO3DkIZXzRjFSz1SA=="],
|
||||
|
||||
"@jimp/plugin-contain": ["@jimp/plugin-contain@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/plugin-blit": "1.6.0", "@jimp/plugin-resize": "1.6.0", "@jimp/types": "1.6.0", "@jimp/utils": "1.6.0", "zod": "^3.23.8" } }, "sha512-oN/n+Vdq/Qg9bB4yOBOxtY9IPAtEfES8J1n9Ddx+XhGBYT1/QTU/JYkGaAkIGoPnyYvmLEDqMz2SGihqlpqfzQ=="],
|
||||
|
||||
"@jimp/plugin-cover": ["@jimp/plugin-cover@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/plugin-crop": "1.6.0", "@jimp/plugin-resize": "1.6.0", "@jimp/types": "1.6.0", "zod": "^3.23.8" } }, "sha512-Iow0h6yqSC269YUJ8HC3Q/MpCi2V55sMlbkkTTx4zPvd8mWZlC0ykrNDeAy9IJegrQ7v5E99rJwmQu25lygKLA=="],
|
||||
|
||||
"@jimp/plugin-crop": ["@jimp/plugin-crop@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/types": "1.6.0", "@jimp/utils": "1.6.0", "zod": "^3.23.8" } }, "sha512-KqZkEhvs+21USdySCUDI+GFa393eDIzbi1smBqkUPTE+pRwSWMAf01D5OC3ZWB+xZsNla93BDS9iCkLHA8wang=="],
|
||||
|
||||
"@jimp/plugin-displace": ["@jimp/plugin-displace@1.6.0", "", { "dependencies": { "@jimp/types": "1.6.0", "@jimp/utils": "1.6.0", "zod": "^3.23.8" } }, "sha512-4Y10X9qwr5F+Bo5ME356XSACEF55485j5nGdiyJ9hYzjQP9nGgxNJaZ4SAOqpd+k5sFaIeD7SQ0Occ26uIng5Q=="],
|
||||
|
||||
"@jimp/plugin-dither": ["@jimp/plugin-dither@1.6.0", "", { "dependencies": { "@jimp/types": "1.6.0" } }, "sha512-600d1RxY0pKwgyU0tgMahLNKsqEcxGdbgXadCiVCoGd6V6glyCvkNrnnwC0n5aJ56Htkj88PToSdF88tNVZEEQ=="],
|
||||
|
||||
"@jimp/plugin-fisheye": ["@jimp/plugin-fisheye@1.6.0", "", { "dependencies": { "@jimp/types": "1.6.0", "@jimp/utils": "1.6.0", "zod": "^3.23.8" } }, "sha512-E5QHKWSCBFtpgZarlmN3Q6+rTQxjirFqo44ohoTjzYVrDI6B6beXNnPIThJgPr0Y9GwfzgyarKvQuQuqCnnfbA=="],
|
||||
|
||||
"@jimp/plugin-flip": ["@jimp/plugin-flip@1.6.0", "", { "dependencies": { "@jimp/types": "1.6.0", "zod": "^3.23.8" } }, "sha512-/+rJVDuBIVOgwoyVkBjUFHtP+wmW0r+r5OQ2GpatQofToPVbJw1DdYWXlwviSx7hvixTWLKVgRWQ5Dw862emDg=="],
|
||||
|
||||
"@jimp/plugin-hash": ["@jimp/plugin-hash@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/js-bmp": "1.6.0", "@jimp/js-jpeg": "1.6.0", "@jimp/js-png": "1.6.0", "@jimp/js-tiff": "1.6.0", "@jimp/plugin-color": "1.6.0", "@jimp/plugin-resize": "1.6.0", "@jimp/types": "1.6.0", "@jimp/utils": "1.6.0", "any-base": "^1.1.0" } }, "sha512-wWzl0kTpDJgYVbZdajTf+4NBSKvmI3bRI8q6EH9CVeIHps9VWVsUvEyb7rpbcwVLWYuzDtP2R0lTT6WeBNQH9Q=="],
|
||||
|
||||
"@jimp/plugin-mask": ["@jimp/plugin-mask@1.6.0", "", { "dependencies": { "@jimp/types": "1.6.0", "zod": "^3.23.8" } }, "sha512-Cwy7ExSJMZszvkad8NV8o/Z92X2kFUFM8mcDAhNVxU0Q6tA0op2UKRJY51eoK8r6eds/qak3FQkXakvNabdLnA=="],
|
||||
|
||||
"@jimp/plugin-print": ["@jimp/plugin-print@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/js-jpeg": "1.6.0", "@jimp/js-png": "1.6.0", "@jimp/plugin-blit": "1.6.0", "@jimp/types": "1.6.0", "parse-bmfont-ascii": "^1.0.6", "parse-bmfont-binary": "^1.0.6", "parse-bmfont-xml": "^1.1.6", "simple-xml-to-json": "^1.2.2", "zod": "^3.23.8" } }, "sha512-zarTIJi8fjoGMSI/M3Xh5yY9T65p03XJmPsuNet19K/Q7mwRU6EV2pfj+28++2PV2NJ+htDF5uecAlnGyxFN2A=="],
|
||||
|
||||
"@jimp/plugin-quantize": ["@jimp/plugin-quantize@1.6.0", "", { "dependencies": { "image-q": "^4.0.0", "zod": "^3.23.8" } }, "sha512-EmzZ/s9StYQwbpG6rUGBCisc3f64JIhSH+ncTJd+iFGtGo0YvSeMdAd+zqgiHpfZoOL54dNavZNjF4otK+mvlg=="],
|
||||
|
||||
"@jimp/plugin-resize": ["@jimp/plugin-resize@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/types": "1.6.0", "zod": "^3.23.8" } }, "sha512-uSUD1mqXN9i1SGSz5ov3keRZ7S9L32/mAQG08wUwZiEi5FpbV0K8A8l1zkazAIZi9IJzLlTauRNU41Mi8IF9fA=="],
|
||||
|
||||
"@jimp/plugin-rotate": ["@jimp/plugin-rotate@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/plugin-crop": "1.6.0", "@jimp/plugin-resize": "1.6.0", "@jimp/types": "1.6.0", "@jimp/utils": "1.6.0", "zod": "^3.23.8" } }, "sha512-JagdjBLnUZGSG4xjCLkIpQOZZ3Mjbg8aGCCi4G69qR+OjNpOeGI7N2EQlfK/WE8BEHOW5vdjSyglNqcYbQBWRw=="],
|
||||
|
||||
"@jimp/plugin-threshold": ["@jimp/plugin-threshold@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/plugin-color": "1.6.0", "@jimp/plugin-hash": "1.6.0", "@jimp/types": "1.6.0", "@jimp/utils": "1.6.0", "zod": "^3.23.8" } }, "sha512-M59m5dzLoHOVWdM41O8z9SyySzcDn43xHseOH0HavjsfQsT56GGCC4QzU1banJidbUrePhzoEdS42uFE8Fei8w=="],
|
||||
|
||||
"@jimp/types": ["@jimp/types@1.6.0", "", { "dependencies": { "zod": "^3.23.8" } }, "sha512-7UfRsiKo5GZTAATxm2qQ7jqmUXP0DxTArztllTcYdyw6Xi5oT4RaoXynVtCD4UyLK5gJgkZJcwonoijrhYFKfg=="],
|
||||
|
||||
"@jimp/utils": ["@jimp/utils@1.6.0", "", { "dependencies": { "@jimp/types": "1.6.0", "tinycolor2": "^1.6.0" } }, "sha512-gqFTGEosKbOkYF/WFj26jMHOI5OH2jeP1MmC/zbK6BF6VJBf8rIC5898dPfSzZEbSA0wbbV5slbntWVc5PKLFA=="],
|
||||
|
||||
"@jsquash/webp": ["@jsquash/webp@1.5.0", "", { "dependencies": { "wasm-feature-detect": "^1.2.11" } }, "sha512-KggLoj2MnRSfIqTeKe1EmbljTX2vuV7mh79k89PCL1pyqiDULcPM1L47twxXt0hkb68F70bXiL31MxsuoZtKFw=="],
|
||||
|
||||
"@tokenizer/token": ["@tokenizer/token@0.3.0", "", {}, "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="],
|
||||
|
||||
"@types/debug": ["@types/debug@4.1.12", "", { "dependencies": { "@types/ms": "*" } }, "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ=="],
|
||||
|
||||
"@types/mdast": ["@types/mdast@4.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA=="],
|
||||
|
||||
"@types/ms": ["@types/ms@2.1.0", "", {}, "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA=="],
|
||||
|
||||
"@types/node": ["@types/node@16.9.1", "", {}, "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g=="],
|
||||
|
||||
"@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="],
|
||||
|
||||
"abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="],
|
||||
|
||||
"ansi-colors": ["ansi-colors@4.1.3", "", {}, "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw=="],
|
||||
|
||||
"any-base": ["any-base@1.1.0", "", {}, "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg=="],
|
||||
|
||||
"argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="],
|
||||
|
||||
"await-to-js": ["await-to-js@3.0.0", "", {}, "sha512-zJAaP9zxTcvTHRlejau3ZOY4V7SRpiByf3/dxx2uyKxxor19tpmpV2QRsTKikckwhaPmr2dVpxxMr7jOCYVp5g=="],
|
||||
|
||||
"bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="],
|
||||
|
||||
"baoyu-chrome-cdp": ["baoyu-chrome-cdp@file:vendor/baoyu-chrome-cdp", {}],
|
||||
|
||||
"baoyu-md": ["baoyu-md@file:vendor/baoyu-md", { "dependencies": { "fflate": "^0.8.2", "front-matter": "^4.0.2", "highlight.js": "^11.11.1", "juice": "^11.0.1", "marked": "^15.0.6", "reading-time": "^1.5.0", "remark-cjk-friendly": "^1.1.0", "remark-parse": "^11.0.0", "remark-stringify": "^11.0.0", "unified": "^11.0.5" } }],
|
||||
|
||||
"base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="],
|
||||
|
||||
"bmp-ts": ["bmp-ts@1.0.9", "", {}, "sha512-cTEHk2jLrPyi+12M3dhpEbnnPOsaZuq7C45ylbbQIiWgDFZq4UVYPEY5mlqjvsj/6gJv9qX5sa+ebDzLXT28Vw=="],
|
||||
|
||||
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
||||
|
||||
"buffer": ["buffer@6.0.3", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA=="],
|
||||
|
||||
"character-entities": ["character-entities@2.0.2", "", {}, "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ=="],
|
||||
|
||||
"cheerio": ["cheerio@1.0.0", "", { "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", "domutils": "^3.1.0", "encoding-sniffer": "^0.2.0", "htmlparser2": "^9.1.0", "parse5": "^7.1.2", "parse5-htmlparser2-tree-adapter": "^7.0.0", "parse5-parser-stream": "^7.1.2", "undici": "^6.19.5", "whatwg-mimetype": "^4.0.0" } }, "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww=="],
|
||||
@@ -66,22 +142,40 @@
|
||||
|
||||
"esprima": ["esprima@4.0.1", "", { "bin": { "esparse": "./bin/esparse.js", "esvalidate": "./bin/esvalidate.js" } }, "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A=="],
|
||||
|
||||
"event-target-shim": ["event-target-shim@5.0.1", "", {}, "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="],
|
||||
|
||||
"events": ["events@3.3.0", "", {}, "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="],
|
||||
|
||||
"exif-parser": ["exif-parser@0.1.12", "", {}, "sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw=="],
|
||||
|
||||
"extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="],
|
||||
|
||||
"fflate": ["fflate@0.8.2", "", {}, "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A=="],
|
||||
|
||||
"file-type": ["file-type@16.5.4", "", { "dependencies": { "readable-web-to-node-stream": "^3.0.0", "strtok3": "^6.2.4", "token-types": "^4.1.1" } }, "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw=="],
|
||||
|
||||
"front-matter": ["front-matter@4.0.2", "", { "dependencies": { "js-yaml": "^3.13.1" } }, "sha512-I8ZuJ/qG92NWX8i5x1Y8qyj3vizhXS31OxjKDu3LKP+7/qBgfIKValiZIEwoVoJKUHlhWtYrktkxV1XsX+pPlg=="],
|
||||
|
||||
"get-east-asian-width": ["get-east-asian-width@1.5.0", "", {}, "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA=="],
|
||||
|
||||
"gifwrap": ["gifwrap@0.10.1", "", { "dependencies": { "image-q": "^4.0.0", "omggif": "^1.0.10" } }, "sha512-2760b1vpJHNmLzZ/ubTtNnEx5WApN/PYWJvXvgS+tL1egTTthayFYIQQNi136FLEDcN/IyEY2EcGpIITD6eYUw=="],
|
||||
|
||||
"highlight.js": ["highlight.js@11.11.1", "", {}, "sha512-Xwwo44whKBVCYoliBQwaPvtd/2tYFkRQtXDWj1nackaV2JPXx3L0+Jvd8/qCJ2p+ML0/XVkJ2q+Mr+UVdpJK5w=="],
|
||||
|
||||
"htmlparser2": ["htmlparser2@9.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.1.0", "entities": "^4.5.0" } }, "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ=="],
|
||||
|
||||
"iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="],
|
||||
|
||||
"ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="],
|
||||
|
||||
"image-q": ["image-q@4.0.0", "", { "dependencies": { "@types/node": "16.9.1" } }, "sha512-PfJGVgIfKQJuq3s0tTDOKtztksibuUEbJQIYT3by6wctQo+Rdlh7ef4evJ5NCdxY4CfMbvFkocEwbl4BF8RlJw=="],
|
||||
|
||||
"is-plain-obj": ["is-plain-obj@4.1.0", "", {}, "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg=="],
|
||||
|
||||
"jimp": ["jimp@1.6.0", "", { "dependencies": { "@jimp/core": "1.6.0", "@jimp/diff": "1.6.0", "@jimp/js-bmp": "1.6.0", "@jimp/js-gif": "1.6.0", "@jimp/js-jpeg": "1.6.0", "@jimp/js-png": "1.6.0", "@jimp/js-tiff": "1.6.0", "@jimp/plugin-blit": "1.6.0", "@jimp/plugin-blur": "1.6.0", "@jimp/plugin-circle": "1.6.0", "@jimp/plugin-color": "1.6.0", "@jimp/plugin-contain": "1.6.0", "@jimp/plugin-cover": "1.6.0", "@jimp/plugin-crop": "1.6.0", "@jimp/plugin-displace": "1.6.0", "@jimp/plugin-dither": "1.6.0", "@jimp/plugin-fisheye": "1.6.0", "@jimp/plugin-flip": "1.6.0", "@jimp/plugin-hash": "1.6.0", "@jimp/plugin-mask": "1.6.0", "@jimp/plugin-print": "1.6.0", "@jimp/plugin-quantize": "1.6.0", "@jimp/plugin-resize": "1.6.0", "@jimp/plugin-rotate": "1.6.0", "@jimp/plugin-threshold": "1.6.0", "@jimp/types": "1.6.0", "@jimp/utils": "1.6.0" } }, "sha512-YcwCHw1kiqEeI5xRpDlPPBGL2EOpBKLwO4yIBJcXWHPj5PnA5urGq0jbyhM5KoNpypQ6VboSoxc9D8HyfvngSg=="],
|
||||
|
||||
"jpeg-js": ["jpeg-js@0.4.4", "", {}, "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg=="],
|
||||
|
||||
"js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="],
|
||||
|
||||
"juice": ["juice@11.1.1", "", { "dependencies": { "cheerio": "1.0.0", "commander": "^12.1.0", "entities": "^7.0.0", "mensch": "^0.3.4", "slick": "^1.12.2", "web-resource-inliner": "^8.0.0" }, "bin": { "juice": "bin/juice" } }, "sha512-4SBfZqKcc6DrIS+5b/WiGoWaZsdUPBH+e6SbRlNjJpaIRtfoBhYReAtobIEW6mcLeFFDXLBJMuZwkJLkBJjs2w=="],
|
||||
@@ -146,18 +240,40 @@
|
||||
|
||||
"micromark-util-types": ["micromark-util-types@2.0.2", "", {}, "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA=="],
|
||||
|
||||
"mime": ["mime@2.6.0", "", { "bin": { "mime": "cli.js" } }, "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg=="],
|
||||
"mime": ["mime@3.0.0", "", { "bin": { "mime": "cli.js" } }, "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A=="],
|
||||
|
||||
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
|
||||
|
||||
"nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
|
||||
|
||||
"omggif": ["omggif@1.0.10", "", {}, "sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw=="],
|
||||
|
||||
"pako": ["pako@1.0.11", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="],
|
||||
|
||||
"parse-bmfont-ascii": ["parse-bmfont-ascii@1.0.6", "", {}, "sha512-U4RrVsUFCleIOBsIGYOMKjn9PavsGOXxbvYGtMOEfnId0SVNsgehXh1DxUdVPLoxd5mvcEtvmKs2Mmf0Mpa1ZA=="],
|
||||
|
||||
"parse-bmfont-binary": ["parse-bmfont-binary@1.0.6", "", {}, "sha512-GxmsRea0wdGdYthjuUeWTMWPqm2+FAd4GI8vCvhgJsFnoGhTrLhXDDupwTo7rXVAgaLIGoVHDZS9p/5XbSqeWA=="],
|
||||
|
||||
"parse-bmfont-xml": ["parse-bmfont-xml@1.1.6", "", { "dependencies": { "xml-parse-from-string": "^1.0.0", "xml2js": "^0.5.0" } }, "sha512-0cEliVMZEhrFDwMh4SxIyVJpqYoOWDJ9P895tFuS+XuNzI5UBmBk5U5O4KuJdTnZpSBI4LFA2+ZiJaiwfSwlMA=="],
|
||||
|
||||
"parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="],
|
||||
|
||||
"parse5-htmlparser2-tree-adapter": ["parse5-htmlparser2-tree-adapter@7.1.0", "", { "dependencies": { "domhandler": "^5.0.3", "parse5": "^7.0.0" } }, "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g=="],
|
||||
|
||||
"parse5-parser-stream": ["parse5-parser-stream@7.1.2", "", { "dependencies": { "parse5": "^7.0.0" } }, "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow=="],
|
||||
|
||||
"peek-readable": ["peek-readable@4.1.0", "", {}, "sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg=="],
|
||||
|
||||
"pixelmatch": ["pixelmatch@5.3.0", "", { "dependencies": { "pngjs": "^6.0.0" }, "bin": { "pixelmatch": "bin/pixelmatch" } }, "sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q=="],
|
||||
|
||||
"pngjs": ["pngjs@7.0.0", "", {}, "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow=="],
|
||||
|
||||
"process": ["process@0.11.10", "", {}, "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A=="],
|
||||
|
||||
"readable-stream": ["readable-stream@4.7.0", "", { "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", "events": "^3.3.0", "process": "^0.11.10", "string_decoder": "^1.3.0" } }, "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg=="],
|
||||
|
||||
"readable-web-to-node-stream": ["readable-web-to-node-stream@3.0.4", "", { "dependencies": { "readable-stream": "^4.7.0" } }, "sha512-9nX56alTf5bwXQ3ZDipHJhusu9NTQJ/CVPtb/XHAJCXihZeitfJvIRS4GqQ/mfIoOE3IelHMrpayVrosdHBuLw=="],
|
||||
|
||||
"reading-time": ["reading-time@1.5.0", "", {}, "sha512-onYyVhBNr4CmAxFsKS7bz+uTLRakypIe4R+5A824vBSkQy/hB3fZepoVEf8OVAxzLvK+H/jm9TzpI3ETSm64Kg=="],
|
||||
|
||||
"remark-cjk-friendly": ["remark-cjk-friendly@1.2.3", "", { "dependencies": { "micromark-extension-cjk-friendly": "1.2.3" }, "peerDependencies": { "@types/mdast": "^4.0.0", "unified": "^11.0.0" }, "optionalPeers": ["@types/mdast"] }, "sha512-UvAgxwlNk+l9Oqgl/9MWK2eWRS7zgBW/nXX9AthV7nd/3lNejF138E7Xbmk9Zs4WjTJGs721r7fAEc7tNFoH7g=="],
|
||||
@@ -166,12 +282,26 @@
|
||||
|
||||
"remark-stringify": ["remark-stringify@11.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-to-markdown": "^2.0.0", "unified": "^11.0.0" } }, "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw=="],
|
||||
|
||||
"safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="],
|
||||
|
||||
"safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
|
||||
|
||||
"sax": ["sax@1.6.0", "", {}, "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA=="],
|
||||
|
||||
"simple-xml-to-json": ["simple-xml-to-json@1.2.4", "", {}, "sha512-3MY16e0ocMHL7N1ufpdObURGyX+lCo0T/A+y6VCwosLdH1HSda4QZl1Sdt/O+2qWp48WFi26XEp5rF0LoaL0Dg=="],
|
||||
|
||||
"slick": ["slick@1.12.2", "", {}, "sha512-4qdtOGcBjral6YIBCWJ0ljFSKNLz9KkhbWtuGvUyRowl1kxfuE1x/Z/aJcaiilpb3do9bl5K7/1h9XC5wWpY/A=="],
|
||||
|
||||
"sprintf-js": ["sprintf-js@1.0.3", "", {}, "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="],
|
||||
|
||||
"string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="],
|
||||
|
||||
"strtok3": ["strtok3@6.3.0", "", { "dependencies": { "@tokenizer/token": "^0.3.0", "peek-readable": "^4.1.0" } }, "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw=="],
|
||||
|
||||
"tinycolor2": ["tinycolor2@1.6.0", "", {}, "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw=="],
|
||||
|
||||
"token-types": ["token-types@4.2.1", "", { "dependencies": { "@tokenizer/token": "^0.3.0", "ieee754": "^1.2.1" } }, "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ=="],
|
||||
|
||||
"trough": ["trough@2.2.0", "", {}, "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw=="],
|
||||
|
||||
"undici": ["undici@6.24.0", "", {}, "sha512-lVLNosgqo5EkGqh5XUDhGfsMSoO8K0BAN0TyJLvwNRSl4xWGZlCVYsAIpa/OpA3TvmnM01GWcoKmc3ZWo5wKKA=="],
|
||||
@@ -186,18 +316,30 @@
|
||||
|
||||
"unist-util-visit-parents": ["unist-util-visit-parents@6.0.2", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-is": "^6.0.0" } }, "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ=="],
|
||||
|
||||
"utif2": ["utif2@4.1.0", "", { "dependencies": { "pako": "^1.0.11" } }, "sha512-+oknB9FHrJ7oW7A2WZYajOcv4FcDR4CfoGB0dPNfxbi4GO05RRnFmt5oa23+9w32EanrYcSJWspUiJkLMs+37w=="],
|
||||
|
||||
"valid-data-url": ["valid-data-url@3.0.1", "", {}, "sha512-jOWVmzVceKlVVdwjNSenT4PbGghU0SBIizAev8ofZVgivk/TVHXSbNL8LP6M3spZvkR9/QolkyJavGSX5Cs0UA=="],
|
||||
|
||||
"vfile": ["vfile@6.0.3", "", { "dependencies": { "@types/unist": "^3.0.0", "vfile-message": "^4.0.0" } }, "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q=="],
|
||||
|
||||
"vfile-message": ["vfile-message@4.0.3", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw=="],
|
||||
|
||||
"wasm-feature-detect": ["wasm-feature-detect@1.8.0", "", {}, "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ=="],
|
||||
|
||||
"web-resource-inliner": ["web-resource-inliner@8.0.0", "", { "dependencies": { "ansi-colors": "^4.1.1", "escape-goat": "^3.0.0", "htmlparser2": "^9.1.0", "mime": "^2.4.6", "valid-data-url": "^3.0.0" } }, "sha512-Ezr98sqXW/+OCGoUEXuOKVR+oVFlSdn1tIySEEJdiSAw4IjrW8hQkwARSSBJTSB5Us5dnytDgL0ZDliAYBhaNA=="],
|
||||
|
||||
"whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="],
|
||||
|
||||
"whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="],
|
||||
|
||||
"xml-parse-from-string": ["xml-parse-from-string@1.0.1", "", {}, "sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g=="],
|
||||
|
||||
"xml2js": ["xml2js@0.5.0", "", { "dependencies": { "sax": ">=0.6.0", "xmlbuilder": "~11.0.0" } }, "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA=="],
|
||||
|
||||
"xmlbuilder": ["xmlbuilder@11.0.1", "", {}, "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="],
|
||||
|
||||
"zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
|
||||
|
||||
"zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="],
|
||||
|
||||
"dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
|
||||
@@ -205,5 +347,9 @@
|
||||
"htmlparser2/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
|
||||
|
||||
"parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
|
||||
|
||||
"pixelmatch/pngjs": ["pngjs@6.0.0", "", {}, "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg=="],
|
||||
|
||||
"web-resource-inliner/mime": ["mime@2.6.0", "", { "bin": { "mime": "cli.js" } }, "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg=="],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@jsquash/webp": "^1.5.0",
|
||||
"baoyu-chrome-cdp": "file:./vendor/baoyu-chrome-cdp",
|
||||
"baoyu-md": "file:./vendor/baoyu-md"
|
||||
"baoyu-md": "file:./vendor/baoyu-md",
|
||||
"jimp": "^1.6.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,12 +9,26 @@ import { COLOR_PRESETS, FONT_FAMILY_MAP } from "./constants.ts";
|
||||
import {
|
||||
buildMarkdownDocumentMeta,
|
||||
formatTimestamp,
|
||||
renderMarkdownDocument,
|
||||
resolveColorToken,
|
||||
resolveFontFamilyToken,
|
||||
resolveMarkdownStyle,
|
||||
resolveRenderOptions,
|
||||
} from "./document.ts";
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, `\\$&`);
|
||||
}
|
||||
|
||||
function findInlineStyle(html: string, tagName: string, text: string): string {
|
||||
const pattern = new RegExp(
|
||||
`<${tagName}[^>]*style="([^"]*)"[^>]*>${escapeRegExp(text)}</${tagName}>`,
|
||||
);
|
||||
const match = html.match(pattern);
|
||||
assert.ok(match, `Expected inline style for <${tagName}>${text}</${tagName}>`);
|
||||
return match![1]!;
|
||||
}
|
||||
|
||||
function useCwd(t: TestContext, cwd: string): void {
|
||||
const previous = process.cwd();
|
||||
process.chdir(cwd);
|
||||
@@ -138,3 +152,23 @@ keep_title: true
|
||||
assert.equal(explicit.fontSize, "18px");
|
||||
assert.equal(explicit.keepTitle, false);
|
||||
});
|
||||
|
||||
test("renderMarkdownDocument layers default rules into grace theme before CSS inlining", async () => {
|
||||
const { html } = await renderMarkdownDocument(
|
||||
`## Section\n\nParagraph with **bold** text.`,
|
||||
{ keepTitle: true, theme: "grace" },
|
||||
);
|
||||
|
||||
const h2Style = findInlineStyle(html, "h2", "Section");
|
||||
assert.match(h2Style, /background: #92617E/);
|
||||
assert.match(h2Style, /box-shadow: 0 4px 6px rgba\(0, 0, 0, 0\.1\)/);
|
||||
|
||||
const pMatch = html.match(/<p[^>]*style="([^"]*)"[^>]*>/);
|
||||
assert.ok(pMatch, "Expected inline style on <p> tag");
|
||||
assert.match(pMatch![1]!, /color:/);
|
||||
|
||||
const strongPattern = /<strong[^>]*style="([^"]*)"[^>]*>bold<\/strong>/;
|
||||
const strongMatch = html.match(strongPattern);
|
||||
assert.ok(strongMatch, "Expected inline style for <strong>bold</strong>");
|
||||
assert.match(strongMatch![1]!, /font-weight:/);
|
||||
});
|
||||
|
||||
@@ -59,6 +59,17 @@ test("normalizeCssText and normalizeInlineCss replace variables and strip declar
|
||||
assert.doesNotMatch(normalizedHtml, /var\(--md-primary-color\)/);
|
||||
});
|
||||
|
||||
test("normalizeInlineCss removes quoted custom property values without leaving fragments behind", () => {
|
||||
const normalizedHtml = normalizeInlineCss(
|
||||
`<html style="--md-font-family: Menlo, Monaco, 'Courier New', monospace; color: var(--md-primary-color)"></html>`,
|
||||
DEFAULT_STYLE,
|
||||
);
|
||||
|
||||
assert.match(normalizedHtml, /style=" color: #0F4C81"/);
|
||||
assert.doesNotMatch(normalizedHtml, /Courier New/);
|
||||
assert.doesNotMatch(normalizedHtml, /--md-font-family/);
|
||||
});
|
||||
|
||||
test("HTML structure helpers hoist nested lists and remove the first heading", () => {
|
||||
const nestedList = `<ul><li>Parent<ul><li>Child</li></ul></li></ul>`;
|
||||
assert.equal(
|
||||
|
||||
@@ -100,13 +100,13 @@ export function normalizeCssText(cssText: string, style: StyleConfig = DEFAULT_S
|
||||
.replace(/var\(--md-accent-color\)/g, style.accentColor)
|
||||
.replace(/var\(--md-container-bg\)/g, style.containerBg)
|
||||
.replace(/hsl\(var\(--foreground\)\)/g, "#3f3f3f")
|
||||
.replace(/--md-primary-color:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-font-family:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-font-size:\s*[^;"']+;?/g, "")
|
||||
.replace(/--blockquote-background:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-accent-color:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-container-bg:\s*[^;"']+;?/g, "")
|
||||
.replace(/--foreground:\s*[^;"']+;?/g, "");
|
||||
.replace(/--md-primary-color:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-font-family:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-font-size:\s*[^;]+;?/g, "")
|
||||
.replace(/--blockquote-background:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-accent-color:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-container-bg:\s*[^;]+;?/g, "")
|
||||
.replace(/--foreground:\s*[^;]+;?/g, "");
|
||||
}
|
||||
|
||||
export function normalizeInlineCss(html: string, style: StyleConfig = DEFAULT_STYLE): string {
|
||||
|
||||
@@ -6,6 +6,7 @@ import type { ThemeName } from "./types.js";
|
||||
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
export const THEME_DIR = path.resolve(SCRIPT_DIR, "themes");
|
||||
const FALLBACK_THEMES: ThemeName[] = ["default", "grace", "simple"];
|
||||
const THEMES_EXTENDING_DEFAULT = new Set<ThemeName>(["grace", "simple"]);
|
||||
|
||||
function stripOutputScope(cssContent: string): string {
|
||||
let css = cssContent;
|
||||
@@ -41,6 +42,7 @@ export function loadThemeCss(theme: ThemeName): {
|
||||
themeCss: string;
|
||||
} {
|
||||
const basePath = path.join(THEME_DIR, "base.css");
|
||||
const defaultThemePath = path.join(THEME_DIR, "default.css");
|
||||
const themePath = path.join(THEME_DIR, `${theme}.css`);
|
||||
|
||||
if (!fs.existsSync(basePath)) {
|
||||
@@ -51,9 +53,18 @@ export function loadThemeCss(theme: ThemeName): {
|
||||
throw new Error(`Missing theme CSS for "${theme}": ${themePath}`);
|
||||
}
|
||||
|
||||
const layeredThemeCss: string[] = [];
|
||||
if (theme !== "default" && THEMES_EXTENDING_DEFAULT.has(theme)) {
|
||||
if (!fs.existsSync(defaultThemePath)) {
|
||||
throw new Error(`Missing default theme CSS: ${defaultThemePath}`);
|
||||
}
|
||||
layeredThemeCss.push(fs.readFileSync(defaultThemePath, "utf-8"));
|
||||
}
|
||||
layeredThemeCss.push(fs.readFileSync(themePath, "utf-8"));
|
||||
|
||||
return {
|
||||
baseCss: fs.readFileSync(basePath, "utf-8"),
|
||||
themeCss: fs.readFileSync(themePath, "utf-8"),
|
||||
themeCss: layeredThemeCss.join("\n"),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,11 @@ import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { loadWechatExtendConfig, resolveAccount, loadCredentials } from "./wechat-extend-config.ts";
|
||||
import {
|
||||
type WechatUploadAsset,
|
||||
prepareWechatBodyImageUpload,
|
||||
needsWechatBodyImageProcessing,
|
||||
} from "./wechat-image-processor.ts";
|
||||
|
||||
interface AccessTokenResponse {
|
||||
access_token?: string;
|
||||
@@ -52,10 +57,10 @@ interface ArticleOptions {
|
||||
}
|
||||
|
||||
const TOKEN_URL = "https://api.weixin.qq.com/cgi-bin/token";
|
||||
const UPLOAD_URL = "https://api.weixin.qq.com/cgi-bin/material/add_material";
|
||||
const UPLOAD_BODY_IMG_URL = "https://api.weixin.qq.com/cgi-bin/media/uploadimg";
|
||||
const UPLOAD_MATERIAL_URL = "https://api.weixin.qq.com/cgi-bin/material/add_material";
|
||||
const DRAFT_URL = "https://api.weixin.qq.com/cgi-bin/draft/add";
|
||||
|
||||
|
||||
async function fetchAccessToken(appId: string, appSecret: string): Promise<string> {
|
||||
const url = `${TOKEN_URL}?grant_type=client_credential&appid=${appId}&secret=${appSecret}`;
|
||||
const res = await fetch(url);
|
||||
@@ -72,14 +77,20 @@ async function fetchAccessToken(appId: string, appSecret: string): Promise<strin
|
||||
return data.access_token;
|
||||
}
|
||||
|
||||
async function uploadImage(
|
||||
function toHttpsUrl(url: string | undefined): string {
|
||||
if (!url) return "";
|
||||
return url.startsWith("http://") ? url.replace(/^http:\/\//i, "https://") : url;
|
||||
}
|
||||
|
||||
async function loadUploadAsset(
|
||||
imagePath: string,
|
||||
accessToken: string,
|
||||
baseDir?: string
|
||||
): Promise<UploadResponse> {
|
||||
baseDir?: string,
|
||||
): Promise<WechatUploadAsset> {
|
||||
let fileBuffer: Buffer;
|
||||
let filename: string;
|
||||
let contentType: string;
|
||||
let fileSize = 0;
|
||||
let fileExt = "";
|
||||
|
||||
if (imagePath.startsWith("http://") || imagePath.startsWith("https://")) {
|
||||
const response = await fetch(imagePath);
|
||||
@@ -91,8 +102,10 @@ async function uploadImage(
|
||||
throw new Error(`Remote image is empty: ${imagePath}`);
|
||||
}
|
||||
fileBuffer = Buffer.from(buffer);
|
||||
fileSize = buffer.byteLength;
|
||||
const urlPath = imagePath.split("?")[0];
|
||||
filename = path.basename(urlPath) || "image.jpg";
|
||||
fileExt = path.extname(filename).toLowerCase();
|
||||
contentType = response.headers.get("content-type") || "image/jpeg";
|
||||
} else {
|
||||
const resolvedPath = path.isAbsolute(imagePath)
|
||||
@@ -106,19 +119,85 @@ async function uploadImage(
|
||||
if (stats.size === 0) {
|
||||
throw new Error(`Local image is empty: ${resolvedPath}`);
|
||||
}
|
||||
fileSize = stats.size;
|
||||
fileBuffer = fs.readFileSync(resolvedPath);
|
||||
filename = path.basename(resolvedPath);
|
||||
const ext = path.extname(filename).toLowerCase();
|
||||
fileExt = path.extname(filename).toLowerCase();
|
||||
const mimeTypes: Record<string, string> = {
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".png": "image/png",
|
||||
".gif": "image/gif",
|
||||
".webp": "image/webp",
|
||||
".bmp": "image/bmp",
|
||||
".tiff": "image/tiff",
|
||||
".tif": "image/tiff",
|
||||
".svg": "image/svg+xml",
|
||||
".ico": "image/x-icon",
|
||||
};
|
||||
contentType = mimeTypes[ext] || "image/jpeg";
|
||||
contentType = mimeTypes[fileExt] || "image/jpeg";
|
||||
}
|
||||
|
||||
return {
|
||||
buffer: fileBuffer,
|
||||
filename,
|
||||
contentType,
|
||||
fileExt,
|
||||
fileSize,
|
||||
};
|
||||
}
|
||||
|
||||
async function uploadImage(
|
||||
imagePath: string,
|
||||
accessToken: string,
|
||||
baseDir?: string,
|
||||
uploadType: "body" | "material" = "body"
|
||||
): Promise<UploadResponse> {
|
||||
const asset = await loadUploadAsset(imagePath, baseDir);
|
||||
let uploadAsset = asset;
|
||||
|
||||
if (uploadType === "body" && needsWechatBodyImageProcessing(asset)) {
|
||||
const prepared = await prepareWechatBodyImageUpload(asset);
|
||||
uploadAsset = {
|
||||
...asset,
|
||||
buffer: prepared.buffer,
|
||||
filename: prepared.filename,
|
||||
contentType: prepared.contentType,
|
||||
fileExt: path.extname(prepared.filename).toLowerCase(),
|
||||
fileSize: prepared.buffer.length,
|
||||
};
|
||||
const note = prepared.processingNotes.join(", ");
|
||||
console.error(`[wechat-api] Processed ${asset.filename} for body upload: ${note}`);
|
||||
}
|
||||
|
||||
const result = await uploadToWechat(
|
||||
uploadAsset.buffer,
|
||||
uploadAsset.filename,
|
||||
uploadAsset.contentType,
|
||||
accessToken,
|
||||
uploadType,
|
||||
);
|
||||
|
||||
// media/uploadimg 接口只返回 URL,material/add_material 返回 media_id
|
||||
if (uploadType === "body") {
|
||||
return {
|
||||
url: toHttpsUrl(result.url),
|
||||
media_id: "",
|
||||
} as UploadResponse;
|
||||
} else {
|
||||
result.url = toHttpsUrl(result.url);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
// 实际的微信上传函数
|
||||
async function uploadToWechat(
|
||||
fileBuffer: Buffer,
|
||||
filename: string,
|
||||
contentType: string,
|
||||
accessToken: string,
|
||||
uploadType: "body" | "material"
|
||||
): Promise<UploadResponse> {
|
||||
const boundary = `----WebKitFormBoundary${Date.now().toString(16)}`;
|
||||
const header = [
|
||||
`--${boundary}`,
|
||||
@@ -133,7 +212,8 @@ async function uploadImage(
|
||||
const footerBuffer = Buffer.from(footer, "utf-8");
|
||||
const body = Buffer.concat([headerBuffer, fileBuffer, footerBuffer]);
|
||||
|
||||
const url = `${UPLOAD_URL}?access_token=${accessToken}&type=image`;
|
||||
const uploadUrl = uploadType === "body" ? UPLOAD_BODY_IMG_URL : UPLOAD_MATERIAL_URL;
|
||||
const url = `${uploadUrl}?type=image&access_token=${accessToken}`;
|
||||
const res = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
@@ -147,10 +227,6 @@ async function uploadImage(
|
||||
throw new Error(`Upload failed ${data.errcode}: ${data.errmsg}`);
|
||||
}
|
||||
|
||||
if (data.url?.startsWith("http://")) {
|
||||
data.url = data.url.replace(/^http:\/\//i, "https://");
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
@@ -159,17 +235,19 @@ async function uploadImagesInHtml(
|
||||
accessToken: string,
|
||||
baseDir: string,
|
||||
contentImages: ImageInfo[] = [],
|
||||
): Promise<{ html: string; firstMediaId: string; allMediaIds: string[] }> {
|
||||
articleType: ArticleType = "news",
|
||||
collectNewsCoverFallback: boolean = false,
|
||||
): Promise<{ html: string; firstCoverMediaId: string; imageMediaIds: string[] }> {
|
||||
const imgRegex = /<img[^>]*\ssrc=["']([^"']+)["'][^>]*>/gi;
|
||||
const matches = [...html.matchAll(imgRegex)];
|
||||
|
||||
if (matches.length === 0 && contentImages.length === 0) {
|
||||
return { html, firstMediaId: "", allMediaIds: [] };
|
||||
return { html, firstCoverMediaId: "", imageMediaIds: [] };
|
||||
}
|
||||
|
||||
let firstMediaId = "";
|
||||
let firstCoverMediaId = "";
|
||||
let updatedHtml = html;
|
||||
const allMediaIds: string[] = [];
|
||||
const imageMediaIds: string[] = [];
|
||||
const uploadedBySource = new Map<string, UploadResponse>();
|
||||
|
||||
for (const match of matches) {
|
||||
@@ -177,8 +255,13 @@ async function uploadImagesInHtml(
|
||||
if (!src) continue;
|
||||
|
||||
if (src.startsWith("https://mmbiz.qpic.cn")) {
|
||||
if (!firstMediaId) {
|
||||
firstMediaId = src;
|
||||
if (collectNewsCoverFallback && !firstCoverMediaId) {
|
||||
try {
|
||||
const coverResp = await uploadImage(src, accessToken, baseDir, "material");
|
||||
firstCoverMediaId = coverResp.media_id;
|
||||
} catch (err) {
|
||||
console.error(`[wechat-api] Failed to reuse existing WeChat image as cover: ${src}`, err);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@@ -186,20 +269,31 @@ async function uploadImagesInHtml(
|
||||
const localPathMatch = fullTag.match(/data-local-path=["']([^"']+)["']/);
|
||||
const imagePath = localPathMatch ? localPathMatch[1]! : src;
|
||||
|
||||
console.error(`[wechat-api] Uploading image: ${imagePath}`);
|
||||
console.error(`[wechat-api] Uploading body image: ${imagePath}`);
|
||||
try {
|
||||
let resp = uploadedBySource.get(imagePath);
|
||||
if (!resp) {
|
||||
resp = await uploadImage(imagePath, accessToken, baseDir);
|
||||
// 正文图片使用 media/uploadimg 接口获取 URL
|
||||
resp = await uploadImage(imagePath, accessToken, baseDir, "body");
|
||||
uploadedBySource.set(imagePath, resp);
|
||||
}
|
||||
const newTag = fullTag
|
||||
.replace(/\ssrc=["'][^"']+["']/, ` src="${resp.url}"`)
|
||||
.replace(/\sdata-local-path=["'][^"']+["']/, "");
|
||||
updatedHtml = updatedHtml.replace(fullTag, newTag);
|
||||
allMediaIds.push(resp.media_id);
|
||||
if (!firstMediaId) {
|
||||
firstMediaId = resp.media_id;
|
||||
const shouldUploadMaterial = articleType === "newspic" || (collectNewsCoverFallback && !firstCoverMediaId);
|
||||
if (shouldUploadMaterial) {
|
||||
let materialResp = uploadedBySource.get(`${imagePath}:material`);
|
||||
if (!materialResp) {
|
||||
materialResp = await uploadImage(imagePath, accessToken, baseDir, "material");
|
||||
uploadedBySource.set(`${imagePath}:material`, materialResp);
|
||||
}
|
||||
if (articleType === "newspic" && materialResp.media_id) {
|
||||
imageMediaIds.push(materialResp.media_id);
|
||||
}
|
||||
if (collectNewsCoverFallback && !firstCoverMediaId && materialResp.media_id) {
|
||||
firstCoverMediaId = materialResp.media_id;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[wechat-api] Failed to upload ${imagePath}:`, err);
|
||||
@@ -210,27 +304,38 @@ async function uploadImagesInHtml(
|
||||
if (!updatedHtml.includes(image.placeholder)) continue;
|
||||
|
||||
const imagePath = image.localPath || image.originalPath;
|
||||
console.error(`[wechat-api] Uploading placeholder image: ${imagePath}`);
|
||||
console.error(`[wechat-api] Uploading body image: ${imagePath}`);
|
||||
|
||||
try {
|
||||
let resp = uploadedBySource.get(imagePath);
|
||||
if (!resp) {
|
||||
resp = await uploadImage(imagePath, accessToken, baseDir);
|
||||
// 正文图片使用 media/uploadimg 接口获取 URL
|
||||
resp = await uploadImage(imagePath, accessToken, baseDir, "body");
|
||||
uploadedBySource.set(imagePath, resp);
|
||||
}
|
||||
|
||||
const replacementTag = `<img src="${resp.url}" style="display: block; width: 100%; margin: 1.5em auto;">`;
|
||||
updatedHtml = replaceAllPlaceholders(updatedHtml, image.placeholder, replacementTag);
|
||||
allMediaIds.push(resp.media_id);
|
||||
if (!firstMediaId) {
|
||||
firstMediaId = resp.media_id;
|
||||
const shouldUploadMaterial = articleType === "newspic" || (collectNewsCoverFallback && !firstCoverMediaId);
|
||||
if (shouldUploadMaterial) {
|
||||
let materialResp = uploadedBySource.get(`${imagePath}:material`);
|
||||
if (!materialResp) {
|
||||
materialResp = await uploadImage(imagePath, accessToken, baseDir, "material");
|
||||
uploadedBySource.set(`${imagePath}:material`, materialResp);
|
||||
}
|
||||
if (articleType === "newspic" && materialResp.media_id) {
|
||||
imageMediaIds.push(materialResp.media_id);
|
||||
}
|
||||
if (collectNewsCoverFallback && !firstCoverMediaId && materialResp.media_id) {
|
||||
firstCoverMediaId = materialResp.media_id;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[wechat-api] Failed to upload placeholder ${image.placeholder}:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
return { html: updatedHtml, firstMediaId, allMediaIds };
|
||||
return { html: updatedHtml, firstCoverMediaId, imageMediaIds };
|
||||
}
|
||||
|
||||
async function publishToDraft(
|
||||
@@ -345,7 +450,7 @@ function renderMarkdownWithPlaceholders(
|
||||
|
||||
function replaceAllPlaceholders(html: string, placeholder: string, replacement: string): string {
|
||||
const escapedPlaceholder = placeholder.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
return html.replace(new RegExp(escapedPlaceholder, "g"), replacement);
|
||||
return html.replace(new RegExp(escapedPlaceholder + "(?!\\d)", "g"), replacement);
|
||||
}
|
||||
|
||||
function extractHtmlContent(htmlPath: string): string {
|
||||
@@ -589,19 +694,13 @@ async function main(): Promise<void> {
|
||||
}
|
||||
|
||||
const creds = loadCredentials(resolved);
|
||||
for (const skippedSource of creds.skippedSources) {
|
||||
console.error(`[wechat-api] Skipped incomplete credential source: ${skippedSource}`);
|
||||
}
|
||||
console.error(`[wechat-api] Credentials source: ${creds.source}`);
|
||||
console.error("[wechat-api] Fetching access token...");
|
||||
const accessToken = await fetchAccessToken(creds.appId, creds.appSecret);
|
||||
|
||||
console.error("[wechat-api] Uploading images...");
|
||||
const { html: processedHtml, firstMediaId, allMediaIds } = await uploadImagesInHtml(
|
||||
htmlContent,
|
||||
accessToken,
|
||||
baseDir,
|
||||
contentImages,
|
||||
);
|
||||
htmlContent = processedHtml;
|
||||
|
||||
let thumbMediaId = "";
|
||||
const rawCoverPath = args.cover ||
|
||||
frontmatter.coverImage ||
|
||||
frontmatter.featureImage ||
|
||||
@@ -610,19 +709,31 @@ async function main(): Promise<void> {
|
||||
const coverPath = rawCoverPath && !path.isAbsolute(rawCoverPath) && args.cover
|
||||
? path.resolve(process.cwd(), rawCoverPath)
|
||||
: rawCoverPath;
|
||||
const needNewsCoverFallback = args.articleType === "news" && !coverPath;
|
||||
|
||||
console.error("[wechat-api] Uploading body images...");
|
||||
const { html: processedHtml, firstCoverMediaId, imageMediaIds } = await uploadImagesInHtml(
|
||||
htmlContent,
|
||||
accessToken,
|
||||
baseDir,
|
||||
contentImages,
|
||||
args.articleType,
|
||||
needNewsCoverFallback,
|
||||
);
|
||||
htmlContent = processedHtml;
|
||||
|
||||
let thumbMediaId = "";
|
||||
|
||||
if (coverPath) {
|
||||
console.error(`[wechat-api] Uploading cover: ${coverPath}`);
|
||||
const coverResp = await uploadImage(coverPath, accessToken, baseDir);
|
||||
// 封面图片使用 material/add_material 接口
|
||||
const coverResp = await uploadImage(coverPath, accessToken, baseDir, "material");
|
||||
thumbMediaId = coverResp.media_id;
|
||||
} else if (firstMediaId) {
|
||||
if (firstMediaId.startsWith("https://")) {
|
||||
console.error(`[wechat-api] Uploading first image as cover: ${firstMediaId}`);
|
||||
const coverResp = await uploadImage(firstMediaId, accessToken, baseDir);
|
||||
thumbMediaId = coverResp.media_id;
|
||||
} else {
|
||||
thumbMediaId = firstMediaId;
|
||||
}
|
||||
console.error(`[wechat-api] Cover uploaded successfully, media_id: ${thumbMediaId}`);
|
||||
} else if (firstCoverMediaId && args.articleType === "news") {
|
||||
// news 类型没有封面时,使用第一张正文图的 media_id 作为封面(兜底逻辑)
|
||||
thumbMediaId = firstCoverMediaId;
|
||||
console.error(`[wechat-api] Using first body image as cover (fallback), media_id: ${thumbMediaId}`);
|
||||
}
|
||||
|
||||
if (args.articleType === "news" && !thumbMediaId) {
|
||||
@@ -630,7 +741,7 @@ async function main(): Promise<void> {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (args.articleType === "newspic" && allMediaIds.length === 0) {
|
||||
if (args.articleType === "newspic" && imageMediaIds.length === 0) {
|
||||
console.error("Error: newspic requires at least one image in content.");
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -643,7 +754,7 @@ async function main(): Promise<void> {
|
||||
content: htmlContent,
|
||||
thumbMediaId,
|
||||
articleType: args.articleType,
|
||||
imageMediaIds: args.articleType === "newspic" ? allMediaIds : undefined,
|
||||
imageMediaIds: args.articleType === "newspic" ? imageMediaIds : undefined,
|
||||
needOpenComment: resolved.need_open_comment,
|
||||
onlyFansCanComment: resolved.only_fans_can_comment,
|
||||
}, accessToken);
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import test, { type TestContext } from "node:test";
|
||||
|
||||
import { loadCredentials } from "./wechat-extend-config.ts";
|
||||
|
||||
function useCwd(t: TestContext, cwd: string): void {
|
||||
const previous = process.cwd();
|
||||
process.chdir(cwd);
|
||||
t.after(() => {
|
||||
process.chdir(previous);
|
||||
});
|
||||
}
|
||||
|
||||
function useHome(t: TestContext, home: string): void {
|
||||
const previous = process.env.HOME;
|
||||
process.env.HOME = home;
|
||||
t.after(() => {
|
||||
if (previous === undefined) {
|
||||
delete process.env.HOME;
|
||||
return;
|
||||
}
|
||||
process.env.HOME = previous;
|
||||
});
|
||||
}
|
||||
|
||||
function useWechatEnv(
|
||||
t: TestContext,
|
||||
values: Partial<Record<"WECHAT_APP_ID" | "WECHAT_APP_SECRET", string | undefined>>,
|
||||
): void {
|
||||
const previous = {
|
||||
WECHAT_APP_ID: process.env.WECHAT_APP_ID,
|
||||
WECHAT_APP_SECRET: process.env.WECHAT_APP_SECRET,
|
||||
};
|
||||
|
||||
if (values.WECHAT_APP_ID === undefined) {
|
||||
delete process.env.WECHAT_APP_ID;
|
||||
} else {
|
||||
process.env.WECHAT_APP_ID = values.WECHAT_APP_ID;
|
||||
}
|
||||
|
||||
if (values.WECHAT_APP_SECRET === undefined) {
|
||||
delete process.env.WECHAT_APP_SECRET;
|
||||
} else {
|
||||
process.env.WECHAT_APP_SECRET = values.WECHAT_APP_SECRET;
|
||||
}
|
||||
|
||||
t.after(() => {
|
||||
if (previous.WECHAT_APP_ID === undefined) {
|
||||
delete process.env.WECHAT_APP_ID;
|
||||
} else {
|
||||
process.env.WECHAT_APP_ID = previous.WECHAT_APP_ID;
|
||||
}
|
||||
|
||||
if (previous.WECHAT_APP_SECRET === undefined) {
|
||||
delete process.env.WECHAT_APP_SECRET;
|
||||
} else {
|
||||
process.env.WECHAT_APP_SECRET = previous.WECHAT_APP_SECRET;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function makeTempDir(prefix: string): Promise<string> {
|
||||
return fs.mkdtemp(path.join(os.tmpdir(), prefix));
|
||||
}
|
||||
|
||||
async function writeEnvFile(root: string, content: string): Promise<void> {
|
||||
const envPath = path.join(root, ".baoyu-skills", ".env");
|
||||
await fs.mkdir(path.dirname(envPath), { recursive: true });
|
||||
await fs.writeFile(envPath, content);
|
||||
}
|
||||
|
||||
test("loadCredentials selects the first complete source without mixing values across sources", async (t) => {
|
||||
const cwdRoot = await makeTempDir("wechat-creds-cwd-");
|
||||
const homeRoot = await makeTempDir("wechat-creds-home-");
|
||||
|
||||
useCwd(t, cwdRoot);
|
||||
useHome(t, homeRoot);
|
||||
useWechatEnv(t, {
|
||||
WECHAT_APP_ID: undefined,
|
||||
WECHAT_APP_SECRET: "stale-secret-from-process-env",
|
||||
});
|
||||
|
||||
await writeEnvFile(cwdRoot, "WECHAT_APP_ID=cwd-app-id\nWECHAT_APP_SECRET=cwd-app-secret\n");
|
||||
await writeEnvFile(homeRoot, "WECHAT_APP_ID=home-app-id\nWECHAT_APP_SECRET=home-app-secret\n");
|
||||
|
||||
const credentials = loadCredentials();
|
||||
|
||||
assert.equal(credentials.appId, "cwd-app-id");
|
||||
assert.equal(credentials.appSecret, "cwd-app-secret");
|
||||
assert.equal(credentials.source, "<cwd>/.baoyu-skills/.env");
|
||||
assert.deepEqual(credentials.skippedSources, [
|
||||
"process.env missing WECHAT_APP_ID",
|
||||
]);
|
||||
});
|
||||
|
||||
test("loadCredentials prefers a complete process.env pair over lower-priority files", async (t) => {
|
||||
const cwdRoot = await makeTempDir("wechat-creds-cwd-");
|
||||
const homeRoot = await makeTempDir("wechat-creds-home-");
|
||||
|
||||
useCwd(t, cwdRoot);
|
||||
useHome(t, homeRoot);
|
||||
useWechatEnv(t, {
|
||||
WECHAT_APP_ID: "env-app-id",
|
||||
WECHAT_APP_SECRET: "env-app-secret",
|
||||
});
|
||||
|
||||
await writeEnvFile(cwdRoot, "WECHAT_APP_ID=cwd-app-id\nWECHAT_APP_SECRET=cwd-app-secret\n");
|
||||
await writeEnvFile(homeRoot, "WECHAT_APP_ID=home-app-id\nWECHAT_APP_SECRET=home-app-secret\n");
|
||||
|
||||
const credentials = loadCredentials();
|
||||
|
||||
assert.equal(credentials.appId, "env-app-id");
|
||||
assert.equal(credentials.appSecret, "env-app-secret");
|
||||
assert.equal(credentials.source, "process.env");
|
||||
assert.deepEqual(credentials.skippedSources, []);
|
||||
});
|
||||
|
||||
test("loadCredentials reports skipped incomplete sources when no complete pair exists", async (t) => {
|
||||
const cwdRoot = await makeTempDir("wechat-creds-cwd-");
|
||||
const homeRoot = await makeTempDir("wechat-creds-home-");
|
||||
|
||||
useCwd(t, cwdRoot);
|
||||
useHome(t, homeRoot);
|
||||
useWechatEnv(t, {
|
||||
WECHAT_APP_ID: "env-app-id",
|
||||
WECHAT_APP_SECRET: undefined,
|
||||
});
|
||||
|
||||
await writeEnvFile(cwdRoot, "WECHAT_APP_SECRET=cwd-app-secret\n");
|
||||
|
||||
assert.throws(
|
||||
() => loadCredentials(),
|
||||
/Incomplete credential sources skipped:\n- process\.env missing WECHAT_APP_SECRET\n- <cwd>\/\.baoyu-skills\/\.env missing WECHAT_APP_ID/,
|
||||
);
|
||||
});
|
||||
@@ -196,48 +196,116 @@ function aliasToEnvKey(alias: string): string {
|
||||
return alias.toUpperCase().replace(/-/g, "_");
|
||||
}
|
||||
|
||||
export function loadCredentials(account?: ResolvedAccount): { appId: string; appSecret: string } {
|
||||
if (account?.app_id && account?.app_secret) {
|
||||
return { appId: account.app_id, appSecret: account.app_secret };
|
||||
interface CredentialSource {
|
||||
name: string;
|
||||
appIdKey: string;
|
||||
appSecretKey: string;
|
||||
appId?: string;
|
||||
appSecret?: string;
|
||||
}
|
||||
|
||||
export interface LoadedCredentials {
|
||||
appId: string;
|
||||
appSecret: string;
|
||||
source: string;
|
||||
skippedSources: string[];
|
||||
}
|
||||
|
||||
function normalizeCredentialValue(value?: string): string | undefined {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function describeMissingKeys(source: CredentialSource): string {
|
||||
const missingKeys: string[] = [];
|
||||
if (!source.appId) missingKeys.push(source.appIdKey);
|
||||
if (!source.appSecret) missingKeys.push(source.appSecretKey);
|
||||
return `${source.name} missing ${missingKeys.join(" and ")}`;
|
||||
}
|
||||
|
||||
function buildCredentialSource(
|
||||
name: string,
|
||||
values: Record<string, string | undefined>,
|
||||
appIdKey: string,
|
||||
appSecretKey: string,
|
||||
): CredentialSource {
|
||||
return {
|
||||
name,
|
||||
appIdKey,
|
||||
appSecretKey,
|
||||
appId: normalizeCredentialValue(values[appIdKey]),
|
||||
appSecret: normalizeCredentialValue(values[appSecretKey]),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveCredentialSource(
|
||||
sources: CredentialSource[],
|
||||
account?: ResolvedAccount,
|
||||
): LoadedCredentials {
|
||||
const skippedSources: string[] = [];
|
||||
|
||||
for (const source of sources) {
|
||||
if (source.appId && source.appSecret) {
|
||||
return {
|
||||
appId: source.appId,
|
||||
appSecret: source.appSecret,
|
||||
source: source.name,
|
||||
skippedSources,
|
||||
};
|
||||
}
|
||||
|
||||
if (source.appId || source.appSecret) {
|
||||
skippedSources.push(describeMissingKeys(source));
|
||||
}
|
||||
}
|
||||
|
||||
const hint = account?.alias ? ` (account: ${account.alias})` : "";
|
||||
const partialHint = skippedSources.length > 0
|
||||
? `\nIncomplete credential sources skipped:\n- ${skippedSources.join("\n- ")}`
|
||||
: "";
|
||||
|
||||
throw new Error(
|
||||
`Missing WECHAT_APP_ID or WECHAT_APP_SECRET${hint}.\n` +
|
||||
"Set via EXTEND.md account config, environment variables, or .baoyu-skills/.env file." +
|
||||
partialHint
|
||||
);
|
||||
}
|
||||
|
||||
export function loadCredentials(account?: ResolvedAccount): LoadedCredentials {
|
||||
const cwdEnvPath = path.join(process.cwd(), ".baoyu-skills", ".env");
|
||||
const homeEnvPath = path.join(os.homedir(), ".baoyu-skills", ".env");
|
||||
const cwdEnv = loadEnvFile(cwdEnvPath);
|
||||
const homeEnv = loadEnvFile(homeEnvPath);
|
||||
|
||||
const sources: CredentialSource[] = [];
|
||||
|
||||
if (account?.app_id || account?.app_secret) {
|
||||
sources.push({
|
||||
name: account.alias ? `EXTEND.md account "${account.alias}"` : "EXTEND.md account config",
|
||||
appIdKey: "app_id",
|
||||
appSecretKey: "app_secret",
|
||||
appId: normalizeCredentialValue(account.app_id),
|
||||
appSecret: normalizeCredentialValue(account.app_secret),
|
||||
});
|
||||
}
|
||||
|
||||
const prefix = account?.alias ? `WECHAT_${aliasToEnvKey(account.alias)}_` : "";
|
||||
|
||||
let appId = "";
|
||||
let appSecret = "";
|
||||
|
||||
if (prefix) {
|
||||
appId = process.env[`${prefix}APP_ID`]
|
||||
|| cwdEnv[`${prefix}APP_ID`]
|
||||
|| homeEnv[`${prefix}APP_ID`]
|
||||
|| "";
|
||||
appSecret = process.env[`${prefix}APP_SECRET`]
|
||||
|| cwdEnv[`${prefix}APP_SECRET`]
|
||||
|| homeEnv[`${prefix}APP_SECRET`]
|
||||
|| "";
|
||||
}
|
||||
|
||||
if (!appId) {
|
||||
appId = process.env.WECHAT_APP_ID || cwdEnv.WECHAT_APP_ID || homeEnv.WECHAT_APP_ID || "";
|
||||
}
|
||||
if (!appSecret) {
|
||||
appSecret = process.env.WECHAT_APP_SECRET || cwdEnv.WECHAT_APP_SECRET || homeEnv.WECHAT_APP_SECRET || "";
|
||||
}
|
||||
|
||||
if (!appId || !appSecret) {
|
||||
const hint = account?.alias ? ` (account: ${account.alias})` : "";
|
||||
throw new Error(
|
||||
`Missing WECHAT_APP_ID or WECHAT_APP_SECRET${hint}.\n` +
|
||||
"Set via EXTEND.md account config, environment variables, or .baoyu-skills/.env file."
|
||||
const prefixedKeyLabel = `${prefix}APP_ID/${prefix}APP_SECRET`;
|
||||
sources.push(
|
||||
buildCredentialSource(`process.env (${prefixedKeyLabel})`, process.env, `${prefix}APP_ID`, `${prefix}APP_SECRET`),
|
||||
buildCredentialSource(`<cwd>/.baoyu-skills/.env (${prefixedKeyLabel})`, cwdEnv, `${prefix}APP_ID`, `${prefix}APP_SECRET`),
|
||||
buildCredentialSource(`~/.baoyu-skills/.env (${prefixedKeyLabel})`, homeEnv, `${prefix}APP_ID`, `${prefix}APP_SECRET`),
|
||||
);
|
||||
}
|
||||
|
||||
return { appId, appSecret };
|
||||
sources.push(
|
||||
buildCredentialSource("process.env", process.env, "WECHAT_APP_ID", "WECHAT_APP_SECRET"),
|
||||
buildCredentialSource("<cwd>/.baoyu-skills/.env", cwdEnv, "WECHAT_APP_ID", "WECHAT_APP_SECRET"),
|
||||
buildCredentialSource("~/.baoyu-skills/.env", homeEnv, "WECHAT_APP_ID", "WECHAT_APP_SECRET"),
|
||||
);
|
||||
|
||||
return resolveCredentialSource(sources, account);
|
||||
}
|
||||
|
||||
export function listAccounts(config: WechatExtendConfig): string[] {
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Jimp, JimpMime } from "jimp";
|
||||
import decodeWebp, { init as initWebpDecode } from "@jsquash/webp/decode.js";
|
||||
|
||||
export interface WechatUploadAsset {
|
||||
buffer: Buffer;
|
||||
filename: string;
|
||||
contentType: string;
|
||||
fileExt: string;
|
||||
fileSize: number;
|
||||
}
|
||||
|
||||
export interface PreparedWechatUploadAsset {
|
||||
buffer: Buffer;
|
||||
filename: string;
|
||||
contentType: string;
|
||||
wasProcessed: boolean;
|
||||
processingNotes: string[];
|
||||
}
|
||||
|
||||
export const WECHAT_BODY_IMAGE_MAX_SIZE = 1024 * 1024; // 1MB
|
||||
export const WECHAT_BODY_IMAGE_UNSUPPORTED_FORMATS = new Set([
|
||||
".gif",
|
||||
".webp",
|
||||
".bmp",
|
||||
".tiff",
|
||||
".tif",
|
||||
".svg",
|
||||
".ico",
|
||||
]);
|
||||
|
||||
const BODY_UPLOAD_ALLOWED_MIME_TYPES = new Set([
|
||||
JimpMime.jpeg,
|
||||
JimpMime.png,
|
||||
]);
|
||||
|
||||
const MIME_TO_EXT: Record<string, string> = {
|
||||
"image/jpeg": ".jpg",
|
||||
"image/png": ".png",
|
||||
"image/gif": ".gif",
|
||||
"image/webp": ".webp",
|
||||
"image/bmp": ".bmp",
|
||||
"image/x-ms-bmp": ".bmp",
|
||||
"image/tiff": ".tiff",
|
||||
"image/svg+xml": ".svg",
|
||||
"image/x-icon": ".ico",
|
||||
"image/vnd.microsoft.icon": ".ico",
|
||||
};
|
||||
|
||||
const JPEG_QUALITY_STEPS = [82, 74, 66, 58, 50, 42, 34];
|
||||
const MAX_WIDTH_STEPS = [2560, 2048, 1600, 1280, 1024, 800, 640, 480];
|
||||
|
||||
let webpDecoderReady: Promise<void> | undefined;
|
||||
|
||||
type JimpImage = Awaited<ReturnType<typeof Jimp.read>>;
|
||||
|
||||
function normalizeMimeType(contentType: string): string {
|
||||
return contentType.split(";")[0]!.trim().toLowerCase();
|
||||
}
|
||||
|
||||
function extFromMimeType(contentType: string): string {
|
||||
return MIME_TO_EXT[normalizeMimeType(contentType)] || "";
|
||||
}
|
||||
|
||||
function ensureFileExt(asset: WechatUploadAsset): string {
|
||||
return asset.fileExt || extFromMimeType(asset.contentType);
|
||||
}
|
||||
|
||||
function basenameWithoutExt(filename: string): string {
|
||||
const base = path.basename(filename, path.extname(filename));
|
||||
return base || "image";
|
||||
}
|
||||
|
||||
function renameWithExt(filename: string, ext: string): string {
|
||||
return `${basenameWithoutExt(filename)}${ext}`;
|
||||
}
|
||||
|
||||
export function needsWechatBodyImageProcessing(asset: WechatUploadAsset): boolean {
|
||||
if (asset.fileSize > WECHAT_BODY_IMAGE_MAX_SIZE) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const normalizedMimeType = normalizeMimeType(asset.contentType);
|
||||
if (BODY_UPLOAD_ALLOWED_MIME_TYPES.has(normalizedMimeType)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const fileExt = ensureFileExt(asset);
|
||||
return WECHAT_BODY_IMAGE_UNSUPPORTED_FORMATS.has(fileExt) || !fileExt;
|
||||
}
|
||||
|
||||
async function ensureWebpDecoder(): Promise<void> {
|
||||
if (!webpDecoderReady) {
|
||||
webpDecoderReady = (async () => {
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const wasmPath = path.resolve(__dirname, "node_modules/@jsquash/webp/codec/dec/webp_dec.wasm");
|
||||
const wasmModule = await WebAssembly.compile(await fs.readFile(wasmPath));
|
||||
await initWebpDecode(wasmModule, {});
|
||||
})();
|
||||
}
|
||||
|
||||
await webpDecoderReady;
|
||||
}
|
||||
|
||||
async function loadImageForProcessing(asset: WechatUploadAsset): Promise<JimpImage> {
|
||||
const fileExt = ensureFileExt(asset);
|
||||
const normalizedMimeType = normalizeMimeType(asset.contentType);
|
||||
|
||||
if (fileExt === ".webp" || normalizedMimeType === "image/webp") {
|
||||
await ensureWebpDecoder();
|
||||
const decoded = await decodeWebp(asset.buffer);
|
||||
return new Jimp({
|
||||
data: Buffer.from(decoded.data.buffer, decoded.data.byteOffset, decoded.data.byteLength),
|
||||
width: decoded.width,
|
||||
height: decoded.height,
|
||||
});
|
||||
}
|
||||
|
||||
if (fileExt === ".svg" || fileExt === ".ico") {
|
||||
throw new Error(`Cannot convert ${fileExt} image for WeChat body upload; provide a PNG or JPG instead.`);
|
||||
}
|
||||
|
||||
return Jimp.read(asset.buffer);
|
||||
}
|
||||
|
||||
function imageHasTransparency(image: JimpImage): boolean {
|
||||
const { data } = image.bitmap;
|
||||
for (let i = 3; i < data.length; i += 4) {
|
||||
if (data[i] !== 255) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function buildCandidateWidths(width: number): number[] {
|
||||
const candidates = new Set<number>([width]);
|
||||
|
||||
for (const maxWidth of MAX_WIDTH_STEPS) {
|
||||
if (width > maxWidth) {
|
||||
candidates.add(maxWidth);
|
||||
}
|
||||
}
|
||||
|
||||
return [...candidates].sort((a, b) => b - a);
|
||||
}
|
||||
|
||||
function resizeToWidth(image: JimpImage, width: number): JimpImage {
|
||||
const cloned = image.clone();
|
||||
if (width < image.bitmap.width) {
|
||||
cloned.resize({ w: width });
|
||||
}
|
||||
return cloned;
|
||||
}
|
||||
|
||||
function flattenOnWhite(image: JimpImage): JimpImage {
|
||||
const flattened = new Jimp({
|
||||
width: image.bitmap.width,
|
||||
height: image.bitmap.height,
|
||||
color: 0xffffffff,
|
||||
});
|
||||
flattened.composite(image, 0, 0);
|
||||
return flattened;
|
||||
}
|
||||
|
||||
async function encodePng(image: JimpImage): Promise<Buffer> {
|
||||
return image.getBuffer(JimpMime.png);
|
||||
}
|
||||
|
||||
async function encodeJpeg(image: JimpImage, quality: number): Promise<Buffer> {
|
||||
const jpegSource = imageHasTransparency(image) ? flattenOnWhite(image) : image;
|
||||
return jpegSource.getBuffer(JimpMime.jpeg, { quality });
|
||||
}
|
||||
|
||||
function buildProcessingNotes(asset: WechatUploadAsset): string[] {
|
||||
const notes: string[] = [];
|
||||
const fileExt = ensureFileExt(asset);
|
||||
|
||||
if (fileExt && WECHAT_BODY_IMAGE_UNSUPPORTED_FORMATS.has(fileExt)) {
|
||||
notes.push(`converted unsupported ${fileExt} source`);
|
||||
}
|
||||
|
||||
if (asset.fileSize > WECHAT_BODY_IMAGE_MAX_SIZE) {
|
||||
notes.push(`compressed ${(asset.fileSize / 1024 / 1024).toFixed(2)}MB source below 1MB`);
|
||||
}
|
||||
|
||||
if (notes.length === 0) {
|
||||
notes.push("re-encoded for WeChat body upload");
|
||||
}
|
||||
|
||||
return notes;
|
||||
}
|
||||
|
||||
export async function prepareWechatBodyImageUpload(
|
||||
asset: WechatUploadAsset,
|
||||
): Promise<PreparedWechatUploadAsset> {
|
||||
if (!needsWechatBodyImageProcessing(asset)) {
|
||||
return {
|
||||
buffer: asset.buffer,
|
||||
filename: asset.filename,
|
||||
contentType: asset.contentType,
|
||||
wasProcessed: false,
|
||||
processingNotes: [],
|
||||
};
|
||||
}
|
||||
|
||||
const image = await loadImageForProcessing(asset);
|
||||
const widths = buildCandidateWidths(image.bitmap.width);
|
||||
const preferPng = imageHasTransparency(image) || ensureFileExt(asset) === ".png";
|
||||
const processingNotes = buildProcessingNotes(asset);
|
||||
|
||||
for (const width of widths) {
|
||||
const resized = resizeToWidth(image, width);
|
||||
|
||||
if (preferPng) {
|
||||
const pngBuffer = await encodePng(resized);
|
||||
if (pngBuffer.length <= WECHAT_BODY_IMAGE_MAX_SIZE) {
|
||||
return {
|
||||
buffer: pngBuffer,
|
||||
filename: renameWithExt(asset.filename, ".png"),
|
||||
contentType: JimpMime.png,
|
||||
wasProcessed: true,
|
||||
processingNotes: width < image.bitmap.width
|
||||
? [...processingNotes, `resized to ${width}px wide`]
|
||||
: processingNotes,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
for (const quality of JPEG_QUALITY_STEPS) {
|
||||
const jpegBuffer = await encodeJpeg(resized, quality);
|
||||
if (jpegBuffer.length <= WECHAT_BODY_IMAGE_MAX_SIZE) {
|
||||
const notes = [...processingNotes, `encoded as JPEG (${quality} quality)`];
|
||||
if (width < image.bitmap.width) {
|
||||
notes.push(`resized to ${width}px wide`);
|
||||
}
|
||||
return {
|
||||
buffer: jpegBuffer,
|
||||
filename: renameWithExt(asset.filename, ".jpg"),
|
||||
contentType: JimpMime.jpeg,
|
||||
wasProcessed: true,
|
||||
processingNotes: notes,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Unable to reduce ${asset.filename} below 1MB for WeChat body upload.`);
|
||||
}
|
||||
@@ -123,6 +123,8 @@ ${BUN_X} {baseDir}/scripts/weibo-article.ts article.md --cover ./cover.jpg
|
||||
- Title: 32 characters max (truncated with warning if longer)
|
||||
- Summary/导语: 44 characters max (auto-regenerated from content if longer)
|
||||
|
||||
**Markdown-to-HTML**: Do NOT pass any `--theme` parameter when converting markdown to HTML. Use the default theme (no theme argument).
|
||||
|
||||
**Article Workflow**:
|
||||
1. Opens `https://card.weibo.com/article/v3/editor`
|
||||
2. Clicks "写文章" button, waits for editor to become editable
|
||||
|
||||
@@ -9,12 +9,26 @@ import { COLOR_PRESETS, FONT_FAMILY_MAP } from "./constants.ts";
|
||||
import {
|
||||
buildMarkdownDocumentMeta,
|
||||
formatTimestamp,
|
||||
renderMarkdownDocument,
|
||||
resolveColorToken,
|
||||
resolveFontFamilyToken,
|
||||
resolveMarkdownStyle,
|
||||
resolveRenderOptions,
|
||||
} from "./document.ts";
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, `\\$&`);
|
||||
}
|
||||
|
||||
function findInlineStyle(html: string, tagName: string, text: string): string {
|
||||
const pattern = new RegExp(
|
||||
`<${tagName}[^>]*style="([^"]*)"[^>]*>${escapeRegExp(text)}</${tagName}>`,
|
||||
);
|
||||
const match = html.match(pattern);
|
||||
assert.ok(match, `Expected inline style for <${tagName}>${text}</${tagName}>`);
|
||||
return match![1]!;
|
||||
}
|
||||
|
||||
function useCwd(t: TestContext, cwd: string): void {
|
||||
const previous = process.cwd();
|
||||
process.chdir(cwd);
|
||||
@@ -138,3 +152,23 @@ keep_title: true
|
||||
assert.equal(explicit.fontSize, "18px");
|
||||
assert.equal(explicit.keepTitle, false);
|
||||
});
|
||||
|
||||
test("renderMarkdownDocument layers default rules into grace theme before CSS inlining", async () => {
|
||||
const { html } = await renderMarkdownDocument(
|
||||
`## Section\n\nParagraph with **bold** text.`,
|
||||
{ keepTitle: true, theme: "grace" },
|
||||
);
|
||||
|
||||
const h2Style = findInlineStyle(html, "h2", "Section");
|
||||
assert.match(h2Style, /background: #92617E/);
|
||||
assert.match(h2Style, /box-shadow: 0 4px 6px rgba\(0, 0, 0, 0\.1\)/);
|
||||
|
||||
const pMatch = html.match(/<p[^>]*style="([^"]*)"[^>]*>/);
|
||||
assert.ok(pMatch, "Expected inline style on <p> tag");
|
||||
assert.match(pMatch![1]!, /color:/);
|
||||
|
||||
const strongPattern = /<strong[^>]*style="([^"]*)"[^>]*>bold<\/strong>/;
|
||||
const strongMatch = html.match(strongPattern);
|
||||
assert.ok(strongMatch, "Expected inline style for <strong>bold</strong>");
|
||||
assert.match(strongMatch![1]!, /font-weight:/);
|
||||
});
|
||||
|
||||
@@ -59,6 +59,17 @@ test("normalizeCssText and normalizeInlineCss replace variables and strip declar
|
||||
assert.doesNotMatch(normalizedHtml, /var\(--md-primary-color\)/);
|
||||
});
|
||||
|
||||
test("normalizeInlineCss removes quoted custom property values without leaving fragments behind", () => {
|
||||
const normalizedHtml = normalizeInlineCss(
|
||||
`<html style="--md-font-family: Menlo, Monaco, 'Courier New', monospace; color: var(--md-primary-color)"></html>`,
|
||||
DEFAULT_STYLE,
|
||||
);
|
||||
|
||||
assert.match(normalizedHtml, /style=" color: #0F4C81"/);
|
||||
assert.doesNotMatch(normalizedHtml, /Courier New/);
|
||||
assert.doesNotMatch(normalizedHtml, /--md-font-family/);
|
||||
});
|
||||
|
||||
test("HTML structure helpers hoist nested lists and remove the first heading", () => {
|
||||
const nestedList = `<ul><li>Parent<ul><li>Child</li></ul></li></ul>`;
|
||||
assert.equal(
|
||||
|
||||
@@ -100,13 +100,13 @@ export function normalizeCssText(cssText: string, style: StyleConfig = DEFAULT_S
|
||||
.replace(/var\(--md-accent-color\)/g, style.accentColor)
|
||||
.replace(/var\(--md-container-bg\)/g, style.containerBg)
|
||||
.replace(/hsl\(var\(--foreground\)\)/g, "#3f3f3f")
|
||||
.replace(/--md-primary-color:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-font-family:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-font-size:\s*[^;"']+;?/g, "")
|
||||
.replace(/--blockquote-background:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-accent-color:\s*[^;"']+;?/g, "")
|
||||
.replace(/--md-container-bg:\s*[^;"']+;?/g, "")
|
||||
.replace(/--foreground:\s*[^;"']+;?/g, "");
|
||||
.replace(/--md-primary-color:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-font-family:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-font-size:\s*[^;]+;?/g, "")
|
||||
.replace(/--blockquote-background:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-accent-color:\s*[^;]+;?/g, "")
|
||||
.replace(/--md-container-bg:\s*[^;]+;?/g, "")
|
||||
.replace(/--foreground:\s*[^;]+;?/g, "");
|
||||
}
|
||||
|
||||
export function normalizeInlineCss(html: string, style: StyleConfig = DEFAULT_STYLE): string {
|
||||
|
||||
@@ -6,6 +6,7 @@ import type { ThemeName } from "./types.js";
|
||||
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
export const THEME_DIR = path.resolve(SCRIPT_DIR, "themes");
|
||||
const FALLBACK_THEMES: ThemeName[] = ["default", "grace", "simple"];
|
||||
const THEMES_EXTENDING_DEFAULT = new Set<ThemeName>(["grace", "simple"]);
|
||||
|
||||
function stripOutputScope(cssContent: string): string {
|
||||
let css = cssContent;
|
||||
@@ -41,6 +42,7 @@ export function loadThemeCss(theme: ThemeName): {
|
||||
themeCss: string;
|
||||
} {
|
||||
const basePath = path.join(THEME_DIR, "base.css");
|
||||
const defaultThemePath = path.join(THEME_DIR, "default.css");
|
||||
const themePath = path.join(THEME_DIR, `${theme}.css`);
|
||||
|
||||
if (!fs.existsSync(basePath)) {
|
||||
@@ -51,9 +53,18 @@ export function loadThemeCss(theme: ThemeName): {
|
||||
throw new Error(`Missing theme CSS for "${theme}": ${themePath}`);
|
||||
}
|
||||
|
||||
const layeredThemeCss: string[] = [];
|
||||
if (theme !== "default" && THEMES_EXTENDING_DEFAULT.has(theme)) {
|
||||
if (!fs.existsSync(defaultThemePath)) {
|
||||
throw new Error(`Missing default theme CSS: ${defaultThemePath}`);
|
||||
}
|
||||
layeredThemeCss.push(fs.readFileSync(defaultThemePath, "utf-8"));
|
||||
}
|
||||
layeredThemeCss.push(fs.readFileSync(themePath, "utf-8"));
|
||||
|
||||
return {
|
||||
baseCss: fs.readFileSync(basePath, "utf-8"),
|
||||
themeCss: fs.readFileSync(themePath, "utf-8"),
|
||||
themeCss: layeredThemeCss.join("\n"),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: baoyu-url-to-markdown
|
||||
description: Fetch any URL and convert to markdown using Chrome CDP. Saves the rendered HTML snapshot alongside the markdown, uses an upgraded Defuddle pipeline with better web-component handling and YouTube transcript extraction, and automatically falls back to the pre-Defuddle HTML-to-Markdown pipeline when needed. If local browser capture fails entirely, it can fall back to the hosted defuddle.md API. Supports two modes - auto-capture on page load, or wait for user signal (for pages requiring login). Use when user wants to save a webpage as markdown.
|
||||
version: 1.58.1
|
||||
version: 1.59.0
|
||||
metadata:
|
||||
openclaw:
|
||||
homepage: https://github.com/JimLiu/baoyu-skills#baoyu-url-to-markdown
|
||||
@@ -30,6 +30,9 @@ Fetches any URL via Chrome CDP, saves the rendered HTML snapshot, and converts i
|
||||
|--------|---------|
|
||||
| `scripts/main.ts` | CLI entry point for URL fetching |
|
||||
| `scripts/html-to-markdown.ts` | Markdown conversion entry point and converter selection |
|
||||
| `scripts/parsers/index.ts` | Unified parser entry: dispatches URL-specific rules before generic converters |
|
||||
| `scripts/parsers/types.ts` | Unified parser interface shared by all rule files |
|
||||
| `scripts/parsers/rules/*.ts` | One file per URL rule, for example X status and X article |
|
||||
| `scripts/defuddle-converter.ts` | Defuddle-based conversion |
|
||||
| `scripts/legacy-converter.ts` | Pre-Defuddle legacy extraction and markdown conversion |
|
||||
| `scripts/markdown-conversion-shared.ts` | Shared metadata parsing and markdown document helpers |
|
||||
@@ -115,10 +118,13 @@ Full reference: [references/config/first-time-setup.md](references/config/first-
|
||||
## Features
|
||||
|
||||
- Chrome CDP for full JavaScript rendering
|
||||
- URL-specific parser layer for sites that need custom HTML rules before generic extraction
|
||||
- Two capture modes: auto or wait-for-user
|
||||
- Save rendered HTML as a sibling `-captured.html` file
|
||||
- Clean markdown output with metadata
|
||||
- Upgraded Defuddle-first markdown conversion with automatic fallback to the pre-Defuddle extractor from git history
|
||||
- X/Twitter pages can use HTML-specific parsing for Tweets and Articles, which improves title/body/media extraction on `x.com` / `twitter.com`
|
||||
- `archive.ph` / related archive mirrors can restore the original URL from `input[name=q]` and prefer `#CONTENT` before falling back to the page body
|
||||
- Materializes shadow DOM content before conversion so web-component pages survive serialization better
|
||||
- YouTube pages can include transcript/caption text in the markdown when YouTube exposes a caption track
|
||||
- If local browser capture fails completely, can fall back to `defuddle.md/<url>` and still save markdown
|
||||
@@ -201,14 +207,16 @@ When `--download-media` is enabled:
|
||||
|
||||
Conversion order:
|
||||
|
||||
1. Try Defuddle first
|
||||
2. For rich pages such as YouTube, prefer Defuddle's extractor-specific output (including transcripts when available) instead of replacing it with the legacy pipeline
|
||||
3. If Defuddle throws, cannot load, returns obviously incomplete markdown, or captures lower-quality content than the legacy pipeline, automatically fall back to the pre-Defuddle extractor
|
||||
4. If the entire local browser capture flow fails before markdown can be produced, try the hosted `https://defuddle.md/<url>` API and save its markdown output directly
|
||||
5. The legacy fallback path uses the older Readability/selector/Next.js-data based HTML-to-Markdown implementation recovered from git history
|
||||
1. Try the URL-specific parser layer first when a site rule matches
|
||||
2. If no specialized parser matches, try Defuddle
|
||||
3. For rich pages such as YouTube, prefer Defuddle's extractor-specific output (including transcripts when available) instead of replacing it with the legacy pipeline
|
||||
4. If Defuddle throws, cannot load, returns obviously incomplete markdown, or captures lower-quality content than the legacy pipeline, automatically fall back to the pre-Defuddle extractor
|
||||
5. If the entire local browser capture flow fails before markdown can be produced, try the hosted `https://defuddle.md/<url>` API and save its markdown output directly
|
||||
6. The legacy fallback path uses the older Readability/selector/Next.js-data based HTML-to-Markdown implementation recovered from git history
|
||||
|
||||
CLI output will show:
|
||||
|
||||
- `Converter: parser:...` when a URL-specific parser succeeded
|
||||
- `Converter: defuddle` when Defuddle succeeds
|
||||
- `Converter: legacy:...` plus `Fallback used: ...` when fallback was needed
|
||||
- `Converter: defuddle-api` when local browser capture failed and the hosted API was used instead
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
scoreMarkdownQuality,
|
||||
shouldCompareWithLegacy,
|
||||
} from "./legacy-converter.js";
|
||||
import { tryUrlRuleParsers } from "./parsers/index.js";
|
||||
|
||||
export type { ConversionResult, PageMetadata };
|
||||
export { createMarkdownDocument, formatMetadataYaml };
|
||||
@@ -105,6 +106,11 @@ export async function extractContent(html: string, url: string): Promise<Convers
|
||||
const capturedAt = new Date().toISOString();
|
||||
const baseMetadata = extractMetadataFromHtml(html, url, capturedAt);
|
||||
|
||||
const specializedResult = tryUrlRuleParsers(html, url, baseMetadata);
|
||||
if (specializedResult) {
|
||||
return specializedResult;
|
||||
}
|
||||
|
||||
const defuddleResult = await tryDefuddleConversion(html, url, baseMetadata);
|
||||
if (defuddleResult.ok) {
|
||||
if (shouldPreferDefuddle(defuddleResult.result)) {
|
||||
|
||||
@@ -521,14 +521,18 @@ turndown.addRule("collapseFigure", {
|
||||
|
||||
turndown.addRule("dropInvisibleAnchors", {
|
||||
filter(node) {
|
||||
return node.nodeName === "A" && !(node as Element).textContent?.trim();
|
||||
return (
|
||||
node.nodeName === "A" &&
|
||||
!(node as Element).textContent?.trim() &&
|
||||
!(node as Element).querySelector("img, video, picture, source")
|
||||
);
|
||||
},
|
||||
replacement() {
|
||||
return "";
|
||||
},
|
||||
});
|
||||
|
||||
function convertHtmlToMarkdown(html: string): string {
|
||||
export function convertHtmlFragmentToMarkdown(html: string): string {
|
||||
if (!html || !html.trim()) return "";
|
||||
|
||||
try {
|
||||
@@ -609,7 +613,7 @@ export function shouldCompareWithLegacy(markdown: string): boolean {
|
||||
export function convertWithLegacyExtractor(html: string, baseMetadata: PageMetadata): ConversionResult {
|
||||
const extracted = extractFromHtml(html);
|
||||
|
||||
let markdown = extracted?.html ? convertHtmlToMarkdown(extracted.html) : "";
|
||||
let markdown = extracted?.html ? convertHtmlFragmentToMarkdown(extracted.html) : "";
|
||||
if (!markdown.trim()) {
|
||||
markdown = extracted?.textContent?.trim() || fallbackPlainText(html);
|
||||
}
|
||||
|
||||
@@ -52,15 +52,71 @@ function parseArgs(argv: string[]): Args {
|
||||
return args;
|
||||
}
|
||||
|
||||
function generateSlug(title: string, url: string): string {
|
||||
const text = title || new URL(url).pathname.replace(/\//g, "-");
|
||||
return text
|
||||
.toLowerCase()
|
||||
const SLUG_STOP_WORDS = new Set([
|
||||
"the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
|
||||
"have", "has", "had", "do", "does", "did", "will", "would", "shall",
|
||||
"should", "may", "might", "must", "can", "could", "to", "of", "in",
|
||||
"for", "on", "with", "at", "by", "from", "as", "into", "through",
|
||||
"during", "before", "after", "above", "below", "between", "out",
|
||||
"off", "over", "under", "again", "further", "then", "once", "here",
|
||||
"there", "when", "where", "why", "how", "all", "both", "each",
|
||||
"few", "more", "most", "other", "some", "such", "no", "nor", "not",
|
||||
"only", "own", "same", "so", "than", "too", "very", "just", "but",
|
||||
"and", "or", "if", "this", "that", "these", "those", "it", "its",
|
||||
"http", "https", "www", "com", "org", "net", "post", "article",
|
||||
]);
|
||||
|
||||
function extractSlugFromContent(content: string): string | null {
|
||||
const body = content.replace(/^---\n[\s\S]*?\n---\n?/, "").slice(0, 1000);
|
||||
const words = body
|
||||
.replace(/[^\w\s-]/g, " ")
|
||||
.replace(/\s+/g, "-")
|
||||
.split(/\s+/)
|
||||
.filter((w) => /^[a-zA-Z]/.test(w) && w.length >= 2 && !SLUG_STOP_WORDS.has(w.toLowerCase()))
|
||||
.map((w) => w.toLowerCase());
|
||||
|
||||
const unique: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const w of words) {
|
||||
if (!seen.has(w)) {
|
||||
seen.add(w);
|
||||
unique.push(w);
|
||||
if (unique.length >= 6) break;
|
||||
}
|
||||
}
|
||||
return unique.length >= 2 ? unique.join("-").slice(0, 50) : null;
|
||||
}
|
||||
|
||||
function generateSlug(title: string, url: string, content?: string): string {
|
||||
const asciiWords = title
|
||||
.replace(/[^\w\s]/g, " ")
|
||||
.split(/\s+/)
|
||||
.filter((w) => /[a-zA-Z]/.test(w) && w.length >= 2 && !SLUG_STOP_WORDS.has(w.toLowerCase()))
|
||||
.map((w) => w.toLowerCase());
|
||||
|
||||
if (asciiWords.length >= 2) {
|
||||
return asciiWords.slice(0, 6).join("-").slice(0, 50);
|
||||
}
|
||||
|
||||
if (content) {
|
||||
const contentSlug = extractSlugFromContent(content);
|
||||
if (contentSlug) return contentSlug;
|
||||
}
|
||||
|
||||
const GENERIC_PATH_SEGMENTS = new Set(["status", "article", "post", "posts", "p", "blog", "news", "articles"]);
|
||||
const parsed = new URL(url);
|
||||
const pathSlug = parsed.pathname
|
||||
.split("/")
|
||||
.filter((s) => s.length > 0 && !/^\d{10,}$/.test(s) && !GENERIC_PATH_SEGMENTS.has(s.toLowerCase()))
|
||||
.join("-")
|
||||
.toLowerCase()
|
||||
.replace(/[^\w-]/g, "-")
|
||||
.replace(/-+/g, "-")
|
||||
.replace(/^-|-$/g, "")
|
||||
.slice(0, 50) || "page";
|
||||
.slice(0, 40);
|
||||
|
||||
const prefix = asciiWords.slice(0, 2).join("-");
|
||||
const combined = prefix ? `${prefix}-${pathSlug}` : pathSlug;
|
||||
return combined.slice(0, 50) || "page";
|
||||
}
|
||||
|
||||
function formatTimestamp(): string {
|
||||
@@ -124,18 +180,18 @@ async function fetchDefuddleApiMarkdown(targetUrl: string): Promise<{ markdown:
|
||||
};
|
||||
}
|
||||
|
||||
async function generateOutputPath(url: string, title: string, outputDir?: string): Promise<string> {
|
||||
async function generateOutputPath(url: string, title: string, outputDir?: string, content?: string): Promise<string> {
|
||||
const domain = new URL(url).hostname.replace(/^www\./, "");
|
||||
const slug = generateSlug(title, url);
|
||||
const slug = generateSlug(title, url, content);
|
||||
const dataDir = outputDir ? path.resolve(outputDir) : resolveUrlToMarkdownDataDir();
|
||||
const basePath = path.join(dataDir, domain, `${slug}.md`);
|
||||
const basePath = path.join(dataDir, domain, slug, `${slug}.md`);
|
||||
|
||||
if (!(await fileExists(basePath))) {
|
||||
return basePath;
|
||||
}
|
||||
|
||||
const timestampSlug = `${slug}-${formatTimestamp()}`;
|
||||
return path.join(dataDir, domain, `${timestampSlug}.md`);
|
||||
return path.join(dataDir, domain, timestampSlug, `${timestampSlug}.md`);
|
||||
}
|
||||
|
||||
async function waitForUserSignal(): Promise<void> {
|
||||
@@ -249,13 +305,12 @@ async function main(): Promise<void> {
|
||||
|
||||
try {
|
||||
const result = await captureUrl(args);
|
||||
outputPath = args.output || await generateOutputPath(args.url, result.metadata.title, args.outputDir);
|
||||
document = createMarkdownDocument(result);
|
||||
outputPath = args.output || await generateOutputPath(args.url, result.metadata.title, args.outputDir, document);
|
||||
const outputDir = path.dirname(outputPath);
|
||||
htmlSnapshotPath = deriveHtmlSnapshotPath(outputPath);
|
||||
await mkdir(outputDir, { recursive: true });
|
||||
await writeFile(htmlSnapshotPath, result.rawHtml, "utf-8");
|
||||
|
||||
document = createMarkdownDocument(result);
|
||||
conversionMethod = result.conversionMethod;
|
||||
fallbackReason = result.fallbackReason;
|
||||
} catch (error) {
|
||||
@@ -265,10 +320,9 @@ async function main(): Promise<void> {
|
||||
|
||||
try {
|
||||
const remoteResult = await fetchDefuddleApiMarkdown(args.url);
|
||||
outputPath = args.output || await generateOutputPath(args.url, remoteResult.title, args.outputDir);
|
||||
await mkdir(path.dirname(outputPath), { recursive: true });
|
||||
|
||||
document = remoteResult.markdown;
|
||||
outputPath = args.output || await generateOutputPath(args.url, remoteResult.title, args.outputDir, document);
|
||||
await mkdir(path.dirname(outputPath), { recursive: true });
|
||||
conversionMethod = "defuddle-api";
|
||||
fallbackReason = `Local browser capture failed: ${primaryError}`;
|
||||
} catch (remoteError) {
|
||||
|
||||
@@ -300,6 +300,24 @@ export function createMarkdownDocument(result: ConversionResult): string {
|
||||
const escapedTitle = result.metadata.title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const titleRegex = new RegExp(`^#\\s+${escapedTitle}\\s*(\\n|$)`, "i");
|
||||
const hasTitle = titleRegex.test(result.markdown.trimStart());
|
||||
const title = result.metadata.title && !hasTitle ? `\n\n# ${result.metadata.title}\n\n` : "\n\n";
|
||||
const firstMeaningfulLine = result.markdown
|
||||
.replace(/\r\n/g, "\n")
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.find((line) => line && !/^!?\[[^\]]*\]\([^)]+\)$/.test(line))
|
||||
?.replace(/^>\s*/, "")
|
||||
?.replace(/^#+\s+/, "")
|
||||
?.trim();
|
||||
const comparableTitle = result.metadata.title.toLowerCase().replace(/(?:\.{3}|…)\s*$/, "");
|
||||
const comparableFirstLine = firstMeaningfulLine?.toLowerCase() ?? "";
|
||||
const titleRepeatsContent =
|
||||
comparableTitle !== "" &&
|
||||
comparableFirstLine !== "" &&
|
||||
(comparableFirstLine === comparableTitle ||
|
||||
comparableFirstLine.startsWith(comparableTitle) ||
|
||||
comparableTitle.startsWith(comparableFirstLine));
|
||||
const title = result.metadata.title && !hasTitle && !titleRepeatsContent
|
||||
? `\n\n# ${result.metadata.title}\n\n`
|
||||
: "\n\n";
|
||||
return yaml + title + result.markdown;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
createMarkdownDocument,
|
||||
extractMetadataFromHtml,
|
||||
} from "../markdown-conversion-shared.js";
|
||||
import { tryUrlRuleParsers } from "./index.js";
|
||||
|
||||
const CAPTURED_AT = "2026-03-22T06:00:00.000Z";
|
||||
|
||||
const ARTICLE_HTML = `<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<body>
|
||||
<div data-testid="twitterArticleReadView">
|
||||
<a href="/dotey/article/2035141635713941927/media/1">
|
||||
<div data-testid="tweetPhoto">
|
||||
<img src="https://pbs.twimg.com/media/article-cover.jpg" alt="Image">
|
||||
</div>
|
||||
</a>
|
||||
<div data-testid="twitter-article-title">Karpathy:"写代码"已经不是对的动词了</div>
|
||||
<div data-testid="User-Name">
|
||||
<a href="/dotey">宝玉 Verified account</a>
|
||||
<a href="/dotey">@dotey</a>
|
||||
<time datetime="2026-03-20T23:49:11.000Z">Mar 20</time>
|
||||
</div>
|
||||
<div data-testid="twitterArticleRichTextView">
|
||||
<p>Andrej Karpathy 说他从 2024 年 12 月起就基本没手写过一行代码。</p>
|
||||
<a href="/dotey/article/2035141635713941927/media/2">
|
||||
<div>
|
||||
<div>
|
||||
<div data-testid="tweetPhoto">
|
||||
<img src="https://pbs.twimg.com/media/article-inline.jpg" alt="Image">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</a>
|
||||
<h2>要点速览</h2>
|
||||
<ul>
|
||||
<li>核心焦虑从 GPU 利用率转向 Token 吞吐量</li>
|
||||
</ul>
|
||||
<blockquote>
|
||||
<p>写代码已经不是对的动词了。</p>
|
||||
</blockquote>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
const STATUS_HTML = `<!doctype html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
<article data-testid="tweet">
|
||||
<div data-testid="User-Name">
|
||||
<a href="/dotey">宝玉 Verified account</a>
|
||||
<a href="/dotey">@dotey</a>
|
||||
<time datetime="2026-03-22T05:33:00.000Z">Mar 22</time>
|
||||
</div>
|
||||
<div data-testid="tweetText">
|
||||
<span>转译:把下面这段加到你的 Codex 自定义指令里,体验会好太多:</span>
|
||||
</div>
|
||||
<div data-testid="tweetPhoto">
|
||||
<img src="https://pbs.twimg.com/media/tweet-main.jpg" alt="Image">
|
||||
</div>
|
||||
<div data-testid="User-Name">
|
||||
<a href="/mattshumer_">Matt Shumer Verified account</a>
|
||||
<a href="/mattshumer_">@mattshumer_</a>
|
||||
<time datetime="2026-03-17T00:00:00.000Z">Mar 17</time>
|
||||
</div>
|
||||
<div data-testid="tweetText">
|
||||
<span>Add this to your Codex custom instructions for a way better experience.</span>
|
||||
</div>
|
||||
</article>
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
const ARCHIVE_HTML = `<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>archive.ph</title>
|
||||
</head>
|
||||
<body>
|
||||
<form>
|
||||
<input
|
||||
type="text"
|
||||
name="q"
|
||||
value="https://www.newscientist.com/article/2520204-major-leap-towards-reanimation-after-death-as-mammals-brain-preserved/"
|
||||
>
|
||||
</form>
|
||||
<div id="HEADER">
|
||||
Archive shell text that should be ignored when CONTENT exists.
|
||||
</div>
|
||||
<div id="CONTENT">
|
||||
<h1>Major leap towards reanimation after death as mammal brain preserved</h1>
|
||||
<p>
|
||||
Researchers say the preserved structure and activity markers suggest a significant step
|
||||
forward in keeping delicate brain tissue viable after clinical death.
|
||||
</p>
|
||||
<p>
|
||||
The archive wrapper should not take precedence over the actual article body when the
|
||||
CONTENT container is available for parsing.
|
||||
</p>
|
||||
<img src="https://cdn.example.com/brain.jpg" alt="Brain tissue">
|
||||
</div>
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
const ARCHIVE_FALLBACK_HTML = `<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>archive.ph</title>
|
||||
</head>
|
||||
<body>
|
||||
<input type="text" name="q" value="https://example.com/fallback-story">
|
||||
<main>
|
||||
<h1>Fallback body parsing still works</h1>
|
||||
<p>
|
||||
When CONTENT is absent, the parser should fall back to the body content instead of
|
||||
returning null or keeping the archive wrapper as the final URL.
|
||||
</p>
|
||||
<p>
|
||||
This ensures archived pages with slightly different layouts still produce usable markdown.
|
||||
</p>
|
||||
</main>
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
function parse(html: string, url: string) {
|
||||
const baseMetadata = extractMetadataFromHtml(html, url, CAPTURED_AT);
|
||||
return tryUrlRuleParsers(html, url, baseMetadata);
|
||||
}
|
||||
|
||||
test("parses archive.ph pages from CONTENT and restores the original URL", () => {
|
||||
const result = parse(ARCHIVE_HTML, "https://archive.ph/SMcX5");
|
||||
|
||||
assert.ok(result);
|
||||
assert.equal(result.conversionMethod, "parser:archive-ph");
|
||||
assert.equal(
|
||||
result.metadata.url,
|
||||
"https://www.newscientist.com/article/2520204-major-leap-towards-reanimation-after-death-as-mammals-brain-preserved/"
|
||||
);
|
||||
assert.equal(
|
||||
result.metadata.title,
|
||||
"Major leap towards reanimation after death as mammal brain preserved"
|
||||
);
|
||||
assert.equal(result.metadata.coverImage, "https://cdn.example.com/brain.jpg");
|
||||
assert.ok(result.markdown.includes("Researchers say the preserved structure"));
|
||||
assert.ok(result.markdown.includes(""));
|
||||
assert.ok(!result.markdown.includes("Archive shell text that should be ignored"));
|
||||
});
|
||||
|
||||
test("falls back to body when archive.ph CONTENT is missing", () => {
|
||||
const result = parse(ARCHIVE_FALLBACK_HTML, "https://archive.ph/fallback");
|
||||
|
||||
assert.ok(result);
|
||||
assert.equal(result.conversionMethod, "parser:archive-ph");
|
||||
assert.equal(result.metadata.url, "https://example.com/fallback-story");
|
||||
assert.equal(result.metadata.title, "Fallback body parsing still works");
|
||||
assert.ok(result.markdown.includes("When CONTENT is absent"));
|
||||
});
|
||||
|
||||
test("parses X article pages from HTML", () => {
|
||||
const result = parse(
|
||||
ARTICLE_HTML,
|
||||
"https://x.com/dotey/article/2035141635713941927"
|
||||
);
|
||||
|
||||
assert.ok(result);
|
||||
assert.equal(result.conversionMethod, "parser:x-article");
|
||||
assert.equal(result.metadata.title, "Karpathy:\"写代码\"已经不是对的动词了");
|
||||
assert.equal(result.metadata.author, "宝玉 (@dotey)");
|
||||
assert.equal(result.metadata.coverImage, "https://pbs.twimg.com/media/article-cover.jpg");
|
||||
assert.equal(result.metadata.published, "2026-03-20T23:49:11.000Z");
|
||||
assert.equal(result.metadata.language, "zh");
|
||||
assert.ok(result.markdown.includes("## 要点速览"));
|
||||
assert.ok(
|
||||
result.markdown.includes(
|
||||
"[](/dotey/article/2035141635713941927/media/2)"
|
||||
)
|
||||
);
|
||||
assert.ok(result.markdown.includes("写代码已经不是对的动词了。"));
|
||||
|
||||
const document = createMarkdownDocument(result);
|
||||
assert.ok(document.includes("# Karpathy:\"写代码\"已经不是对的动词了"));
|
||||
});
|
||||
|
||||
test("parses X status pages from HTML without duplicating the title heading", () => {
|
||||
const result = parse(
|
||||
STATUS_HTML,
|
||||
"https://x.com/dotey/status/2035590649081196710"
|
||||
);
|
||||
|
||||
assert.ok(result);
|
||||
assert.equal(result.conversionMethod, "parser:x-status");
|
||||
assert.equal(result.metadata.author, "宝玉 (@dotey)");
|
||||
assert.equal(result.metadata.coverImage, "https://pbs.twimg.com/media/tweet-main.jpg");
|
||||
assert.equal(result.metadata.language, "zh");
|
||||
assert.ok(result.markdown.includes("转译:把下面这段加到你的 Codex 自定义指令里"));
|
||||
assert.ok(result.markdown.includes("> Quote from Matt Shumer (@mattshumer_)"));
|
||||
assert.ok(result.markdown.includes("!["));
|
||||
|
||||
const document = createMarkdownDocument(result);
|
||||
assert.ok(
|
||||
!document.includes("\n\n# 转译:把下面这段加到你的 Codex 自定义指令里,体验会好太多:\n\n")
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import {
|
||||
isMarkdownUsable,
|
||||
normalizeMarkdown,
|
||||
parseDocument,
|
||||
type ConversionResult,
|
||||
type PageMetadata,
|
||||
} from "../markdown-conversion-shared.js";
|
||||
import { URL_RULE_PARSERS } from "./rules/index.js";
|
||||
import type { UrlRuleParserContext } from "./types.js";
|
||||
|
||||
export type { UrlRuleParser, UrlRuleParserContext } from "./types.js";
|
||||
|
||||
export function tryUrlRuleParsers(
|
||||
html: string,
|
||||
url: string,
|
||||
baseMetadata: PageMetadata
|
||||
): ConversionResult | null {
|
||||
const document = parseDocument(html);
|
||||
const context: UrlRuleParserContext = {
|
||||
html,
|
||||
url,
|
||||
document,
|
||||
baseMetadata,
|
||||
};
|
||||
|
||||
for (const parser of URL_RULE_PARSERS) {
|
||||
if (!parser.supports(context)) continue;
|
||||
|
||||
try {
|
||||
const result = parser.parse(context);
|
||||
if (!result) continue;
|
||||
|
||||
const markdown = normalizeMarkdown(result.markdown);
|
||||
if (!isMarkdownUsable(markdown, html)) continue;
|
||||
|
||||
return {
|
||||
...result,
|
||||
markdown,
|
||||
};
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.warn(`[url-to-markdown] parser ${parser.id} failed: ${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import { convertHtmlFragmentToMarkdown } from "../../legacy-converter.js";
|
||||
import {
|
||||
normalizeMarkdown,
|
||||
pickString,
|
||||
type ConversionResult,
|
||||
} from "../../markdown-conversion-shared.js";
|
||||
import type { UrlRuleParser, UrlRuleParserContext } from "../types.js";
|
||||
|
||||
const ARCHIVE_HOSTS = new Set([
|
||||
"archive.ph",
|
||||
"archive.is",
|
||||
"archive.today",
|
||||
"archive.md",
|
||||
"archive.vn",
|
||||
"archive.li",
|
||||
"archive.fo",
|
||||
]);
|
||||
|
||||
function isArchiveHost(url: string): boolean {
|
||||
try {
|
||||
return ARCHIVE_HOSTS.has(new URL(url).hostname.toLowerCase());
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readOriginalUrl(document: Document): string | undefined {
|
||||
const value = document.querySelector("input[name='q']")?.getAttribute("value")?.trim();
|
||||
if (!value) return undefined;
|
||||
|
||||
try {
|
||||
return new URL(value).href;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function summarize(text: string, maxLength: number): string | undefined {
|
||||
const normalized = text.replace(/\s+/g, " ").trim();
|
||||
if (!normalized) return undefined;
|
||||
if (normalized.length <= maxLength) return normalized;
|
||||
return `${normalized.slice(0, Math.max(0, maxLength - 1)).trimEnd()}…`;
|
||||
}
|
||||
|
||||
function pickContentRoot(document: Document): Element | null {
|
||||
return (
|
||||
document.querySelector("#CONTENT") ??
|
||||
document.querySelector("#content") ??
|
||||
document.body
|
||||
);
|
||||
}
|
||||
|
||||
function pickContentTitle(root: Element, fallbackTitle: string): string {
|
||||
const contentTitle = pickString(
|
||||
root.querySelector("h1")?.textContent,
|
||||
root.querySelector("[itemprop='headline']")?.textContent,
|
||||
root.querySelector("article h2")?.textContent
|
||||
);
|
||||
if (contentTitle) return contentTitle;
|
||||
if (fallbackTitle && !/^archive\./i.test(fallbackTitle.trim())) return fallbackTitle;
|
||||
return "";
|
||||
}
|
||||
|
||||
function parseArchivePage(context: UrlRuleParserContext): ConversionResult | null {
|
||||
const root = pickContentRoot(context.document);
|
||||
if (!root) return null;
|
||||
|
||||
const markdown = normalizeMarkdown(convertHtmlFragmentToMarkdown(root.innerHTML));
|
||||
if (!markdown) return null;
|
||||
|
||||
const originalUrl = readOriginalUrl(context.document) ?? context.baseMetadata.url;
|
||||
const bodyText = root.textContent?.replace(/\s+/g, " ").trim() ?? "";
|
||||
const published = root.querySelector("time[datetime]")?.getAttribute("datetime") ?? undefined;
|
||||
const coverImage = root.querySelector("img[src]")?.getAttribute("src") ?? undefined;
|
||||
|
||||
return {
|
||||
metadata: {
|
||||
...context.baseMetadata,
|
||||
url: originalUrl,
|
||||
title: pickContentTitle(root, context.baseMetadata.title),
|
||||
description: summarize(bodyText, 220) ?? context.baseMetadata.description,
|
||||
published: pickString(published, context.baseMetadata.published) ?? undefined,
|
||||
coverImage: pickString(coverImage, context.baseMetadata.coverImage) ?? undefined,
|
||||
},
|
||||
markdown,
|
||||
rawHtml: context.html,
|
||||
conversionMethod: "parser:archive-ph",
|
||||
};
|
||||
}
|
||||
|
||||
export const archivePhRuleParser: UrlRuleParser = {
|
||||
id: "archive-ph",
|
||||
supports(context) {
|
||||
return isArchiveHost(context.url);
|
||||
},
|
||||
parse: parseArchivePage,
|
||||
};
|
||||
@@ -0,0 +1,10 @@
|
||||
import { archivePhRuleParser } from "./archive-ph.js";
|
||||
import { xArticleRuleParser } from "./x-article.js";
|
||||
import { xStatusRuleParser } from "./x-status.js";
|
||||
import type { UrlRuleParser } from "../types.js";
|
||||
|
||||
export const URL_RULE_PARSERS: UrlRuleParser[] = [
|
||||
archivePhRuleParser,
|
||||
xArticleRuleParser,
|
||||
xStatusRuleParser,
|
||||
];
|
||||
@@ -0,0 +1,137 @@
|
||||
import {
|
||||
normalizeMarkdown,
|
||||
pickString,
|
||||
type ConversionResult,
|
||||
} from "../../markdown-conversion-shared.js";
|
||||
import type { UrlRuleParser, UrlRuleParserContext } from "../types.js";
|
||||
import {
|
||||
cleanText,
|
||||
collectMediaMarkdown,
|
||||
convertXRichTextElementToMarkdown,
|
||||
extractPublishedForCurrentUrl,
|
||||
inferLanguage,
|
||||
isXArticlePath,
|
||||
isXHost,
|
||||
normalizeXMarkdown,
|
||||
parseUrl,
|
||||
pickFirstValidLinkText,
|
||||
sanitizeCoverImage,
|
||||
summarizeText,
|
||||
} from "./x-shared.js";
|
||||
|
||||
function collectArticleMarkdown(root: Element): { markdown: string; mediaUrls: string[] } {
|
||||
const parts: string[] = [];
|
||||
const seenMedia = new Set<string>();
|
||||
const mediaUrls: string[] = [];
|
||||
|
||||
function pushPart(value: string): void {
|
||||
const normalized = normalizeMarkdown(value);
|
||||
if (!normalized) return;
|
||||
parts.push(normalized);
|
||||
}
|
||||
|
||||
function walk(node: Element): void {
|
||||
const testId = node.getAttribute("data-testid");
|
||||
|
||||
if (testId === "twitterArticleRichTextView" || testId === "longformRichTextComponent") {
|
||||
const bodyMedia = collectMediaMarkdown(node, seenMedia);
|
||||
mediaUrls.push(...bodyMedia.urls.filter((url) => !mediaUrls.includes(url)));
|
||||
pushPart(convertXRichTextElementToMarkdown(node));
|
||||
return;
|
||||
}
|
||||
|
||||
if (testId === "tweetPhoto") {
|
||||
const media = collectMediaMarkdown(node, seenMedia);
|
||||
mediaUrls.push(...media.urls.filter((url) => !mediaUrls.includes(url)));
|
||||
for (const line of media.lines) pushPart(line);
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
testId === "twitter-article-title" ||
|
||||
testId === "User-Name" ||
|
||||
testId === "Tweet-User-Avatar" ||
|
||||
testId === "reply" ||
|
||||
testId === "retweet" ||
|
||||
testId === "like" ||
|
||||
testId === "bookmark" ||
|
||||
testId === "caret" ||
|
||||
testId === "app-text-transition-container"
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (node.tagName === "TIME" || node.tagName === "BUTTON") {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const child of Array.from(node.children)) {
|
||||
walk(child);
|
||||
}
|
||||
}
|
||||
|
||||
for (const child of Array.from(root.children)) {
|
||||
walk(child);
|
||||
}
|
||||
|
||||
return {
|
||||
markdown: normalizeXMarkdown(parts.join("\n\n")),
|
||||
mediaUrls,
|
||||
};
|
||||
}
|
||||
|
||||
function parseXArticle(context: UrlRuleParserContext): ConversionResult | null {
|
||||
const articleRoot = context.document.querySelector("[data-testid='twitterArticleReadView']") as Element | null;
|
||||
if (!articleRoot) return null;
|
||||
|
||||
const title = cleanText(
|
||||
context.document.querySelector("[data-testid='twitter-article-title']")?.textContent
|
||||
);
|
||||
const identity = pickFirstValidLinkText(
|
||||
context.document.querySelector("[data-testid='User-Name']")
|
||||
);
|
||||
const published = extractPublishedForCurrentUrl(articleRoot, context.url);
|
||||
const { markdown, mediaUrls } = collectArticleMarkdown(articleRoot);
|
||||
if (!markdown) return null;
|
||||
|
||||
const bodyText = cleanText(
|
||||
context.document.querySelector("[data-testid='twitterArticleRichTextView']")?.textContent ??
|
||||
context.document.querySelector("[data-testid='longformRichTextComponent']")?.textContent
|
||||
);
|
||||
|
||||
return {
|
||||
metadata: {
|
||||
...context.baseMetadata,
|
||||
title: pickString(title, context.baseMetadata.title) ?? "",
|
||||
description: summarizeText(bodyText, 220) ?? context.baseMetadata.description,
|
||||
author: pickString(identity.author, context.baseMetadata.author) ?? undefined,
|
||||
published: pickString(published, context.baseMetadata.published) ?? undefined,
|
||||
coverImage: sanitizeCoverImage(mediaUrls[0], context.baseMetadata.coverImage),
|
||||
language: inferLanguage(bodyText, context.baseMetadata.language),
|
||||
},
|
||||
markdown,
|
||||
rawHtml: context.html,
|
||||
conversionMethod: "parser:x-article",
|
||||
};
|
||||
}
|
||||
|
||||
export const xArticleRuleParser: UrlRuleParser = {
|
||||
id: "x-article",
|
||||
supports(context) {
|
||||
const parsed = parseUrl(context.url);
|
||||
if (!parsed || !isXHost(parsed.hostname)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
isXArticlePath(parsed.pathname) ||
|
||||
Boolean(
|
||||
context.document.querySelector("[data-testid='twitterArticleReadView']") ||
|
||||
context.document.querySelector("[data-testid='twitterArticleRichTextView']")
|
||||
)
|
||||
);
|
||||
},
|
||||
parse(context) {
|
||||
return parseXArticle(context);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,249 @@
|
||||
import { convertHtmlFragmentToMarkdown } from "../../legacy-converter.js";
|
||||
import { normalizeMarkdown } from "../../markdown-conversion-shared.js";
|
||||
|
||||
export const DEFAULT_X_OG_IMAGE = "https://abs.twimg.com/rweb/ssr/default/v2/og/image.png";
|
||||
|
||||
export type MediaResult = {
|
||||
lines: string[];
|
||||
urls: string[];
|
||||
};
|
||||
|
||||
export function isXHost(hostname: string): boolean {
|
||||
const normalized = hostname.toLowerCase();
|
||||
return (
|
||||
normalized === "x.com" ||
|
||||
normalized === "twitter.com" ||
|
||||
normalized.endsWith(".x.com") ||
|
||||
normalized.endsWith(".twitter.com")
|
||||
);
|
||||
}
|
||||
|
||||
export function parseUrl(input: string): URL | null {
|
||||
try {
|
||||
return new URL(input);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function isXStatusPath(pathname: string): boolean {
|
||||
return /^\/[^/]+\/status(?:es)?\/\d+$/i.test(pathname) || /^\/i\/web\/status\/\d+$/i.test(pathname);
|
||||
}
|
||||
|
||||
export function isXArticlePath(pathname: string): boolean {
|
||||
return /^\/[^/]+\/article\/\d+$/i.test(pathname) || /^\/(?:i\/)?article\/\d+$/i.test(pathname);
|
||||
}
|
||||
|
||||
export function cleanText(value: string | null | undefined): string {
|
||||
return (value ?? "").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
export function cleanUserLabel(value: string | null | undefined): string {
|
||||
return cleanText(value).replace(/\bVerified account\b/gi, "").replace(/\s{2,}/g, " ").trim();
|
||||
}
|
||||
|
||||
export function escapeMarkdownAlt(text: string): string {
|
||||
return text.replace(/[\[\]]/g, "\\$&");
|
||||
}
|
||||
|
||||
export function normalizeAlt(text: string | null | undefined): string {
|
||||
const cleaned = cleanText(text);
|
||||
if (!cleaned || /^(image|photo)$/i.test(cleaned)) return "";
|
||||
return escapeMarkdownAlt(cleaned);
|
||||
}
|
||||
|
||||
export function summarizeText(text: string, maxLength: number): string | undefined {
|
||||
const normalized = cleanText(text);
|
||||
if (!normalized) return undefined;
|
||||
return normalized.length > maxLength
|
||||
? `${normalized.slice(0, maxLength - 3)}...`
|
||||
: normalized;
|
||||
}
|
||||
|
||||
export function buildTweetTitle(text: string, fallback: string): string {
|
||||
return summarizeText(text, 80) ?? fallback;
|
||||
}
|
||||
|
||||
export function normalizeXMarkdown(markdown: string): string {
|
||||
return normalizeMarkdown(markdown.replace(/^(#{1,6})\s*\n+([^\n])/gm, "$1 $2"));
|
||||
}
|
||||
|
||||
export function inferLanguage(text: string, fallback?: string): string | undefined {
|
||||
const normalized = cleanText(text);
|
||||
if (!normalized) return fallback;
|
||||
|
||||
const han = (normalized.match(/\p{Script=Han}/gu) || []).length;
|
||||
const hiragana = (normalized.match(/\p{Script=Hiragana}/gu) || []).length;
|
||||
const katakana = (normalized.match(/\p{Script=Katakana}/gu) || []).length;
|
||||
const hangul = (normalized.match(/\p{Script=Hangul}/gu) || []).length;
|
||||
|
||||
if (hangul >= 8) return "ko";
|
||||
if (hiragana + katakana >= 8) return "ja";
|
||||
if (han >= 16) return "zh";
|
||||
return fallback;
|
||||
}
|
||||
|
||||
export function buildQuoteMarkdown(markdown: string, author?: string): string {
|
||||
const normalized = normalizeMarkdown(markdown);
|
||||
if (!normalized) return "";
|
||||
|
||||
const lines = normalized.split("\n");
|
||||
const prefixed = lines.map((line) => (line ? `> ${line}` : ">")).join("\n");
|
||||
const header = author ? `> Quote from ${author}` : "> Quote";
|
||||
return `${header}\n${prefixed}`;
|
||||
}
|
||||
|
||||
export function pickFirstValidLinkText(userNameEl: Element | null | undefined): {
|
||||
name?: string;
|
||||
username?: string;
|
||||
author?: string;
|
||||
} {
|
||||
if (!userNameEl) return {};
|
||||
|
||||
const linkTexts = Array.from(userNameEl.querySelectorAll("a[href]"))
|
||||
.map((link) => cleanUserLabel(link.textContent))
|
||||
.filter(Boolean);
|
||||
|
||||
let username = linkTexts.find((text) => text.startsWith("@"));
|
||||
let name = linkTexts.find((text) => !text.startsWith("@") && !/^(promote|more)$/i.test(text));
|
||||
|
||||
if (!username || !name) {
|
||||
const text = cleanUserLabel(userNameEl.textContent);
|
||||
const fallbackMatch = text.match(/^(.*?)\s*(@[A-Za-z0-9_]+)(?:\s*·.*)?$/);
|
||||
if (fallbackMatch) {
|
||||
name = name ?? cleanText(fallbackMatch[1]);
|
||||
username = username ?? cleanText(fallbackMatch[2]);
|
||||
}
|
||||
}
|
||||
|
||||
const author = name && username ? `${name} (${username})` : username ?? name;
|
||||
return { name, username, author };
|
||||
}
|
||||
|
||||
export function extractPublishedForCurrentUrl(root: ParentNode, url: string): string | undefined {
|
||||
const parsed = parseUrl(url);
|
||||
if (!parsed) return undefined;
|
||||
const currentPath = parsed.pathname.toLowerCase();
|
||||
|
||||
for (const timeElement of root.querySelectorAll("a[href] time[datetime]")) {
|
||||
const href = timeElement.closest("a")?.getAttribute("href");
|
||||
const hrefUrl = href ? parseUrl(href.startsWith("http") ? href : `${parsed.origin}${href}`) : null;
|
||||
if (hrefUrl?.pathname.toLowerCase() === currentPath) {
|
||||
return timeElement.getAttribute("datetime") ?? undefined;
|
||||
}
|
||||
}
|
||||
|
||||
return root.querySelector("time[datetime]")?.getAttribute("datetime") ?? undefined;
|
||||
}
|
||||
|
||||
export function collectMediaMarkdown(root: ParentNode, seen: Set<string>): MediaResult {
|
||||
const lines: string[] = [];
|
||||
const urls: string[] = [];
|
||||
const rootElement = root as Element & {
|
||||
getAttribute?: (name: string) => string | null;
|
||||
};
|
||||
const photoNodes = [
|
||||
...(typeof rootElement.getAttribute === "function" &&
|
||||
rootElement.getAttribute("data-testid") === "tweetPhoto"
|
||||
? [rootElement]
|
||||
: []),
|
||||
...Array.from(root.querySelectorAll("[data-testid='tweetPhoto']")),
|
||||
];
|
||||
|
||||
for (const node of photoNodes) {
|
||||
const img = node.querySelector("img");
|
||||
const imageUrl = img?.getAttribute("src");
|
||||
if (imageUrl && !seen.has(imageUrl)) {
|
||||
seen.add(imageUrl);
|
||||
urls.push(imageUrl);
|
||||
lines.push(``);
|
||||
}
|
||||
|
||||
const video = node.querySelector("video");
|
||||
const posterUrl = video?.getAttribute("poster");
|
||||
if (posterUrl && !seen.has(posterUrl)) {
|
||||
seen.add(posterUrl);
|
||||
urls.push(posterUrl);
|
||||
lines.push(``);
|
||||
}
|
||||
|
||||
const videoUrl = video?.getAttribute("src") ?? video?.querySelector("source")?.getAttribute("src");
|
||||
if (videoUrl && !seen.has(videoUrl)) {
|
||||
seen.add(videoUrl);
|
||||
urls.push(videoUrl);
|
||||
lines.push(`[video](${videoUrl})`);
|
||||
}
|
||||
}
|
||||
|
||||
return { lines, urls };
|
||||
}
|
||||
|
||||
export function materializeTweetPhotoNodes(root: Element): void {
|
||||
for (const photo of Array.from(root.querySelectorAll("[data-testid='tweetPhoto']"))) {
|
||||
const document = photo.ownerDocument;
|
||||
const container = document.createElement("span");
|
||||
|
||||
const img = photo.querySelector("img");
|
||||
const imageUrl = img?.getAttribute("src");
|
||||
if (imageUrl) {
|
||||
const image = document.createElement("img");
|
||||
image.setAttribute("src", imageUrl);
|
||||
const alt = normalizeAlt(img?.getAttribute("alt"));
|
||||
if (alt) {
|
||||
image.setAttribute("alt", alt);
|
||||
}
|
||||
container.appendChild(image);
|
||||
}
|
||||
|
||||
const video = photo.querySelector("video");
|
||||
const posterUrl = video?.getAttribute("poster");
|
||||
if (posterUrl) {
|
||||
const poster = document.createElement("img");
|
||||
poster.setAttribute("src", posterUrl);
|
||||
poster.setAttribute("alt", "video");
|
||||
container.appendChild(poster);
|
||||
}
|
||||
|
||||
const videoUrl = video?.getAttribute("src") ?? video?.querySelector("source")?.getAttribute("src");
|
||||
if (videoUrl) {
|
||||
if (container.childNodes.length > 0) {
|
||||
container.appendChild(document.createTextNode(" "));
|
||||
}
|
||||
const link = document.createElement("a");
|
||||
link.setAttribute("href", videoUrl);
|
||||
link.textContent = "video";
|
||||
container.appendChild(link);
|
||||
}
|
||||
|
||||
if (container.childNodes.length === 0) {
|
||||
photo.remove();
|
||||
continue;
|
||||
}
|
||||
|
||||
photo.replaceWith(container);
|
||||
}
|
||||
}
|
||||
|
||||
function collapseLinkedMediaContainers(root: Element): void {
|
||||
for (const anchor of Array.from(root.querySelectorAll("a[href]"))) {
|
||||
const images = Array.from(anchor.querySelectorAll("img"));
|
||||
if (images.length !== 1) continue;
|
||||
if (cleanText(anchor.textContent)) continue;
|
||||
|
||||
const image = images[0].cloneNode(true);
|
||||
anchor.replaceChildren(image);
|
||||
}
|
||||
}
|
||||
|
||||
export function convertXRichTextElementToMarkdown(node: Element): string {
|
||||
const clone = node.cloneNode(true) as Element;
|
||||
materializeTweetPhotoNodes(clone);
|
||||
collapseLinkedMediaContainers(clone);
|
||||
return normalizeXMarkdown(convertHtmlFragmentToMarkdown(clone.innerHTML));
|
||||
}
|
||||
|
||||
export function sanitizeCoverImage(primary?: string, fallback?: string): string | undefined {
|
||||
if (primary) return primary;
|
||||
if (!fallback || fallback === DEFAULT_X_OG_IMAGE) return undefined;
|
||||
return fallback;
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import type { ConversionResult } from "../../markdown-conversion-shared.js";
|
||||
import type { UrlRuleParser, UrlRuleParserContext } from "../types.js";
|
||||
import {
|
||||
buildQuoteMarkdown,
|
||||
buildTweetTitle,
|
||||
cleanText,
|
||||
collectMediaMarkdown,
|
||||
convertXRichTextElementToMarkdown,
|
||||
extractPublishedForCurrentUrl,
|
||||
inferLanguage,
|
||||
isXHost,
|
||||
isXStatusPath,
|
||||
normalizeXMarkdown,
|
||||
parseUrl,
|
||||
pickFirstValidLinkText,
|
||||
sanitizeCoverImage,
|
||||
summarizeText,
|
||||
} from "./x-shared.js";
|
||||
|
||||
function parseXStatus(context: UrlRuleParserContext): ConversionResult | null {
|
||||
const article = context.document.querySelector("article[data-testid='tweet'], article") as Element | null;
|
||||
if (!article) return null;
|
||||
|
||||
const tweetTextElements = Array.from(article.querySelectorAll("[data-testid='tweetText']")) as Element[];
|
||||
if (tweetTextElements.length === 0) return null;
|
||||
|
||||
const userNameElements = Array.from(article.querySelectorAll("[data-testid='User-Name']")) as Element[];
|
||||
const mainTextElement = tweetTextElements[0];
|
||||
const mainIdentity = pickFirstValidLinkText(userNameElements[0]);
|
||||
const published = extractPublishedForCurrentUrl(article, context.url);
|
||||
const mainMarkdown = normalizeXMarkdown(convertXRichTextElementToMarkdown(mainTextElement));
|
||||
if (!mainMarkdown) return null;
|
||||
|
||||
const parts = [mainMarkdown];
|
||||
const quotedTextElements = tweetTextElements.slice(1);
|
||||
const quotedUserNameElements = userNameElements.slice(1);
|
||||
|
||||
quotedTextElements.forEach((element, index) => {
|
||||
const quoteMarkdown = normalizeXMarkdown(convertXRichTextElementToMarkdown(element));
|
||||
if (!quoteMarkdown) return;
|
||||
const quoteIdentity = pickFirstValidLinkText(quotedUserNameElements[index]);
|
||||
parts.push(buildQuoteMarkdown(quoteMarkdown, quoteIdentity.author));
|
||||
});
|
||||
|
||||
const media = collectMediaMarkdown(article, new Set<string>());
|
||||
if (media.lines.length > 0) {
|
||||
parts.push(media.lines.join("\n\n"));
|
||||
}
|
||||
|
||||
const mainText = cleanText(mainTextElement.textContent);
|
||||
const markdown = normalizeXMarkdown(parts.join("\n\n"));
|
||||
|
||||
return {
|
||||
metadata: {
|
||||
...context.baseMetadata,
|
||||
title: buildTweetTitle(mainText, context.baseMetadata.title),
|
||||
description: summarizeText(mainText, 220) ?? context.baseMetadata.description,
|
||||
author: mainIdentity.author ?? context.baseMetadata.author,
|
||||
published: published ?? context.baseMetadata.published,
|
||||
coverImage: sanitizeCoverImage(media.urls[0], context.baseMetadata.coverImage),
|
||||
language: inferLanguage(mainText, context.baseMetadata.language),
|
||||
},
|
||||
markdown,
|
||||
rawHtml: context.html,
|
||||
conversionMethod: "parser:x-status",
|
||||
};
|
||||
}
|
||||
|
||||
export const xStatusRuleParser: UrlRuleParser = {
|
||||
id: "x-status",
|
||||
supports(context) {
|
||||
const parsed = parseUrl(context.url);
|
||||
if (!parsed || !isXHost(parsed.hostname)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return isXStatusPath(parsed.pathname) && Boolean(context.document.querySelector("[data-testid='tweetText']"));
|
||||
},
|
||||
parse(context): ConversionResult | null {
|
||||
return parseXStatus(context);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,14 @@
|
||||
import type { ConversionResult, PageMetadata } from "../markdown-conversion-shared.js";
|
||||
|
||||
export interface UrlRuleParserContext {
|
||||
html: string;
|
||||
url: string;
|
||||
document: Document;
|
||||
baseMetadata: PageMetadata;
|
||||
}
|
||||
|
||||
export interface UrlRuleParser {
|
||||
id: string;
|
||||
supports(context: UrlRuleParserContext): boolean;
|
||||
parse(context: UrlRuleParserContext): ConversionResult | null;
|
||||
}
|
||||
@@ -151,8 +151,7 @@ From outline entry:
|
||||
```markdown
|
||||
## Watermark
|
||||
|
||||
Include a subtle watermark "{content}" positioned at {position}
|
||||
with approximately {opacity*100}% visibility. The watermark should
|
||||
Include a subtle watermark "{content}" positioned at {position}. The watermark should
|
||||
be legible but not distracting from the main content.
|
||||
```
|
||||
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
---
|
||||
name: baoyu-youtube-transcript
|
||||
description: Downloads YouTube video transcripts/subtitles and cover images by URL or video ID. Supports multiple languages, translation, chapters, and speaker identification. Caches raw data for fast re-formatting. Use when user asks to "get YouTube transcript", "download subtitles", "get captions", "YouTube字幕", "YouTube封面", "视频封面", "video thumbnail", "video cover image", or provides a YouTube URL and wants the transcript/subtitle text or cover image extracted.
|
||||
version: 1.1.0
|
||||
metadata:
|
||||
openclaw:
|
||||
homepage: https://github.com/JimLiu/baoyu-skills#baoyu-youtube-transcript
|
||||
requires:
|
||||
anyBins:
|
||||
- bun
|
||||
- npx
|
||||
---
|
||||
|
||||
# YouTube Transcript
|
||||
|
||||
Downloads transcripts (subtitles/captions) from YouTube videos. Works with both manually created and auto-generated transcripts. No API key or browser required — uses YouTube's InnerTube API directly.
|
||||
|
||||
Fetches video metadata and cover image on first run, caches raw data for fast re-formatting.
|
||||
|
||||
## Script Directory
|
||||
|
||||
Scripts in `scripts/` subdirectory. `{baseDir}` = this SKILL.md's directory path. Resolve `${BUN_X}` runtime: if `bun` installed → `bun`; if `npx` available → `npx -y bun`; else suggest installing bun. Replace `{baseDir}` and `${BUN_X}` with actual values.
|
||||
|
||||
| Script | Purpose |
|
||||
|--------|---------|
|
||||
| `scripts/main.ts` | Transcript download CLI |
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
# Default: markdown with timestamps (English)
|
||||
${BUN_X} {baseDir}/scripts/main.ts <youtube-url-or-id>
|
||||
|
||||
# Specify languages (priority order)
|
||||
${BUN_X} {baseDir}/scripts/main.ts <url> --languages zh,en,ja
|
||||
|
||||
# Without timestamps
|
||||
${BUN_X} {baseDir}/scripts/main.ts <url> --no-timestamps
|
||||
|
||||
# With chapter segmentation
|
||||
${BUN_X} {baseDir}/scripts/main.ts <url> --chapters
|
||||
|
||||
# With speaker identification (requires AI post-processing)
|
||||
${BUN_X} {baseDir}/scripts/main.ts <url> --speakers
|
||||
|
||||
# SRT subtitle file
|
||||
${BUN_X} {baseDir}/scripts/main.ts <url> --format srt
|
||||
|
||||
# Translate transcript
|
||||
${BUN_X} {baseDir}/scripts/main.ts <url> --translate zh-Hans
|
||||
|
||||
# List available transcripts
|
||||
${BUN_X} {baseDir}/scripts/main.ts <url> --list
|
||||
|
||||
# Force re-fetch (ignore cache)
|
||||
${BUN_X} {baseDir}/scripts/main.ts <url> --refresh
|
||||
```
|
||||
|
||||
## Options
|
||||
|
||||
| Option | Description | Default |
|
||||
|--------|-------------|---------|
|
||||
| `<url-or-id>` | YouTube URL or video ID (multiple allowed) | Required |
|
||||
| `--languages <codes>` | Language codes, comma-separated, in priority order | `en` |
|
||||
| `--format <fmt>` | Output format: `text`, `srt` | `text` |
|
||||
| `--translate <code>` | Translate to specified language code | |
|
||||
| `--list` | List available transcripts instead of fetching | |
|
||||
| `--timestamps` | Include `[HH:MM:SS → HH:MM:SS]` timestamps per paragraph | on |
|
||||
| `--no-timestamps` | Disable timestamps | |
|
||||
| `--chapters` | Chapter segmentation from video description | |
|
||||
| `--speakers` | Raw transcript with metadata for speaker identification | |
|
||||
| `--exclude-generated` | Skip auto-generated transcripts | |
|
||||
| `--exclude-manually-created` | Skip manually created transcripts | |
|
||||
| `--refresh` | Force re-fetch, ignore cached data | |
|
||||
| `-o, --output <path>` | Save to specific file path | auto-generated |
|
||||
| `--output-dir <dir>` | Base output directory | `youtube-transcript` |
|
||||
|
||||
## Input Formats
|
||||
|
||||
Accepts any of these as video input:
|
||||
- Full URL: `https://www.youtube.com/watch?v=dQw4w9WgXcQ`
|
||||
- Short URL: `https://youtu.be/dQw4w9WgXcQ`
|
||||
- Embed URL: `https://www.youtube.com/embed/dQw4w9WgXcQ`
|
||||
- Shorts URL: `https://www.youtube.com/shorts/dQw4w9WgXcQ`
|
||||
- Video ID: `dQw4w9WgXcQ`
|
||||
|
||||
## Output Formats
|
||||
|
||||
| Format | Extension | Description |
|
||||
|--------|-----------|-------------|
|
||||
| `text` | `.md` | Markdown with frontmatter (incl. `description`), title heading, summary, optional TOC/cover/timestamps/chapters/speakers |
|
||||
| `srt` | `.srt` | SubRip subtitle format for video players |
|
||||
|
||||
## Output Directory
|
||||
|
||||
```
|
||||
youtube-transcript/
|
||||
├── .index.json # Video ID → directory path mapping (for cache lookup)
|
||||
└── {channel-slug}/{title-full-slug}/
|
||||
├── meta.json # Video metadata (title, channel, description, duration, chapters, etc.)
|
||||
├── transcript-raw.json # Raw transcript snippets from YouTube API (cached)
|
||||
├── transcript-sentences.json # Sentence-segmented transcript (split by punctuation, merged across snippets)
|
||||
├── imgs/
|
||||
│ └── cover.jpg # Video thumbnail
|
||||
├── transcript.md # Markdown transcript (generated from sentences)
|
||||
└── transcript.srt # SRT subtitle (generated from raw snippets, if --format srt)
|
||||
```
|
||||
|
||||
- `{channel-slug}`: Channel name in kebab-case
|
||||
- `{title-full-slug}`: Full video title in kebab-case
|
||||
|
||||
The `--list` mode outputs to stdout only (no file saved).
|
||||
|
||||
## Caching
|
||||
|
||||
On first fetch, the script saves:
|
||||
- `meta.json` — video metadata, chapters, cover image path, language info
|
||||
- `transcript-raw.json` — raw transcript snippets from YouTube API (`{ text, start, duration }[]`)
|
||||
- `transcript-sentences.json` — sentence-segmented transcript (`{ text, start: "HH:mm:ss", end: "HH:mm:ss" }[]`), split by sentence-ending punctuation (`.?!…。?!` etc.), timestamps proportionally allocated by character length, CJK-aware text merging
|
||||
- `imgs/cover.jpg` — video thumbnail
|
||||
|
||||
Subsequent runs for the same video use cached data (no network calls). Use `--refresh` to force re-fetch. If a different language is requested, the cache is automatically refreshed.
|
||||
|
||||
SRT output (`--format srt`) is generated from `transcript-raw.json`. Text/markdown output uses `transcript-sentences.json` for natural sentence boundaries.
|
||||
|
||||
## Workflow
|
||||
|
||||
When user provides a YouTube URL and wants the transcript:
|
||||
|
||||
1. Run with `--list` first if the user hasn't specified a language, to show available options
|
||||
2. **Always single-quote the URL** when running the script — zsh treats `?` as a glob wildcard, so an unquoted YouTube URL causes "no matches found": use `'https://www.youtube.com/watch?v=ID'`
|
||||
3. Default: run with `--chapters --speakers` for the richest output (chapters + speaker identification)
|
||||
3. The script auto-saves cached data + output file and prints the file path
|
||||
4. For `--speakers` mode: after the script saves the raw file, follow the speaker identification workflow below to post-process with speaker labels
|
||||
|
||||
When user only wants a cover image or metadata, running the script with any option will also cache `meta.json` and `imgs/cover.jpg`.
|
||||
|
||||
When re-formatting the same video (e.g., first text then SRT), the cached data is reused — no re-fetch needed.
|
||||
|
||||
## Chapter & Speaker Workflow
|
||||
|
||||
### Chapters (`--chapters`)
|
||||
|
||||
The script parses chapter timestamps from the video description (e.g., `0:00 Introduction`), segments the transcript by chapter boundaries, groups snippets into readable paragraphs, and saves as `.md` with a Table of Contents. No further processing needed.
|
||||
|
||||
If no chapter timestamps exist in the description, the transcript is output as grouped paragraphs without chapter headings.
|
||||
|
||||
### Speaker Identification (`--speakers`)
|
||||
|
||||
Speaker identification requires AI processing. The script outputs a raw `.md` file containing:
|
||||
- YAML frontmatter with video metadata (title, channel, date, cover, description, language)
|
||||
- Video description (for speaker name extraction)
|
||||
- Chapter list from description (if available)
|
||||
- Raw transcript in SRT format (pre-computed start/end timestamps, token-efficient)
|
||||
|
||||
After the script saves the raw file, spawn a sub-agent (use a cheaper model like Sonnet for cost efficiency) to process speaker identification:
|
||||
|
||||
1. Read the saved `.md` file
|
||||
2. Read the prompt template at `{baseDir}/prompts/speaker-transcript.md`
|
||||
3. Process the raw transcript following the prompt:
|
||||
- Identify speakers using video metadata (title → guest, channel → host, description → names)
|
||||
- Detect speaker turns from conversation flow, question-answer patterns, and contextual cues
|
||||
- Segment into chapters (use description chapters if available, else create from topic shifts)
|
||||
- Format with `**Speaker Name:**` labels, paragraph grouping (2-4 sentences), and `[HH:MM:SS → HH:MM:SS]` timestamps
|
||||
4. Overwrite the `.md` file with the processed transcript (keep the YAML frontmatter)
|
||||
|
||||
When `--speakers` is used, `--chapters` is implied — the processed output always includes chapter segmentation.
|
||||
|
||||
## Error Cases
|
||||
|
||||
| Error | Meaning |
|
||||
|-------|---------|
|
||||
| Transcripts disabled | Video has no captions at all |
|
||||
| No transcript found | Requested language not available |
|
||||
| Video unavailable | Video deleted, private, or region-locked |
|
||||
| IP blocked | Too many requests, try again later |
|
||||
| Age restricted | Video requires login for age verification |
|
||||
@@ -0,0 +1,118 @@
|
||||
# Speaker & Chapter Transcript Processing
|
||||
|
||||
You are an expert transcript specialist. Process the raw transcript file (with YAML frontmatter metadata and SRT-formatted transcript) into a structured, verbatim transcript with speaker identification and chapter segmentation.
|
||||
|
||||
## Output Structure
|
||||
|
||||
Produce a single cohesive markdown file containing:
|
||||
1. YAML frontmatter (keep the original frontmatter from the raw file, which includes `description`)
|
||||
2. `# Title` heading (from frontmatter title)
|
||||
3. Description/summary paragraph (from frontmatter `description`)
|
||||
4. Table of Contents
|
||||
5. Cover image (if `cover` exists in frontmatter): `` — right after the ToC
|
||||
6. Full chapter-segmented transcript with speaker labels
|
||||
|
||||
Use the same language as the transcription for the title and ToC.
|
||||
|
||||
## Rules
|
||||
|
||||
### Transcription Fidelity
|
||||
- Preserve every spoken word exactly, including filler words (`um`, `uh`, `like`) and stutters
|
||||
- **NEVER translate.** If the audio mixes languages (e.g., "这个 feature 很酷"), replicate that mix exactly
|
||||
|
||||
### Speaker Identification
|
||||
- **Priority 1: Use metadata.** Analyze the video's title, channel name, and description to identify speakers
|
||||
- **Priority 2: Use transcript content.** Look for introductions, how speakers address each other, contextual cues
|
||||
- **Fallback:** Use consistent generic labels (`**Speaker 1:**`, `**Host:**`, etc.)
|
||||
- **Consistency:** If a speaker's name is revealed later, update ALL previous labels for that speaker
|
||||
|
||||
### Chapter Generation
|
||||
- If the raw file contains a `# Chapters` section, use those as the primary basis for segmenting
|
||||
- Otherwise, create chapters based on significant topic shifts in the conversation
|
||||
|
||||
### Input Format
|
||||
- The `# Transcript` section contains SRT-formatted subtitles with pre-computed start/end timestamps
|
||||
- Each SRT block has: sequence number, `HH:MM:SS,mmm --> HH:MM:SS,mmm` timestamp line, and text
|
||||
- Use the SRT timestamps directly — no need to calculate paragraph start/end times, just merge adjacent blocks
|
||||
|
||||
### Formatting
|
||||
|
||||
**Timestamps:** Use `[HH:MM:SS → HH:MM:SS]` format (start → end) at the end of each paragraph. No milliseconds.
|
||||
|
||||
**Table of Contents:**
|
||||
```
|
||||
## Table of Contents
|
||||
* [HH:MM:SS] Chapter Title
|
||||
```
|
||||
|
||||
**Chapters:**
|
||||
```
|
||||
## Chapter Title [HH:MM:SS]
|
||||
```
|
||||
Two blank lines between chapters.
|
||||
|
||||
**Dialogue Paragraphs:**
|
||||
- First paragraph of a speaker's turn starts with `**Speaker Name:** `
|
||||
- Split long monologues into 2-4 sentence paragraphs separated by blank lines
|
||||
- Subsequent paragraphs from the SAME speaker do NOT repeat the speaker label
|
||||
- Every paragraph ends with exactly ONE timestamp range `[HH:MM:SS → HH:MM:SS]`
|
||||
|
||||
Correct example:
|
||||
```
|
||||
**Jane Doe:** The study focuses on long-term effects of dietary changes. We tracked two groups over five years. [00:00:15 → 00:00:21]
|
||||
|
||||
The first group followed the new regimen, while the second group maintained a traditional diet. [00:00:21 → 00:00:28]
|
||||
|
||||
**Host:** Fascinating. And what did you find? [00:00:28 → 00:00:31]
|
||||
```
|
||||
|
||||
Wrong (multiple timestamps in one paragraph):
|
||||
```
|
||||
**Host:** Welcome back. [00:00:01] Today we have a guest. [00:00:02]
|
||||
```
|
||||
|
||||
**Non-Speech Audio:** On its own line: `[Laughter] [HH:MM:SS]`
|
||||
|
||||
## Example Output
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: "Example Interview"
|
||||
channel: "The Show"
|
||||
date: 2024-04-15
|
||||
url: "https://www.youtube.com/watch?v=xxx"
|
||||
cover: imgs/cover.jpg
|
||||
description: "Jane Doe discusses her groundbreaking five-year study on the long-term effects of dietary changes."
|
||||
language: en
|
||||
---
|
||||
|
||||
# Example Interview
|
||||
|
||||
Jane Doe discusses her groundbreaking five-year study on the long-term effects of dietary changes.
|
||||
|
||||
## Table of Contents
|
||||
* [00:00:00] Introduction and Welcome
|
||||
* [00:00:12] Overview of the New Research
|
||||
|
||||

|
||||
|
||||
|
||||
## Introduction and Welcome [00:00:00]
|
||||
|
||||
**Host:** Welcome back to the show. Today, we have a, uh, very special guest, Jane Doe. [00:00:00 → 00:00:03]
|
||||
|
||||
**Jane Doe:** Thank you for having me. I'm excited to be here and discuss the findings. [00:00:03 → 00:00:07]
|
||||
|
||||
|
||||
## Overview of the New Research [00:00:12]
|
||||
|
||||
**Host:** So, Jane, before we get into the nitty-gritty, could you, you know, give us a brief overview for our audience? [00:00:12 → 00:00:16]
|
||||
|
||||
**Jane Doe:** Of course. The study focuses on the long-term effects of specific dietary changes. It's a bit complicated but essentially we tracked two large groups over a five-year period. [00:00:16 → 00:00:23]
|
||||
|
||||
The first group followed the new regimen, while the second group, our control, maintained a traditional diet. This allowed us to isolate variables effectively. [00:00:23 → 00:00:30]
|
||||
|
||||
[Laughter] [00:00:30]
|
||||
|
||||
**Host:** Fascinating. And what did you find? [00:00:31 → 00:00:33]
|
||||
```
|
||||
@@ -0,0 +1,857 @@
|
||||
#!/usr/bin/env bun
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
|
||||
import { dirname, join, resolve } from "path";
|
||||
|
||||
type Format = "text" | "srt";
|
||||
|
||||
interface Options {
|
||||
videoIds: string[];
|
||||
languages: string[];
|
||||
format: Format;
|
||||
translate: string;
|
||||
list: boolean;
|
||||
excludeGenerated: boolean;
|
||||
excludeManual: boolean;
|
||||
output: string;
|
||||
outputDir: string;
|
||||
timestamps: boolean;
|
||||
chapters: boolean;
|
||||
speakers: boolean;
|
||||
refresh: boolean;
|
||||
}
|
||||
|
||||
interface Snippet {
|
||||
text: string;
|
||||
start: number;
|
||||
duration: number;
|
||||
}
|
||||
|
||||
interface Sentence {
|
||||
text: string;
|
||||
start: string;
|
||||
end: string;
|
||||
}
|
||||
|
||||
interface TranscriptInfo {
|
||||
language: string;
|
||||
languageCode: string;
|
||||
isGenerated: boolean;
|
||||
isTranslatable: boolean;
|
||||
baseUrl: string;
|
||||
translationLanguages: { language: string; languageCode: string }[];
|
||||
}
|
||||
|
||||
interface Chapter {
|
||||
title: string;
|
||||
start: number;
|
||||
end: number;
|
||||
}
|
||||
|
||||
interface VideoMeta {
|
||||
videoId: string;
|
||||
title: string;
|
||||
channel: string;
|
||||
channelId: string;
|
||||
description: string;
|
||||
duration: number;
|
||||
publishDate: string;
|
||||
url: string;
|
||||
coverImage: string;
|
||||
thumbnailUrl: string;
|
||||
language: { code: string; name: string; isGenerated: boolean };
|
||||
chapters: Chapter[];
|
||||
}
|
||||
|
||||
interface VideoResult {
|
||||
videoId: string;
|
||||
title?: string;
|
||||
filePath?: string;
|
||||
content?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
const WATCH_URL = "https://www.youtube.com/watch?v=";
|
||||
const INNERTUBE_URL = "https://www.youtube.com/youtubei/v1/player";
|
||||
const INNERTUBE_CTX = { client: { clientName: "ANDROID", clientVersion: "20.10.38" } };
|
||||
|
||||
function extractVideoId(input: string): string {
|
||||
input = input.replace(/\\/g, "").trim();
|
||||
const patterns = [
|
||||
/(?:youtube\.com\/watch\?.*v=|youtu\.be\/|youtube\.com\/embed\/|youtube\.com\/v\/|youtube\.com\/shorts\/)([a-zA-Z0-9_-]{11})/,
|
||||
/^([a-zA-Z0-9_-]{11})$/,
|
||||
];
|
||||
for (const p of patterns) {
|
||||
const m = input.match(p);
|
||||
if (m) return m[1];
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
function slugify(s: string): string {
|
||||
return s
|
||||
.toLowerCase()
|
||||
.replace(/[^\w\s-]/g, "")
|
||||
.replace(/\s+/g, "-")
|
||||
.replace(/-+/g, "-")
|
||||
.replace(/^-|-$/g, "") || "untitled";
|
||||
}
|
||||
|
||||
function htmlUnescape(s: string): string {
|
||||
return s
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/'/g, "'")
|
||||
.replace(///g, "/")
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n)))
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, n) => String.fromCharCode(parseInt(n, 16)));
|
||||
}
|
||||
|
||||
function stripTags(s: string): string {
|
||||
return s.replace(/<[^>]*>/g, "");
|
||||
}
|
||||
|
||||
function parseTranscriptXml(xml: string): Snippet[] {
|
||||
const snippets: Snippet[] = [];
|
||||
const re = /<text\s+start="([^"]*)"(?:\s+dur="([^"]*)")?[^>]*>([\s\S]*?)<\/text>/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(xml)) !== null) {
|
||||
const raw = m[3];
|
||||
if (!raw) continue;
|
||||
snippets.push({
|
||||
text: htmlUnescape(stripTags(raw)),
|
||||
start: parseFloat(m[1]),
|
||||
duration: parseFloat(m[2] || "0"),
|
||||
});
|
||||
}
|
||||
return snippets;
|
||||
}
|
||||
|
||||
// --- YouTube API ---
|
||||
|
||||
async function fetchHtml(videoId: string): Promise<string> {
|
||||
const r = await fetch(WATCH_URL + videoId, {
|
||||
headers: { "Accept-Language": "en-US", "User-Agent": "Mozilla/5.0" },
|
||||
});
|
||||
if (!r.ok) throw new Error(`HTTP ${r.status} fetching video page`);
|
||||
let html = await r.text();
|
||||
if (html.includes('action="https://consent.youtube.com/s"')) {
|
||||
const cv = html.match(/name="v" value="(.*?)"/);
|
||||
if (!cv) throw new Error("Failed to create consent cookie");
|
||||
const r2 = await fetch(WATCH_URL + videoId, {
|
||||
headers: {
|
||||
"Accept-Language": "en-US",
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
Cookie: `CONSENT=YES+${cv[1]}`,
|
||||
},
|
||||
});
|
||||
if (!r2.ok) throw new Error(`HTTP ${r2.status} fetching video page (consent)`);
|
||||
html = await r2.text();
|
||||
}
|
||||
return html;
|
||||
}
|
||||
|
||||
function extractApiKey(html: string, videoId: string): string {
|
||||
const m = html.match(/"INNERTUBE_API_KEY":\s*"([a-zA-Z0-9_-]+)"/);
|
||||
if (!m) {
|
||||
if (html.includes('class="g-recaptcha"')) throw new Error(`IP blocked for ${videoId} (reCAPTCHA)`);
|
||||
throw new Error(`Cannot extract API key for ${videoId}`);
|
||||
}
|
||||
return m[1];
|
||||
}
|
||||
|
||||
async function fetchInnertubeData(videoId: string, apiKey: string): Promise<any> {
|
||||
const r = await fetch(`${INNERTUBE_URL}?key=${apiKey}`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ context: INNERTUBE_CTX, videoId }),
|
||||
});
|
||||
if (r.status === 429) throw new Error(`IP blocked for ${videoId} (429)`);
|
||||
if (!r.ok) throw new Error(`HTTP ${r.status} from InnerTube API`);
|
||||
return r.json();
|
||||
}
|
||||
|
||||
function assertPlayability(data: any, videoId: string) {
|
||||
const ps = data?.playabilityStatus;
|
||||
if (!ps) return;
|
||||
const status = ps.status;
|
||||
if (status === "OK" || !status) return;
|
||||
const reason = ps.reason || "";
|
||||
if (status === "LOGIN_REQUIRED") {
|
||||
if (reason.includes("bot")) throw new Error(`Request blocked for ${videoId}: bot detected`);
|
||||
if (reason.includes("inappropriate")) throw new Error(`Age restricted: ${videoId}`);
|
||||
}
|
||||
if (status === "ERROR" && reason.includes("unavailable")) {
|
||||
if (videoId.startsWith("http")) throw new Error(`Invalid video ID: pass the ID, not the URL`);
|
||||
throw new Error(`Video unavailable: ${videoId}`);
|
||||
}
|
||||
const subreasons = ps.errorScreen?.playerErrorMessageRenderer?.subreason?.runs?.map((r: any) => r.text).join("") || "";
|
||||
throw new Error(`Video unplayable (${videoId}): ${reason} ${subreasons}`.trim());
|
||||
}
|
||||
|
||||
function extractCaptionsJson(data: any, videoId: string): any {
|
||||
assertPlayability(data, videoId);
|
||||
const cj = data?.captions?.playerCaptionsTracklistRenderer;
|
||||
if (!cj || !cj.captionTracks) throw new Error(`Transcripts disabled for ${videoId}`);
|
||||
return cj;
|
||||
}
|
||||
|
||||
function buildTranscriptList(captionsJson: any): TranscriptInfo[] {
|
||||
const tlLangs = (captionsJson.translationLanguages || []).map((tl: any) => ({
|
||||
language: tl.languageName?.runs?.[0]?.text || tl.languageName?.simpleText || "",
|
||||
languageCode: tl.languageCode,
|
||||
}));
|
||||
return (captionsJson.captionTracks || []).map((t: any) => ({
|
||||
language: t.name?.runs?.[0]?.text || t.name?.simpleText || "",
|
||||
languageCode: t.languageCode,
|
||||
isGenerated: t.kind === "asr",
|
||||
isTranslatable: !!t.isTranslatable,
|
||||
baseUrl: (t.baseUrl || "").replace(/&fmt=srv3/g, ""),
|
||||
translationLanguages: t.isTranslatable ? tlLangs : [],
|
||||
}));
|
||||
}
|
||||
|
||||
function findTranscript(
|
||||
transcripts: TranscriptInfo[],
|
||||
languages: string[],
|
||||
excludeGenerated: boolean,
|
||||
excludeManual: boolean
|
||||
): TranscriptInfo {
|
||||
let filtered = transcripts;
|
||||
if (excludeGenerated) filtered = filtered.filter((t) => !t.isGenerated);
|
||||
if (excludeManual) filtered = filtered.filter((t) => t.isGenerated);
|
||||
for (const lang of languages) {
|
||||
const found = filtered.find((t) => t.languageCode === lang);
|
||||
if (found) return found;
|
||||
}
|
||||
const available = filtered.map((t) => `${t.languageCode} ("${t.language}")`).join(", ");
|
||||
throw new Error(`No transcript found for languages [${languages.join(", ")}]. Available: ${available || "none"}`);
|
||||
}
|
||||
|
||||
async function fetchTranscriptSnippets(info: TranscriptInfo, translateTo?: string): Promise<{ snippets: Snippet[]; language: string; languageCode: string }> {
|
||||
let url = info.baseUrl;
|
||||
let lang = info.language;
|
||||
let langCode = info.languageCode;
|
||||
if (translateTo) {
|
||||
if (!info.isTranslatable) throw new Error(`Transcript ${info.languageCode} is not translatable`);
|
||||
const tl = info.translationLanguages.find((t) => t.languageCode === translateTo);
|
||||
if (!tl) throw new Error(`Translation language ${translateTo} not available`);
|
||||
url += `&tlang=${translateTo}`;
|
||||
lang = tl.language;
|
||||
langCode = translateTo;
|
||||
}
|
||||
const r = await fetch(url, { headers: { "Accept-Language": "en-US" } });
|
||||
if (!r.ok) throw new Error(`HTTP ${r.status} fetching transcript`);
|
||||
return { snippets: parseTranscriptXml(await r.text()), language: lang, languageCode: langCode };
|
||||
}
|
||||
|
||||
// --- Metadata & chapters ---
|
||||
|
||||
function parseChapters(description: string, duration: number = 0): Chapter[] {
|
||||
const raw: { title: string; start: number }[] = [];
|
||||
for (const line of description.split("\n")) {
|
||||
const m = line.trim().match(/^(?:(\d{1,2}):)?(\d{1,2}):(\d{2})\s+(.+)$/);
|
||||
if (m) {
|
||||
const h = m[1] ? parseInt(m[1]) : 0;
|
||||
raw.push({ title: m[4].trim(), start: h * 3600 + parseInt(m[2]) * 60 + parseInt(m[3]) });
|
||||
}
|
||||
}
|
||||
if (raw.length < 2) return [];
|
||||
return raw.map((ch, i) => ({
|
||||
title: ch.title,
|
||||
start: ch.start,
|
||||
end: i < raw.length - 1 ? raw[i + 1].start : Math.max(duration, ch.start),
|
||||
}));
|
||||
}
|
||||
|
||||
function getThumbnailUrls(videoId: string, data: any): string[] {
|
||||
const urls = [
|
||||
`https://i.ytimg.com/vi/${videoId}/maxresdefault.jpg`,
|
||||
`https://i.ytimg.com/vi/${videoId}/hqdefault.jpg`,
|
||||
];
|
||||
const thumbnails = data?.videoDetails?.thumbnail?.thumbnails ||
|
||||
data?.microformat?.playerMicroformatRenderer?.thumbnail?.thumbnails || [];
|
||||
if (thumbnails.length) {
|
||||
const sorted = [...thumbnails].sort((a: any, b: any) => (b.width || 0) - (a.width || 0));
|
||||
for (const t of sorted) if (t.url && !urls.includes(t.url)) urls.push(t.url);
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
function buildVideoMeta(data: any, videoId: string, langInfo: { code: string; name: string; isGenerated: boolean }, chapters: Chapter[]): VideoMeta {
|
||||
const vd = data?.videoDetails || {};
|
||||
const mf = data?.microformat?.playerMicroformatRenderer || {};
|
||||
return {
|
||||
videoId,
|
||||
title: vd.title || mf.title?.simpleText || "",
|
||||
channel: vd.author || mf.ownerChannelName || "",
|
||||
channelId: vd.channelId || mf.externalChannelId || "",
|
||||
description: vd.shortDescription || mf.description?.simpleText || "",
|
||||
duration: parseInt(vd.lengthSeconds || "0"),
|
||||
publishDate: mf.publishDate || mf.uploadDate || "",
|
||||
url: `https://www.youtube.com/watch?v=${videoId}`,
|
||||
coverImage: "",
|
||||
thumbnailUrl: getThumbnailUrls(videoId, data)[0],
|
||||
language: langInfo,
|
||||
chapters,
|
||||
};
|
||||
}
|
||||
|
||||
async function downloadCoverImage(urls: string[], outputPath: string): Promise<boolean> {
|
||||
for (const u of urls) {
|
||||
try {
|
||||
const r = await fetch(u);
|
||||
if (r.ok) {
|
||||
writeFileSync(outputPath, Buffer.from(await r.arrayBuffer()));
|
||||
return true;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function parseSrt(srt: string): Snippet[] {
|
||||
const blocks = srt.trim().split(/\n\n+/);
|
||||
const snippets: Snippet[] = [];
|
||||
for (const block of blocks) {
|
||||
const lines = block.split("\n");
|
||||
if (lines.length < 3) continue;
|
||||
const m = lines[1].match(/(\d{2}):(\d{2}):(\d{2}),(\d{3})\s*-->\s*(\d{2}):(\d{2}):(\d{2}),(\d{3})/);
|
||||
if (!m) continue;
|
||||
const start = parseInt(m[1]) * 3600 + parseInt(m[2]) * 60 + parseInt(m[3]) + parseInt(m[4]) / 1000;
|
||||
const end = parseInt(m[5]) * 3600 + parseInt(m[6]) * 60 + parseInt(m[7]) + parseInt(m[8]) / 1000;
|
||||
snippets.push({ text: lines.slice(2).join(" "), start, duration: end - start });
|
||||
}
|
||||
return snippets;
|
||||
}
|
||||
|
||||
// --- Timestamp formatting ---
|
||||
|
||||
function ts(t: number): string {
|
||||
const h = Math.floor(t / 3600);
|
||||
const m = Math.floor((t % 3600) / 60);
|
||||
const s = Math.floor(t % 60);
|
||||
return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
function tsMs(t: number, sep: string): string {
|
||||
const h = Math.floor(t / 3600);
|
||||
const m = Math.floor((t % 3600) / 60);
|
||||
const s = Math.floor(t % 60);
|
||||
const ms = Math.round((t - Math.floor(t)) * 1000);
|
||||
return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}${sep}${String(ms).padStart(3, "0")}`;
|
||||
}
|
||||
|
||||
// --- Paragraph grouping ---
|
||||
|
||||
interface Paragraph {
|
||||
text: string;
|
||||
start: number;
|
||||
end: number;
|
||||
}
|
||||
|
||||
function groupIntoParagraphs(snippets: Snippet[]): Paragraph[] {
|
||||
if (!snippets.length) return [];
|
||||
const paras: Paragraph[] = [];
|
||||
let buf: Snippet[] = [];
|
||||
for (let i = 0; i < snippets.length; i++) {
|
||||
buf.push(snippets[i]);
|
||||
const last = i === snippets.length - 1;
|
||||
const gap = !last && snippets[i + 1].start - (snippets[i].start + snippets[i].duration) > 1.5;
|
||||
if (last || gap || buf.length >= 8) {
|
||||
const lastS = buf[buf.length - 1];
|
||||
paras.push({ text: buf.map(s => s.text).join(" "), start: buf[0].start, end: lastS.start + lastS.duration });
|
||||
buf = [];
|
||||
}
|
||||
}
|
||||
return paras;
|
||||
}
|
||||
|
||||
// --- Sentence segmentation ---
|
||||
|
||||
const SENTENCE_END_RE = /[.?!…。?!⁈⁇‼‽.]/;
|
||||
|
||||
function isCJK(ch: string): boolean {
|
||||
const code = ch.charCodeAt(0);
|
||||
return (code >= 0x4E00 && code <= 0x9FFF) ||
|
||||
(code >= 0x3040 && code <= 0x309F) ||
|
||||
(code >= 0x30A0 && code <= 0x30FF) ||
|
||||
(code >= 0xAC00 && code <= 0xD7AF) ||
|
||||
(code >= 0x3400 && code <= 0x4DBF) ||
|
||||
(code >= 0xF900 && code <= 0xFAFF);
|
||||
}
|
||||
|
||||
function splitSnippetAtPunctuation(s: Snippet): { text: string; start: number; end: number }[] {
|
||||
const { text, start, duration } = s;
|
||||
const end = start + duration;
|
||||
if (!text.length) return [];
|
||||
|
||||
const splitPoints: number[] = [];
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
if (SENTENCE_END_RE.test(text[i])) {
|
||||
while (i + 1 < text.length && SENTENCE_END_RE.test(text[i + 1])) i++;
|
||||
if (i < text.length - 1) splitPoints.push(i);
|
||||
}
|
||||
}
|
||||
|
||||
if (!splitPoints.length) return [{ text, start, end }];
|
||||
|
||||
const parts: { text: string; start: number; end: number }[] = [];
|
||||
let prev = 0;
|
||||
for (const pos of splitPoints) {
|
||||
const partText = text.slice(prev, pos + 1).trim();
|
||||
if (partText) {
|
||||
parts.push({
|
||||
text: partText,
|
||||
start: start + (prev / text.length) * duration,
|
||||
end: start + ((pos + 1) / text.length) * duration,
|
||||
});
|
||||
}
|
||||
prev = pos + 1;
|
||||
}
|
||||
|
||||
const remaining = text.slice(prev).trim();
|
||||
if (remaining) {
|
||||
parts.push({ text: remaining, start: start + (prev / text.length) * duration, end });
|
||||
}
|
||||
|
||||
return parts;
|
||||
}
|
||||
|
||||
function mergeTexts(texts: string[]): string {
|
||||
if (!texts.length) return "";
|
||||
let result = texts[0];
|
||||
for (let i = 1; i < texts.length; i++) {
|
||||
const next = texts[i];
|
||||
if (!next) continue;
|
||||
const lastChar = result[result.length - 1];
|
||||
const firstChar = next[0];
|
||||
if (isCJK(lastChar) || isCJK(firstChar)) {
|
||||
result += next;
|
||||
} else {
|
||||
result = result.trimEnd() + " " + next.trimStart();
|
||||
}
|
||||
}
|
||||
return result.replace(/ {2,}/g, " ");
|
||||
}
|
||||
|
||||
function segmentIntoSentences(snippets: Snippet[]): Sentence[] {
|
||||
const parts: { text: string; start: number; end: number }[] = [];
|
||||
for (const s of snippets) parts.push(...splitSnippetAtPunctuation(s));
|
||||
|
||||
const sentences: Sentence[] = [];
|
||||
let buf: { text: string; start: number; end: number }[] = [];
|
||||
|
||||
for (const part of parts) {
|
||||
buf.push(part);
|
||||
if (SENTENCE_END_RE.test(part.text[part.text.length - 1])) {
|
||||
sentences.push({
|
||||
text: mergeTexts(buf.map(b => b.text)),
|
||||
start: ts(buf[0].start),
|
||||
end: ts(buf[buf.length - 1].end),
|
||||
});
|
||||
buf = [];
|
||||
}
|
||||
}
|
||||
|
||||
if (buf.length) {
|
||||
sentences.push({
|
||||
text: mergeTexts(buf.map(b => b.text)),
|
||||
start: ts(buf[0].start),
|
||||
end: ts(buf[buf.length - 1].end),
|
||||
});
|
||||
}
|
||||
|
||||
return sentences;
|
||||
}
|
||||
|
||||
function parseTs(t: string): number {
|
||||
const [h, m, s] = t.split(":").map(Number);
|
||||
return h * 3600 + m * 60 + s;
|
||||
}
|
||||
|
||||
function groupSentenceParas(sentences: Sentence[]): Paragraph[] {
|
||||
if (!sentences.length) return [];
|
||||
const paras: Paragraph[] = [];
|
||||
let buf: Sentence[] = [];
|
||||
for (let i = 0; i < sentences.length; i++) {
|
||||
buf.push(sentences[i]);
|
||||
const last = i === sentences.length - 1;
|
||||
const gap = !last && parseTs(sentences[i + 1].start) - parseTs(sentences[i].end) > 2;
|
||||
if (last || gap || buf.length >= 5) {
|
||||
paras.push({
|
||||
text: mergeTexts(buf.map(s => s.text)),
|
||||
start: parseTs(buf[0].start),
|
||||
end: parseTs(buf[buf.length - 1].end),
|
||||
});
|
||||
buf = [];
|
||||
}
|
||||
}
|
||||
return paras;
|
||||
}
|
||||
|
||||
// --- Format functions ---
|
||||
|
||||
function formatSrt(snippets: Snippet[]): string {
|
||||
return snippets
|
||||
.map((s, i) => {
|
||||
const end = i < snippets.length - 1 && snippets[i + 1].start < s.start + s.duration
|
||||
? snippets[i + 1].start
|
||||
: s.start + s.duration;
|
||||
return `${i + 1}\n${tsMs(s.start, ",")} --> ${tsMs(end, ",")}\n${s.text}`;
|
||||
})
|
||||
.join("\n\n") + "\n";
|
||||
}
|
||||
|
||||
function yamlEscape(s: string): string {
|
||||
if (/[:"'{}\[\]#&*!|>%@`\n]/.test(s) || s.trim() !== s) return `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
|
||||
return s;
|
||||
}
|
||||
|
||||
function extractSummary(description: string): string {
|
||||
if (!description) return "";
|
||||
const firstPara = description.split(/\n\s*\n/)[0].trim();
|
||||
const lines = firstPara.split("\n").filter(l => !/^\s*(https?:\/\/|#|@|\d+:\d+)/.test(l) && l.trim());
|
||||
return lines.join(" ").slice(0, 300).trim();
|
||||
}
|
||||
|
||||
function formatMarkdown(sentences: Sentence[], meta: VideoMeta, opts: { timestamps: boolean; chapters: boolean; speakers: boolean }, snippets?: Snippet[]): string {
|
||||
const summary = extractSummary(meta.description);
|
||||
let md = "---\n";
|
||||
md += `title: ${yamlEscape(meta.title)}\n`;
|
||||
md += `channel: ${yamlEscape(meta.channel)}\n`;
|
||||
if (meta.publishDate) md += `date: ${meta.publishDate}\n`;
|
||||
md += `url: ${yamlEscape(meta.url)}\n`;
|
||||
if (meta.coverImage) md += `cover: ${meta.coverImage}\n`;
|
||||
if (summary) md += `description: ${yamlEscape(summary)}\n`;
|
||||
if (meta.language) md += `language: ${meta.language.code}\n`;
|
||||
md += "---\n\n";
|
||||
|
||||
if (opts.speakers) {
|
||||
md += `# ${meta.title}\n\n`;
|
||||
if (summary) md += `${summary}\n\n`;
|
||||
if (meta.description) md += "# Description\n\n" + meta.description.trim() + "\n\n";
|
||||
if (meta.chapters.length) {
|
||||
md += "# Chapters\n\n";
|
||||
for (const ch of meta.chapters) md += `* [${ts(ch.start)}] ${ch.title}\n`;
|
||||
md += "\n";
|
||||
}
|
||||
md += "# Transcript\n\n";
|
||||
md += snippets ? formatSrt(snippets) : "";
|
||||
return md;
|
||||
}
|
||||
|
||||
md += `# ${meta.title}\n\n`;
|
||||
if (summary) md += `${summary}\n\n`;
|
||||
|
||||
const chapters = opts.chapters ? meta.chapters : [];
|
||||
|
||||
if (chapters.length) {
|
||||
md += "## Table of Contents\n\n";
|
||||
for (const ch of chapters) md += opts.timestamps ? `* [${ts(ch.start)}] ${ch.title}\n` : `* ${ch.title}\n`;
|
||||
md += "\n";
|
||||
if (meta.coverImage) md += `\n\n`;
|
||||
md += "\n";
|
||||
for (let i = 0; i < chapters.length; i++) {
|
||||
const nextStart = i < chapters.length - 1 ? chapters[i + 1].start : Infinity;
|
||||
const chSentences = sentences.filter(s => parseTs(s.start) >= chapters[i].start && parseTs(s.start) < nextStart);
|
||||
const paras = groupSentenceParas(chSentences);
|
||||
md += opts.timestamps
|
||||
? `## [${ts(chapters[i].start)}] ${chapters[i].title}\n\n`
|
||||
: `## ${chapters[i].title}\n\n`;
|
||||
for (const p of paras) md += opts.timestamps ? `${p.text} [${ts(p.start)} → ${ts(p.end)}]\n\n` : `${p.text}\n\n`;
|
||||
md += "\n";
|
||||
}
|
||||
} else {
|
||||
const paras = groupSentenceParas(sentences);
|
||||
for (const p of paras) md += opts.timestamps ? `${p.text} [${ts(p.start)} → ${ts(p.end)}]\n\n` : `${p.text}\n\n`;
|
||||
}
|
||||
|
||||
return md.trimEnd() + "\n";
|
||||
}
|
||||
|
||||
function formatListOutput(videoId: string, title: string, transcripts: TranscriptInfo[]): string {
|
||||
const manual = transcripts.filter((t) => !t.isGenerated);
|
||||
const generated = transcripts.filter((t) => t.isGenerated);
|
||||
const tlLangs = transcripts.find((t) => t.translationLanguages.length > 0)?.translationLanguages || [];
|
||||
const fmtList = (list: TranscriptInfo[]) =>
|
||||
list.length ? list.map((t) => ` - ${t.languageCode} ("${t.language}")${t.isTranslatable ? " [TRANSLATABLE]" : ""}`).join("\n") : "None";
|
||||
const fmtTl = tlLangs.length
|
||||
? tlLangs.map((t) => ` - ${t.languageCode} ("${t.language}")`).join("\n")
|
||||
: "None";
|
||||
return `Transcripts for ${videoId}${title ? ` (${title})` : ""}:\n\n(MANUALLY CREATED)\n${fmtList(manual)}\n\n(GENERATED)\n${fmtList(generated)}\n\n(TRANSLATION LANGUAGES)\n${fmtTl}`;
|
||||
}
|
||||
|
||||
// --- File helpers ---
|
||||
|
||||
function ensureDir(p: string) {
|
||||
const dir = dirname(p);
|
||||
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
function resolveBaseDir(outputDir: string): string {
|
||||
return resolve(outputDir || "youtube-transcript");
|
||||
}
|
||||
|
||||
function loadIndex(baseDir: string): Record<string, string> {
|
||||
try { return JSON.parse(readFileSync(join(baseDir, ".index.json"), "utf-8")); } catch { return {}; }
|
||||
}
|
||||
|
||||
function saveIndex(baseDir: string, index: Record<string, string>) {
|
||||
const p = join(baseDir, ".index.json");
|
||||
ensureDir(p);
|
||||
writeFileSync(p, JSON.stringify(index, null, 2));
|
||||
}
|
||||
|
||||
function lookupVideoDir(videoId: string, baseDir: string): string | null {
|
||||
const rel = loadIndex(baseDir)[videoId];
|
||||
if (rel) {
|
||||
const dir = resolve(baseDir, rel);
|
||||
if (existsSync(dir)) return dir;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function registerVideoDir(videoId: string, channelSlug: string, titleSlug: string, baseDir: string): string {
|
||||
const rel = join(channelSlug, titleSlug);
|
||||
const index = loadIndex(baseDir);
|
||||
index[videoId] = rel;
|
||||
saveIndex(baseDir, index);
|
||||
return resolve(baseDir, rel);
|
||||
}
|
||||
|
||||
function hasCachedData(videoDir: string): boolean {
|
||||
return existsSync(join(videoDir, "meta.json")) && existsSync(join(videoDir, "transcript-raw.json"));
|
||||
}
|
||||
|
||||
function loadMeta(videoDir: string): VideoMeta {
|
||||
return JSON.parse(readFileSync(join(videoDir, "meta.json"), "utf-8"));
|
||||
}
|
||||
|
||||
function loadSnippets(videoDir: string): Snippet[] {
|
||||
return JSON.parse(readFileSync(join(videoDir, "transcript-raw.json"), "utf-8"));
|
||||
}
|
||||
|
||||
function loadSentences(videoDir: string): Sentence[] {
|
||||
return JSON.parse(readFileSync(join(videoDir, "transcript-sentences.json"), "utf-8"));
|
||||
}
|
||||
|
||||
// --- Main processing ---
|
||||
|
||||
async function fetchAndCache(videoId: string, baseDir: string, opts: Options): Promise<{ meta: VideoMeta; snippets: Snippet[]; sentences: Sentence[]; videoDir: string }> {
|
||||
const html = await fetchHtml(videoId);
|
||||
const apiKey = extractApiKey(html, videoId);
|
||||
const data = await fetchInnertubeData(videoId, apiKey);
|
||||
const captionsJson = extractCaptionsJson(data, videoId);
|
||||
const transcripts = buildTranscriptList(captionsJson);
|
||||
const info = findTranscript(transcripts, opts.languages, opts.excludeGenerated, opts.excludeManual);
|
||||
const result = await fetchTranscriptSnippets(info, opts.translate || undefined);
|
||||
const description = data?.videoDetails?.shortDescription || "";
|
||||
const duration = parseInt(data?.videoDetails?.lengthSeconds || "0");
|
||||
const chapters = parseChapters(description, duration);
|
||||
const langInfo = { code: result.languageCode, name: result.language, isGenerated: info.isGenerated };
|
||||
const meta = buildVideoMeta(data, videoId, langInfo, chapters);
|
||||
|
||||
const videoDir = registerVideoDir(videoId, slugify(meta.channel), slugify(meta.title), baseDir);
|
||||
ensureDir(join(videoDir, "meta.json"));
|
||||
|
||||
writeFileSync(join(videoDir, "transcript-raw.json"), JSON.stringify(result.snippets, null, 2));
|
||||
|
||||
const sentences = segmentIntoSentences(result.snippets);
|
||||
writeFileSync(join(videoDir, "transcript-sentences.json"), JSON.stringify(sentences, null, 2));
|
||||
|
||||
const imgPath = join(videoDir, "imgs", "cover.jpg");
|
||||
ensureDir(imgPath);
|
||||
const downloaded = await downloadCoverImage(getThumbnailUrls(videoId, data), imgPath);
|
||||
meta.coverImage = downloaded ? "imgs/cover.jpg" : "";
|
||||
|
||||
writeFileSync(join(videoDir, "meta.json"), JSON.stringify(meta, null, 2));
|
||||
|
||||
return { meta, snippets: result.snippets, sentences, videoDir };
|
||||
}
|
||||
|
||||
async function processVideo(videoId: string, opts: Options): Promise<VideoResult> {
|
||||
const baseDir = resolveBaseDir(opts.outputDir);
|
||||
|
||||
// --list: always fetch fresh
|
||||
if (opts.list) {
|
||||
const html = await fetchHtml(videoId);
|
||||
const apiKey = extractApiKey(html, videoId);
|
||||
const data = await fetchInnertubeData(videoId, apiKey);
|
||||
const title = data?.videoDetails?.title || "";
|
||||
const captionsJson = extractCaptionsJson(data, videoId);
|
||||
const transcripts = buildTranscriptList(captionsJson);
|
||||
return { videoId, title, content: formatListOutput(videoId, title, transcripts) };
|
||||
}
|
||||
|
||||
let videoDir = lookupVideoDir(videoId, baseDir);
|
||||
let meta: VideoMeta;
|
||||
let snippets: Snippet[];
|
||||
let sentences: Sentence[];
|
||||
let needsFetch = opts.refresh || !videoDir || !hasCachedData(videoDir);
|
||||
|
||||
if (!needsFetch && videoDir) {
|
||||
meta = loadMeta(videoDir);
|
||||
snippets = loadSnippets(videoDir);
|
||||
sentences = loadSentences(videoDir);
|
||||
const wantLangs = opts.translate ? [opts.translate] : opts.languages;
|
||||
if (!wantLangs.includes(meta.language.code)) needsFetch = true;
|
||||
// Backfill chapter end times for caches created before this field existed
|
||||
if (!needsFetch && meta.chapters.length > 0 && meta.chapters.some((ch: any) => ch.end === undefined)) {
|
||||
for (let i = 0; i < meta.chapters.length; i++) {
|
||||
meta.chapters[i].end = i < meta.chapters.length - 1
|
||||
? meta.chapters[i + 1].start
|
||||
: Math.max(meta.duration, meta.chapters[i].start);
|
||||
}
|
||||
try { writeFileSync(join(videoDir, "meta.json"), JSON.stringify(meta, null, 2)); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
if (needsFetch) {
|
||||
const result = await fetchAndCache(videoId, baseDir, opts);
|
||||
meta = result.meta;
|
||||
snippets = result.snippets;
|
||||
sentences = result.sentences;
|
||||
videoDir = result.videoDir;
|
||||
} else {
|
||||
meta = meta!;
|
||||
snippets = snippets!;
|
||||
sentences = sentences!;
|
||||
}
|
||||
|
||||
let content: string;
|
||||
let ext: string;
|
||||
|
||||
if (opts.format === "srt") {
|
||||
content = formatSrt(snippets);
|
||||
ext = "srt";
|
||||
} else {
|
||||
content = formatMarkdown(sentences, meta, {
|
||||
timestamps: opts.timestamps,
|
||||
chapters: opts.chapters,
|
||||
speakers: opts.speakers,
|
||||
}, snippets);
|
||||
ext = "md";
|
||||
}
|
||||
|
||||
const filePath = opts.output ? resolve(opts.output) : join(videoDir!, `transcript.${ext}`);
|
||||
ensureDir(filePath);
|
||||
writeFileSync(filePath, content);
|
||||
|
||||
return { videoId, title: meta.title, filePath };
|
||||
}
|
||||
|
||||
// --- CLI ---
|
||||
|
||||
function printHelp() {
|
||||
console.log(`Usage: bun main.ts <video-url-or-id> [options]
|
||||
|
||||
Options:
|
||||
--languages <codes> Language codes, comma-separated (default: en)
|
||||
--format <fmt> Output format: text, srt (default: text)
|
||||
--translate <code> Translate to language code
|
||||
--list List available transcripts
|
||||
--timestamps Include timestamps (default: on)
|
||||
--no-timestamps Disable timestamps
|
||||
--chapters Chapter segmentation from description
|
||||
--speakers Raw transcript with metadata for speaker identification
|
||||
--exclude-generated Skip auto-generated transcripts
|
||||
--exclude-manually-created Skip manually created transcripts
|
||||
--refresh Force re-fetch (ignore cache)
|
||||
-o, --output <path> Save to specific file path
|
||||
--output-dir <dir> Base output directory (default: youtube-transcript)
|
||||
-h, --help Show help`);
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): Options | null {
|
||||
const opts: Options = {
|
||||
videoIds: [],
|
||||
languages: ["en"],
|
||||
format: "text",
|
||||
translate: "",
|
||||
list: false,
|
||||
excludeGenerated: false,
|
||||
excludeManual: false,
|
||||
output: "",
|
||||
outputDir: "",
|
||||
timestamps: true,
|
||||
chapters: false,
|
||||
speakers: false,
|
||||
refresh: false,
|
||||
};
|
||||
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i];
|
||||
if (arg === "-h" || arg === "--help") {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
} else if (arg === "--languages") {
|
||||
const v = argv[++i];
|
||||
if (v) opts.languages = v.split(",").map((s) => s.trim());
|
||||
} else if (arg === "--format") {
|
||||
const v = argv[++i]?.toLowerCase();
|
||||
if (v === "text" || v === "srt") opts.format = v;
|
||||
else {
|
||||
console.error(`Invalid format: ${v}. Use: text, srt`);
|
||||
return null;
|
||||
}
|
||||
} else if (arg === "--translate") {
|
||||
opts.translate = argv[++i] || "";
|
||||
} else if (arg === "--list" || arg === "--list-transcripts") {
|
||||
opts.list = true;
|
||||
} else if (arg === "--timestamps" || arg === "-t") {
|
||||
opts.timestamps = true;
|
||||
} else if (arg === "--no-timestamps") {
|
||||
opts.timestamps = false;
|
||||
} else if (arg === "--chapters") {
|
||||
opts.chapters = true;
|
||||
} else if (arg === "--speakers") {
|
||||
opts.speakers = true;
|
||||
} else if (arg === "--exclude-generated") {
|
||||
opts.excludeGenerated = true;
|
||||
} else if (arg === "--exclude-manually-created") {
|
||||
opts.excludeManual = true;
|
||||
} else if (arg === "--refresh") {
|
||||
opts.refresh = true;
|
||||
} else if (arg === "-o" || arg === "--output") {
|
||||
opts.output = argv[++i] || "";
|
||||
} else if (arg === "--output-dir") {
|
||||
opts.outputDir = argv[++i] || "";
|
||||
} else if (!arg.startsWith("-")) {
|
||||
opts.videoIds.push(extractVideoId(arg));
|
||||
}
|
||||
}
|
||||
|
||||
if (opts.videoIds.length === 0) {
|
||||
console.error("Error: At least one video URL or ID required");
|
||||
printHelp();
|
||||
return null;
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const opts = parseArgs(process.argv.slice(2));
|
||||
if (!opts) process.exit(1);
|
||||
|
||||
if (opts.excludeGenerated && opts.excludeManual) {
|
||||
console.error("Error: Cannot exclude both generated and manually created transcripts");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
for (const videoId of opts.videoIds) {
|
||||
try {
|
||||
const r = await processVideo(videoId, opts);
|
||||
if (r.error) console.error(`Error (${r.videoId}): ${r.error}`);
|
||||
else if (r.filePath) console.log(r.filePath);
|
||||
else if (r.content) console.log(r.content);
|
||||
} catch (e) {
|
||||
console.error(`Error (${videoId}): ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in New Issue
Block a user