mirror of
https://github.com/JimLiu/baoyu-skills.git
synced 2026-07-12 05:51:44 +08:00
Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| dad8f3a800 | |||
| 35298d7c9d | |||
| f22374ab62 | |||
| d4e80b1bc3 | |||
| a5761dc71a | |||
| a5189dff37 | |||
| 39fe872bf3 | |||
| 52813504f8 | |||
| a4d4108cd1 | |||
| d7e763f1f5 | |||
| 097c09c59b | |||
| e4cd8bfefc | |||
| 3dc5f2e06f | |||
| e5d6c8ec68 | |||
| 6a4b312146 | |||
| 2d6fe533eb | |||
| f53af25e65 | |||
| c7e32b4590 | |||
| 8d973f2bc5 | |||
| ba20cf89f2 |
@@ -6,48 +6,32 @@
|
|||||||
},
|
},
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"description": "Skills shared by Baoyu for improving daily work efficiency",
|
"description": "Skills shared by Baoyu for improving daily work efficiency",
|
||||||
"version": "1.76.1"
|
"version": "1.79.2"
|
||||||
},
|
},
|
||||||
"plugins": [
|
"plugins": [
|
||||||
{
|
{
|
||||||
"name": "content-skills",
|
"name": "baoyu-skills",
|
||||||
"description": "Content generation and publishing skills",
|
"description": "Content generation, AI backends, and utility tools for daily work efficiency",
|
||||||
"source": "./",
|
"source": "./",
|
||||||
"strict": true,
|
"strict": true,
|
||||||
"skills": [
|
"skills": [
|
||||||
"./skills/baoyu-xhs-images",
|
|
||||||
"./skills/baoyu-post-to-x",
|
|
||||||
"./skills/baoyu-post-to-wechat",
|
|
||||||
"./skills/baoyu-post-to-weibo",
|
|
||||||
"./skills/baoyu-article-illustrator",
|
"./skills/baoyu-article-illustrator",
|
||||||
"./skills/baoyu-cover-image",
|
|
||||||
"./skills/baoyu-slide-deck",
|
|
||||||
"./skills/baoyu-comic",
|
"./skills/baoyu-comic",
|
||||||
"./skills/baoyu-infographic"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "ai-generation-skills",
|
|
||||||
"description": "AI-powered generation backends",
|
|
||||||
"source": "./",
|
|
||||||
"strict": true,
|
|
||||||
"skills": [
|
|
||||||
"./skills/baoyu-danger-gemini-web",
|
|
||||||
"./skills/baoyu-image-gen"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "utility-skills",
|
|
||||||
"description": "Utility tools for content processing",
|
|
||||||
"source": "./",
|
|
||||||
"strict": true,
|
|
||||||
"skills": [
|
|
||||||
"./skills/baoyu-danger-x-to-markdown",
|
|
||||||
"./skills/baoyu-compress-image",
|
"./skills/baoyu-compress-image",
|
||||||
"./skills/baoyu-url-to-markdown",
|
"./skills/baoyu-cover-image",
|
||||||
|
"./skills/baoyu-danger-gemini-web",
|
||||||
|
"./skills/baoyu-danger-x-to-markdown",
|
||||||
"./skills/baoyu-format-markdown",
|
"./skills/baoyu-format-markdown",
|
||||||
|
"./skills/baoyu-image-gen",
|
||||||
|
"./skills/baoyu-infographic",
|
||||||
"./skills/baoyu-markdown-to-html",
|
"./skills/baoyu-markdown-to-html",
|
||||||
|
"./skills/baoyu-post-to-weibo",
|
||||||
|
"./skills/baoyu-post-to-wechat",
|
||||||
|
"./skills/baoyu-post-to-x",
|
||||||
|
"./skills/baoyu-slide-deck",
|
||||||
"./skills/baoyu-translate",
|
"./skills/baoyu-translate",
|
||||||
|
"./skills/baoyu-url-to-markdown",
|
||||||
|
"./skills/baoyu-xhs-images",
|
||||||
"./skills/baoyu-youtube-transcript"
|
"./skills/baoyu-youtube-transcript"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,49 @@
|
|||||||
|
|
||||||
English | [中文](./CHANGELOG.zh.md)
|
English | [中文](./CHANGELOG.zh.md)
|
||||||
|
|
||||||
|
## 1.79.2 - 2026-03-23
|
||||||
|
|
||||||
|
### Fixes
|
||||||
|
- `baoyu-cover-image`: simplify reference image handling — use `--ref` when model supports it, only create description files for models without reference image support
|
||||||
|
- `baoyu-post-to-weibo`: add no-theme rule for article markdown-to-HTML conversion
|
||||||
|
|
||||||
|
### Tests
|
||||||
|
- Fix Node-compatible parser tests and add parser test dependencies
|
||||||
|
|
||||||
|
## 1.79.1 - 2026-03-23
|
||||||
|
|
||||||
|
### Fixes
|
||||||
|
- Consolidate to single plugin to prevent duplicate skill registration (by @TyrealQ)
|
||||||
|
- `baoyu-article-illustrator`: remove opacity parameter from watermark prompt
|
||||||
|
- `baoyu-comic`: fix Doraemon naming spacing and remove opacity from watermark prompt
|
||||||
|
- `baoyu-xhs-images`: remove opacity from watermark prompt and fix CJK spacing
|
||||||
|
|
||||||
|
### Documentation
|
||||||
|
- Update project documentation to reflect single-plugin architecture
|
||||||
|
|
||||||
|
## 1.79.0 - 2026-03-22
|
||||||
|
|
||||||
|
### Features
|
||||||
|
- `baoyu-post-to-wechat`: improve credential loading with multi-source resolution, priority ordering, and diagnostics for skipped incomplete sources
|
||||||
|
|
||||||
|
## 1.78.0 - 2026-03-22
|
||||||
|
|
||||||
|
### Features
|
||||||
|
- `baoyu-url-to-markdown`: add URL-specific parser layer for X/Twitter and archive.ph sites
|
||||||
|
- `baoyu-url-to-markdown`: improved slug generation with stop words removal and subdirectory output structure
|
||||||
|
|
||||||
|
### Fixes
|
||||||
|
- `baoyu-url-to-markdown`: preserve anchor elements containing media in legacy converter
|
||||||
|
- `baoyu-url-to-markdown`: smarter title deduplication to avoid redundant headings
|
||||||
|
|
||||||
|
## 1.77.0 - 2026-03-22
|
||||||
|
|
||||||
|
### Features
|
||||||
|
- `baoyu-youtube-transcript`: add end times to chapter data (by @jzOcb)
|
||||||
|
|
||||||
|
### Fixes
|
||||||
|
- `sync-clawhub`: skip failed skills instead of aborting
|
||||||
|
|
||||||
## 1.76.1 - 2026-03-21
|
## 1.76.1 - 2026-03-21
|
||||||
|
|
||||||
### Documentation
|
### Documentation
|
||||||
|
|||||||
@@ -2,6 +2,49 @@
|
|||||||
|
|
||||||
[English](./CHANGELOG.md) | 中文
|
[English](./CHANGELOG.md) | 中文
|
||||||
|
|
||||||
|
## 1.79.2 - 2026-03-23
|
||||||
|
|
||||||
|
### 修复
|
||||||
|
- `baoyu-cover-image`:简化参考图片处理流程 — 模型支持 `--ref` 时直接传递,仅在模型不支持参考图时创建描述文件
|
||||||
|
- `baoyu-post-to-weibo`:文章 Markdown 转 HTML 时不传递 --theme 参数
|
||||||
|
|
||||||
|
### 测试
|
||||||
|
- 修复 Node 兼容的解析器测试,添加解析器测试依赖
|
||||||
|
|
||||||
|
## 1.79.1 - 2026-03-23
|
||||||
|
|
||||||
|
### 修复
|
||||||
|
- 合并为单一插件,防止 skill 重复注册 (by @TyrealQ)
|
||||||
|
- `baoyu-article-illustrator`:移除水印提示词中的不透明度参数
|
||||||
|
- `baoyu-comic`:修正哆啦 A 梦命名间距,移除水印不透明度参数
|
||||||
|
- `baoyu-xhs-images`:移除水印不透明度参数,修正中英文间距
|
||||||
|
|
||||||
|
### 文档
|
||||||
|
- 更新项目文档以反映单一插件架构
|
||||||
|
|
||||||
|
## 1.79.0 - 2026-03-22
|
||||||
|
|
||||||
|
### 新功能
|
||||||
|
- `baoyu-post-to-wechat`:改进凭据加载机制,支持多来源优先级解析,并提供不完整凭据来源的诊断信息
|
||||||
|
|
||||||
|
## 1.78.0 - 2026-03-22
|
||||||
|
|
||||||
|
### 新功能
|
||||||
|
- `baoyu-url-to-markdown`:新增 URL 专用解析层,支持 X/Twitter 和 archive.ph 站点的定制化 HTML 提取
|
||||||
|
- `baoyu-url-to-markdown`:改进 slug 生成算法,去除停用词并采用子目录输出结构
|
||||||
|
|
||||||
|
### 修复
|
||||||
|
- `baoyu-url-to-markdown`:旧版转换器保留包含媒体元素的锚标签
|
||||||
|
- `baoyu-url-to-markdown`:更智能的标题去重,避免重复添加标题
|
||||||
|
|
||||||
|
## 1.77.0 - 2026-03-22
|
||||||
|
|
||||||
|
### 新功能
|
||||||
|
- `baoyu-youtube-transcript`:为章节数据添加结束时间 (by @jzOcb)
|
||||||
|
|
||||||
|
### 修复
|
||||||
|
- `sync-clawhub`:跳过失败的技能而不是中止同步
|
||||||
|
|
||||||
## 1.76.1 - 2026-03-21
|
## 1.76.1 - 2026-03-21
|
||||||
|
|
||||||
### 文档
|
### 文档
|
||||||
|
|||||||
@@ -1,16 +1,16 @@
|
|||||||
# CLAUDE.md
|
# CLAUDE.md
|
||||||
|
|
||||||
Claude Code marketplace plugin providing AI-powered content generation skills. Version: **1.76.0**.
|
Claude Code marketplace plugin providing AI-powered content generation skills. Version: **1.79.2**.
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
Skills organized into three categories in `.claude-plugin/marketplace.json` (defines plugin metadata, version, and skill paths):
|
Skills are exposed through the single `baoyu-skills` plugin in `.claude-plugin/marketplace.json` (which defines plugin metadata, version, and skill paths). The repo docs still group them into three logical areas:
|
||||||
|
|
||||||
| Category | Description |
|
| Group | Description |
|
||||||
|----------|-------------|
|
|-------|-------------|
|
||||||
| `content-skills` | Generate or publish content (images, slides, comics, posts) |
|
| Content Skills | Generate or publish content (images, slides, comics, posts) |
|
||||||
| `ai-generation-skills` | AI generation backends |
|
| AI Generation Skills | AI generation backends |
|
||||||
| `utility-skills` | Content processing (conversion, compression, translation) |
|
| Utility Skills | Content processing (conversion, compression, translation) |
|
||||||
|
|
||||||
Each skill contains `SKILL.md` (YAML front matter + docs), optional `scripts/`, `references/`, `prompts/`.
|
Each skill contains `SKILL.md` (YAML front matter + docs), optional `scripts/`, `references/`, `prompts/`.
|
||||||
|
|
||||||
|
|||||||
@@ -52,16 +52,14 @@ Run the following command in Claude Code:
|
|||||||
|
|
||||||
1. Select **Browse and install plugins**
|
1. Select **Browse and install plugins**
|
||||||
2. Select **baoyu-skills**
|
2. Select **baoyu-skills**
|
||||||
3. Select the plugin(s) you want to install
|
3. Select the **baoyu-skills** plugin
|
||||||
4. Select **Install now**
|
4. Select **Install now**
|
||||||
|
|
||||||
**Option 2: Direct Install**
|
**Option 2: Direct Install**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Install specific plugin
|
# Install the marketplace's single plugin
|
||||||
/plugin install content-skills@baoyu-skills
|
/plugin install baoyu-skills@baoyu-skills
|
||||||
/plugin install ai-generation-skills@baoyu-skills
|
|
||||||
/plugin install utility-skills@baoyu-skills
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**Option 3: Ask the Agent**
|
**Option 3: Ask the Agent**
|
||||||
@@ -70,13 +68,13 @@ Simply tell Claude Code:
|
|||||||
|
|
||||||
> Please install Skills from github.com/JimLiu/baoyu-skills
|
> Please install Skills from github.com/JimLiu/baoyu-skills
|
||||||
|
|
||||||
### Available Plugins
|
### Available Plugin
|
||||||
|
|
||||||
| Plugin | Description | Skills |
|
The marketplace now exposes a single plugin so each skill is registered exactly once.
|
||||||
|--------|-------------|--------|
|
|
||||||
| **content-skills** | Content generation and publishing | [xhs-images](#baoyu-xhs-images), [infographic](#baoyu-infographic), [cover-image](#baoyu-cover-image), [slide-deck](#baoyu-slide-deck), [comic](#baoyu-comic), [article-illustrator](#baoyu-article-illustrator), [post-to-x](#baoyu-post-to-x), [post-to-wechat](#baoyu-post-to-wechat), [post-to-weibo](#baoyu-post-to-weibo) |
|
| Plugin | Description | Includes |
|
||||||
| **ai-generation-skills** | AI-powered generation backends | [image-gen](#baoyu-image-gen), [danger-gemini-web](#baoyu-danger-gemini-web) |
|
|--------|-------------|----------|
|
||||||
| **utility-skills** | Utility tools for content processing | [youtube-transcript](#baoyu-youtube-transcript), [url-to-markdown](#baoyu-url-to-markdown), [danger-x-to-markdown](#baoyu-danger-x-to-markdown), [compress-image](#baoyu-compress-image), [format-markdown](#baoyu-format-markdown), [markdown-to-html](#baoyu-markdown-to-html), [translate](#baoyu-translate) |
|
| **baoyu-skills** | Content generation, AI backends, and utility tools for daily work efficiency | All skills in this repository, organized below as Content Skills, AI Generation Skills, and Utility Skills |
|
||||||
|
|
||||||
## Update Skills
|
## Update Skills
|
||||||
|
|
||||||
|
|||||||
+7
-9
@@ -52,16 +52,14 @@ clawhub install baoyu-markdown-to-html
|
|||||||
|
|
||||||
1. 选择 **Browse and install plugins**
|
1. 选择 **Browse and install plugins**
|
||||||
2. 选择 **baoyu-skills**
|
2. 选择 **baoyu-skills**
|
||||||
3. 选择要安装的插件
|
3. 选择 **baoyu-skills** 插件
|
||||||
4. 选择 **Install now**
|
4. 选择 **Install now**
|
||||||
|
|
||||||
**方式二:直接安装**
|
**方式二:直接安装**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 安装指定插件
|
# 安装 marketplace 中唯一的插件
|
||||||
/plugin install content-skills@baoyu-skills
|
/plugin install baoyu-skills@baoyu-skills
|
||||||
/plugin install ai-generation-skills@baoyu-skills
|
|
||||||
/plugin install utility-skills@baoyu-skills
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**方式三:告诉 Agent**
|
**方式三:告诉 Agent**
|
||||||
@@ -72,11 +70,11 @@ clawhub install baoyu-markdown-to-html
|
|||||||
|
|
||||||
### 可用插件
|
### 可用插件
|
||||||
|
|
||||||
| 插件 | 说明 | 包含技能 |
|
现在 marketplace 只暴露一个插件,这样每个 skill 只会注册一次。
|
||||||
|
|
||||||
|
| 插件 | 说明 | 包含内容 |
|
||||||
|------|------|----------|
|
|------|------|----------|
|
||||||
| **content-skills** | 内容生成和发布 | [xhs-images](#baoyu-xhs-images), [infographic](#baoyu-infographic), [cover-image](#baoyu-cover-image), [slide-deck](#baoyu-slide-deck), [comic](#baoyu-comic), [article-illustrator](#baoyu-article-illustrator), [post-to-x](#baoyu-post-to-x), [post-to-wechat](#baoyu-post-to-wechat), [post-to-weibo](#baoyu-post-to-weibo) |
|
| **baoyu-skills** | 提供内容生成、AI 后端和日常效率工具技能 | 仓库中的全部 skills,仍按下方的内容技能、AI 生成技能、工具技能三个分类展示 |
|
||||||
| **ai-generation-skills** | AI 生成后端 | [image-gen](#baoyu-image-gen), [danger-gemini-web](#baoyu-danger-gemini-web) |
|
|
||||||
| **utility-skills** | 内容处理工具 | [youtube-transcript](#baoyu-youtube-transcript), [url-to-markdown](#baoyu-url-to-markdown), [danger-x-to-markdown](#baoyu-danger-x-to-markdown), [compress-image](#baoyu-compress-image), [format-markdown](#baoyu-format-markdown), [markdown-to-html](#baoyu-markdown-to-html), [translate](#baoyu-translate) |
|
|
||||||
|
|
||||||
## 更新技能
|
## 更新技能
|
||||||
|
|
||||||
|
|||||||
+11
-9
@@ -34,20 +34,22 @@ metadata:
|
|||||||
1. Create `skills/baoyu-<name>/SKILL.md` with YAML front matter
|
1. Create `skills/baoyu-<name>/SKILL.md` with YAML front matter
|
||||||
2. Add TypeScript in `skills/baoyu-<name>/scripts/` (if applicable)
|
2. Add TypeScript in `skills/baoyu-<name>/scripts/` (if applicable)
|
||||||
3. Add prompt templates in `skills/baoyu-<name>/prompts/` if needed
|
3. Add prompt templates in `skills/baoyu-<name>/prompts/` if needed
|
||||||
4. Register in `marketplace.json` under appropriate category
|
4. Register the skill in `.claude-plugin/marketplace.json` under the `baoyu-skills` plugin entry
|
||||||
5. Add Script Directory section to SKILL.md if skill has scripts
|
5. Add Script Directory section to SKILL.md if skill has scripts
|
||||||
6. Add openclaw metadata to frontmatter
|
6. Add openclaw metadata to frontmatter
|
||||||
|
|
||||||
## Category Selection
|
## Skill Grouping
|
||||||
|
|
||||||
| If your skill... | Use category |
|
All skills are registered under the single `baoyu-skills` plugin. Use these logical groups when deciding where the skill should appear in the docs:
|
||||||
|------------------|--------------|
|
|
||||||
| Generates visual content (images, slides, comics) | `content-skills` |
|
|
||||||
| Publishes to platforms (X, WeChat, Weibo) | `content-skills` |
|
|
||||||
| Provides AI generation backend | `ai-generation-skills` |
|
|
||||||
| Converts or processes content | `utility-skills` |
|
|
||||||
|
|
||||||
New category: add plugin object to `marketplace.json` with `name`, `description`, `skills[]`.
|
| If your skill... | Use group |
|
||||||
|
|------------------|-----------|
|
||||||
|
| Generates visual content (images, slides, comics) | Content Skills |
|
||||||
|
| Publishes to platforms (X, WeChat, Weibo) | Content Skills |
|
||||||
|
| Provides AI generation backend | AI Generation Skills |
|
||||||
|
| Converts or processes content | Utility Skills |
|
||||||
|
|
||||||
|
If you add a new logical group, update the docs that present grouped skills, but keep the skill registered under the single `baoyu-skills` plugin entry.
|
||||||
|
|
||||||
## Writing Descriptions
|
## Writing Descriptions
|
||||||
|
|
||||||
|
|||||||
Generated
+105
-1
@@ -9,7 +9,11 @@
|
|||||||
"packages/*"
|
"packages/*"
|
||||||
],
|
],
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"tsx": "^4.20.5"
|
"@mozilla/readability": "^0.6.0",
|
||||||
|
"linkedom": "^0.18.12",
|
||||||
|
"tsx": "^4.20.5",
|
||||||
|
"turndown": "^7.2.2",
|
||||||
|
"turndown-plugin-gfm": "^1.0.2"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@esbuild/aix-ppc64": {
|
"node_modules/@esbuild/aix-ppc64": {
|
||||||
@@ -454,6 +458,23 @@
|
|||||||
"node": ">=18"
|
"node": ">=18"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/@mixmark-io/domino": {
|
||||||
|
"version": "2.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",
|
||||||
|
"integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "BSD-2-Clause"
|
||||||
|
},
|
||||||
|
"node_modules/@mozilla/readability": {
|
||||||
|
"version": "0.6.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.6.0.tgz",
|
||||||
|
"integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=14.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@types/debug": {
|
"node_modules/@types/debug": {
|
||||||
"version": "4.1.12",
|
"version": "4.1.12",
|
||||||
"resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz",
|
"resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz",
|
||||||
@@ -615,6 +636,13 @@
|
|||||||
"url": "https://github.com/sponsors/fb55"
|
"url": "https://github.com/sponsors/fb55"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/cssom": {
|
||||||
|
"version": "0.5.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
|
||||||
|
"integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/debug": {
|
"node_modules/debug": {
|
||||||
"version": "4.4.3",
|
"version": "4.4.3",
|
||||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
||||||
@@ -896,6 +924,13 @@
|
|||||||
"node": ">=12.0.0"
|
"node": ">=12.0.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/html-escaper": {
|
||||||
|
"version": "3.0.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz",
|
||||||
|
"integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/htmlparser2": {
|
"node_modules/htmlparser2": {
|
||||||
"version": "9.1.0",
|
"version": "9.1.0",
|
||||||
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz",
|
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz",
|
||||||
@@ -984,6 +1019,51 @@
|
|||||||
"node": ">=18.17"
|
"node": ">=18.17"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/linkedom": {
|
||||||
|
"version": "0.18.12",
|
||||||
|
"resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.12.tgz",
|
||||||
|
"integrity": "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "ISC",
|
||||||
|
"dependencies": {
|
||||||
|
"css-select": "^5.1.0",
|
||||||
|
"cssom": "^0.5.0",
|
||||||
|
"html-escaper": "^3.0.3",
|
||||||
|
"htmlparser2": "^10.0.0",
|
||||||
|
"uhyphen": "^0.2.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=16"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"canvas": ">= 2"
|
||||||
|
},
|
||||||
|
"peerDependenciesMeta": {
|
||||||
|
"canvas": {
|
||||||
|
"optional": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/linkedom/node_modules/htmlparser2": {
|
||||||
|
"version": "10.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz",
|
||||||
|
"integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==",
|
||||||
|
"dev": true,
|
||||||
|
"funding": [
|
||||||
|
"https://github.com/fb55/htmlparser2?sponsor=1",
|
||||||
|
{
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/fb55"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"domelementtype": "^2.3.0",
|
||||||
|
"domhandler": "^5.0.3",
|
||||||
|
"domutils": "^3.2.2",
|
||||||
|
"entities": "^7.0.1"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/longest-streak": {
|
"node_modules/longest-streak": {
|
||||||
"version": "3.1.0",
|
"version": "3.1.0",
|
||||||
"resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz",
|
"resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz",
|
||||||
@@ -1768,6 +1848,30 @@
|
|||||||
"fsevents": "~2.3.3"
|
"fsevents": "~2.3.3"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/turndown": {
|
||||||
|
"version": "7.2.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.2.tgz",
|
||||||
|
"integrity": "sha512-1F7db8BiExOKxjSMU2b7if62D/XOyQyZbPKq/nUwopfgnHlqXHqQ0lvfUTeUIr1lZJzOPFn43dODyMSIfvWRKQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@mixmark-io/domino": "^2.2.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/turndown-plugin-gfm": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
|
"node_modules/uhyphen": {
|
||||||
|
"version": "0.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz",
|
||||||
|
"integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "ISC"
|
||||||
|
},
|
||||||
"node_modules/undici": {
|
"node_modules/undici": {
|
||||||
"version": "6.24.0",
|
"version": "6.24.0",
|
||||||
"resolved": "https://registry.npmjs.org/undici/-/undici-6.24.0.tgz",
|
"resolved": "https://registry.npmjs.org/undici/-/undici-6.24.0.tgz",
|
||||||
|
|||||||
@@ -10,6 +10,10 @@
|
|||||||
"test:coverage": "node --import tsx --experimental-test-coverage --test"
|
"test:coverage": "node --import tsx --experimental-test-coverage --test"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
"@mozilla/readability": "^0.6.0",
|
||||||
|
"linkedom": "^0.18.12",
|
||||||
|
"turndown": "^7.2.2",
|
||||||
|
"turndown-plugin-gfm": "^1.0.2",
|
||||||
"tsx": "^4.20.5"
|
"tsx": "^4.20.5"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+24
-11
@@ -151,6 +151,9 @@ async function main() {
|
|||||||
.map((tag) => tag.trim())
|
.map((tag) => tag.trim())
|
||||||
.filter(Boolean);
|
.filter(Boolean);
|
||||||
|
|
||||||
|
let succeeded = 0;
|
||||||
|
const failed = [];
|
||||||
|
|
||||||
for (const candidate of actionable) {
|
for (const candidate of actionable) {
|
||||||
const version =
|
const version =
|
||||||
candidate.status === "new"
|
candidate.status === "new"
|
||||||
@@ -158,20 +161,30 @@ async function main() {
|
|||||||
: bumpSemver(candidate.latestVersion, options.bump);
|
: bumpSemver(candidate.latestVersion, options.bump);
|
||||||
|
|
||||||
console.log(`Publishing ${candidate.slug}@${version}`);
|
console.log(`Publishing ${candidate.slug}@${version}`);
|
||||||
const files = await listTextFiles(candidate.folder);
|
try {
|
||||||
await publishSkill({
|
const files = await listTextFiles(candidate.folder);
|
||||||
registry,
|
await publishSkill({
|
||||||
token: config.token,
|
registry,
|
||||||
skill: candidate,
|
token: config.token,
|
||||||
files,
|
skill: candidate,
|
||||||
version,
|
files,
|
||||||
changelog: options.changelog,
|
version,
|
||||||
tags,
|
changelog: options.changelog,
|
||||||
});
|
tags,
|
||||||
|
});
|
||||||
|
succeeded++;
|
||||||
|
} catch (err) {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
console.error(`SKIPPED ${candidate.slug}: ${msg}`);
|
||||||
|
failed.push(candidate.slug);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log("");
|
console.log("");
|
||||||
console.log(`Uploaded ${actionable.length} skill(s).`);
|
console.log(`Uploaded ${succeeded}/${actionable.length} skill(s).`);
|
||||||
|
if (failed.length > 0) {
|
||||||
|
console.log(`Failed (${failed.length}): ${failed.join(", ")}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseArgs(argv) {
|
function parseArgs(argv) {
|
||||||
|
|||||||
@@ -280,5 +280,5 @@ TEXTURE: Halftone transitions between sides
|
|||||||
If watermark enabled in preferences, append:
|
If watermark enabled in preferences, append:
|
||||||
|
|
||||||
```
|
```
|
||||||
Include a subtle watermark "[content]" positioned at [position] with approximately [opacity*100]% visibility.
|
Include a subtle watermark "[content]" positioned at [position].
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -278,7 +278,7 @@ Create storyboard and character definitions using the confirmed style from Step
|
|||||||
| Role | Character | Visual Description |
|
| Role | Character | Visual Description |
|
||||||
|------|-----------|-------------------|
|
|------|-----------|-------------------|
|
||||||
| Student | 大雄 (Nobita) | Japanese boy, 10yo, round glasses, black hair parted in middle, yellow shirt, navy shorts |
|
| Student | 大雄 (Nobita) | Japanese boy, 10yo, round glasses, black hair parted in middle, yellow shirt, navy shorts |
|
||||||
| Mentor | 哆啦A梦 (Doraemon) | Round blue robot cat, big white eyes, red nose, whiskers, white belly with 4D pocket, golden bell, no ears |
|
| Mentor | 哆啦 A 梦 (Doraemon) | Round blue robot cat, big white eyes, red nose, whiskers, white belly with 4D pocket, golden bell, no ears |
|
||||||
| Challenge | 胖虎 (Gian) | Stocky boy, rough features, small eyes, orange shirt |
|
| Challenge | 胖虎 (Gian) | Stocky boy, rough features, small eyes, orange shirt |
|
||||||
| Support | 静香 (Shizuka) | Cute girl, black short hair, pink dress, gentle expression |
|
| Support | 静香 (Shizuka) | Cute girl, black short hair, pink dress, gentle expression |
|
||||||
|
|
||||||
@@ -359,8 +359,7 @@ Art: [art style] | Tone: [tone] | Layout: [layout type]
|
|||||||
**Watermark Application** (if enabled in preferences):
|
**Watermark Application** (if enabled in preferences):
|
||||||
Add to each prompt:
|
Add to each prompt:
|
||||||
```
|
```
|
||||||
Include a subtle watermark "[content]" positioned at [position]
|
Include a subtle watermark "[content]" positioned at [position]. The watermark should
|
||||||
with approximately [opacity*100]% visibility. The watermark should
|
|
||||||
be legible but not distracting from the comic panels and storytelling.
|
be legible but not distracting from the comic panels and storytelling.
|
||||||
Ensure watermark does not overlap speech bubbles or key action.
|
Ensure watermark does not overlap speech bubbles or key action.
|
||||||
```
|
```
|
||||||
@@ -452,8 +451,8 @@ When skill does NOT support reference images, create combined prompt files:
|
|||||||
|
|
||||||
## Character Reference (maintain consistency)
|
## Character Reference (maintain consistency)
|
||||||
[Copy relevant sections from characters/characters.md here]
|
[Copy relevant sections from characters/characters.md here]
|
||||||
- 大雄: Japanese boy, round glasses, yellow shirt, navy shorts...
|
- 大雄:Japanese boy, round glasses, yellow shirt, navy shorts...
|
||||||
- 哆啦A梦: Round blue robot cat, white belly, red nose, golden bell...
|
- 哆啦 A 梦:Round blue robot cat, white belly, red nose, golden bell...
|
||||||
|
|
||||||
## Page Content
|
## Page Content
|
||||||
[Original page prompt here]
|
[Original page prompt here]
|
||||||
|
|||||||
@@ -162,15 +162,14 @@ if (Test-Path "$HOME/.baoyu-skills/baoyu-cover-image/EXTEND.md") { "user" }
|
|||||||
5. **Detect language**: Compare source, user input, EXTEND.md preference
|
5. **Detect language**: Compare source, user input, EXTEND.md preference
|
||||||
6. **Determine output directory**: Per File Structure rules
|
6. **Determine output directory**: Per File Structure rules
|
||||||
|
|
||||||
**⚠️ People in Reference Images — MUST follow all 3 rules:**
|
**⚠️ People in Reference Images:**
|
||||||
|
|
||||||
If reference images contain **people** who should appear in the cover:
|
If reference images contain **people** who should appear in the cover:
|
||||||
|
|
||||||
1. **`usage: direct`** — MUST set in refs description file. NEVER use `style` or `palette` when people need to appear
|
- **Model supports `--ref`** (default): Copy image to `refs/`, pass via `--ref` at generation. No description file needed — the model sees the face directly.
|
||||||
2. **Per-character description** — MUST describe each person's distinctive features (hair, glasses, skin tone, clothing) in `refs/ref-NN-{slug}.md`. Vague descriptions like "a man" will fail
|
- **Model does NOT support `--ref`** (Jimeng, Seedream 3.0): Create `refs/ref-NN-{slug}.md` with per-character description (hair, glasses, skin tone, clothing). Embed as MUST/REQUIRED instructions in prompt text.
|
||||||
3. **`--ref` flag** — MUST pass reference image via `--ref` in Step 4 so the model sees actual faces
|
|
||||||
|
|
||||||
See [reference-images.md § Character Analysis](references/workflow/reference-images.md) for description format.
|
See [reference-images.md](references/workflow/reference-images.md) for full decision table.
|
||||||
|
|
||||||
### Step 2: Confirm Options ⚠️
|
### Step 2: Confirm Options ⚠️
|
||||||
|
|
||||||
|
|||||||
@@ -16,17 +16,24 @@ Guide for processing user-provided reference images in cover generation.
|
|||||||
|
|
||||||
**If user provides file path**:
|
**If user provides file path**:
|
||||||
1. Copy to `refs/ref-NN-{slug}.{ext}` (NN = 01, 02, ...)
|
1. Copy to `refs/ref-NN-{slug}.{ext}` (NN = 01, 02, ...)
|
||||||
2. Create description: `refs/ref-NN-{slug}.md`
|
2. **Only** create description file `refs/ref-NN-{slug}.md` when model does NOT support `--ref` (see below)
|
||||||
3. Verify files exist before proceeding
|
3. Verify image file exists before proceeding
|
||||||
|
|
||||||
**Description File Format**:
|
**When to create description file**:
|
||||||
|
|
||||||
|
| Situation | Action |
|
||||||
|
|-----------|--------|
|
||||||
|
| Model supports `--ref` (Google, OpenAI, OpenRouter, Replicate, Seedream 4.0+) | Copy image only. **No description file needed.** Pass via `--ref` at generation. |
|
||||||
|
| Model does NOT support `--ref` (Jimeng, Seedream 3.0) | Copy image + create description file. Embed description in prompt text. |
|
||||||
|
|
||||||
|
**Description File Format** (only when needed):
|
||||||
```yaml
|
```yaml
|
||||||
---
|
---
|
||||||
ref_id: NN
|
ref_id: NN
|
||||||
filename: ref-NN-{slug}.{ext}
|
filename: ref-NN-{slug}.{ext}
|
||||||
usage: direct | style | palette
|
usage: direct | style | palette
|
||||||
---
|
---
|
||||||
[User's description or auto-generated description]
|
[Character or style description to embed in prompt]
|
||||||
```
|
```
|
||||||
|
|
||||||
| Usage | When to Use |
|
| Usage | When to Use |
|
||||||
|
|||||||
@@ -694,6 +694,10 @@ async function main(): Promise<void> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const creds = loadCredentials(resolved);
|
const creds = loadCredentials(resolved);
|
||||||
|
for (const skippedSource of creds.skippedSources) {
|
||||||
|
console.error(`[wechat-api] Skipped incomplete credential source: ${skippedSource}`);
|
||||||
|
}
|
||||||
|
console.error(`[wechat-api] Credentials source: ${creds.source}`);
|
||||||
console.error("[wechat-api] Fetching access token...");
|
console.error("[wechat-api] Fetching access token...");
|
||||||
const accessToken = await fetchAccessToken(creds.appId, creds.appSecret);
|
const accessToken = await fetchAccessToken(creds.appId, creds.appSecret);
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,139 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import fs from "node:fs/promises";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import process from "node:process";
|
||||||
|
import test, { type TestContext } from "node:test";
|
||||||
|
|
||||||
|
import { loadCredentials } from "./wechat-extend-config.ts";
|
||||||
|
|
||||||
|
function useCwd(t: TestContext, cwd: string): void {
|
||||||
|
const previous = process.cwd();
|
||||||
|
process.chdir(cwd);
|
||||||
|
t.after(() => {
|
||||||
|
process.chdir(previous);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function useHome(t: TestContext, home: string): void {
|
||||||
|
const previous = process.env.HOME;
|
||||||
|
process.env.HOME = home;
|
||||||
|
t.after(() => {
|
||||||
|
if (previous === undefined) {
|
||||||
|
delete process.env.HOME;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
process.env.HOME = previous;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function useWechatEnv(
|
||||||
|
t: TestContext,
|
||||||
|
values: Partial<Record<"WECHAT_APP_ID" | "WECHAT_APP_SECRET", string | undefined>>,
|
||||||
|
): void {
|
||||||
|
const previous = {
|
||||||
|
WECHAT_APP_ID: process.env.WECHAT_APP_ID,
|
||||||
|
WECHAT_APP_SECRET: process.env.WECHAT_APP_SECRET,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (values.WECHAT_APP_ID === undefined) {
|
||||||
|
delete process.env.WECHAT_APP_ID;
|
||||||
|
} else {
|
||||||
|
process.env.WECHAT_APP_ID = values.WECHAT_APP_ID;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (values.WECHAT_APP_SECRET === undefined) {
|
||||||
|
delete process.env.WECHAT_APP_SECRET;
|
||||||
|
} else {
|
||||||
|
process.env.WECHAT_APP_SECRET = values.WECHAT_APP_SECRET;
|
||||||
|
}
|
||||||
|
|
||||||
|
t.after(() => {
|
||||||
|
if (previous.WECHAT_APP_ID === undefined) {
|
||||||
|
delete process.env.WECHAT_APP_ID;
|
||||||
|
} else {
|
||||||
|
process.env.WECHAT_APP_ID = previous.WECHAT_APP_ID;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (previous.WECHAT_APP_SECRET === undefined) {
|
||||||
|
delete process.env.WECHAT_APP_SECRET;
|
||||||
|
} else {
|
||||||
|
process.env.WECHAT_APP_SECRET = previous.WECHAT_APP_SECRET;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function makeTempDir(prefix: string): Promise<string> {
|
||||||
|
return fs.mkdtemp(path.join(os.tmpdir(), prefix));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeEnvFile(root: string, content: string): Promise<void> {
|
||||||
|
const envPath = path.join(root, ".baoyu-skills", ".env");
|
||||||
|
await fs.mkdir(path.dirname(envPath), { recursive: true });
|
||||||
|
await fs.writeFile(envPath, content);
|
||||||
|
}
|
||||||
|
|
||||||
|
test("loadCredentials selects the first complete source without mixing values across sources", async (t) => {
|
||||||
|
const cwdRoot = await makeTempDir("wechat-creds-cwd-");
|
||||||
|
const homeRoot = await makeTempDir("wechat-creds-home-");
|
||||||
|
|
||||||
|
useCwd(t, cwdRoot);
|
||||||
|
useHome(t, homeRoot);
|
||||||
|
useWechatEnv(t, {
|
||||||
|
WECHAT_APP_ID: undefined,
|
||||||
|
WECHAT_APP_SECRET: "stale-secret-from-process-env",
|
||||||
|
});
|
||||||
|
|
||||||
|
await writeEnvFile(cwdRoot, "WECHAT_APP_ID=cwd-app-id\nWECHAT_APP_SECRET=cwd-app-secret\n");
|
||||||
|
await writeEnvFile(homeRoot, "WECHAT_APP_ID=home-app-id\nWECHAT_APP_SECRET=home-app-secret\n");
|
||||||
|
|
||||||
|
const credentials = loadCredentials();
|
||||||
|
|
||||||
|
assert.equal(credentials.appId, "cwd-app-id");
|
||||||
|
assert.equal(credentials.appSecret, "cwd-app-secret");
|
||||||
|
assert.equal(credentials.source, "<cwd>/.baoyu-skills/.env");
|
||||||
|
assert.deepEqual(credentials.skippedSources, [
|
||||||
|
"process.env missing WECHAT_APP_ID",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("loadCredentials prefers a complete process.env pair over lower-priority files", async (t) => {
|
||||||
|
const cwdRoot = await makeTempDir("wechat-creds-cwd-");
|
||||||
|
const homeRoot = await makeTempDir("wechat-creds-home-");
|
||||||
|
|
||||||
|
useCwd(t, cwdRoot);
|
||||||
|
useHome(t, homeRoot);
|
||||||
|
useWechatEnv(t, {
|
||||||
|
WECHAT_APP_ID: "env-app-id",
|
||||||
|
WECHAT_APP_SECRET: "env-app-secret",
|
||||||
|
});
|
||||||
|
|
||||||
|
await writeEnvFile(cwdRoot, "WECHAT_APP_ID=cwd-app-id\nWECHAT_APP_SECRET=cwd-app-secret\n");
|
||||||
|
await writeEnvFile(homeRoot, "WECHAT_APP_ID=home-app-id\nWECHAT_APP_SECRET=home-app-secret\n");
|
||||||
|
|
||||||
|
const credentials = loadCredentials();
|
||||||
|
|
||||||
|
assert.equal(credentials.appId, "env-app-id");
|
||||||
|
assert.equal(credentials.appSecret, "env-app-secret");
|
||||||
|
assert.equal(credentials.source, "process.env");
|
||||||
|
assert.deepEqual(credentials.skippedSources, []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("loadCredentials reports skipped incomplete sources when no complete pair exists", async (t) => {
|
||||||
|
const cwdRoot = await makeTempDir("wechat-creds-cwd-");
|
||||||
|
const homeRoot = await makeTempDir("wechat-creds-home-");
|
||||||
|
|
||||||
|
useCwd(t, cwdRoot);
|
||||||
|
useHome(t, homeRoot);
|
||||||
|
useWechatEnv(t, {
|
||||||
|
WECHAT_APP_ID: "env-app-id",
|
||||||
|
WECHAT_APP_SECRET: undefined,
|
||||||
|
});
|
||||||
|
|
||||||
|
await writeEnvFile(cwdRoot, "WECHAT_APP_SECRET=cwd-app-secret\n");
|
||||||
|
|
||||||
|
assert.throws(
|
||||||
|
() => loadCredentials(),
|
||||||
|
/Incomplete credential sources skipped:\n- process\.env missing WECHAT_APP_SECRET\n- <cwd>\/\.baoyu-skills\/\.env missing WECHAT_APP_ID/,
|
||||||
|
);
|
||||||
|
});
|
||||||
@@ -196,48 +196,116 @@ function aliasToEnvKey(alias: string): string {
|
|||||||
return alias.toUpperCase().replace(/-/g, "_");
|
return alias.toUpperCase().replace(/-/g, "_");
|
||||||
}
|
}
|
||||||
|
|
||||||
export function loadCredentials(account?: ResolvedAccount): { appId: string; appSecret: string } {
|
interface CredentialSource {
|
||||||
if (account?.app_id && account?.app_secret) {
|
name: string;
|
||||||
return { appId: account.app_id, appSecret: account.app_secret };
|
appIdKey: string;
|
||||||
|
appSecretKey: string;
|
||||||
|
appId?: string;
|
||||||
|
appSecret?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LoadedCredentials {
|
||||||
|
appId: string;
|
||||||
|
appSecret: string;
|
||||||
|
source: string;
|
||||||
|
skippedSources: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeCredentialValue(value?: string): string | undefined {
|
||||||
|
const trimmed = value?.trim();
|
||||||
|
return trimmed ? trimmed : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function describeMissingKeys(source: CredentialSource): string {
|
||||||
|
const missingKeys: string[] = [];
|
||||||
|
if (!source.appId) missingKeys.push(source.appIdKey);
|
||||||
|
if (!source.appSecret) missingKeys.push(source.appSecretKey);
|
||||||
|
return `${source.name} missing ${missingKeys.join(" and ")}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildCredentialSource(
|
||||||
|
name: string,
|
||||||
|
values: Record<string, string | undefined>,
|
||||||
|
appIdKey: string,
|
||||||
|
appSecretKey: string,
|
||||||
|
): CredentialSource {
|
||||||
|
return {
|
||||||
|
name,
|
||||||
|
appIdKey,
|
||||||
|
appSecretKey,
|
||||||
|
appId: normalizeCredentialValue(values[appIdKey]),
|
||||||
|
appSecret: normalizeCredentialValue(values[appSecretKey]),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveCredentialSource(
|
||||||
|
sources: CredentialSource[],
|
||||||
|
account?: ResolvedAccount,
|
||||||
|
): LoadedCredentials {
|
||||||
|
const skippedSources: string[] = [];
|
||||||
|
|
||||||
|
for (const source of sources) {
|
||||||
|
if (source.appId && source.appSecret) {
|
||||||
|
return {
|
||||||
|
appId: source.appId,
|
||||||
|
appSecret: source.appSecret,
|
||||||
|
source: source.name,
|
||||||
|
skippedSources,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (source.appId || source.appSecret) {
|
||||||
|
skippedSources.push(describeMissingKeys(source));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const hint = account?.alias ? ` (account: ${account.alias})` : "";
|
||||||
|
const partialHint = skippedSources.length > 0
|
||||||
|
? `\nIncomplete credential sources skipped:\n- ${skippedSources.join("\n- ")}`
|
||||||
|
: "";
|
||||||
|
|
||||||
|
throw new Error(
|
||||||
|
`Missing WECHAT_APP_ID or WECHAT_APP_SECRET${hint}.\n` +
|
||||||
|
"Set via EXTEND.md account config, environment variables, or .baoyu-skills/.env file." +
|
||||||
|
partialHint
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function loadCredentials(account?: ResolvedAccount): LoadedCredentials {
|
||||||
const cwdEnvPath = path.join(process.cwd(), ".baoyu-skills", ".env");
|
const cwdEnvPath = path.join(process.cwd(), ".baoyu-skills", ".env");
|
||||||
const homeEnvPath = path.join(os.homedir(), ".baoyu-skills", ".env");
|
const homeEnvPath = path.join(os.homedir(), ".baoyu-skills", ".env");
|
||||||
const cwdEnv = loadEnvFile(cwdEnvPath);
|
const cwdEnv = loadEnvFile(cwdEnvPath);
|
||||||
const homeEnv = loadEnvFile(homeEnvPath);
|
const homeEnv = loadEnvFile(homeEnvPath);
|
||||||
|
|
||||||
|
const sources: CredentialSource[] = [];
|
||||||
|
|
||||||
|
if (account?.app_id || account?.app_secret) {
|
||||||
|
sources.push({
|
||||||
|
name: account.alias ? `EXTEND.md account "${account.alias}"` : "EXTEND.md account config",
|
||||||
|
appIdKey: "app_id",
|
||||||
|
appSecretKey: "app_secret",
|
||||||
|
appId: normalizeCredentialValue(account.app_id),
|
||||||
|
appSecret: normalizeCredentialValue(account.app_secret),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const prefix = account?.alias ? `WECHAT_${aliasToEnvKey(account.alias)}_` : "";
|
const prefix = account?.alias ? `WECHAT_${aliasToEnvKey(account.alias)}_` : "";
|
||||||
|
|
||||||
let appId = "";
|
|
||||||
let appSecret = "";
|
|
||||||
|
|
||||||
if (prefix) {
|
if (prefix) {
|
||||||
appId = process.env[`${prefix}APP_ID`]
|
const prefixedKeyLabel = `${prefix}APP_ID/${prefix}APP_SECRET`;
|
||||||
|| cwdEnv[`${prefix}APP_ID`]
|
sources.push(
|
||||||
|| homeEnv[`${prefix}APP_ID`]
|
buildCredentialSource(`process.env (${prefixedKeyLabel})`, process.env, `${prefix}APP_ID`, `${prefix}APP_SECRET`),
|
||||||
|| "";
|
buildCredentialSource(`<cwd>/.baoyu-skills/.env (${prefixedKeyLabel})`, cwdEnv, `${prefix}APP_ID`, `${prefix}APP_SECRET`),
|
||||||
appSecret = process.env[`${prefix}APP_SECRET`]
|
buildCredentialSource(`~/.baoyu-skills/.env (${prefixedKeyLabel})`, homeEnv, `${prefix}APP_ID`, `${prefix}APP_SECRET`),
|
||||||
|| cwdEnv[`${prefix}APP_SECRET`]
|
|
||||||
|| homeEnv[`${prefix}APP_SECRET`]
|
|
||||||
|| "";
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!appId) {
|
|
||||||
appId = process.env.WECHAT_APP_ID || cwdEnv.WECHAT_APP_ID || homeEnv.WECHAT_APP_ID || "";
|
|
||||||
}
|
|
||||||
if (!appSecret) {
|
|
||||||
appSecret = process.env.WECHAT_APP_SECRET || cwdEnv.WECHAT_APP_SECRET || homeEnv.WECHAT_APP_SECRET || "";
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!appId || !appSecret) {
|
|
||||||
const hint = account?.alias ? ` (account: ${account.alias})` : "";
|
|
||||||
throw new Error(
|
|
||||||
`Missing WECHAT_APP_ID or WECHAT_APP_SECRET${hint}.\n` +
|
|
||||||
"Set via EXTEND.md account config, environment variables, or .baoyu-skills/.env file."
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
return { appId, appSecret };
|
sources.push(
|
||||||
|
buildCredentialSource("process.env", process.env, "WECHAT_APP_ID", "WECHAT_APP_SECRET"),
|
||||||
|
buildCredentialSource("<cwd>/.baoyu-skills/.env", cwdEnv, "WECHAT_APP_ID", "WECHAT_APP_SECRET"),
|
||||||
|
buildCredentialSource("~/.baoyu-skills/.env", homeEnv, "WECHAT_APP_ID", "WECHAT_APP_SECRET"),
|
||||||
|
);
|
||||||
|
|
||||||
|
return resolveCredentialSource(sources, account);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function listAccounts(config: WechatExtendConfig): string[] {
|
export function listAccounts(config: WechatExtendConfig): string[] {
|
||||||
|
|||||||
@@ -123,6 +123,8 @@ ${BUN_X} {baseDir}/scripts/weibo-article.ts article.md --cover ./cover.jpg
|
|||||||
- Title: 32 characters max (truncated with warning if longer)
|
- Title: 32 characters max (truncated with warning if longer)
|
||||||
- Summary/导语: 44 characters max (auto-regenerated from content if longer)
|
- Summary/导语: 44 characters max (auto-regenerated from content if longer)
|
||||||
|
|
||||||
|
**Markdown-to-HTML**: Do NOT pass any `--theme` parameter when converting markdown to HTML. Use the default theme (no theme argument).
|
||||||
|
|
||||||
**Article Workflow**:
|
**Article Workflow**:
|
||||||
1. Opens `https://card.weibo.com/article/v3/editor`
|
1. Opens `https://card.weibo.com/article/v3/editor`
|
||||||
2. Clicks "写文章" button, waits for editor to become editable
|
2. Clicks "写文章" button, waits for editor to become editable
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
---
|
---
|
||||||
name: baoyu-url-to-markdown
|
name: baoyu-url-to-markdown
|
||||||
description: Fetch any URL and convert to markdown using Chrome CDP. Saves the rendered HTML snapshot alongside the markdown, uses an upgraded Defuddle pipeline with better web-component handling and YouTube transcript extraction, and automatically falls back to the pre-Defuddle HTML-to-Markdown pipeline when needed. If local browser capture fails entirely, it can fall back to the hosted defuddle.md API. Supports two modes - auto-capture on page load, or wait for user signal (for pages requiring login). Use when user wants to save a webpage as markdown.
|
description: Fetch any URL and convert to markdown using Chrome CDP. Saves the rendered HTML snapshot alongside the markdown, uses an upgraded Defuddle pipeline with better web-component handling and YouTube transcript extraction, and automatically falls back to the pre-Defuddle HTML-to-Markdown pipeline when needed. If local browser capture fails entirely, it can fall back to the hosted defuddle.md API. Supports two modes - auto-capture on page load, or wait for user signal (for pages requiring login). Use when user wants to save a webpage as markdown.
|
||||||
version: 1.58.1
|
version: 1.59.0
|
||||||
metadata:
|
metadata:
|
||||||
openclaw:
|
openclaw:
|
||||||
homepage: https://github.com/JimLiu/baoyu-skills#baoyu-url-to-markdown
|
homepage: https://github.com/JimLiu/baoyu-skills#baoyu-url-to-markdown
|
||||||
@@ -30,6 +30,9 @@ Fetches any URL via Chrome CDP, saves the rendered HTML snapshot, and converts i
|
|||||||
|--------|---------|
|
|--------|---------|
|
||||||
| `scripts/main.ts` | CLI entry point for URL fetching |
|
| `scripts/main.ts` | CLI entry point for URL fetching |
|
||||||
| `scripts/html-to-markdown.ts` | Markdown conversion entry point and converter selection |
|
| `scripts/html-to-markdown.ts` | Markdown conversion entry point and converter selection |
|
||||||
|
| `scripts/parsers/index.ts` | Unified parser entry: dispatches URL-specific rules before generic converters |
|
||||||
|
| `scripts/parsers/types.ts` | Unified parser interface shared by all rule files |
|
||||||
|
| `scripts/parsers/rules/*.ts` | One file per URL rule, for example X status and X article |
|
||||||
| `scripts/defuddle-converter.ts` | Defuddle-based conversion |
|
| `scripts/defuddle-converter.ts` | Defuddle-based conversion |
|
||||||
| `scripts/legacy-converter.ts` | Pre-Defuddle legacy extraction and markdown conversion |
|
| `scripts/legacy-converter.ts` | Pre-Defuddle legacy extraction and markdown conversion |
|
||||||
| `scripts/markdown-conversion-shared.ts` | Shared metadata parsing and markdown document helpers |
|
| `scripts/markdown-conversion-shared.ts` | Shared metadata parsing and markdown document helpers |
|
||||||
@@ -115,10 +118,13 @@ Full reference: [references/config/first-time-setup.md](references/config/first-
|
|||||||
## Features
|
## Features
|
||||||
|
|
||||||
- Chrome CDP for full JavaScript rendering
|
- Chrome CDP for full JavaScript rendering
|
||||||
|
- URL-specific parser layer for sites that need custom HTML rules before generic extraction
|
||||||
- Two capture modes: auto or wait-for-user
|
- Two capture modes: auto or wait-for-user
|
||||||
- Save rendered HTML as a sibling `-captured.html` file
|
- Save rendered HTML as a sibling `-captured.html` file
|
||||||
- Clean markdown output with metadata
|
- Clean markdown output with metadata
|
||||||
- Upgraded Defuddle-first markdown conversion with automatic fallback to the pre-Defuddle extractor from git history
|
- Upgraded Defuddle-first markdown conversion with automatic fallback to the pre-Defuddle extractor from git history
|
||||||
|
- X/Twitter pages can use HTML-specific parsing for Tweets and Articles, which improves title/body/media extraction on `x.com` / `twitter.com`
|
||||||
|
- `archive.ph` / related archive mirrors can restore the original URL from `input[name=q]` and prefer `#CONTENT` before falling back to the page body
|
||||||
- Materializes shadow DOM content before conversion so web-component pages survive serialization better
|
- Materializes shadow DOM content before conversion so web-component pages survive serialization better
|
||||||
- YouTube pages can include transcript/caption text in the markdown when YouTube exposes a caption track
|
- YouTube pages can include transcript/caption text in the markdown when YouTube exposes a caption track
|
||||||
- If local browser capture fails completely, can fall back to `defuddle.md/<url>` and still save markdown
|
- If local browser capture fails completely, can fall back to `defuddle.md/<url>` and still save markdown
|
||||||
@@ -201,14 +207,16 @@ When `--download-media` is enabled:
|
|||||||
|
|
||||||
Conversion order:
|
Conversion order:
|
||||||
|
|
||||||
1. Try Defuddle first
|
1. Try the URL-specific parser layer first when a site rule matches
|
||||||
2. For rich pages such as YouTube, prefer Defuddle's extractor-specific output (including transcripts when available) instead of replacing it with the legacy pipeline
|
2. If no specialized parser matches, try Defuddle
|
||||||
3. If Defuddle throws, cannot load, returns obviously incomplete markdown, or captures lower-quality content than the legacy pipeline, automatically fall back to the pre-Defuddle extractor
|
3. For rich pages such as YouTube, prefer Defuddle's extractor-specific output (including transcripts when available) instead of replacing it with the legacy pipeline
|
||||||
4. If the entire local browser capture flow fails before markdown can be produced, try the hosted `https://defuddle.md/<url>` API and save its markdown output directly
|
4. If Defuddle throws, cannot load, returns obviously incomplete markdown, or captures lower-quality content than the legacy pipeline, automatically fall back to the pre-Defuddle extractor
|
||||||
5. The legacy fallback path uses the older Readability/selector/Next.js-data based HTML-to-Markdown implementation recovered from git history
|
5. If the entire local browser capture flow fails before markdown can be produced, try the hosted `https://defuddle.md/<url>` API and save its markdown output directly
|
||||||
|
6. The legacy fallback path uses the older Readability/selector/Next.js-data based HTML-to-Markdown implementation recovered from git history
|
||||||
|
|
||||||
CLI output will show:
|
CLI output will show:
|
||||||
|
|
||||||
|
- `Converter: parser:...` when a URL-specific parser succeeded
|
||||||
- `Converter: defuddle` when Defuddle succeeds
|
- `Converter: defuddle` when Defuddle succeeds
|
||||||
- `Converter: legacy:...` plus `Fallback used: ...` when fallback was needed
|
- `Converter: legacy:...` plus `Fallback used: ...` when fallback was needed
|
||||||
- `Converter: defuddle-api` when local browser capture failed and the hosted API was used instead
|
- `Converter: defuddle-api` when local browser capture failed and the hosted API was used instead
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import {
|
|||||||
scoreMarkdownQuality,
|
scoreMarkdownQuality,
|
||||||
shouldCompareWithLegacy,
|
shouldCompareWithLegacy,
|
||||||
} from "./legacy-converter.js";
|
} from "./legacy-converter.js";
|
||||||
|
import { tryUrlRuleParsers } from "./parsers/index.js";
|
||||||
|
|
||||||
export type { ConversionResult, PageMetadata };
|
export type { ConversionResult, PageMetadata };
|
||||||
export { createMarkdownDocument, formatMetadataYaml };
|
export { createMarkdownDocument, formatMetadataYaml };
|
||||||
@@ -105,6 +106,11 @@ export async function extractContent(html: string, url: string): Promise<Convers
|
|||||||
const capturedAt = new Date().toISOString();
|
const capturedAt = new Date().toISOString();
|
||||||
const baseMetadata = extractMetadataFromHtml(html, url, capturedAt);
|
const baseMetadata = extractMetadataFromHtml(html, url, capturedAt);
|
||||||
|
|
||||||
|
const specializedResult = tryUrlRuleParsers(html, url, baseMetadata);
|
||||||
|
if (specializedResult) {
|
||||||
|
return specializedResult;
|
||||||
|
}
|
||||||
|
|
||||||
const defuddleResult = await tryDefuddleConversion(html, url, baseMetadata);
|
const defuddleResult = await tryDefuddleConversion(html, url, baseMetadata);
|
||||||
if (defuddleResult.ok) {
|
if (defuddleResult.ok) {
|
||||||
if (shouldPreferDefuddle(defuddleResult.result)) {
|
if (shouldPreferDefuddle(defuddleResult.result)) {
|
||||||
|
|||||||
@@ -521,14 +521,18 @@ turndown.addRule("collapseFigure", {
|
|||||||
|
|
||||||
turndown.addRule("dropInvisibleAnchors", {
|
turndown.addRule("dropInvisibleAnchors", {
|
||||||
filter(node) {
|
filter(node) {
|
||||||
return node.nodeName === "A" && !(node as Element).textContent?.trim();
|
return (
|
||||||
|
node.nodeName === "A" &&
|
||||||
|
!(node as Element).textContent?.trim() &&
|
||||||
|
!(node as Element).querySelector("img, video, picture, source")
|
||||||
|
);
|
||||||
},
|
},
|
||||||
replacement() {
|
replacement() {
|
||||||
return "";
|
return "";
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
function convertHtmlToMarkdown(html: string): string {
|
export function convertHtmlFragmentToMarkdown(html: string): string {
|
||||||
if (!html || !html.trim()) return "";
|
if (!html || !html.trim()) return "";
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -609,7 +613,7 @@ export function shouldCompareWithLegacy(markdown: string): boolean {
|
|||||||
export function convertWithLegacyExtractor(html: string, baseMetadata: PageMetadata): ConversionResult {
|
export function convertWithLegacyExtractor(html: string, baseMetadata: PageMetadata): ConversionResult {
|
||||||
const extracted = extractFromHtml(html);
|
const extracted = extractFromHtml(html);
|
||||||
|
|
||||||
let markdown = extracted?.html ? convertHtmlToMarkdown(extracted.html) : "";
|
let markdown = extracted?.html ? convertHtmlFragmentToMarkdown(extracted.html) : "";
|
||||||
if (!markdown.trim()) {
|
if (!markdown.trim()) {
|
||||||
markdown = extracted?.textContent?.trim() || fallbackPlainText(html);
|
markdown = extracted?.textContent?.trim() || fallbackPlainText(html);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -52,15 +52,71 @@ function parseArgs(argv: string[]): Args {
|
|||||||
return args;
|
return args;
|
||||||
}
|
}
|
||||||
|
|
||||||
function generateSlug(title: string, url: string): string {
|
const SLUG_STOP_WORDS = new Set([
|
||||||
const text = title || new URL(url).pathname.replace(/\//g, "-");
|
"the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
|
||||||
return text
|
"have", "has", "had", "do", "does", "did", "will", "would", "shall",
|
||||||
|
"should", "may", "might", "must", "can", "could", "to", "of", "in",
|
||||||
|
"for", "on", "with", "at", "by", "from", "as", "into", "through",
|
||||||
|
"during", "before", "after", "above", "below", "between", "out",
|
||||||
|
"off", "over", "under", "again", "further", "then", "once", "here",
|
||||||
|
"there", "when", "where", "why", "how", "all", "both", "each",
|
||||||
|
"few", "more", "most", "other", "some", "such", "no", "nor", "not",
|
||||||
|
"only", "own", "same", "so", "than", "too", "very", "just", "but",
|
||||||
|
"and", "or", "if", "this", "that", "these", "those", "it", "its",
|
||||||
|
"http", "https", "www", "com", "org", "net", "post", "article",
|
||||||
|
]);
|
||||||
|
|
||||||
|
function extractSlugFromContent(content: string): string | null {
|
||||||
|
const body = content.replace(/^---\n[\s\S]*?\n---\n?/, "").slice(0, 1000);
|
||||||
|
const words = body
|
||||||
|
.replace(/[^\w\s-]/g, " ")
|
||||||
|
.split(/\s+/)
|
||||||
|
.filter((w) => /^[a-zA-Z]/.test(w) && w.length >= 2 && !SLUG_STOP_WORDS.has(w.toLowerCase()))
|
||||||
|
.map((w) => w.toLowerCase());
|
||||||
|
|
||||||
|
const unique: string[] = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
for (const w of words) {
|
||||||
|
if (!seen.has(w)) {
|
||||||
|
seen.add(w);
|
||||||
|
unique.push(w);
|
||||||
|
if (unique.length >= 6) break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return unique.length >= 2 ? unique.join("-").slice(0, 50) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function generateSlug(title: string, url: string, content?: string): string {
|
||||||
|
const asciiWords = title
|
||||||
|
.replace(/[^\w\s]/g, " ")
|
||||||
|
.split(/\s+/)
|
||||||
|
.filter((w) => /[a-zA-Z]/.test(w) && w.length >= 2 && !SLUG_STOP_WORDS.has(w.toLowerCase()))
|
||||||
|
.map((w) => w.toLowerCase());
|
||||||
|
|
||||||
|
if (asciiWords.length >= 2) {
|
||||||
|
return asciiWords.slice(0, 6).join("-").slice(0, 50);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (content) {
|
||||||
|
const contentSlug = extractSlugFromContent(content);
|
||||||
|
if (contentSlug) return contentSlug;
|
||||||
|
}
|
||||||
|
|
||||||
|
const GENERIC_PATH_SEGMENTS = new Set(["status", "article", "post", "posts", "p", "blog", "news", "articles"]);
|
||||||
|
const parsed = new URL(url);
|
||||||
|
const pathSlug = parsed.pathname
|
||||||
|
.split("/")
|
||||||
|
.filter((s) => s.length > 0 && !/^\d{10,}$/.test(s) && !GENERIC_PATH_SEGMENTS.has(s.toLowerCase()))
|
||||||
|
.join("-")
|
||||||
.toLowerCase()
|
.toLowerCase()
|
||||||
.replace(/[^\w\s-]/g, "")
|
.replace(/[^\w-]/g, "-")
|
||||||
.replace(/\s+/g, "-")
|
|
||||||
.replace(/-+/g, "-")
|
.replace(/-+/g, "-")
|
||||||
.replace(/^-|-$/g, "")
|
.replace(/^-|-$/g, "")
|
||||||
.slice(0, 50) || "page";
|
.slice(0, 40);
|
||||||
|
|
||||||
|
const prefix = asciiWords.slice(0, 2).join("-");
|
||||||
|
const combined = prefix ? `${prefix}-${pathSlug}` : pathSlug;
|
||||||
|
return combined.slice(0, 50) || "page";
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatTimestamp(): string {
|
function formatTimestamp(): string {
|
||||||
@@ -124,18 +180,18 @@ async function fetchDefuddleApiMarkdown(targetUrl: string): Promise<{ markdown:
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async function generateOutputPath(url: string, title: string, outputDir?: string): Promise<string> {
|
async function generateOutputPath(url: string, title: string, outputDir?: string, content?: string): Promise<string> {
|
||||||
const domain = new URL(url).hostname.replace(/^www\./, "");
|
const domain = new URL(url).hostname.replace(/^www\./, "");
|
||||||
const slug = generateSlug(title, url);
|
const slug = generateSlug(title, url, content);
|
||||||
const dataDir = outputDir ? path.resolve(outputDir) : resolveUrlToMarkdownDataDir();
|
const dataDir = outputDir ? path.resolve(outputDir) : resolveUrlToMarkdownDataDir();
|
||||||
const basePath = path.join(dataDir, domain, `${slug}.md`);
|
const basePath = path.join(dataDir, domain, slug, `${slug}.md`);
|
||||||
|
|
||||||
if (!(await fileExists(basePath))) {
|
if (!(await fileExists(basePath))) {
|
||||||
return basePath;
|
return basePath;
|
||||||
}
|
}
|
||||||
|
|
||||||
const timestampSlug = `${slug}-${formatTimestamp()}`;
|
const timestampSlug = `${slug}-${formatTimestamp()}`;
|
||||||
return path.join(dataDir, domain, `${timestampSlug}.md`);
|
return path.join(dataDir, domain, timestampSlug, `${timestampSlug}.md`);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function waitForUserSignal(): Promise<void> {
|
async function waitForUserSignal(): Promise<void> {
|
||||||
@@ -249,13 +305,12 @@ async function main(): Promise<void> {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await captureUrl(args);
|
const result = await captureUrl(args);
|
||||||
outputPath = args.output || await generateOutputPath(args.url, result.metadata.title, args.outputDir);
|
document = createMarkdownDocument(result);
|
||||||
|
outputPath = args.output || await generateOutputPath(args.url, result.metadata.title, args.outputDir, document);
|
||||||
const outputDir = path.dirname(outputPath);
|
const outputDir = path.dirname(outputPath);
|
||||||
htmlSnapshotPath = deriveHtmlSnapshotPath(outputPath);
|
htmlSnapshotPath = deriveHtmlSnapshotPath(outputPath);
|
||||||
await mkdir(outputDir, { recursive: true });
|
await mkdir(outputDir, { recursive: true });
|
||||||
await writeFile(htmlSnapshotPath, result.rawHtml, "utf-8");
|
await writeFile(htmlSnapshotPath, result.rawHtml, "utf-8");
|
||||||
|
|
||||||
document = createMarkdownDocument(result);
|
|
||||||
conversionMethod = result.conversionMethod;
|
conversionMethod = result.conversionMethod;
|
||||||
fallbackReason = result.fallbackReason;
|
fallbackReason = result.fallbackReason;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -265,10 +320,9 @@ async function main(): Promise<void> {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
const remoteResult = await fetchDefuddleApiMarkdown(args.url);
|
const remoteResult = await fetchDefuddleApiMarkdown(args.url);
|
||||||
outputPath = args.output || await generateOutputPath(args.url, remoteResult.title, args.outputDir);
|
|
||||||
await mkdir(path.dirname(outputPath), { recursive: true });
|
|
||||||
|
|
||||||
document = remoteResult.markdown;
|
document = remoteResult.markdown;
|
||||||
|
outputPath = args.output || await generateOutputPath(args.url, remoteResult.title, args.outputDir, document);
|
||||||
|
await mkdir(path.dirname(outputPath), { recursive: true });
|
||||||
conversionMethod = "defuddle-api";
|
conversionMethod = "defuddle-api";
|
||||||
fallbackReason = `Local browser capture failed: ${primaryError}`;
|
fallbackReason = `Local browser capture failed: ${primaryError}`;
|
||||||
} catch (remoteError) {
|
} catch (remoteError) {
|
||||||
|
|||||||
@@ -300,6 +300,24 @@ export function createMarkdownDocument(result: ConversionResult): string {
|
|||||||
const escapedTitle = result.metadata.title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
const escapedTitle = result.metadata.title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||||
const titleRegex = new RegExp(`^#\\s+${escapedTitle}\\s*(\\n|$)`, "i");
|
const titleRegex = new RegExp(`^#\\s+${escapedTitle}\\s*(\\n|$)`, "i");
|
||||||
const hasTitle = titleRegex.test(result.markdown.trimStart());
|
const hasTitle = titleRegex.test(result.markdown.trimStart());
|
||||||
const title = result.metadata.title && !hasTitle ? `\n\n# ${result.metadata.title}\n\n` : "\n\n";
|
const firstMeaningfulLine = result.markdown
|
||||||
|
.replace(/\r\n/g, "\n")
|
||||||
|
.split("\n")
|
||||||
|
.map((line) => line.trim())
|
||||||
|
.find((line) => line && !/^!?\[[^\]]*\]\([^)]+\)$/.test(line))
|
||||||
|
?.replace(/^>\s*/, "")
|
||||||
|
?.replace(/^#+\s+/, "")
|
||||||
|
?.trim();
|
||||||
|
const comparableTitle = result.metadata.title.toLowerCase().replace(/(?:\.{3}|…)\s*$/, "");
|
||||||
|
const comparableFirstLine = firstMeaningfulLine?.toLowerCase() ?? "";
|
||||||
|
const titleRepeatsContent =
|
||||||
|
comparableTitle !== "" &&
|
||||||
|
comparableFirstLine !== "" &&
|
||||||
|
(comparableFirstLine === comparableTitle ||
|
||||||
|
comparableFirstLine.startsWith(comparableTitle) ||
|
||||||
|
comparableTitle.startsWith(comparableFirstLine));
|
||||||
|
const title = result.metadata.title && !hasTitle && !titleRepeatsContent
|
||||||
|
? `\n\n# ${result.metadata.title}\n\n`
|
||||||
|
: "\n\n";
|
||||||
return yaml + title + result.markdown;
|
return yaml + title + result.markdown;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,206 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import test from "node:test";
|
||||||
|
|
||||||
|
import {
|
||||||
|
createMarkdownDocument,
|
||||||
|
extractMetadataFromHtml,
|
||||||
|
} from "../markdown-conversion-shared.js";
|
||||||
|
import { tryUrlRuleParsers } from "./index.js";
|
||||||
|
|
||||||
|
const CAPTURED_AT = "2026-03-22T06:00:00.000Z";
|
||||||
|
|
||||||
|
const ARTICLE_HTML = `<!doctype html>
|
||||||
|
<html lang="zh-CN">
|
||||||
|
<body>
|
||||||
|
<div data-testid="twitterArticleReadView">
|
||||||
|
<a href="/dotey/article/2035141635713941927/media/1">
|
||||||
|
<div data-testid="tweetPhoto">
|
||||||
|
<img src="https://pbs.twimg.com/media/article-cover.jpg" alt="Image">
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
<div data-testid="twitter-article-title">Karpathy:"写代码"已经不是对的动词了</div>
|
||||||
|
<div data-testid="User-Name">
|
||||||
|
<a href="/dotey">宝玉 Verified account</a>
|
||||||
|
<a href="/dotey">@dotey</a>
|
||||||
|
<time datetime="2026-03-20T23:49:11.000Z">Mar 20</time>
|
||||||
|
</div>
|
||||||
|
<div data-testid="twitterArticleRichTextView">
|
||||||
|
<p>Andrej Karpathy 说他从 2024 年 12 月起就基本没手写过一行代码。</p>
|
||||||
|
<a href="/dotey/article/2035141635713941927/media/2">
|
||||||
|
<div>
|
||||||
|
<div>
|
||||||
|
<div data-testid="tweetPhoto">
|
||||||
|
<img src="https://pbs.twimg.com/media/article-inline.jpg" alt="Image">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
<h2>要点速览</h2>
|
||||||
|
<ul>
|
||||||
|
<li>核心焦虑从 GPU 利用率转向 Token 吞吐量</li>
|
||||||
|
</ul>
|
||||||
|
<blockquote>
|
||||||
|
<p>写代码已经不是对的动词了。</p>
|
||||||
|
</blockquote>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>`;
|
||||||
|
|
||||||
|
const STATUS_HTML = `<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<body>
|
||||||
|
<article data-testid="tweet">
|
||||||
|
<div data-testid="User-Name">
|
||||||
|
<a href="/dotey">宝玉 Verified account</a>
|
||||||
|
<a href="/dotey">@dotey</a>
|
||||||
|
<time datetime="2026-03-22T05:33:00.000Z">Mar 22</time>
|
||||||
|
</div>
|
||||||
|
<div data-testid="tweetText">
|
||||||
|
<span>转译:把下面这段加到你的 Codex 自定义指令里,体验会好太多:</span>
|
||||||
|
</div>
|
||||||
|
<div data-testid="tweetPhoto">
|
||||||
|
<img src="https://pbs.twimg.com/media/tweet-main.jpg" alt="Image">
|
||||||
|
</div>
|
||||||
|
<div data-testid="User-Name">
|
||||||
|
<a href="/mattshumer_">Matt Shumer Verified account</a>
|
||||||
|
<a href="/mattshumer_">@mattshumer_</a>
|
||||||
|
<time datetime="2026-03-17T00:00:00.000Z">Mar 17</time>
|
||||||
|
</div>
|
||||||
|
<div data-testid="tweetText">
|
||||||
|
<span>Add this to your Codex custom instructions for a way better experience.</span>
|
||||||
|
</div>
|
||||||
|
</article>
|
||||||
|
</body>
|
||||||
|
</html>`;
|
||||||
|
|
||||||
|
const ARCHIVE_HTML = `<!doctype html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>archive.ph</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<form>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
name="q"
|
||||||
|
value="https://www.newscientist.com/article/2520204-major-leap-towards-reanimation-after-death-as-mammals-brain-preserved/"
|
||||||
|
>
|
||||||
|
</form>
|
||||||
|
<div id="HEADER">
|
||||||
|
Archive shell text that should be ignored when CONTENT exists.
|
||||||
|
</div>
|
||||||
|
<div id="CONTENT">
|
||||||
|
<h1>Major leap towards reanimation after death as mammal brain preserved</h1>
|
||||||
|
<p>
|
||||||
|
Researchers say the preserved structure and activity markers suggest a significant step
|
||||||
|
forward in keeping delicate brain tissue viable after clinical death.
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
The archive wrapper should not take precedence over the actual article body when the
|
||||||
|
CONTENT container is available for parsing.
|
||||||
|
</p>
|
||||||
|
<img src="https://cdn.example.com/brain.jpg" alt="Brain tissue">
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>`;
|
||||||
|
|
||||||
|
const ARCHIVE_FALLBACK_HTML = `<!doctype html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>archive.ph</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<input type="text" name="q" value="https://example.com/fallback-story">
|
||||||
|
<main>
|
||||||
|
<h1>Fallback body parsing still works</h1>
|
||||||
|
<p>
|
||||||
|
When CONTENT is absent, the parser should fall back to the body content instead of
|
||||||
|
returning null or keeping the archive wrapper as the final URL.
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
This ensures archived pages with slightly different layouts still produce usable markdown.
|
||||||
|
</p>
|
||||||
|
</main>
|
||||||
|
</body>
|
||||||
|
</html>`;
|
||||||
|
|
||||||
|
function parse(html: string, url: string) {
|
||||||
|
const baseMetadata = extractMetadataFromHtml(html, url, CAPTURED_AT);
|
||||||
|
return tryUrlRuleParsers(html, url, baseMetadata);
|
||||||
|
}
|
||||||
|
|
||||||
|
test("parses archive.ph pages from CONTENT and restores the original URL", () => {
|
||||||
|
const result = parse(ARCHIVE_HTML, "https://archive.ph/SMcX5");
|
||||||
|
|
||||||
|
assert.ok(result);
|
||||||
|
assert.equal(result.conversionMethod, "parser:archive-ph");
|
||||||
|
assert.equal(
|
||||||
|
result.metadata.url,
|
||||||
|
"https://www.newscientist.com/article/2520204-major-leap-towards-reanimation-after-death-as-mammals-brain-preserved/"
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
result.metadata.title,
|
||||||
|
"Major leap towards reanimation after death as mammal brain preserved"
|
||||||
|
);
|
||||||
|
assert.equal(result.metadata.coverImage, "https://cdn.example.com/brain.jpg");
|
||||||
|
assert.ok(result.markdown.includes("Researchers say the preserved structure"));
|
||||||
|
assert.ok(result.markdown.includes(""));
|
||||||
|
assert.ok(!result.markdown.includes("Archive shell text that should be ignored"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("falls back to body when archive.ph CONTENT is missing", () => {
|
||||||
|
const result = parse(ARCHIVE_FALLBACK_HTML, "https://archive.ph/fallback");
|
||||||
|
|
||||||
|
assert.ok(result);
|
||||||
|
assert.equal(result.conversionMethod, "parser:archive-ph");
|
||||||
|
assert.equal(result.metadata.url, "https://example.com/fallback-story");
|
||||||
|
assert.equal(result.metadata.title, "Fallback body parsing still works");
|
||||||
|
assert.ok(result.markdown.includes("When CONTENT is absent"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("parses X article pages from HTML", () => {
|
||||||
|
const result = parse(
|
||||||
|
ARTICLE_HTML,
|
||||||
|
"https://x.com/dotey/article/2035141635713941927"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.ok(result);
|
||||||
|
assert.equal(result.conversionMethod, "parser:x-article");
|
||||||
|
assert.equal(result.metadata.title, "Karpathy:\"写代码\"已经不是对的动词了");
|
||||||
|
assert.equal(result.metadata.author, "宝玉 (@dotey)");
|
||||||
|
assert.equal(result.metadata.coverImage, "https://pbs.twimg.com/media/article-cover.jpg");
|
||||||
|
assert.equal(result.metadata.published, "2026-03-20T23:49:11.000Z");
|
||||||
|
assert.equal(result.metadata.language, "zh");
|
||||||
|
assert.ok(result.markdown.includes("## 要点速览"));
|
||||||
|
assert.ok(
|
||||||
|
result.markdown.includes(
|
||||||
|
"[](/dotey/article/2035141635713941927/media/2)"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert.ok(result.markdown.includes("写代码已经不是对的动词了。"));
|
||||||
|
|
||||||
|
const document = createMarkdownDocument(result);
|
||||||
|
assert.ok(document.includes("# Karpathy:\"写代码\"已经不是对的动词了"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("parses X status pages from HTML without duplicating the title heading", () => {
|
||||||
|
const result = parse(
|
||||||
|
STATUS_HTML,
|
||||||
|
"https://x.com/dotey/status/2035590649081196710"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.ok(result);
|
||||||
|
assert.equal(result.conversionMethod, "parser:x-status");
|
||||||
|
assert.equal(result.metadata.author, "宝玉 (@dotey)");
|
||||||
|
assert.equal(result.metadata.coverImage, "https://pbs.twimg.com/media/tweet-main.jpg");
|
||||||
|
assert.equal(result.metadata.language, "zh");
|
||||||
|
assert.ok(result.markdown.includes("转译:把下面这段加到你的 Codex 自定义指令里"));
|
||||||
|
assert.ok(result.markdown.includes("> Quote from Matt Shumer (@mattshumer_)"));
|
||||||
|
assert.ok(result.markdown.includes("!["));
|
||||||
|
|
||||||
|
const document = createMarkdownDocument(result);
|
||||||
|
assert.ok(
|
||||||
|
!document.includes("\n\n# 转译:把下面这段加到你的 Codex 自定义指令里,体验会好太多:\n\n")
|
||||||
|
);
|
||||||
|
});
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
import {
|
||||||
|
isMarkdownUsable,
|
||||||
|
normalizeMarkdown,
|
||||||
|
parseDocument,
|
||||||
|
type ConversionResult,
|
||||||
|
type PageMetadata,
|
||||||
|
} from "../markdown-conversion-shared.js";
|
||||||
|
import { URL_RULE_PARSERS } from "./rules/index.js";
|
||||||
|
import type { UrlRuleParserContext } from "./types.js";
|
||||||
|
|
||||||
|
export type { UrlRuleParser, UrlRuleParserContext } from "./types.js";
|
||||||
|
|
||||||
|
export function tryUrlRuleParsers(
|
||||||
|
html: string,
|
||||||
|
url: string,
|
||||||
|
baseMetadata: PageMetadata
|
||||||
|
): ConversionResult | null {
|
||||||
|
const document = parseDocument(html);
|
||||||
|
const context: UrlRuleParserContext = {
|
||||||
|
html,
|
||||||
|
url,
|
||||||
|
document,
|
||||||
|
baseMetadata,
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const parser of URL_RULE_PARSERS) {
|
||||||
|
if (!parser.supports(context)) continue;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = parser.parse(context);
|
||||||
|
if (!result) continue;
|
||||||
|
|
||||||
|
const markdown = normalizeMarkdown(result.markdown);
|
||||||
|
if (!isMarkdownUsable(markdown, html)) continue;
|
||||||
|
|
||||||
|
return {
|
||||||
|
...result,
|
||||||
|
markdown,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
console.warn(`[url-to-markdown] parser ${parser.id} failed: ${message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
import { convertHtmlFragmentToMarkdown } from "../../legacy-converter.js";
|
||||||
|
import {
|
||||||
|
normalizeMarkdown,
|
||||||
|
pickString,
|
||||||
|
type ConversionResult,
|
||||||
|
} from "../../markdown-conversion-shared.js";
|
||||||
|
import type { UrlRuleParser, UrlRuleParserContext } from "../types.js";
|
||||||
|
|
||||||
|
const ARCHIVE_HOSTS = new Set([
|
||||||
|
"archive.ph",
|
||||||
|
"archive.is",
|
||||||
|
"archive.today",
|
||||||
|
"archive.md",
|
||||||
|
"archive.vn",
|
||||||
|
"archive.li",
|
||||||
|
"archive.fo",
|
||||||
|
]);
|
||||||
|
|
||||||
|
function isArchiveHost(url: string): boolean {
|
||||||
|
try {
|
||||||
|
return ARCHIVE_HOSTS.has(new URL(url).hostname.toLowerCase());
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function readOriginalUrl(document: Document): string | undefined {
|
||||||
|
const value = document.querySelector("input[name='q']")?.getAttribute("value")?.trim();
|
||||||
|
if (!value) return undefined;
|
||||||
|
|
||||||
|
try {
|
||||||
|
return new URL(value).href;
|
||||||
|
} catch {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function summarize(text: string, maxLength: number): string | undefined {
|
||||||
|
const normalized = text.replace(/\s+/g, " ").trim();
|
||||||
|
if (!normalized) return undefined;
|
||||||
|
if (normalized.length <= maxLength) return normalized;
|
||||||
|
return `${normalized.slice(0, Math.max(0, maxLength - 1)).trimEnd()}…`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function pickContentRoot(document: Document): Element | null {
|
||||||
|
return (
|
||||||
|
document.querySelector("#CONTENT") ??
|
||||||
|
document.querySelector("#content") ??
|
||||||
|
document.body
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function pickContentTitle(root: Element, fallbackTitle: string): string {
|
||||||
|
const contentTitle = pickString(
|
||||||
|
root.querySelector("h1")?.textContent,
|
||||||
|
root.querySelector("[itemprop='headline']")?.textContent,
|
||||||
|
root.querySelector("article h2")?.textContent
|
||||||
|
);
|
||||||
|
if (contentTitle) return contentTitle;
|
||||||
|
if (fallbackTitle && !/^archive\./i.test(fallbackTitle.trim())) return fallbackTitle;
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseArchivePage(context: UrlRuleParserContext): ConversionResult | null {
|
||||||
|
const root = pickContentRoot(context.document);
|
||||||
|
if (!root) return null;
|
||||||
|
|
||||||
|
const markdown = normalizeMarkdown(convertHtmlFragmentToMarkdown(root.innerHTML));
|
||||||
|
if (!markdown) return null;
|
||||||
|
|
||||||
|
const originalUrl = readOriginalUrl(context.document) ?? context.baseMetadata.url;
|
||||||
|
const bodyText = root.textContent?.replace(/\s+/g, " ").trim() ?? "";
|
||||||
|
const published = root.querySelector("time[datetime]")?.getAttribute("datetime") ?? undefined;
|
||||||
|
const coverImage = root.querySelector("img[src]")?.getAttribute("src") ?? undefined;
|
||||||
|
|
||||||
|
return {
|
||||||
|
metadata: {
|
||||||
|
...context.baseMetadata,
|
||||||
|
url: originalUrl,
|
||||||
|
title: pickContentTitle(root, context.baseMetadata.title),
|
||||||
|
description: summarize(bodyText, 220) ?? context.baseMetadata.description,
|
||||||
|
published: pickString(published, context.baseMetadata.published) ?? undefined,
|
||||||
|
coverImage: pickString(coverImage, context.baseMetadata.coverImage) ?? undefined,
|
||||||
|
},
|
||||||
|
markdown,
|
||||||
|
rawHtml: context.html,
|
||||||
|
conversionMethod: "parser:archive-ph",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export const archivePhRuleParser: UrlRuleParser = {
|
||||||
|
id: "archive-ph",
|
||||||
|
supports(context) {
|
||||||
|
return isArchiveHost(context.url);
|
||||||
|
},
|
||||||
|
parse: parseArchivePage,
|
||||||
|
};
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
import { archivePhRuleParser } from "./archive-ph.js";
|
||||||
|
import { xArticleRuleParser } from "./x-article.js";
|
||||||
|
import { xStatusRuleParser } from "./x-status.js";
|
||||||
|
import type { UrlRuleParser } from "../types.js";
|
||||||
|
|
||||||
|
export const URL_RULE_PARSERS: UrlRuleParser[] = [
|
||||||
|
archivePhRuleParser,
|
||||||
|
xArticleRuleParser,
|
||||||
|
xStatusRuleParser,
|
||||||
|
];
|
||||||
@@ -0,0 +1,137 @@
|
|||||||
|
import {
|
||||||
|
normalizeMarkdown,
|
||||||
|
pickString,
|
||||||
|
type ConversionResult,
|
||||||
|
} from "../../markdown-conversion-shared.js";
|
||||||
|
import type { UrlRuleParser, UrlRuleParserContext } from "../types.js";
|
||||||
|
import {
|
||||||
|
cleanText,
|
||||||
|
collectMediaMarkdown,
|
||||||
|
convertXRichTextElementToMarkdown,
|
||||||
|
extractPublishedForCurrentUrl,
|
||||||
|
inferLanguage,
|
||||||
|
isXArticlePath,
|
||||||
|
isXHost,
|
||||||
|
normalizeXMarkdown,
|
||||||
|
parseUrl,
|
||||||
|
pickFirstValidLinkText,
|
||||||
|
sanitizeCoverImage,
|
||||||
|
summarizeText,
|
||||||
|
} from "./x-shared.js";
|
||||||
|
|
||||||
|
function collectArticleMarkdown(root: Element): { markdown: string; mediaUrls: string[] } {
|
||||||
|
const parts: string[] = [];
|
||||||
|
const seenMedia = new Set<string>();
|
||||||
|
const mediaUrls: string[] = [];
|
||||||
|
|
||||||
|
function pushPart(value: string): void {
|
||||||
|
const normalized = normalizeMarkdown(value);
|
||||||
|
if (!normalized) return;
|
||||||
|
parts.push(normalized);
|
||||||
|
}
|
||||||
|
|
||||||
|
function walk(node: Element): void {
|
||||||
|
const testId = node.getAttribute("data-testid");
|
||||||
|
|
||||||
|
if (testId === "twitterArticleRichTextView" || testId === "longformRichTextComponent") {
|
||||||
|
const bodyMedia = collectMediaMarkdown(node, seenMedia);
|
||||||
|
mediaUrls.push(...bodyMedia.urls.filter((url) => !mediaUrls.includes(url)));
|
||||||
|
pushPart(convertXRichTextElementToMarkdown(node));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (testId === "tweetPhoto") {
|
||||||
|
const media = collectMediaMarkdown(node, seenMedia);
|
||||||
|
mediaUrls.push(...media.urls.filter((url) => !mediaUrls.includes(url)));
|
||||||
|
for (const line of media.lines) pushPart(line);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
testId === "twitter-article-title" ||
|
||||||
|
testId === "User-Name" ||
|
||||||
|
testId === "Tweet-User-Avatar" ||
|
||||||
|
testId === "reply" ||
|
||||||
|
testId === "retweet" ||
|
||||||
|
testId === "like" ||
|
||||||
|
testId === "bookmark" ||
|
||||||
|
testId === "caret" ||
|
||||||
|
testId === "app-text-transition-container"
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (node.tagName === "TIME" || node.tagName === "BUTTON") {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const child of Array.from(node.children)) {
|
||||||
|
walk(child);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const child of Array.from(root.children)) {
|
||||||
|
walk(child);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
markdown: normalizeXMarkdown(parts.join("\n\n")),
|
||||||
|
mediaUrls,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseXArticle(context: UrlRuleParserContext): ConversionResult | null {
|
||||||
|
const articleRoot = context.document.querySelector("[data-testid='twitterArticleReadView']") as Element | null;
|
||||||
|
if (!articleRoot) return null;
|
||||||
|
|
||||||
|
const title = cleanText(
|
||||||
|
context.document.querySelector("[data-testid='twitter-article-title']")?.textContent
|
||||||
|
);
|
||||||
|
const identity = pickFirstValidLinkText(
|
||||||
|
context.document.querySelector("[data-testid='User-Name']")
|
||||||
|
);
|
||||||
|
const published = extractPublishedForCurrentUrl(articleRoot, context.url);
|
||||||
|
const { markdown, mediaUrls } = collectArticleMarkdown(articleRoot);
|
||||||
|
if (!markdown) return null;
|
||||||
|
|
||||||
|
const bodyText = cleanText(
|
||||||
|
context.document.querySelector("[data-testid='twitterArticleRichTextView']")?.textContent ??
|
||||||
|
context.document.querySelector("[data-testid='longformRichTextComponent']")?.textContent
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
metadata: {
|
||||||
|
...context.baseMetadata,
|
||||||
|
title: pickString(title, context.baseMetadata.title) ?? "",
|
||||||
|
description: summarizeText(bodyText, 220) ?? context.baseMetadata.description,
|
||||||
|
author: pickString(identity.author, context.baseMetadata.author) ?? undefined,
|
||||||
|
published: pickString(published, context.baseMetadata.published) ?? undefined,
|
||||||
|
coverImage: sanitizeCoverImage(mediaUrls[0], context.baseMetadata.coverImage),
|
||||||
|
language: inferLanguage(bodyText, context.baseMetadata.language),
|
||||||
|
},
|
||||||
|
markdown,
|
||||||
|
rawHtml: context.html,
|
||||||
|
conversionMethod: "parser:x-article",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export const xArticleRuleParser: UrlRuleParser = {
|
||||||
|
id: "x-article",
|
||||||
|
supports(context) {
|
||||||
|
const parsed = parseUrl(context.url);
|
||||||
|
if (!parsed || !isXHost(parsed.hostname)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
isXArticlePath(parsed.pathname) ||
|
||||||
|
Boolean(
|
||||||
|
context.document.querySelector("[data-testid='twitterArticleReadView']") ||
|
||||||
|
context.document.querySelector("[data-testid='twitterArticleRichTextView']")
|
||||||
|
)
|
||||||
|
);
|
||||||
|
},
|
||||||
|
parse(context) {
|
||||||
|
return parseXArticle(context);
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,249 @@
|
|||||||
|
import { convertHtmlFragmentToMarkdown } from "../../legacy-converter.js";
|
||||||
|
import { normalizeMarkdown } from "../../markdown-conversion-shared.js";
|
||||||
|
|
||||||
|
export const DEFAULT_X_OG_IMAGE = "https://abs.twimg.com/rweb/ssr/default/v2/og/image.png";
|
||||||
|
|
||||||
|
export type MediaResult = {
|
||||||
|
lines: string[];
|
||||||
|
urls: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export function isXHost(hostname: string): boolean {
|
||||||
|
const normalized = hostname.toLowerCase();
|
||||||
|
return (
|
||||||
|
normalized === "x.com" ||
|
||||||
|
normalized === "twitter.com" ||
|
||||||
|
normalized.endsWith(".x.com") ||
|
||||||
|
normalized.endsWith(".twitter.com")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseUrl(input: string): URL | null {
|
||||||
|
try {
|
||||||
|
return new URL(input);
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isXStatusPath(pathname: string): boolean {
|
||||||
|
return /^\/[^/]+\/status(?:es)?\/\d+$/i.test(pathname) || /^\/i\/web\/status\/\d+$/i.test(pathname);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isXArticlePath(pathname: string): boolean {
|
||||||
|
return /^\/[^/]+\/article\/\d+$/i.test(pathname) || /^\/(?:i\/)?article\/\d+$/i.test(pathname);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function cleanText(value: string | null | undefined): string {
|
||||||
|
return (value ?? "").replace(/\s+/g, " ").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function cleanUserLabel(value: string | null | undefined): string {
|
||||||
|
return cleanText(value).replace(/\bVerified account\b/gi, "").replace(/\s{2,}/g, " ").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function escapeMarkdownAlt(text: string): string {
|
||||||
|
return text.replace(/[\[\]]/g, "\\$&");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeAlt(text: string | null | undefined): string {
|
||||||
|
const cleaned = cleanText(text);
|
||||||
|
if (!cleaned || /^(image|photo)$/i.test(cleaned)) return "";
|
||||||
|
return escapeMarkdownAlt(cleaned);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function summarizeText(text: string, maxLength: number): string | undefined {
|
||||||
|
const normalized = cleanText(text);
|
||||||
|
if (!normalized) return undefined;
|
||||||
|
return normalized.length > maxLength
|
||||||
|
? `${normalized.slice(0, maxLength - 3)}...`
|
||||||
|
: normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildTweetTitle(text: string, fallback: string): string {
|
||||||
|
return summarizeText(text, 80) ?? fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeXMarkdown(markdown: string): string {
|
||||||
|
return normalizeMarkdown(markdown.replace(/^(#{1,6})\s*\n+([^\n])/gm, "$1 $2"));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function inferLanguage(text: string, fallback?: string): string | undefined {
|
||||||
|
const normalized = cleanText(text);
|
||||||
|
if (!normalized) return fallback;
|
||||||
|
|
||||||
|
const han = (normalized.match(/\p{Script=Han}/gu) || []).length;
|
||||||
|
const hiragana = (normalized.match(/\p{Script=Hiragana}/gu) || []).length;
|
||||||
|
const katakana = (normalized.match(/\p{Script=Katakana}/gu) || []).length;
|
||||||
|
const hangul = (normalized.match(/\p{Script=Hangul}/gu) || []).length;
|
||||||
|
|
||||||
|
if (hangul >= 8) return "ko";
|
||||||
|
if (hiragana + katakana >= 8) return "ja";
|
||||||
|
if (han >= 16) return "zh";
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildQuoteMarkdown(markdown: string, author?: string): string {
|
||||||
|
const normalized = normalizeMarkdown(markdown);
|
||||||
|
if (!normalized) return "";
|
||||||
|
|
||||||
|
const lines = normalized.split("\n");
|
||||||
|
const prefixed = lines.map((line) => (line ? `> ${line}` : ">")).join("\n");
|
||||||
|
const header = author ? `> Quote from ${author}` : "> Quote";
|
||||||
|
return `${header}\n${prefixed}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function pickFirstValidLinkText(userNameEl: Element | null | undefined): {
|
||||||
|
name?: string;
|
||||||
|
username?: string;
|
||||||
|
author?: string;
|
||||||
|
} {
|
||||||
|
if (!userNameEl) return {};
|
||||||
|
|
||||||
|
const linkTexts = Array.from(userNameEl.querySelectorAll("a[href]"))
|
||||||
|
.map((link) => cleanUserLabel(link.textContent))
|
||||||
|
.filter(Boolean);
|
||||||
|
|
||||||
|
let username = linkTexts.find((text) => text.startsWith("@"));
|
||||||
|
let name = linkTexts.find((text) => !text.startsWith("@") && !/^(promote|more)$/i.test(text));
|
||||||
|
|
||||||
|
if (!username || !name) {
|
||||||
|
const text = cleanUserLabel(userNameEl.textContent);
|
||||||
|
const fallbackMatch = text.match(/^(.*?)\s*(@[A-Za-z0-9_]+)(?:\s*·.*)?$/);
|
||||||
|
if (fallbackMatch) {
|
||||||
|
name = name ?? cleanText(fallbackMatch[1]);
|
||||||
|
username = username ?? cleanText(fallbackMatch[2]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const author = name && username ? `${name} (${username})` : username ?? name;
|
||||||
|
return { name, username, author };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function extractPublishedForCurrentUrl(root: ParentNode, url: string): string | undefined {
|
||||||
|
const parsed = parseUrl(url);
|
||||||
|
if (!parsed) return undefined;
|
||||||
|
const currentPath = parsed.pathname.toLowerCase();
|
||||||
|
|
||||||
|
for (const timeElement of root.querySelectorAll("a[href] time[datetime]")) {
|
||||||
|
const href = timeElement.closest("a")?.getAttribute("href");
|
||||||
|
const hrefUrl = href ? parseUrl(href.startsWith("http") ? href : `${parsed.origin}${href}`) : null;
|
||||||
|
if (hrefUrl?.pathname.toLowerCase() === currentPath) {
|
||||||
|
return timeElement.getAttribute("datetime") ?? undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return root.querySelector("time[datetime]")?.getAttribute("datetime") ?? undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function collectMediaMarkdown(root: ParentNode, seen: Set<string>): MediaResult {
|
||||||
|
const lines: string[] = [];
|
||||||
|
const urls: string[] = [];
|
||||||
|
const rootElement = root as Element & {
|
||||||
|
getAttribute?: (name: string) => string | null;
|
||||||
|
};
|
||||||
|
const photoNodes = [
|
||||||
|
...(typeof rootElement.getAttribute === "function" &&
|
||||||
|
rootElement.getAttribute("data-testid") === "tweetPhoto"
|
||||||
|
? [rootElement]
|
||||||
|
: []),
|
||||||
|
...Array.from(root.querySelectorAll("[data-testid='tweetPhoto']")),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const node of photoNodes) {
|
||||||
|
const img = node.querySelector("img");
|
||||||
|
const imageUrl = img?.getAttribute("src");
|
||||||
|
if (imageUrl && !seen.has(imageUrl)) {
|
||||||
|
seen.add(imageUrl);
|
||||||
|
urls.push(imageUrl);
|
||||||
|
lines.push(``);
|
||||||
|
}
|
||||||
|
|
||||||
|
const video = node.querySelector("video");
|
||||||
|
const posterUrl = video?.getAttribute("poster");
|
||||||
|
if (posterUrl && !seen.has(posterUrl)) {
|
||||||
|
seen.add(posterUrl);
|
||||||
|
urls.push(posterUrl);
|
||||||
|
lines.push(``);
|
||||||
|
}
|
||||||
|
|
||||||
|
const videoUrl = video?.getAttribute("src") ?? video?.querySelector("source")?.getAttribute("src");
|
||||||
|
if (videoUrl && !seen.has(videoUrl)) {
|
||||||
|
seen.add(videoUrl);
|
||||||
|
urls.push(videoUrl);
|
||||||
|
lines.push(`[video](${videoUrl})`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { lines, urls };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function materializeTweetPhotoNodes(root: Element): void {
|
||||||
|
for (const photo of Array.from(root.querySelectorAll("[data-testid='tweetPhoto']"))) {
|
||||||
|
const document = photo.ownerDocument;
|
||||||
|
const container = document.createElement("span");
|
||||||
|
|
||||||
|
const img = photo.querySelector("img");
|
||||||
|
const imageUrl = img?.getAttribute("src");
|
||||||
|
if (imageUrl) {
|
||||||
|
const image = document.createElement("img");
|
||||||
|
image.setAttribute("src", imageUrl);
|
||||||
|
const alt = normalizeAlt(img?.getAttribute("alt"));
|
||||||
|
if (alt) {
|
||||||
|
image.setAttribute("alt", alt);
|
||||||
|
}
|
||||||
|
container.appendChild(image);
|
||||||
|
}
|
||||||
|
|
||||||
|
const video = photo.querySelector("video");
|
||||||
|
const posterUrl = video?.getAttribute("poster");
|
||||||
|
if (posterUrl) {
|
||||||
|
const poster = document.createElement("img");
|
||||||
|
poster.setAttribute("src", posterUrl);
|
||||||
|
poster.setAttribute("alt", "video");
|
||||||
|
container.appendChild(poster);
|
||||||
|
}
|
||||||
|
|
||||||
|
const videoUrl = video?.getAttribute("src") ?? video?.querySelector("source")?.getAttribute("src");
|
||||||
|
if (videoUrl) {
|
||||||
|
if (container.childNodes.length > 0) {
|
||||||
|
container.appendChild(document.createTextNode(" "));
|
||||||
|
}
|
||||||
|
const link = document.createElement("a");
|
||||||
|
link.setAttribute("href", videoUrl);
|
||||||
|
link.textContent = "video";
|
||||||
|
container.appendChild(link);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (container.childNodes.length === 0) {
|
||||||
|
photo.remove();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
photo.replaceWith(container);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function collapseLinkedMediaContainers(root: Element): void {
|
||||||
|
for (const anchor of Array.from(root.querySelectorAll("a[href]"))) {
|
||||||
|
const images = Array.from(anchor.querySelectorAll("img"));
|
||||||
|
if (images.length !== 1) continue;
|
||||||
|
if (cleanText(anchor.textContent)) continue;
|
||||||
|
|
||||||
|
const image = images[0].cloneNode(true);
|
||||||
|
anchor.replaceChildren(image);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function convertXRichTextElementToMarkdown(node: Element): string {
|
||||||
|
const clone = node.cloneNode(true) as Element;
|
||||||
|
materializeTweetPhotoNodes(clone);
|
||||||
|
collapseLinkedMediaContainers(clone);
|
||||||
|
return normalizeXMarkdown(convertHtmlFragmentToMarkdown(clone.innerHTML));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function sanitizeCoverImage(primary?: string, fallback?: string): string | undefined {
|
||||||
|
if (primary) return primary;
|
||||||
|
if (!fallback || fallback === DEFAULT_X_OG_IMAGE) return undefined;
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
import type { ConversionResult } from "../../markdown-conversion-shared.js";
|
||||||
|
import type { UrlRuleParser, UrlRuleParserContext } from "../types.js";
|
||||||
|
import {
|
||||||
|
buildQuoteMarkdown,
|
||||||
|
buildTweetTitle,
|
||||||
|
cleanText,
|
||||||
|
collectMediaMarkdown,
|
||||||
|
convertXRichTextElementToMarkdown,
|
||||||
|
extractPublishedForCurrentUrl,
|
||||||
|
inferLanguage,
|
||||||
|
isXHost,
|
||||||
|
isXStatusPath,
|
||||||
|
normalizeXMarkdown,
|
||||||
|
parseUrl,
|
||||||
|
pickFirstValidLinkText,
|
||||||
|
sanitizeCoverImage,
|
||||||
|
summarizeText,
|
||||||
|
} from "./x-shared.js";
|
||||||
|
|
||||||
|
function parseXStatus(context: UrlRuleParserContext): ConversionResult | null {
|
||||||
|
const article = context.document.querySelector("article[data-testid='tweet'], article") as Element | null;
|
||||||
|
if (!article) return null;
|
||||||
|
|
||||||
|
const tweetTextElements = Array.from(article.querySelectorAll("[data-testid='tweetText']")) as Element[];
|
||||||
|
if (tweetTextElements.length === 0) return null;
|
||||||
|
|
||||||
|
const userNameElements = Array.from(article.querySelectorAll("[data-testid='User-Name']")) as Element[];
|
||||||
|
const mainTextElement = tweetTextElements[0];
|
||||||
|
const mainIdentity = pickFirstValidLinkText(userNameElements[0]);
|
||||||
|
const published = extractPublishedForCurrentUrl(article, context.url);
|
||||||
|
const mainMarkdown = normalizeXMarkdown(convertXRichTextElementToMarkdown(mainTextElement));
|
||||||
|
if (!mainMarkdown) return null;
|
||||||
|
|
||||||
|
const parts = [mainMarkdown];
|
||||||
|
const quotedTextElements = tweetTextElements.slice(1);
|
||||||
|
const quotedUserNameElements = userNameElements.slice(1);
|
||||||
|
|
||||||
|
quotedTextElements.forEach((element, index) => {
|
||||||
|
const quoteMarkdown = normalizeXMarkdown(convertXRichTextElementToMarkdown(element));
|
||||||
|
if (!quoteMarkdown) return;
|
||||||
|
const quoteIdentity = pickFirstValidLinkText(quotedUserNameElements[index]);
|
||||||
|
parts.push(buildQuoteMarkdown(quoteMarkdown, quoteIdentity.author));
|
||||||
|
});
|
||||||
|
|
||||||
|
const media = collectMediaMarkdown(article, new Set<string>());
|
||||||
|
if (media.lines.length > 0) {
|
||||||
|
parts.push(media.lines.join("\n\n"));
|
||||||
|
}
|
||||||
|
|
||||||
|
const mainText = cleanText(mainTextElement.textContent);
|
||||||
|
const markdown = normalizeXMarkdown(parts.join("\n\n"));
|
||||||
|
|
||||||
|
return {
|
||||||
|
metadata: {
|
||||||
|
...context.baseMetadata,
|
||||||
|
title: buildTweetTitle(mainText, context.baseMetadata.title),
|
||||||
|
description: summarizeText(mainText, 220) ?? context.baseMetadata.description,
|
||||||
|
author: mainIdentity.author ?? context.baseMetadata.author,
|
||||||
|
published: published ?? context.baseMetadata.published,
|
||||||
|
coverImage: sanitizeCoverImage(media.urls[0], context.baseMetadata.coverImage),
|
||||||
|
language: inferLanguage(mainText, context.baseMetadata.language),
|
||||||
|
},
|
||||||
|
markdown,
|
||||||
|
rawHtml: context.html,
|
||||||
|
conversionMethod: "parser:x-status",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export const xStatusRuleParser: UrlRuleParser = {
|
||||||
|
id: "x-status",
|
||||||
|
supports(context) {
|
||||||
|
const parsed = parseUrl(context.url);
|
||||||
|
if (!parsed || !isXHost(parsed.hostname)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return isXStatusPath(parsed.pathname) && Boolean(context.document.querySelector("[data-testid='tweetText']"));
|
||||||
|
},
|
||||||
|
parse(context): ConversionResult | null {
|
||||||
|
return parseXStatus(context);
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
import type { ConversionResult, PageMetadata } from "../markdown-conversion-shared.js";
|
||||||
|
|
||||||
|
export interface UrlRuleParserContext {
|
||||||
|
html: string;
|
||||||
|
url: string;
|
||||||
|
document: Document;
|
||||||
|
baseMetadata: PageMetadata;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UrlRuleParser {
|
||||||
|
id: string;
|
||||||
|
supports(context: UrlRuleParserContext): boolean;
|
||||||
|
parse(context: UrlRuleParserContext): ConversionResult | null;
|
||||||
|
}
|
||||||
@@ -151,8 +151,7 @@ From outline entry:
|
|||||||
```markdown
|
```markdown
|
||||||
## Watermark
|
## Watermark
|
||||||
|
|
||||||
Include a subtle watermark "{content}" positioned at {position}
|
Include a subtle watermark "{content}" positioned at {position}. The watermark should
|
||||||
with approximately {opacity*100}% visibility. The watermark should
|
|
||||||
be legible but not distracting from the main content.
|
be legible but not distracting from the main content.
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -295,19 +294,19 @@ Create a Xiaohongshu (Little Red Book) style infographic following these guideli
|
|||||||
## Content
|
## Content
|
||||||
|
|
||||||
**Position**: Content (Page 3 of 6)
|
**Position**: Content (Page 3 of 6)
|
||||||
**Core Message**: ChatGPT使用技巧
|
**Core Message**: ChatGPT 使用技巧
|
||||||
|
|
||||||
**Text Content**:
|
**Text Content**:
|
||||||
- Title: 「ChatGPT」
|
- Title: 「ChatGPT」
|
||||||
- Subtitle: 最强AI助手
|
- Subtitle: 最强 AI 助手
|
||||||
- Points:
|
- Points:
|
||||||
- 写文案:给出框架,秒出初稿
|
- 写文案:给出框架,秒出初稿
|
||||||
- 改文章:润色、翻译、总结
|
- 改文章:润色、翻译、总结
|
||||||
- 编程:写代码、找bug
|
- 编程:写代码、找 bug
|
||||||
- 学习:解释概念、出题练习
|
- 学习:解释概念、出题练习
|
||||||
|
|
||||||
**Visual Concept**:
|
**Visual Concept**:
|
||||||
ChatGPT logo居中,四周放射状展示功能点
|
ChatGPT logo 居中,四周放射状展示功能点
|
||||||
深色科技背景,霓虹绿点缀
|
深色科技背景,霓虹绿点缀
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ interface TranscriptInfo {
|
|||||||
interface Chapter {
|
interface Chapter {
|
||||||
title: string;
|
title: string;
|
||||||
start: number;
|
start: number;
|
||||||
|
end: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface VideoMeta {
|
interface VideoMeta {
|
||||||
@@ -249,16 +250,21 @@ async function fetchTranscriptSnippets(info: TranscriptInfo, translateTo?: strin
|
|||||||
|
|
||||||
// --- Metadata & chapters ---
|
// --- Metadata & chapters ---
|
||||||
|
|
||||||
function parseChapters(description: string): Chapter[] {
|
function parseChapters(description: string, duration: number = 0): Chapter[] {
|
||||||
const chapters: Chapter[] = [];
|
const raw: { title: string; start: number }[] = [];
|
||||||
for (const line of description.split("\n")) {
|
for (const line of description.split("\n")) {
|
||||||
const m = line.trim().match(/^(?:(\d{1,2}):)?(\d{1,2}):(\d{2})\s+(.+)$/);
|
const m = line.trim().match(/^(?:(\d{1,2}):)?(\d{1,2}):(\d{2})\s+(.+)$/);
|
||||||
if (m) {
|
if (m) {
|
||||||
const h = m[1] ? parseInt(m[1]) : 0;
|
const h = m[1] ? parseInt(m[1]) : 0;
|
||||||
chapters.push({ title: m[4].trim(), start: h * 3600 + parseInt(m[2]) * 60 + parseInt(m[3]) });
|
raw.push({ title: m[4].trim(), start: h * 3600 + parseInt(m[2]) * 60 + parseInt(m[3]) });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return chapters.length >= 2 ? chapters : [];
|
if (raw.length < 2) return [];
|
||||||
|
return raw.map((ch, i) => ({
|
||||||
|
title: ch.title,
|
||||||
|
start: ch.start,
|
||||||
|
end: i < raw.length - 1 ? raw[i + 1].start : Math.max(duration, ch.start),
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
function getThumbnailUrls(videoId: string, data: any): string[] {
|
function getThumbnailUrls(videoId: string, data: any): string[] {
|
||||||
@@ -644,7 +650,8 @@ async function fetchAndCache(videoId: string, baseDir: string, opts: Options): P
|
|||||||
const info = findTranscript(transcripts, opts.languages, opts.excludeGenerated, opts.excludeManual);
|
const info = findTranscript(transcripts, opts.languages, opts.excludeGenerated, opts.excludeManual);
|
||||||
const result = await fetchTranscriptSnippets(info, opts.translate || undefined);
|
const result = await fetchTranscriptSnippets(info, opts.translate || undefined);
|
||||||
const description = data?.videoDetails?.shortDescription || "";
|
const description = data?.videoDetails?.shortDescription || "";
|
||||||
const chapters = parseChapters(description);
|
const duration = parseInt(data?.videoDetails?.lengthSeconds || "0");
|
||||||
|
const chapters = parseChapters(description, duration);
|
||||||
const langInfo = { code: result.languageCode, name: result.language, isGenerated: info.isGenerated };
|
const langInfo = { code: result.languageCode, name: result.language, isGenerated: info.isGenerated };
|
||||||
const meta = buildVideoMeta(data, videoId, langInfo, chapters);
|
const meta = buildVideoMeta(data, videoId, langInfo, chapters);
|
||||||
|
|
||||||
@@ -692,6 +699,15 @@ async function processVideo(videoId: string, opts: Options): Promise<VideoResult
|
|||||||
sentences = loadSentences(videoDir);
|
sentences = loadSentences(videoDir);
|
||||||
const wantLangs = opts.translate ? [opts.translate] : opts.languages;
|
const wantLangs = opts.translate ? [opts.translate] : opts.languages;
|
||||||
if (!wantLangs.includes(meta.language.code)) needsFetch = true;
|
if (!wantLangs.includes(meta.language.code)) needsFetch = true;
|
||||||
|
// Backfill chapter end times for caches created before this field existed
|
||||||
|
if (!needsFetch && meta.chapters.length > 0 && meta.chapters.some((ch: any) => ch.end === undefined)) {
|
||||||
|
for (let i = 0; i < meta.chapters.length; i++) {
|
||||||
|
meta.chapters[i].end = i < meta.chapters.length - 1
|
||||||
|
? meta.chapters[i + 1].start
|
||||||
|
: Math.max(meta.duration, meta.chapters[i].start);
|
||||||
|
}
|
||||||
|
try { writeFileSync(join(videoDir, "meta.json"), JSON.stringify(meta, null, 2)); } catch {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (needsFetch) {
|
if (needsFetch) {
|
||||||
|
|||||||
Reference in New Issue
Block a user