Compare commits

...

26 Commits

Author SHA1 Message Date
Jim Liu 宝玉 c1e1526c84 chore: release v1.63.0 2026-03-13 00:24:32 -05:00
Jim Liu 宝玉 0279fa403d feat(baoyu-url-to-markdown): add defuddle.md API fallback, YouTube transcripts, and modular converter architecture 2026-03-13 00:22:03 -05:00
Jim Liu 宝玉 994e47d1be docs: fix Claude Code marketplace repo casing 2026-03-12 12:46:25 -05:00
Jim Liu 宝玉 6a71d5080d chore: release v1.62.0 2026-03-12 01:51:18 -05:00
Jim Liu 宝玉 1e6e6637ac feat(baoyu-infographic): support flexible aspect ratios
Accept custom W:H ratios (e.g., 3:4, 4:3, 2.35:1) in addition to
named presets (landscape/portrait/square).
2026-03-12 01:50:36 -05:00
Jim Liu 宝玉 873b60aee5 docs(wechat): replace credential-like placeholders 2026-03-12 00:53:33 -05:00
Jim Liu 宝玉 b88ac59133 fix: set strict mode on plugins to prevent duplicated slash commands
Closes #79
2026-03-11 23:29:27 -05:00
Jim Liu 宝玉 d889c04084 chore: release v1.61.0 2026-03-11 23:24:43 -05:00
Jim Liu 宝玉 5276fae6bd feat(baoyu-post-to-wechat): add multi-account support
Support multiple WeChat Official Accounts via EXTEND.md accounts block.
Each account gets isolated Chrome profile, credential resolution chain
(inline → prefixed env → unprefixed env), and --account CLI arg.
2026-03-11 23:24:02 -05:00
Jim Liu 宝玉 c0941f8089 fix: exclude out/dist/build dirs and bun.lockb from skill release files 2026-03-11 22:55:47 -05:00
Jim Liu 宝玉 42b8b1fc99 fix: use proper MIME types in skill publish to fix ClawhHub rejection 2026-03-11 22:46:06 -05:00
Jim Liu 宝玉 53c788eb3b chore: release v1.60.0 2026-03-11 22:22:03 -05:00
Jim Liu 宝玉 c79066e96e fix(baoyu-post-to-x): add missing fs import in x-article 2026-03-11 22:18:11 -05:00
Jim Liu 宝玉 05dba5c320 refactor: publish skills directly from synced vendor 2026-03-11 21:36:19 -05:00
Jim Liu 宝玉 270a9af804 refactor: simplify CLAUDE.md, move detailed docs to docs/
CLAUDE.md reduced from 621 to 76 lines (↓88%). Detailed guidelines
moved to on-demand reference files under docs/.
2026-03-11 21:01:59 -05:00
Jim Liu 宝玉 3bba18c1fe build: commit vendored shared skill packages 2026-03-11 20:45:25 -05:00
Jim Liu 宝玉 069c5dc7d7 refactor: unify skill cdp and release artifacts 2026-03-11 19:38:59 -05:00
Jim Liu 宝玉 00bf946403 支持复用已有 Chrome CDP 实例,修复端口检测顺序问题 2026-03-11 17:24:18 -05:00
Jim Liu 宝玉 1cb54420e0 chore: release v1.59.1 2026-03-11 17:08:33 -05:00
Jim Liu 宝玉 6363bd83e2 refactor(scripts): replace clawhub CLI with local sync-clawhub.mjs 2026-03-11 16:45:57 -05:00
Jim Liu 宝玉 7b8247544d refactor(baoyu-post-to-weibo): add entry point guard to md-to-html.ts 2026-03-11 16:45:56 -05:00
Jim Liu 宝玉 a9576ebc67 fix(baoyu-post-to-x): remove disable-blink-features Chrome flag 2026-03-11 16:45:55 -05:00
Jim Liu 宝玉 79ca378229 fix(baoyu-translate): improve annotation density and style preset passing
- Add short text (<5 sentences) annotation reduction rule to translator's notes
- Explicitly pass resolved style preset to 02-prompt.md assembly in all modes
2026-03-11 16:45:51 -05:00
Jim Liu 宝玉 e1a1fe23cb Merge pull request #77 from jackL1020/claude/claude-md-mmlh0po6953zhhbf-D8rFo
docs: Update CLAUDE.md with v1.59.0 features and skill registry
2026-03-11 13:40:27 -05:00
Claude 3d85a7e663 docs: fix marketplace.json path (it's in .claude-plugin/, not root)
https://claude.ai/code/session_01DREj1A7KKhNVE1RXjmg9ZH
2026-03-11 03:28:56 +00:00
Claude 10aabb39f8 docs: update CLAUDE.md to reflect v1.59.0 codebase state
- Add 8 missing skills to architecture tree (baoyu-post-to-weibo,
  baoyu-infographic, baoyu-image-gen, baoyu-url-to-markdown,
  baoyu-format-markdown, baoyu-markdown-to-html, baoyu-translate)
- Fix marketplace.json path (root, not .claude-plugin/)
- Rename SKILL_DIR → baseDir in Script Directory template (v1.57.0 change)
- Add ClawHub/OpenClaw publishing section with sync-clawhub.sh usage
- Add openclaw frontmatter template and requirement for new skills
- Document baoyu-image-gen batch parallel generation capability
- Update Project Overview to mention all AI providers
- Expand Key Dependencies and Choosing a Category tables
- Add image-gen API key requirement note

https://claude.ai/code/session_01DREj1A7KKhNVE1RXjmg9ZH
2026-03-11 03:28:14 +00:00
76 changed files with 7285 additions and 3111 deletions
+4 -4
View File
@@ -6,14 +6,14 @@
},
"metadata": {
"description": "Skills shared by Baoyu for improving daily work efficiency",
"version": "1.59.0"
"version": "1.63.0"
},
"plugins": [
{
"name": "content-skills",
"description": "Content generation and publishing skills",
"source": "./",
"strict": false,
"strict": true,
"skills": [
"./skills/baoyu-xhs-images",
"./skills/baoyu-post-to-x",
@@ -30,7 +30,7 @@
"name": "ai-generation-skills",
"description": "AI-powered generation backends",
"source": "./",
"strict": false,
"strict": true,
"skills": [
"./skills/baoyu-danger-gemini-web",
"./skills/baoyu-image-gen"
@@ -40,7 +40,7 @@
"name": "utility-skills",
"description": "Utility tools for content processing",
"source": "./",
"strict": false,
"strict": true,
"skills": [
"./skills/baoyu-danger-x-to-markdown",
"./skills/baoyu-compress-image",
+29
View File
@@ -35,6 +35,7 @@ Just run `/release-skills` - auto-detects your project configuration.
### Step 1: Detect Project Configuration
1. Check for `.releaserc.yml` (optional config override)
- If present, inspect whether it defines release hooks
2. Auto-detect version file by scanning (priority order):
- `package.json` (Node.js)
- `pyproject.toml` (Python)
@@ -48,6 +49,34 @@ Just run `/release-skills` - auto-detects your project configuration.
4. Identify language of each changelog by filename suffix
5. Display detected configuration
**Project Hook Contract**:
If `.releaserc.yml` defines `release.hooks`, keep the release workflow generic and delegate project-specific packaging/publishing to those hooks.
Supported hooks:
| Hook | Purpose | Expected Responsibility |
|------|---------|-------------------------|
| `prepare_artifact` | Make one target releasable | Validate the target is self-contained, sync/embed local dependencies, optionally stage extra files |
| `publish_artifact` | Publish one releasable target | Upload the prepared target (or a staged directory if the project uses one), attach version/changelog/tags |
Supported placeholders:
| Placeholder | Meaning |
|-------------|---------|
| `{project_root}` | Absolute path to repository root |
| `{target}` | Absolute path to the module/skill being released |
| `{artifact_dir}` | Absolute path to a temporary staging directory for this target, when the project uses one |
| `{version}` | Version selected by the release workflow |
| `{dry_run}` | `true` or `false` |
| `{release_notes_file}` | Absolute path to a UTF-8 file containing release notes/changelog text |
Execution rules:
- Keep the skill generic: do not hardcode registry/package-manager/project layout details into this SKILL.
- If `prepare_artifact` exists, run it once per target before publish-related checks that need the final releasable target state.
- Write release notes to a temp file and pass that file path to `publish_artifact`; do not inline multiline changelog text into shell commands.
- If hooks are absent, fall back to the default project-agnostic release workflow.
**Language Detection Rules**:
Changelog files follow the pattern `CHANGELOG_{LANG}.md` or `CHANGELOG.{lang}.md`, where `{lang}` / `{LANG}` is a language or region code.
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
REPO_ROOT=$(git rev-parse --show-toplevel)
cd "$REPO_ROOT"
node scripts/sync-shared-skill-packages.mjs --repo-root "$REPO_ROOT" --enforce-clean
+2
View File
@@ -164,3 +164,5 @@ posts/
# ClawHub local state (current and legacy directory names from the official CLI)
.clawhub/
.clawdhub/
.release-artifacts/
.worktrees/
+6
View File
@@ -0,0 +1,6 @@
release:
target_globs:
- skills/*
hooks:
prepare_artifact: node scripts/sync-shared-skill-packages.mjs --repo-root "{project_root}" --target "{target}"
publish_artifact: node scripts/publish-skill.mjs --skill-dir "{target}" --version "{version}" --changelog-file "{release_notes_file}" --dry-run "{dry_run}"
+60
View File
@@ -2,6 +2,66 @@
English | [中文](./CHANGELOG.zh.md)
## 1.63.0 - 2026-03-13
### Features
- `baoyu-url-to-markdown`: add hosted `defuddle.md` API fallback when local browser capture fails
- `baoyu-url-to-markdown`: extract YouTube transcript/caption text into markdown output
- `baoyu-url-to-markdown`: materialize shadow DOM content for better web-component page conversion
- `baoyu-url-to-markdown`: include language hint in markdown front matter when available
### Refactor
- `baoyu-url-to-markdown`: split monolithic converter into defuddle, legacy, and shared modules
### Documentation
- Fix Claude Code marketplace repo casing in READMEs
## 1.62.0 - 2026-03-12
### Features
- `baoyu-infographic`: support flexible aspect ratios with custom W:H values (e.g., 3:4, 4:3, 2.35:1) in addition to named presets
### Fixes
- Set strict mode on plugins to prevent duplicated slash commands
### Documentation
- `baoyu-post-to-wechat`: replace credential-like placeholders
## 1.61.0 - 2026-03-11
### Features
- `baoyu-post-to-wechat`: add multi-account support with `--account` CLI arg, EXTEND.md accounts block, isolated Chrome profiles, and credential resolution chain
### Fixes
- Exclude `out/dist/build` dirs and `bun.lockb` from skill release files
- Use proper MIME types in skill publish to fix ClawhHub rejection
## 1.60.0 - 2026-03-11
### Features
- `baoyu-url-to-markdown`: support reusing existing Chrome CDP instances and fix port detection order
### Fixes
- `baoyu-post-to-x`: add missing `fs` import in x-article
### Refactor
- Unify all CDP skills to use shared `baoyu-chrome-cdp` package with vendored copies
- Simplify CLAUDE.md, move detailed documentation to `docs/` directory
- Publish skills directly from synced vendor, removing separate artifact preparation step
## 1.59.1 - 2026-03-11
### Fixes
- `baoyu-translate`: improve short text annotation density rule and add explicit style preset passing to 02-prompt.md
- `baoyu-post-to-x`: remove `--disable-blink-features=AutomationControlled` Chrome flag
### Refactor
- `baoyu-post-to-weibo`: add entry point guard to md-to-html.ts for module import compatibility
- Replace clawhub CLI with local sync-clawhub.mjs script
### Documentation
- Update CLAUDE.md to reflect v1.59.0 codebase state (by @jackL1020)
## 1.59.0 - 2026-03-09
### Features
+60
View File
@@ -2,6 +2,66 @@
[English](./CHANGELOG.md) | 中文
## 1.63.0 - 2026-03-13
### 新功能
- `baoyu-url-to-markdown`:本地浏览器抓取失败时自动回退到 `defuddle.md` 托管 API
- `baoyu-url-to-markdown`:将 YouTube 字幕/文字记录提取到 Markdown 输出中
- `baoyu-url-to-markdown`:转换前展开 Shadow DOM 内容,提升 Web Component 页面的转换质量
- `baoyu-url-to-markdown`Markdown front matter 中包含语言标识(如有)
### 重构
- `baoyu-url-to-markdown`:将单体转换器拆分为 defuddle、legacy 和 shared 三个模块
### 文档
- 修复 README 中 Claude Code marketplace 仓库名大小写
## 1.62.0 - 2026-03-12
### 新功能
- `baoyu-infographic`:支持灵活宽高比,可使用自定义 W:H 值(如 3:4、4:3、2.35:1),同时保留预设名称
### 修复
- 设置插件严格模式,防止重复注册斜杠命令
### 文档
- `baoyu-post-to-wechat`:替换类似凭证的占位符
## 1.61.0 - 2026-03-11
### 新功能
- `baoyu-post-to-wechat`:新增多账号支持,通过 `--account` 参数选择账号,EXTEND.md 支持 accounts 配置块,每个账号独立 Chrome 配置目录和凭证解析链
### 修复
- 排除 `out/dist/build` 目录和 `bun.lockb` 文件,避免打包到技能发布文件中
- 修复技能发布时 MIME 类型不正确导致 ClawhHub 拒绝的问题
## 1.60.0 - 2026-03-11
### 新功能
- `baoyu-url-to-markdown`:支持复用已有 Chrome CDP 实例,修复端口检测顺序问题
### 修复
- `baoyu-post-to-x`:补充 x-article 缺失的 `fs` 导入
### 重构
- 统一所有 CDP 技能使用共享 `baoyu-chrome-cdp` 包,各技能内置 vendor 副本
- 精简 CLAUDE.md,将详细文档移至 `docs/` 目录
- 从 synced vendor 直接发布技能,移除单独的 artifact 准备步骤
## 1.59.1 - 2026-03-11
### 修复
- `baoyu-translate`:改进短文本注释密度规则,补充风格预设到 02-prompt.md 的显式传递
- `baoyu-post-to-x`:移除 `--disable-blink-features=AutomationControlled` Chrome 启动参数
### 重构
- `baoyu-post-to-weibo`:为 md-to-html.ts 添加入口守卫,支持模块导入
- 使用本地 sync-clawhub.mjs 脚本替代 clawhub CLI
### 文档
- 更新 CLAUDE.md 以反映 v1.59.0 代码库状态 (by @jackL1020)
## 1.59.0 - 2026-03-09
### 新功能
+37 -494
View File
@@ -1,533 +1,76 @@
# CLAUDE.md
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Project Overview
Claude Code marketplace plugin providing AI-powered content generation skills. Skills use Gemini Web API (reverse-engineered) for text/image generation and Chrome CDP for browser automation.
Claude Code marketplace plugin providing AI-powered content generation skills. Version: **1.63.0**.
## Architecture
Skills are organized into three plugin categories in `marketplace.json`:
Skills organized into three categories in `.claude-plugin/marketplace.json` (defines plugin metadata, version, and skill paths):
```
skills/
├── [content-skills] # Content generation and publishing
│ ├── baoyu-xhs-images/ # Xiaohongshu infographic series (1-10 images)
│ ├── baoyu-cover-image/ # Article cover images (2.35:1 aspect)
│ ├── baoyu-slide-deck/ # Presentation slides with outlines
│ ├── baoyu-article-illustrator/ # Smart illustration placement
│ ├── baoyu-comic/ # Knowledge comics (Logicomix/Ohmsha style)
│ ├── baoyu-post-to-x/ # X/Twitter posting automation
│ └── baoyu-post-to-wechat/ # WeChat Official Account posting
├── [ai-generation-skills] # AI-powered generation backends
│ └── baoyu-danger-gemini-web/ # Gemini API wrapper (text + image gen)
└── [utility-skills] # Utility tools for content processing
├── baoyu-danger-x-to-markdown/ # X/Twitter content to markdown
└── baoyu-compress-image/ # Image compression
```
**Plugin Categories**:
| Category | Description |
|----------|-------------|
| `content-skills` | Skills that generate or publish content (images, slides, comics, posts) |
| `ai-generation-skills` | Backend skills providing AI generation capabilities |
| `utility-skills` | Helper tools for content processing (conversion, compression) |
| `content-skills` | Generate or publish content (images, slides, comics, posts) |
| `ai-generation-skills` | AI generation backends |
| `utility-skills` | Content processing (conversion, compression, translation) |
Each skill contains:
- `SKILL.md` - YAML front matter (name, description) + documentation
- `scripts/` - TypeScript implementations
- `prompts/system.md` - AI generation guidelines (optional)
Each skill contains `SKILL.md` (YAML front matter + docs), optional `scripts/`, `references/`, `prompts/`.
Top-level `scripts/` contains repo maintenance utilities (sync, hooks, publish).
## Running Skills
All scripts are TypeScript, executed via Bun runtime (no build step).
### Runtime Detection (`${BUN_X}`)
Before running any script, the agent MUST detect the runtime **once per session** and set `${BUN_X}`:
TypeScript via Bun (no build step). Detect runtime once per session:
```bash
# Detect runtime (run once, reuse result)
if command -v bun &>/dev/null; then
BUN_X="bun"
elif command -v npx &>/dev/null; then
BUN_X="npx -y bun"
else
echo "Error: Neither bun nor npx found. Install bun: brew install oven-sh/bun/bun (macOS) or npm install -g bun"
exit 1
fi
if command -v bun &>/dev/null; then BUN_X="bun"
elif command -v npx &>/dev/null; then BUN_X="npx -y bun"
else echo "Error: install bun: brew install oven-sh/bun/bun or npm install -g bun"; exit 1; fi
```
| Priority | Condition | `${BUN_X}` value | Notes |
|----------|-----------|-------------------|-------|
| 1 | `bun` installed | `bun` | Fastest, native execution |
| 2 | `npx` available | `npx -y bun` | Downloads bun on first run via npm |
| 3 | Neither found | Error + install guide | `brew install oven-sh/bun/bun` (macOS) or `npm install -g bun` |
### Script Execution
```bash
${BUN_X} skills/<skill>/scripts/main.ts [options]
```
Examples:
```bash
# Text generation
${BUN_X} skills/baoyu-danger-gemini-web/scripts/main.ts "Hello"
# Image generation
${BUN_X} skills/baoyu-danger-gemini-web/scripts/main.ts --prompt "A cat" --image cat.png
# From prompt files
${BUN_X} skills/baoyu-danger-gemini-web/scripts/main.ts --promptfiles system.md content.md --image out.png
```
Execute: `${BUN_X} skills/<skill>/scripts/main.ts [options]`
## Key Dependencies
- **Bun**: TypeScript runtime (native `bun` preferred, fallback `npx -y bun`)
- **Chrome**: Required for `baoyu-danger-gemini-web` auth and `baoyu-post-to-x` automation
- **npm packages (per skill)**: Some skill subprojects include `package.json`/lockfiles and require dependency installation in their own `scripts/` directories
- **Bun**: TypeScript runtime (`bun` preferred, fallback `npx -y bun`)
- **Chrome**: Required for CDP-based skills (gemini-web, post-to-x/wechat/weibo, url-to-markdown). All CDP skills share a single profile, override via `BAOYU_CHROME_PROFILE_DIR` env var. Platform paths: [docs/chrome-profile.md](docs/chrome-profile.md)
- **Image generation APIs**: `baoyu-image-gen` requires API key (OpenAI, Google, DashScope, or Replicate) configured in EXTEND.md
- **Gemini Web auth**: Browser cookies (first run opens Chrome for login, `--login` to refresh)
## Chrome Profile (Unified)
## Security
All skills that use Chrome CDP share a **single** profile directory. Do NOT create per-skill profiles.
| Platform | Default Path |
|----------|-------------|
| macOS | `~/Library/Application Support/baoyu-skills/chrome-profile` |
| Linux | `$XDG_DATA_HOME/baoyu-skills/chrome-profile` (fallback `~/.local/share/baoyu-skills/chrome-profile`) |
| Windows | `%APPDATA%/baoyu-skills/chrome-profile` |
| WSL | Windows home `/.local/share/baoyu-skills/chrome-profile` |
**Environment variable override**: `BAOYU_CHROME_PROFILE_DIR` (takes priority, all skills respect it).
Each skill also accepts its own legacy env var as fallback (e.g., `X_BROWSER_PROFILE_DIR`), but new skills should only use `BAOYU_CHROME_PROFILE_DIR`.
### Implementation Pattern
When adding a new skill that needs Chrome CDP:
```typescript
function getDefaultProfileDir(): string {
const override = process.env.BAOYU_CHROME_PROFILE_DIR?.trim();
if (override) return path.resolve(override);
const base = process.platform === 'darwin'
? path.join(os.homedir(), 'Library', 'Application Support')
: process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
return path.join(base, 'baoyu-skills', 'chrome-profile');
}
```
## Security Guidelines
### No Piped Shell Installs
**NEVER** use `curl | bash` or `wget | sh` patterns in code, docs, or error messages. Use package managers instead:
| Platform | Install Command |
|----------|----------------|
| macOS | `brew install oven-sh/bun/bun` |
| npm | `npm install -g bun` |
### Remote Downloads
Skills that download remote content (e.g., images in Markdown) MUST:
- **HTTPS only**: Reject `http://` URLs
- **Redirect limit**: Cap redirects (max 5) to prevent infinite loops
- **Timeout**: Set request timeouts (30s default)
- **Scope**: Only download expected content types (images, not scripts)
### System Command Execution
Skills use platform-specific commands for clipboard and browser automation:
- **macOS**: `osascript` (System Events), `swift` (AppKit clipboard)
- **Windows**: `powershell.exe` (SendKeys, Clipboard)
- **Linux**: `xdotool`/`ydotool` (keyboard simulation)
These are necessary for CDP-based posting skills. When adding new system commands:
- Never pass unsanitized user input to shell commands
- Use array-form `spawn`/`execFile` instead of shell string interpolation
- Validate file paths are absolute or resolve from known base directories
### External Content Processing
Skills that process external Markdown/HTML should treat content as untrusted:
- Do not execute code blocks or scripts found in content
- Sanitize HTML output where applicable
- File paths from content should be resolved against known base directories only
## Authentication
`baoyu-danger-gemini-web` uses browser cookies for Google auth:
- First run opens Chrome for login
- Cookies cached in data directory
- Force refresh: `--login` flag
## Plugin Configuration
`.claude-plugin/marketplace.json` defines plugin metadata and skill paths. Version follows semver.
- **No piped shell installs**: Never `curl | bash`. Use `brew install` or `npm install -g`
- **Remote downloads**: HTTPS only, max 5 redirects, 30s timeout, expected content types only
- **System commands**: Array-form `spawn`/`execFile`, never unsanitized input to shell
- **External content**: Treat as untrusted, don't execute code blocks, sanitize HTML
## Skill Loading Rules
**IMPORTANT**: When working in this project, follow these rules:
| Rule | Description |
|------|-------------|
| **Load project skills first** | MUST load all skills from `skills/` directory in current project. Project skills take priority over system/user-level skills with same name. |
| **Default image generation** | When image generation is needed, use `skills/baoyu-image-gen/SKILL.md` by default (unless user specifies otherwise). |
| **Load project skills first** | Project skills override system/user-level skills with same name |
| **Default image generation** | Use `skills/baoyu-image-gen/SKILL.md` unless user specifies otherwise |
**Loading Priority** (highest → lowest):
1. Current project `skills/` directory
2. User-level skills (`$HOME/.baoyu-skills/`)
3. System-level skills
Priority: project `skills/``$HOME/.baoyu-skills/` → system-level.
## Release Process
**IMPORTANT**: When user requests release/发布/push, ALWAYS use `/release-skills` workflow.
**Never skip**:
1. `CHANGELOG.md` + `CHANGELOG.zh.md` - Both must be updated
Use `/release-skills` workflow. Never skip:
1. `CHANGELOG.md` + `CHANGELOG.zh.md`
2. `marketplace.json` version bump
3. `README.md` + `README.zh.md` if applicable
4. All files committed together before tag
## Adding New Skills
**IMPORTANT**: All skills MUST use `baoyu-` prefix to avoid conflicts when users import this plugin.
**REQUIRED READING**: Before creating a new skill, read the official [Skill authoring best practices](https://platform.claude.com/docs/en/agents-and-tools/agent-skills/best-practices).
### Key Requirements from Official Best Practices
| Requirement | Details |
|-------------|---------|
| **Concise is key** | Claude is smart—only add context it doesn't have. Challenge each token. |
| **name field** | Max 64 chars, lowercase letters/numbers/hyphens only, no "anthropic"/"claude" |
| **description field** | Max 1024 chars, non-empty, MUST be third person, include what + when to use |
| **SKILL.md body** | Keep under 500 lines; use separate files for additional content |
| **Naming convention** | Gerund form preferred (e.g., `processing-pdfs`), but `baoyu-` prefix required here |
| **References** | Keep one level deep from SKILL.md; avoid nested references |
| **No time-sensitive info** | Avoid dates/versions that become outdated |
### Steps
1. Create `skills/baoyu-<name>/SKILL.md` with YAML front matter
- Directory name: `baoyu-<name>`
- SKILL.md `name` field: `baoyu-<name>`
2. Add TypeScript in `skills/baoyu-<name>/scripts/`
3. Add prompt templates in `skills/baoyu-<name>/prompts/` if needed
4. **Choose the appropriate category** and register in `marketplace.json`:
- `content-skills`: For content generation/publishing (images, slides, posts)
- `ai-generation-skills`: For AI backend capabilities
- `utility-skills`: For helper tools (conversion, compression)
- If none fit, create a new category with descriptive name
5. **Add Script Directory section** to SKILL.md (see template below)
### Choosing a Category
| If your skill... | Use category |
|------------------|--------------|
| Generates visual content (images, slides, comics) | `content-skills` |
| Publishes to platforms (X, WeChat, etc.) | `content-skills` |
| Provides AI generation backend | `ai-generation-skills` |
| Converts or processes content | `utility-skills` |
| Compresses or optimizes files | `utility-skills` |
**Creating a new category**: If the skill doesn't fit existing categories, add a new plugin object to `marketplace.json` with:
- `name`: Descriptive kebab-case name (e.g., `analytics-skills`)
- `description`: Brief description of the category
- `skills`: Array with the skill path
### Writing Effective Descriptions
**MUST write in third person** (not "I can help you" or "You can use this"):
```yaml
# Good
description: Generates Xiaohongshu infographic series from content. Use when user asks for "小红书图片", "XHS images", or "RedNote infographics".
# Bad
description: I can help you create Xiaohongshu images
description: You can use this to generate XHS content
```
Include both **what** the skill does and **when** to use it (triggers/keywords).
### Script Directory Template
Every SKILL.md with scripts MUST include this section after Usage:
```markdown
## Script Directory
**Important**: All scripts are located in the `scripts/` subdirectory of this skill.
**Agent Execution Instructions**:
1. Determine this SKILL.md file's directory path as `SKILL_DIR`
2. Script path = `${SKILL_DIR}/scripts/<script-name>.ts`
3. Resolve `${BUN_X}` runtime: if `bun` installed → `bun`; if `npx` available → `npx -y bun`; else suggest installing bun
4. Replace all `${SKILL_DIR}` and `${BUN_X}` in this document with actual values
**Script Reference**:
| Script | Purpose |
|--------|---------|
| `scripts/main.ts` | Main entry point |
| `scripts/other.ts` | Other functionality |
```
When referencing scripts in workflow sections, use `${BUN_X} ${SKILL_DIR}/scripts/<name>.ts` so agents can resolve the correct runtime and path.
### Progressive Disclosure
For skills with extensive content, use separate reference files:
```
skills/baoyu-example/
├── SKILL.md # Main instructions (<500 lines)
├── references/
│ ├── styles.md # Style definitions (loaded as needed)
│ └── examples.md # Usage examples (loaded as needed)
└── scripts/
└── main.ts # Executable script
```
In SKILL.md, link to reference files (one level deep only):
```markdown
**Available styles**: See [references/styles.md](references/styles.md)
**Examples**: See [references/examples.md](references/examples.md)
```
## Code Style
- TypeScript throughout, no comments
- Async/await patterns
- Short variable names
- Type-safe interfaces
TypeScript, no comments, async/await, short variable names, type-safe interfaces.
## Image Generation Guidelines
## Adding New Skills
Skills that require image generation MUST delegate to available image generation skills rather than implementing their own.
All skills MUST use `baoyu-` prefix. Details: [docs/creating-skills.md](docs/creating-skills.md)
### Image Generation Skill Selection
## Reference Docs
**Default**: Use `skills/baoyu-image-gen/SKILL.md` (unless user specifies otherwise).
1. Read `skills/baoyu-image-gen/SKILL.md` for parameters and capabilities
2. If user explicitly requests a different skill, check `skills/` directory for alternatives
3. Only ask user to choose when they haven't specified and multiple viable options exist
### Generation Flow Template
Use this template when implementing image generation in skills:
```markdown
### Step N: Generate Images
**Skill Selection**:
1. Check available image generation skills (e.g., `baoyu-danger-gemini-web`)
2. Read selected skill's SKILL.md for parameter reference
3. If multiple skills available, ask user to choose
**Generation Flow**:
1. Call selected image generation skill with:
- Prompt file path (or inline prompt)
- Output image path
- Any skill-specific parameters (refer to skill's SKILL.md)
2. Generate images sequentially (one at a time)
3. After each image, output progress: "Generated X/N"
4. On failure, auto-retry once before reporting error
```
### Output Path Convention
Each session creates an independent directory. Even the same source file generates a new directory per session.
**Output Directory**:
```
<skill-suffix>/<topic-slug>/
```
- `<skill-suffix>`: Skill name suffix (e.g., `xhs-images`, `cover-image`, `slide-deck`, `comic`)
- `<topic-slug>`: Generated from content topic (2-4 words, kebab-case)
- Example: `xhs-images/ai-future-trends/`
**Slug Generation**:
1. Extract main topic from content (2-4 words, kebab-case)
2. Example: "Introduction to Machine Learning" → `intro-machine-learning`
**Conflict Resolution**:
If `<skill-suffix>/<topic-slug>/` already exists:
- Append timestamp: `<topic-slug>-YYYYMMDD-HHMMSS`
- Example: `ai-future` exists → `ai-future-20260118-143052`
**Source Files**:
- Copy all sources to `<skill-suffix>/<topic-slug>/` with naming: `source-{slug}.{ext}`
- Multiple sources supported: text, images, files from conversation
- Examples:
- `source-article.md` (main text content)
- `source-reference.png` (image from conversation)
- `source-data.csv` (additional file)
- Original source files remain unchanged
### Image Naming Convention
Image filenames MUST include meaningful slugs for readability:
**Format**: `NN-{type}-[slug].png`
- `NN`: Two-digit sequence number (01, 02, ...)
- `{type}`: Image type (cover, content, page, slide, illustration, etc.)
- `[slug]`: Descriptive kebab-case slug derived from content
**Examples**:
```
01-cover-ai-future.png
02-content-key-benefits.png
03-page-enigma-machine.png
04-slide-architecture-overview.png
```
**Slug Rules**:
- Derived from image purpose or content (kebab-case)
- Must be unique within the output directory
- 2-5 words, concise but descriptive
- When content changes significantly, update slug accordingly
### Best Practices
- Always read the image generation skill's SKILL.md before calling
- Pass parameters exactly as documented in the skill
- Handle failures gracefully with retry logic
- Provide clear progress feedback to user
## Style Maintenance (baoyu-comic)
When adding, updating, or deleting styles for `baoyu-comic`, follow this workflow:
### Adding a New Style
1. **Create style definition**: `skills/baoyu-comic/references/styles/<style-name>.md`
2. **Update SKILL.md**:
- Add style to `--style` options table
- Add auto-selection entry if applicable
3. **Generate showcase image**:
```bash
${BUN_X} skills/baoyu-danger-gemini-web/scripts/main.ts \
--prompt "A single comic book page in <style-name> style showing [appropriate scene]. Features: [style characteristics from style definition]. 3:4 portrait aspect ratio comic page." \
--image screenshots/comic-styles/<style-name>.png
```
4. **Compress to WebP**:
```bash
${BUN_X} skills/baoyu-compress-image/scripts/main.ts screenshots/comic-styles/<style-name>.png
```
5. **Update both READMEs** (`README.md` and `README.zh.md`):
- Add style to `--style` options
- Add row to style description table
- Add image to style preview grid
### Updating an Existing Style
1. **Update style definition**: `skills/baoyu-comic/references/styles/<style-name>.md`
2. **Regenerate showcase image** (if visual characteristics changed):
- Follow steps 3-4 from "Adding a New Style"
3. **Update READMEs** if description changed
### Deleting a Style
1. **Delete style definition**: `skills/baoyu-comic/references/styles/<style-name>.md`
2. **Delete showcase image**: `screenshots/comic-styles/<style-name>.webp`
3. **Update SKILL.md**:
- Remove from `--style` options
- Remove auto-selection entry
4. **Update both READMEs**:
- Remove from `--style` options
- Remove from style description table
- Remove from style preview grid
### Style Preview Grid Format
READMEs use 3-column tables for style previews:
```markdown
| | | |
|:---:|:---:|:---:|
| ![style1](./screenshots/comic-styles/style1.webp) | ![style2](./screenshots/comic-styles/style2.webp) | ![style3](./screenshots/comic-styles/style3.webp) |
| style1 | style2 | style3 |
```
## Extension Support
Every SKILL.md MUST include two parts for extension support:
### 1. Load Preferences Section (in Step 1 or as "Preferences" section)
For skills with workflows, add as Step 1.1. For utility skills, add as "Preferences (EXTEND.md)" section before Usage:
```markdown
**1.1 Load Preferences (EXTEND.md)**
Check EXTEND.md existence (priority order):
\`\`\`bash
# macOS, Linux, WSL, Git Bash
test -f .baoyu-skills/<skill-name>/EXTEND.md && echo "project"
test -f "${XDG_CONFIG_HOME:-$HOME/.config}/baoyu-skills/<skill-name>/EXTEND.md" && echo "xdg"
test -f "$HOME/.baoyu-skills/<skill-name>/EXTEND.md" && echo "user"
\`\`\`
\`\`\`powershell
# PowerShell (Windows)
if (Test-Path .baoyu-skills/<skill-name>/EXTEND.md) { "project" }
$xdg = if ($env:XDG_CONFIG_HOME) { $env:XDG_CONFIG_HOME } else { "$HOME/.config" }
if (Test-Path "$xdg/baoyu-skills/<skill-name>/EXTEND.md") { "xdg" }
if (Test-Path "$HOME/.baoyu-skills/<skill-name>/EXTEND.md") { "user" }
\`\`\`
┌────────────────────────────────────────────────────────┬──────────────────────────┐
│ Path │ Location │
├────────────────────────────────────────────────────────┼──────────────────────────┤
│ .baoyu-skills/<skill-name>/EXTEND.md │ Project directory │
├────────────────────────────────────────────────────────┼──────────────────────────┤
│ $XDG_CONFIG_HOME/baoyu-skills/<skill-name>/EXTEND.md │ XDG config (~/.config) │
├────────────────────────────────────────────────────────┼──────────────────────────┤
│ $HOME/.baoyu-skills/<skill-name>/EXTEND.md │ User home (legacy) │
└────────────────────────────────────────────────────────┴──────────────────────────┘
┌───────────┬───────────────────────────────────────────────────────────────────────────┐
│ Result │ Action │
├───────────┼───────────────────────────────────────────────────────────────────────────┤
│ Found │ Read, parse, display summary │
├───────────┼───────────────────────────────────────────────────────────────────────────┤
│ Not found │ Ask user with AskUserQuestion (see references/config/first-time-setup.md) │
└───────────┴───────────────────────────────────────────────────────────────────────────┘
**EXTEND.md Supports**: [List supported configuration options for this skill]
Schema: `references/config/preferences-schema.md`
```
### 2. Extension Support Section (at the end)
Simplified section that references the preferences section:
```markdown
## Extension Support
Custom configurations via EXTEND.md. See **Step 1.1** for paths and supported options.
```
Or for utility skills without workflow steps:
```markdown
## Extension Support
Custom configurations via EXTEND.md. See **Preferences** section for paths and supported options.
```
**Notes**:
- Replace `<skill-name>` with actual skill name (e.g., `baoyu-cover-image`)
- Use `$HOME` instead of `~` for cross-platform compatibility (macOS/Linux/WSL/PowerShell)
- `$XDG_CONFIG_HOME` defaults to `~/.config` when unset
- Use `test -f` (Bash) or `Test-Path` (PowerShell) for explicit file existence check
- ASCII tables for clear visual formatting
| Topic | File |
|-------|------|
| Image generation guidelines | [docs/image-generation.md](docs/image-generation.md) |
| Chrome profile platform paths | [docs/chrome-profile.md](docs/chrome-profile.md) |
| Comic style maintenance | [docs/comic-style-maintenance.md](docs/comic-style-maintenance.md) |
| ClawHub/OpenClaw publishing | [docs/publishing.md](docs/publishing.md) |
+45 -3
View File
@@ -43,7 +43,7 @@ Publishing to ClawHub releases the published skill under `MIT-0`, per ClawHub's
Run the following command in Claude Code:
```bash
/plugin marketplace add jimliu/baoyu-skills
/plugin marketplace add JimLiu/baoyu-skills
```
### Install Skills
@@ -169,8 +169,9 @@ Generate professional infographics with 20 layout types and 17 visual styles. An
# Specify both
/baoyu-infographic path/to/content.md --layout funnel --style corporate-memphis
# With aspect ratio
# With aspect ratio (named preset or custom W:H)
/baoyu-infographic path/to/content.md --aspect portrait
/baoyu-infographic path/to/content.md --aspect 3:4
```
**Options**:
@@ -178,7 +179,7 @@ Generate professional infographics with 20 layout types and 17 visual styles. An
|--------|-------------|
| `--layout <name>` | Information layout (20 options) |
| `--style <name>` | Visual style (17 options, default: craft-handmade) |
| `--aspect <ratio>` | landscape (16:9), portrait (9:16), square (1:1) |
| `--aspect <ratio>` | Named: landscape (16:9), portrait (9:16), square (1:1). Custom: any W:H ratio (e.g., 3:4, 4:3, 2.35:1) |
| `--lang <code>` | Output language (en, zh, ja, etc.) |
**Layouts** (information structure):
@@ -581,6 +582,47 @@ To obtain credentials:
**Browser Method** (no API setup needed): Requires Google Chrome. First run opens browser for QR code login (session preserved).
**Multi-Account Support**: Manage multiple WeChat Official Accounts via `EXTEND.md`:
```bash
mkdir -p .baoyu-skills/baoyu-post-to-wechat
```
Create `.baoyu-skills/baoyu-post-to-wechat/EXTEND.md`:
```yaml
# Global settings (shared across all accounts)
default_theme: default
default_color: blue
# Account list
accounts:
- name: My Tech Blog
alias: tech-blog
default: false
default_publish_method: api
default_author: Author Name
need_open_comment: 1
only_fans_can_comment: 0
app_id: your_wechat_app_id
app_secret: your_wechat_app_secret
- name: AI Newsletter
alias: ai-news
default_publish_method: browser
default_author: AI Newsletter
need_open_comment: 1
only_fans_can_comment: 0
```
| Accounts configured | Behavior |
|---------------------|----------|
| No `accounts` block | Single-account mode (backward compatible) |
| 1 account | Auto-select, no prompt |
| 2+ accounts | Prompt to select, or use `--account <alias>` |
| 1 account has `default: true` | Pre-selected as default |
Each account gets an isolated Chrome profile for independent login sessions (browser method). API credentials can be set inline in EXTEND.md or via `.env` with alias-prefixed keys (e.g., `WECHAT_TECH_BLOG_APP_ID`).
#### baoyu-post-to-weibo
Post content to Weibo (微博). Supports regular posts with text, images, and videos, and headline articles (头条文章) with Markdown input. Uses real Chrome with CDP to bypass anti-automation.
+45 -3
View File
@@ -43,7 +43,7 @@ clawhub install baoyu-markdown-to-html
在 Claude Code 中运行:
```bash
/plugin marketplace add jimliu/baoyu-skills
/plugin marketplace add JimLiu/baoyu-skills
```
### 安装技能
@@ -169,8 +169,9 @@ clawhub install baoyu-markdown-to-html
# 同时指定布局和风格
/baoyu-infographic path/to/content.md --layout funnel --style corporate-memphis
# 指定比例
# 指定比例(预设名称或自定义 W:H
/baoyu-infographic path/to/content.md --aspect portrait
/baoyu-infographic path/to/content.md --aspect 3:4
```
**选项**
@@ -178,7 +179,7 @@ clawhub install baoyu-markdown-to-html
|------|------|
| `--layout <name>` | 信息布局(20 种选项) |
| `--style <name>` | 视觉风格(17 种选项,默认:craft-handmade |
| `--aspect <ratio>` | landscape (16:9)、portrait (9:16)、square (1:1) |
| `--aspect <ratio>` | 预设:landscape (16:9)、portrait (9:16)、square (1:1)。自定义:任意 W:H 比例(如 3:4、4:3、2.35:1 |
| `--lang <code>` | 输出语言(en、zh、ja 等) |
**布局**(信息结构):
@@ -581,6 +582,47 @@ WECHAT_APP_SECRET=你的AppSecret
**浏览器方式**(无需 API 配置):需已安装 Google Chrome,首次运行需扫码登录(登录状态会保存)
**多账号支持**:通过 `EXTEND.md` 管理多个微信公众号:
```bash
mkdir -p .baoyu-skills/baoyu-post-to-wechat
```
创建 `.baoyu-skills/baoyu-post-to-wechat/EXTEND.md`
```yaml
# 全局设置(所有账号共享)
default_theme: default
default_color: blue
# 账号列表
accounts:
- name: 宝玉的技术分享
alias: baoyu
default: false
default_publish_method: api
default_author: 宝玉
need_open_comment: 1
only_fans_can_comment: 0
app_id: 你的微信AppID
app_secret: 你的微信AppSecret
- name: AI 工具集
alias: ai-tools
default_publish_method: browser
default_author: AI 工具集
need_open_comment: 1
only_fans_can_comment: 0
```
| 账号配置情况 | 行为 |
|-------------|------|
| 无 `accounts` 块 | 单账号模式(向后兼容) |
| 1 个账号 | 自动选择,无需提示 |
| 2+ 个账号 | 提示选择,或使用 `--account <别名>` |
| 某账号设置 `default: true` | 预选为默认账号 |
每个账号拥有独立的 Chrome 配置目录,保证浏览器方式下的登录会话互不干扰。API 凭证可在 EXTEND.md 中直接配置,也可通过 `.env` 文件使用别名前缀的环境变量(如 `WECHAT_BAOYU_APP_ID`)。
#### baoyu-post-to-weibo
发布内容到微博。支持文字、图片、视频发布和头条文章(长篇 Markdown)。使用真实 Chrome + CDP 绕过反自动化检测。
+27
View File
@@ -0,0 +1,27 @@
# Chrome Profile
All CDP skills share a single profile directory. Do NOT create per-skill profiles.
Override: `BAOYU_CHROME_PROFILE_DIR` env var (takes priority over all defaults).
| Platform | Default Path |
|----------|-------------|
| macOS | `~/Library/Application Support/baoyu-skills/chrome-profile` |
| Linux | `$XDG_DATA_HOME/baoyu-skills/chrome-profile` (fallback `~/.local/share/`) |
| Windows | `%APPDATA%/baoyu-skills/chrome-profile` |
| WSL | Windows home `/.local/share/baoyu-skills/chrome-profile` |
New skills: use `BAOYU_CHROME_PROFILE_DIR` only (not per-skill env vars like `X_BROWSER_PROFILE_DIR`).
## Implementation Pattern
```typescript
function getDefaultProfileDir(): string {
const override = process.env.BAOYU_CHROME_PROFILE_DIR?.trim();
if (override) return path.resolve(override);
const base = process.platform === 'darwin'
? path.join(os.homedir(), 'Library', 'Application Support')
: process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
return path.join(base, 'baoyu-skills', 'chrome-profile');
}
```
+35
View File
@@ -0,0 +1,35 @@
# Style Maintenance (baoyu-comic)
## Adding a New Style
1. Create style definition: `skills/baoyu-comic/references/styles/<style-name>.md`
2. Update SKILL.md: add to `--style` options table + auto-selection entry
3. Generate showcase image:
```bash
${BUN_X} skills/baoyu-danger-gemini-web/scripts/main.ts \
--prompt "A single comic book page in <style-name> style showing [scene]. Features: [characteristics]. 3:4 portrait aspect ratio comic page." \
--image screenshots/comic-styles/<style-name>.png
```
4. Compress: `${BUN_X} skills/baoyu-compress-image/scripts/main.ts screenshots/comic-styles/<style-name>.png`
5. Update both READMEs (`README.md` + `README.zh.md`): add style to options, description table, preview grid
## Updating an Existing Style
1. Update style definition in `references/styles/`
2. Regenerate showcase image if visual characteristics changed (steps 3-4 above)
3. Update READMEs if description changed
## Deleting a Style
1. Delete style definition + showcase image (`.webp`)
2. Remove from SKILL.md `--style` options + auto-selection
3. Remove from both READMEs (options, description table, preview grid)
## Style Preview Grid Format
```markdown
| | | |
|:---:|:---:|:---:|
| ![style1](./screenshots/comic-styles/style1.webp) | ![style2](./screenshots/comic-styles/style2.webp) | ![style3](./screenshots/comic-styles/style3.webp) |
| style1 | style2 | style3 |
```
+135
View File
@@ -0,0 +1,135 @@
# Creating New Skills
**REQUIRED READING**: [Skill authoring best practices](https://platform.claude.com/docs/en/agents-and-tools/agent-skills/best-practices)
## Key Requirements
| Requirement | Details |
|-------------|---------|
| **Prefix** | All skills MUST use `baoyu-` prefix |
| **name field** | Max 64 chars, lowercase letters/numbers/hyphens only, no "anthropic"/"claude" |
| **description** | Max 1024 chars, third person, include what + when to use |
| **SKILL.md body** | Keep under 500 lines; use `references/` for additional content |
| **References** | One level deep from SKILL.md; avoid nested references |
## SKILL.md Frontmatter Template
```yaml
---
name: baoyu-<name>
description: <Third-person description. What it does + when to use it.>
version: <semver matching marketplace.json>
metadata:
openclaw:
homepage: https://github.com/JimLiu/baoyu-skills#baoyu-<name>
requires: # include only if skill has scripts
anyBins:
- bun
- npx
---
```
## Steps
1. Create `skills/baoyu-<name>/SKILL.md` with YAML front matter
2. Add TypeScript in `skills/baoyu-<name>/scripts/` (if applicable)
3. Add prompt templates in `skills/baoyu-<name>/prompts/` if needed
4. Register in `marketplace.json` under appropriate category
5. Add Script Directory section to SKILL.md if skill has scripts
6. Add openclaw metadata to frontmatter
## Category Selection
| If your skill... | Use category |
|------------------|--------------|
| Generates visual content (images, slides, comics) | `content-skills` |
| Publishes to platforms (X, WeChat, Weibo) | `content-skills` |
| Provides AI generation backend | `ai-generation-skills` |
| Converts or processes content | `utility-skills` |
New category: add plugin object to `marketplace.json` with `name`, `description`, `skills[]`.
## Writing Descriptions
**MUST write in third person**:
```yaml
# Good
description: Generates Xiaohongshu infographic series from content. Use when user asks for "小红书图片", "XHS images".
# Bad
description: I can help you create Xiaohongshu images
```
## Script Directory Template
Every SKILL.md with scripts MUST include:
```markdown
## Script Directory
**Important**: All scripts are located in the `scripts/` subdirectory of this skill.
**Agent Execution Instructions**:
1. Determine this SKILL.md file's directory path as `{baseDir}`
2. Script path = `{baseDir}/scripts/<script-name>.ts`
3. Resolve `${BUN_X}` runtime: if `bun` installed → `bun`; if `npx` available → `npx -y bun`; else suggest installing bun
4. Replace all `{baseDir}` and `${BUN_X}` in this document with actual values
**Script Reference**:
| Script | Purpose |
|--------|---------|
| `scripts/main.ts` | Main entry point |
```
## Progressive Disclosure
For skills with extensive content:
```
skills/baoyu-example/
├── SKILL.md # Main instructions (<500 lines)
├── references/
│ ├── styles.md # Loaded as needed
│ └── examples.md # Loaded as needed
└── scripts/
└── main.ts
```
Link from SKILL.md (one level deep only):
```markdown
**Available styles**: See [references/styles.md](references/styles.md)
```
## Extension Support (EXTEND.md)
Every SKILL.md MUST include EXTEND.md loading. Add as Step 1.1 (workflow skills) or "Preferences" section (utility skills):
```markdown
**1.1 Load Preferences (EXTEND.md)**
Check EXTEND.md existence (priority order):
\`\`\`bash
test -f .baoyu-skills/<skill-name>/EXTEND.md && echo "project"
test -f "${XDG_CONFIG_HOME:-$HOME/.config}/baoyu-skills/<skill-name>/EXTEND.md" && echo "xdg"
test -f "$HOME/.baoyu-skills/<skill-name>/EXTEND.md" && echo "user"
\`\`\`
| Path | Location |
|------|----------|
| `.baoyu-skills/<skill-name>/EXTEND.md` | Project directory |
| `$XDG_CONFIG_HOME/baoyu-skills/<skill-name>/EXTEND.md` | XDG config (~/.config) |
| `$HOME/.baoyu-skills/<skill-name>/EXTEND.md` | User home (legacy) |
| Result | Action |
|--------|--------|
| Found | Read, parse, display summary |
| Not found | Ask user with AskUserQuestion |
```
End of SKILL.md should include:
```markdown
## Extension Support
Custom configurations via EXTEND.md. See **Step 1.1** for paths and supported options.
```
+53
View File
@@ -0,0 +1,53 @@
# Image Generation Guidelines
Skills that require image generation MUST delegate to available image generation skills.
## Skill Selection
**Default**: `skills/baoyu-image-gen/SKILL.md` (unless user specifies otherwise).
1. Read skill's SKILL.md for parameters and capabilities
2. If user requests different skill, check `skills/` for alternatives
3. Only ask user when multiple viable options exist
## Generation Flow Template
```markdown
### Step N: Generate Images
**Skill Selection**:
1. Check available skills (`baoyu-image-gen` default, or `baoyu-danger-gemini-web`)
2. Read selected skill's SKILL.md for parameters
3. If multiple skills available, ask user to choose
**Generation Flow**:
1. Call skill with prompt, output path, and skill-specific parameters
2. Generate sequentially by default (batch parallel only when user has multiple prompts)
3. Output progress: "Generated X/N"
4. On failure, auto-retry once before reporting error
```
**Batch Parallel** (`baoyu-image-gen` only): concurrent workers with per-provider throttling via `batch.max_workers` in EXTEND.md.
## Output Path Convention
**Output Directory**: `<skill-suffix>/<topic-slug>/`
- `<skill-suffix>`: e.g., `xhs-images`, `cover-image`, `slide-deck`, `comic`
- `<topic-slug>`: 2-4 words, kebab-case from content topic
- Conflict: append timestamp `<topic-slug>-YYYYMMDD-HHMMSS`
**Source Files**: Copy to output dir as `source-{slug}.{ext}`
## Image Naming Convention
**Format**: `NN-{type}-[slug].png`
- `NN`: Two-digit sequence (01, 02, ...)
- `{type}`: cover, content, page, slide, illustration, etc.
- `[slug]`: 2-5 word kebab-case descriptor, unique within directory
Examples:
```
01-cover-ai-future.png
02-content-key-benefits.png
03-slide-architecture-overview.png
```
+39
View File
@@ -0,0 +1,39 @@
# ClawHub / OpenClaw Publishing
## OpenClaw Metadata
Skills include `metadata.openclaw` in YAML front matter:
```yaml
metadata:
openclaw:
homepage: https://github.com/JimLiu/baoyu-skills#<skill-name>
requires: # only for skills with scripts
anyBins:
- bun
- npx
```
## Publishing Commands
```bash
bash scripts/sync-clawhub.sh # sync all skills
bash scripts/sync-clawhub.sh <skill> # sync one skill
```
Release hooks are configured via `.releaserc.yml`. This repo does not stage a separate release directory: release prep only syncs `packages/` into each skill's committed `scripts/vendor/`, and publish reads the skill directory directly.
## Shared Workspace Packages
`packages/` is the **only** source of truth. Never edit `skills/*/scripts/vendor/` directly.
Current package: `baoyu-chrome-cdp` (Chrome CDP utilities), consumed by 6 skills (`baoyu-danger-gemini-web`, `baoyu-danger-x-to-markdown`, `baoyu-post-to-wechat`, `baoyu-post-to-weibo`, `baoyu-post-to-x`, `baoyu-url-to-markdown`).
**How it works**: Sync script copies packages into each consuming skill's `vendor/` directory and rewrites dependency specs to `file:./vendor/<name>`. Vendor copies are committed to git, making skills self-contained.
**Update workflow**:
1. Edit package under `packages/`
2. Run `node scripts/sync-shared-skill-packages.mjs`
3. Commit synced `vendor/`, `package.json`, and `bun.lock` together
**Git hook**: Run `node scripts/install-git-hooks.mjs` once to enable the `pre-push` hook. It re-syncs and blocks push if vendor copies are stale (`--enforce-clean`).
+9
View File
@@ -0,0 +1,9 @@
{
"name": "baoyu-chrome-cdp",
"private": true,
"version": "0.1.0",
"type": "module",
"exports": {
".": "./src/index.ts"
}
}
+408
View File
@@ -0,0 +1,408 @@
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import process from "node:process";
export type PlatformCandidates = {
darwin?: string[];
win32?: string[];
default: string[];
};
type PendingRequest = {
resolve: (value: unknown) => void;
reject: (error: Error) => void;
timer: ReturnType<typeof setTimeout> | null;
};
type CdpSendOptions = {
sessionId?: string;
timeoutMs?: number;
};
type FetchJsonOptions = {
timeoutMs?: number;
};
type FindChromeExecutableOptions = {
candidates: PlatformCandidates;
envNames?: string[];
};
type ResolveSharedChromeProfileDirOptions = {
envNames?: string[];
appDataDirName?: string;
profileDirName?: string;
wslWindowsHome?: string | null;
};
type FindExistingChromeDebugPortOptions = {
profileDir: string;
timeoutMs?: number;
};
type LaunchChromeOptions = {
chromePath: string;
profileDir: string;
port: number;
url?: string;
headless?: boolean;
extraArgs?: string[];
};
type ChromeTargetInfo = {
targetId: string;
url: string;
type: string;
};
type OpenPageSessionOptions = {
cdp: CdpConnection;
reusing: boolean;
url: string;
matchTarget: (target: ChromeTargetInfo) => boolean;
enablePage?: boolean;
enableRuntime?: boolean;
enableDom?: boolean;
enableNetwork?: boolean;
activateTarget?: boolean;
};
export type PageSession = {
sessionId: string;
targetId: string;
};
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export async function getFreePort(fixedEnvName?: string): Promise<number> {
const fixed = fixedEnvName ? Number.parseInt(process.env[fixedEnvName] ?? "", 10) : NaN;
if (Number.isInteger(fixed) && fixed > 0) return fixed;
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Unable to allocate a free TCP port.")));
return;
}
const port = address.port;
server.close((err) => {
if (err) reject(err);
else resolve(port);
});
});
});
}
export function findChromeExecutable(options: FindChromeExecutableOptions): string | undefined {
for (const envName of options.envNames ?? []) {
const override = process.env[envName]?.trim();
if (override && fs.existsSync(override)) return override;
}
const candidates = process.platform === "darwin"
? options.candidates.darwin ?? options.candidates.default
: process.platform === "win32"
? options.candidates.win32 ?? options.candidates.default
: options.candidates.default;
for (const candidate of candidates) {
if (fs.existsSync(candidate)) return candidate;
}
return undefined;
}
export function resolveSharedChromeProfileDir(options: ResolveSharedChromeProfileDirOptions = {}): string {
for (const envName of options.envNames ?? []) {
const override = process.env[envName]?.trim();
if (override) return path.resolve(override);
}
const appDataDirName = options.appDataDirName ?? "baoyu-skills";
const profileDirName = options.profileDirName ?? "chrome-profile";
if (options.wslWindowsHome) {
return path.join(options.wslWindowsHome, ".local", "share", appDataDirName, profileDirName);
}
const base = process.platform === "darwin"
? path.join(os.homedir(), "Library", "Application Support")
: process.platform === "win32"
? (process.env.APPDATA ?? path.join(os.homedir(), "AppData", "Roaming"))
: (process.env.XDG_DATA_HOME ?? path.join(os.homedir(), ".local", "share"));
return path.join(base, appDataDirName, profileDirName);
}
async function fetchWithTimeout(url: string, timeoutMs?: number): Promise<Response> {
if (!timeoutMs || timeoutMs <= 0) return await fetch(url, { redirect: "follow" });
const ctl = new AbortController();
const timer = setTimeout(() => ctl.abort(), timeoutMs);
try {
return await fetch(url, { redirect: "follow", signal: ctl.signal });
} finally {
clearTimeout(timer);
}
}
async function fetchJson<T = unknown>(url: string, options: FetchJsonOptions = {}): Promise<T> {
const response = await fetchWithTimeout(url, options.timeoutMs);
if (!response.ok) {
throw new Error(`Request failed: ${response.status} ${response.statusText}`);
}
return await response.json() as T;
}
async function isDebugPortReady(port: number, timeoutMs = 3_000): Promise<boolean> {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
`http://127.0.0.1:${port}/json/version`,
{ timeoutMs }
);
return !!version.webSocketDebuggerUrl;
} catch {
return false;
}
}
export async function findExistingChromeDebugPort(options: FindExistingChromeDebugPortOptions): Promise<number | null> {
const timeoutMs = options.timeoutMs ?? 3_000;
const portFile = path.join(options.profileDir, "DevToolsActivePort");
try {
const content = fs.readFileSync(portFile, "utf-8");
const [portLine] = content.split(/\r?\n/);
const port = Number.parseInt(portLine?.trim() ?? "", 10);
if (port > 0 && await isDebugPortReady(port, timeoutMs)) return port;
} catch {}
if (process.platform === "win32") return null;
try {
const result = spawnSync("ps", ["aux"], { encoding: "utf-8", timeout: 5_000 });
if (result.status !== 0 || !result.stdout) return null;
const lines = result.stdout
.split("\n")
.filter((line) => line.includes(options.profileDir) && line.includes("--remote-debugging-port="));
for (const line of lines) {
const portMatch = line.match(/--remote-debugging-port=(\d+)/);
const port = Number.parseInt(portMatch?.[1] ?? "", 10);
if (port > 0 && await isDebugPortReady(port, timeoutMs)) return port;
}
} catch {}
return null;
}
export async function waitForChromeDebugPort(
port: number,
timeoutMs: number,
options?: { includeLastError?: boolean }
): Promise<string> {
const start = Date.now();
let lastError: unknown = null;
while (Date.now() - start < timeoutMs) {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
`http://127.0.0.1:${port}/json/version`,
{ timeoutMs: 5_000 }
);
if (version.webSocketDebuggerUrl) return version.webSocketDebuggerUrl;
lastError = new Error("Missing webSocketDebuggerUrl");
} catch (error) {
lastError = error;
}
await sleep(200);
}
if (options?.includeLastError && lastError) {
throw new Error(
`Chrome debug port not ready: ${lastError instanceof Error ? lastError.message : String(lastError)}`
);
}
throw new Error("Chrome debug port not ready");
}
export class CdpConnection {
private ws: WebSocket;
private nextId = 0;
private pending = new Map<number, PendingRequest>();
private eventHandlers = new Map<string, Set<(params: unknown) => void>>();
private defaultTimeoutMs: number;
private constructor(ws: WebSocket, defaultTimeoutMs = 15_000) {
this.ws = ws;
this.defaultTimeoutMs = defaultTimeoutMs;
this.ws.addEventListener("message", (event) => {
try {
const data = typeof event.data === "string"
? event.data
: new TextDecoder().decode(event.data as ArrayBuffer);
const msg = JSON.parse(data) as {
id?: number;
method?: string;
params?: unknown;
result?: unknown;
error?: { message?: string };
};
if (msg.method) {
const handlers = this.eventHandlers.get(msg.method);
if (handlers) {
handlers.forEach((handler) => handler(msg.params));
}
}
if (msg.id) {
const pending = this.pending.get(msg.id);
if (pending) {
this.pending.delete(msg.id);
if (pending.timer) clearTimeout(pending.timer);
if (msg.error?.message) pending.reject(new Error(msg.error.message));
else pending.resolve(msg.result);
}
}
} catch {}
});
this.ws.addEventListener("close", () => {
for (const [id, pending] of this.pending.entries()) {
this.pending.delete(id);
if (pending.timer) clearTimeout(pending.timer);
pending.reject(new Error("CDP connection closed."));
}
});
}
static async connect(
url: string,
timeoutMs: number,
options?: { defaultTimeoutMs?: number }
): Promise<CdpConnection> {
const ws = new WebSocket(url);
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error("CDP connection timeout.")), timeoutMs);
ws.addEventListener("open", () => {
clearTimeout(timer);
resolve();
});
ws.addEventListener("error", () => {
clearTimeout(timer);
reject(new Error("CDP connection failed."));
});
});
return new CdpConnection(ws, options?.defaultTimeoutMs ?? 15_000);
}
on(method: string, handler: (params: unknown) => void): void {
if (!this.eventHandlers.has(method)) {
this.eventHandlers.set(method, new Set());
}
this.eventHandlers.get(method)?.add(handler);
}
off(method: string, handler: (params: unknown) => void): void {
this.eventHandlers.get(method)?.delete(handler);
}
async send<T = unknown>(method: string, params?: Record<string, unknown>, options?: CdpSendOptions): Promise<T> {
const id = ++this.nextId;
const message: Record<string, unknown> = { id, method };
if (params) message.params = params;
if (options?.sessionId) message.sessionId = options.sessionId;
const timeoutMs = options?.timeoutMs ?? this.defaultTimeoutMs;
const result = await new Promise<unknown>((resolve, reject) => {
const timer = timeoutMs > 0
? setTimeout(() => {
this.pending.delete(id);
reject(new Error(`CDP timeout: ${method}`));
}, timeoutMs)
: null;
this.pending.set(id, { resolve, reject, timer });
this.ws.send(JSON.stringify(message));
});
return result as T;
}
close(): void {
try {
this.ws.close();
} catch {}
}
}
export async function launchChrome(options: LaunchChromeOptions): Promise<ChildProcess> {
await fs.promises.mkdir(options.profileDir, { recursive: true });
const args = [
`--remote-debugging-port=${options.port}`,
`--user-data-dir=${options.profileDir}`,
"--no-first-run",
"--no-default-browser-check",
...(options.extraArgs ?? []),
];
if (options.headless) args.push("--headless=new");
if (options.url) args.push(options.url);
return spawn(options.chromePath, args, { stdio: "ignore" });
}
export function killChrome(chrome: ChildProcess): void {
try {
chrome.kill("SIGTERM");
} catch {}
setTimeout(() => {
if (!chrome.killed) {
try {
chrome.kill("SIGKILL");
} catch {}
}
}, 2_000).unref?.();
}
export async function openPageSession(options: OpenPageSessionOptions): Promise<PageSession> {
let targetId: string;
if (options.reusing) {
const created = await options.cdp.send<{ targetId: string }>("Target.createTarget", { url: options.url });
targetId = created.targetId;
} else {
const targets = await options.cdp.send<{ targetInfos: ChromeTargetInfo[] }>("Target.getTargets");
const existing = targets.targetInfos.find(options.matchTarget);
if (existing) {
targetId = existing.targetId;
} else {
const created = await options.cdp.send<{ targetId: string }>("Target.createTarget", { url: options.url });
targetId = created.targetId;
}
}
const { sessionId } = await options.cdp.send<{ sessionId: string }>(
"Target.attachToTarget",
{ targetId, flatten: true }
);
if (options.activateTarget ?? true) {
await options.cdp.send("Target.activateTarget", { targetId });
}
if (options.enablePage) await options.cdp.send("Page.enable", {}, { sessionId });
if (options.enableRuntime) await options.cdp.send("Runtime.enable", {}, { sessionId });
if (options.enableDom) await options.cdp.send("DOM.enable", {}, { sessionId });
if (options.enableNetwork) await options.cdp.send("Network.enable", {}, { sessionId });
return { sessionId, targetId };
}
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env node
import { spawnSync } from "node:child_process";
import path from "node:path";
async function main() {
const repoRoot = path.resolve(process.cwd());
const hooksPath = path.join(repoRoot, ".githooks");
const result = spawnSync("git", ["config", "core.hooksPath", hooksPath], {
cwd: repoRoot,
stdio: "inherit",
});
if (result.status !== 0) {
throw new Error("Failed to configure core.hooksPath");
}
console.log(`Configured git hooks path: ${hooksPath}`);
}
main().catch((error) => {
console.error(error instanceof Error ? error.message : String(error));
process.exit(1);
});
+75
View File
@@ -0,0 +1,75 @@
import fs from "node:fs/promises";
import path from "node:path";
const PACKAGE_DEPENDENCY_SECTIONS = [
"dependencies",
"optionalDependencies",
"peerDependencies",
"devDependencies",
];
const SKIPPED_DIRS = new Set([".git", ".clawhub", ".clawdhub", "node_modules", "out", "dist", "build"]);
const SKIPPED_FILES = new Set([".DS_Store", "bun.lockb"]);
export async function listReleaseFiles(root) {
const resolvedRoot = path.resolve(root);
const files = [];
async function walk(folder) {
const entries = await fs.readdir(folder, { withFileTypes: true });
for (const entry of entries) {
if (entry.isDirectory() && SKIPPED_DIRS.has(entry.name)) continue;
if (entry.isFile() && SKIPPED_FILES.has(entry.name)) continue;
const fullPath = path.join(folder, entry.name);
if (entry.isDirectory()) {
await walk(fullPath);
continue;
}
if (!entry.isFile()) continue;
const relPath = path.relative(resolvedRoot, fullPath).split(path.sep).join("/");
const bytes = await fs.readFile(fullPath);
files.push({ relPath, bytes });
}
}
await walk(resolvedRoot);
files.sort((left, right) => left.relPath.localeCompare(right.relPath));
return files;
}
export async function validateSelfContainedRelease(root) {
const files = await listReleaseFiles(root);
for (const file of files.filter((entry) => path.posix.basename(entry.relPath) === "package.json")) {
const packageDir = path.resolve(root, fromPosixRel(path.posix.dirname(file.relPath)));
const packageJson = JSON.parse(file.bytes.toString("utf8"));
for (const section of PACKAGE_DEPENDENCY_SECTIONS) {
const dependencies = packageJson[section];
if (!dependencies || typeof dependencies !== "object") continue;
for (const [name, spec] of Object.entries(dependencies)) {
if (typeof spec !== "string" || !spec.startsWith("file:")) continue;
const targetDir = path.resolve(packageDir, spec.slice(5));
if (!isWithinRoot(root, targetDir)) {
throw new Error(
`Release target is not self-contained: ${file.relPath} depends on ${name} via ${spec}`,
);
}
await fs.access(targetDir).catch(() => {
throw new Error(`Missing local dependency for release: ${file.relPath} -> ${spec}`);
});
}
}
}
}
function fromPosixRel(relPath) {
return relPath === "." ? "." : relPath.split("/").join(path.sep);
}
function isWithinRoot(root, target) {
const resolvedRoot = path.resolve(root);
const relative = path.relative(resolvedRoot, path.resolve(target));
return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
}
+280
View File
@@ -0,0 +1,280 @@
import { spawnSync } from "node:child_process";
import { existsSync } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
const PACKAGE_DEPENDENCY_SECTIONS = [
"dependencies",
"optionalDependencies",
"peerDependencies",
"devDependencies",
];
const SKIPPED_DIRS = new Set([".git", ".clawhub", ".clawdhub", "node_modules"]);
const SKIPPED_FILES = new Set([".DS_Store"]);
export async function syncSharedSkillPackages(repoRoot, options = {}) {
const root = path.resolve(repoRoot);
const workspacePackages = await discoverWorkspacePackages(root);
const targetConsumerDirs = normalizeTargetConsumerDirs(root, options.targets ?? []);
const consumers = await discoverSkillScriptPackages(root, targetConsumerDirs);
const runtime = options.install === false ? null : resolveBunRuntime();
const managedPaths = new Set();
const packageDirs = [];
for (const consumer of consumers) {
const result = await syncConsumerPackage({
consumer,
root,
workspacePackages,
runtime,
});
if (!result) continue;
packageDirs.push(consumer.dir);
for (const managedPath of result.managedPaths) {
managedPaths.add(managedPath);
}
}
return {
packageDirs,
managedPaths: [...managedPaths].sort(),
};
}
function normalizeTargetConsumerDirs(repoRoot, targets) {
if (!targets || targets.length === 0) return null;
const consumerDirs = new Set();
for (const target of targets) {
if (!target) continue;
const resolvedTarget = path.resolve(repoRoot, target);
if (path.basename(resolvedTarget) === "scripts") {
consumerDirs.add(resolvedTarget);
continue;
}
consumerDirs.add(path.join(resolvedTarget, "scripts"));
}
return consumerDirs;
}
export function ensureManagedPathsClean(repoRoot, managedPaths) {
if (managedPaths.length === 0) return;
const result = spawnSync("git", ["status", "--porcelain", "--", ...managedPaths], {
cwd: repoRoot,
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
});
if (result.status !== 0) {
throw new Error(result.stderr.trim() || "Failed to inspect git status for managed paths");
}
const output = result.stdout.trim();
if (!output) return;
throw new Error(
[
"Shared skill package sync produced uncommitted managed changes.",
"Review and commit these files before pushing:",
output,
].join("\n"),
);
}
async function syncConsumerPackage({ consumer, root, workspacePackages, runtime }) {
const packageJsonPath = path.join(consumer.dir, "package.json");
const packageJson = JSON.parse(await fs.readFile(packageJsonPath, "utf8"));
const localDeps = collectLocalDependencies(packageJson, workspacePackages);
if (localDeps.length === 0) {
return null;
}
const vendorRoot = path.join(consumer.dir, "vendor");
await fs.rm(vendorRoot, { recursive: true, force: true });
for (const name of localDeps) {
const sourceDir = workspacePackages.get(name);
if (!sourceDir) continue;
await syncPackageTree({
sourceDir,
targetDir: path.join(vendorRoot, name),
workspacePackages,
});
}
rewriteLocalDependencySpecs(packageJson, localDeps);
await writeJson(packageJsonPath, packageJson);
if (runtime) {
runInstall(runtime, consumer.dir);
}
const managedPaths = [
path.relative(root, packageJsonPath).split(path.sep).join("/"),
path.relative(root, path.join(consumer.dir, "bun.lock")).split(path.sep).join("/"),
path.relative(root, vendorRoot).split(path.sep).join("/"),
];
return { managedPaths };
}
async function syncPackageTree({ sourceDir, targetDir, workspacePackages }) {
await fs.rm(targetDir, { recursive: true, force: true });
await fs.mkdir(targetDir, { recursive: true });
const sourcePackageJsonPath = path.join(sourceDir, "package.json");
const packageJson = JSON.parse(await fs.readFile(sourcePackageJsonPath, "utf8"));
const localDeps = collectLocalDependencies(packageJson, workspacePackages);
const entries = await fs.readdir(sourceDir, { withFileTypes: true });
for (const entry of entries) {
if (SKIPPED_DIRS.has(entry.name) || SKIPPED_FILES.has(entry.name)) continue;
const sourcePath = path.join(sourceDir, entry.name);
const targetPath = path.join(targetDir, entry.name);
if (entry.isDirectory()) {
await copyDirectory(sourcePath, targetPath);
continue;
}
if (!entry.isFile() || entry.name === "package.json") continue;
await fs.mkdir(path.dirname(targetPath), { recursive: true });
await fs.copyFile(sourcePath, targetPath);
}
for (const name of localDeps) {
const nestedSourceDir = workspacePackages.get(name);
if (!nestedSourceDir) continue;
await syncPackageTree({
sourceDir: nestedSourceDir,
targetDir: path.join(targetDir, "vendor", name),
workspacePackages,
});
}
rewriteLocalDependencySpecs(packageJson, localDeps);
await writeJson(path.join(targetDir, "package.json"), packageJson);
}
async function copyDirectory(sourceDir, targetDir) {
await fs.mkdir(targetDir, { recursive: true });
const entries = await fs.readdir(sourceDir, { withFileTypes: true });
for (const entry of entries) {
if (SKIPPED_DIRS.has(entry.name) || SKIPPED_FILES.has(entry.name)) continue;
const sourcePath = path.join(sourceDir, entry.name);
const targetPath = path.join(targetDir, entry.name);
if (entry.isDirectory()) {
await copyDirectory(sourcePath, targetPath);
continue;
}
if (!entry.isFile()) continue;
await fs.mkdir(path.dirname(targetPath), { recursive: true });
await fs.copyFile(sourcePath, targetPath);
}
}
async function discoverWorkspacePackages(repoRoot) {
const packagesRoot = path.join(repoRoot, "packages");
const map = new Map();
if (!existsSync(packagesRoot)) return map;
const entries = await fs.readdir(packagesRoot, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isDirectory()) continue;
const packageJsonPath = path.join(packagesRoot, entry.name, "package.json");
if (!existsSync(packageJsonPath)) continue;
const packageJson = JSON.parse(await fs.readFile(packageJsonPath, "utf8"));
if (!packageJson.name) continue;
map.set(packageJson.name, path.join(packagesRoot, entry.name));
}
return map;
}
async function discoverSkillScriptPackages(repoRoot, targetConsumerDirs = null) {
const skillsRoot = path.join(repoRoot, "skills");
const consumers = [];
const skillEntries = await fs.readdir(skillsRoot, { withFileTypes: true });
for (const entry of skillEntries) {
if (!entry.isDirectory()) continue;
const scriptsDir = path.join(skillsRoot, entry.name, "scripts");
if (targetConsumerDirs && !targetConsumerDirs.has(path.resolve(scriptsDir))) continue;
const packageJsonPath = path.join(scriptsDir, "package.json");
if (!existsSync(packageJsonPath)) continue;
consumers.push({ dir: scriptsDir, packageJsonPath });
}
return consumers.sort((left, right) => left.dir.localeCompare(right.dir));
}
function collectLocalDependencies(packageJson, workspacePackages) {
const localDeps = [];
for (const section of PACKAGE_DEPENDENCY_SECTIONS) {
const dependencies = packageJson[section];
if (!dependencies || typeof dependencies !== "object") continue;
for (const name of Object.keys(dependencies)) {
if (!workspacePackages.has(name)) continue;
localDeps.push(name);
}
}
return [...new Set(localDeps)].sort();
}
function rewriteLocalDependencySpecs(packageJson, localDeps) {
for (const section of PACKAGE_DEPENDENCY_SECTIONS) {
const dependencies = packageJson[section];
if (!dependencies || typeof dependencies !== "object") continue;
for (const name of localDeps) {
if (!(name in dependencies)) continue;
dependencies[name] = `file:./vendor/${name}`;
}
}
}
async function writeJson(filePath, value) {
await fs.mkdir(path.dirname(filePath), { recursive: true });
await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`);
}
function resolveBunRuntime() {
if (commandExists("bun")) {
return { command: "bun", args: [] };
}
if (commandExists("npx")) {
return { command: "npx", args: ["-y", "bun"] };
}
throw new Error(
"Neither bun nor npx is installed. Install bun with `brew install oven-sh/bun/bun` or `npm install -g bun`.",
);
}
function commandExists(command) {
const result = spawnSync("sh", ["-lc", `command -v ${command}`], {
stdio: "ignore",
});
return result.status === 0;
}
function runInstall(runtime, cwd) {
const result = spawnSync(runtime.command, [...runtime.args, "install"], {
cwd,
stdio: "inherit",
});
if (result.status !== 0) {
throw new Error(`Failed to refresh Bun dependencies in ${cwd}`);
}
}
+311
View File
@@ -0,0 +1,311 @@
#!/usr/bin/env node
import fs from "node:fs/promises";
import { existsSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { listReleaseFiles, validateSelfContainedRelease } from "./lib/release-files.mjs";
const DEFAULT_REGISTRY = "https://clawhub.ai";
async function main() {
const options = parseArgs(process.argv.slice(2));
if (!options.skillDir || !options.version) {
throw new Error("--skill-dir and --version are required");
}
const skillDir = path.resolve(options.skillDir);
const skill = buildSkillEntry(skillDir, options.slug, options.displayName);
const changelog = options.changelogFile
? await fs.readFile(path.resolve(options.changelogFile), "utf8")
: "";
await validateSelfContainedRelease(skillDir);
const files = await listReleaseFiles(skillDir);
if (files.length === 0) {
throw new Error(`Skill directory is empty: ${skillDir}`);
}
if (options.dryRun) {
console.log(`Dry run: would publish ${skill.slug}@${options.version}`);
console.log(`Skill: ${skillDir}`);
console.log(`Files: ${files.length}`);
return;
}
const config = await readClawhubConfig();
const registry = (
options.registry ||
process.env.CLAWHUB_REGISTRY ||
process.env.CLAWDHUB_REGISTRY ||
config.registry ||
DEFAULT_REGISTRY
).replace(/\/+$/, "");
if (!config.token) {
throw new Error("Not logged in. Run: clawhub login");
}
await apiJson(registry, config.token, "/api/v1/whoami");
const tags = options.tags
.split(",")
.map((tag) => tag.trim())
.filter(Boolean);
await publishSkill({
registry,
token: config.token,
skill,
files,
version: options.version,
changelog,
tags,
});
}
function parseArgs(argv) {
const options = {
skillDir: "",
version: "",
changelogFile: "",
registry: "",
tags: "latest",
dryRun: false,
slug: "",
displayName: "",
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--skill-dir") {
options.skillDir = argv[index + 1] ?? "";
index += 1;
continue;
}
if (arg === "--version") {
options.version = argv[index + 1] ?? "";
index += 1;
continue;
}
if (arg === "--changelog-file") {
options.changelogFile = argv[index + 1] ?? "";
index += 1;
continue;
}
if (arg === "--registry") {
options.registry = argv[index + 1] ?? "";
index += 1;
continue;
}
if (arg === "--tags") {
options.tags = argv[index + 1] ?? "latest";
index += 1;
continue;
}
if (arg === "--slug") {
options.slug = argv[index + 1] ?? "";
index += 1;
continue;
}
if (arg === "--display-name") {
options.displayName = argv[index + 1] ?? "";
index += 1;
continue;
}
if (arg === "--dry-run") {
const next = argv[index + 1];
if (next && !next.startsWith("-")) {
options.dryRun = parseBoolean(next);
index += 1;
} else {
options.dryRun = true;
}
continue;
}
if (arg === "-h" || arg === "--help") {
printUsage();
process.exit(0);
}
throw new Error(`Unknown argument: ${arg}`);
}
return options;
}
function printUsage() {
console.log(`Usage: publish-skill.mjs --skill-dir <dir> --version <semver> [options]
Options:
--skill-dir <dir> Skill directory to publish
--version <semver> Version to publish
--changelog-file <file> Release notes file
--registry <url> Override registry base URL
--tags <tags> Comma-separated tags (default: latest)
--slug <value> Override slug
--display-name <value> Override display name
--dry-run Print publish plan without network calls
-h, --help Show help`);
}
function buildSkillEntry(folder, slugOverride, displayNameOverride) {
const base = path.basename(folder);
return {
folder,
slug: slugOverride || sanitizeSlug(base),
displayName: displayNameOverride || titleCase(base),
};
}
async function readClawhubConfig() {
const configPath = getConfigPath();
try {
return JSON.parse(await fs.readFile(configPath, "utf8"));
} catch {
return {};
}
}
function getConfigPath() {
const override =
process.env.CLAWHUB_CONFIG_PATH?.trim() || process.env.CLAWDHUB_CONFIG_PATH?.trim();
if (override) {
return path.resolve(override);
}
const home = os.homedir();
if (process.platform === "darwin") {
const clawhub = path.join(home, "Library", "Application Support", "clawhub", "config.json");
const clawdhub = path.join(home, "Library", "Application Support", "clawdhub", "config.json");
return pathForExistingConfig(clawhub, clawdhub);
}
const xdg = process.env.XDG_CONFIG_HOME;
if (xdg) {
const clawhub = path.join(xdg, "clawhub", "config.json");
const clawdhub = path.join(xdg, "clawdhub", "config.json");
return pathForExistingConfig(clawhub, clawdhub);
}
if (process.platform === "win32" && process.env.APPDATA) {
const clawhub = path.join(process.env.APPDATA, "clawhub", "config.json");
const clawdhub = path.join(process.env.APPDATA, "clawdhub", "config.json");
return pathForExistingConfig(clawhub, clawdhub);
}
const clawhub = path.join(home, ".config", "clawhub", "config.json");
const clawdhub = path.join(home, ".config", "clawdhub", "config.json");
return pathForExistingConfig(clawhub, clawdhub);
}
function pathForExistingConfig(primary, legacy) {
if (existsSync(primary)) return path.resolve(primary);
if (existsSync(legacy)) return path.resolve(legacy);
return path.resolve(primary);
}
async function publishSkill({ registry, token, skill, files, version, changelog, tags }) {
const form = new FormData();
form.set(
"payload",
JSON.stringify({
slug: skill.slug,
displayName: skill.displayName,
version,
changelog,
tags,
acceptLicenseTerms: true,
}),
);
for (const file of files) {
form.append("files", new Blob([file.bytes], { type: mimeType(file.relPath) }), file.relPath);
}
const response = await fetch(`${registry}/api/v1/skills`, {
method: "POST",
headers: {
Accept: "application/json",
Authorization: `Bearer ${token}`,
},
body: form,
});
const text = await response.text();
if (!response.ok) {
throw new Error(text || `Publish failed for ${skill.slug} (HTTP ${response.status})`);
}
const result = text ? JSON.parse(text) : {};
console.log(`OK. Published ${skill.slug}@${version}${result.versionId ? ` (${result.versionId})` : ""}`);
}
async function apiJson(registry, token, requestPath) {
const response = await fetch(`${registry}${requestPath}`, {
headers: {
Accept: "application/json",
Authorization: `Bearer ${token}`,
},
});
const text = await response.text();
let body = null;
try {
body = text ? JSON.parse(text) : null;
} catch {
body = { message: text };
}
if (response.status < 200 || response.status >= 300) {
throw new Error(body?.message || `HTTP ${response.status}`);
}
return body;
}
function sanitizeSlug(value) {
return value
.trim()
.toLowerCase()
.replace(/[^a-z0-9-]+/g, "-")
.replace(/^-+/, "")
.replace(/-+$/, "")
.replace(/--+/g, "-");
}
function titleCase(value) {
return value
.trim()
.replace(/[-_]+/g, " ")
.replace(/\s+/g, " ")
.replace(/\b\w/g, (char) => char.toUpperCase());
}
const MIME_MAP = {
".md": "text/markdown",
".ts": "text/plain",
".js": "text/javascript",
".mjs": "text/javascript",
".json": "application/json",
".yml": "text/yaml",
".yaml": "text/yaml",
".txt": "text/plain",
".html": "text/html",
".css": "text/css",
".xml": "text/xml",
".svg": "image/svg+xml",
};
function mimeType(relPath) {
const ext = path.extname(relPath).toLowerCase();
return MIME_MAP[ext] || "text/plain";
}
function parseBoolean(value) {
return ["1", "true", "yes", "on"].includes(String(value).trim().toLowerCase());
}
main().catch((error) => {
console.error(error instanceof Error ? error.message : String(error));
process.exit(1);
});
+528
View File
@@ -0,0 +1,528 @@
#!/usr/bin/env node
import crypto from "node:crypto";
import fs from "node:fs/promises";
import { existsSync } from "node:fs";
import path from "node:path";
import os from "node:os";
const DEFAULT_REGISTRY = "https://clawhub.ai";
const TEXT_EXTENSIONS = new Set([
"md",
"mdx",
"txt",
"json",
"json5",
"yaml",
"yml",
"toml",
"js",
"cjs",
"mjs",
"ts",
"tsx",
"jsx",
"py",
"sh",
"rb",
"go",
"rs",
"swift",
"kt",
"java",
"cs",
"cpp",
"c",
"h",
"hpp",
"sql",
"csv",
"ini",
"cfg",
"env",
"xml",
"html",
"css",
"scss",
"sass",
"svg",
]);
async function main() {
const options = parseArgs(process.argv.slice(2));
const config = await readClawhubConfig();
const registry = (
process.env.CLAWHUB_REGISTRY ||
process.env.CLAWDHUB_REGISTRY ||
config.registry ||
DEFAULT_REGISTRY
).replace(/\/+$/, "");
if (!config.token) {
throw new Error("Not logged in. Run: clawhub login");
}
await apiJson(registry, config.token, "/api/v1/whoami");
const roots = options.roots.length > 0 ? options.roots : [path.resolve("skills")];
const skills = await findSkills(roots);
if (skills.length === 0) {
throw new Error("No skills found.");
}
console.log("ClawHub sync");
console.log(`Roots with skills: ${roots.join(", ")}`);
const locals = await mapWithConcurrency(skills, options.concurrency, async (skill) => {
const files = await listTextFiles(skill.folder);
const fingerprint = buildFingerprint(files);
return {
...skill,
fileCount: files.length,
fingerprint,
};
});
const candidates = await mapWithConcurrency(locals, options.concurrency, async (skill) => {
const query = new URLSearchParams({
slug: skill.slug,
hash: skill.fingerprint,
});
const { status, body } = await apiJsonWithStatus(
registry,
config.token,
`/api/v1/resolve?${query.toString()}`
);
if (status === 404) {
return {
...skill,
status: "new",
latestVersion: null,
matchVersion: null,
};
}
if (status !== 200) {
throw new Error(body?.message || `Resolve failed for ${skill.slug} (HTTP ${status})`);
}
const latestVersion = body?.latestVersion?.version ?? null;
const matchVersion = body?.match?.version ?? null;
if (!latestVersion) {
return {
...skill,
status: "new",
latestVersion: null,
matchVersion: null,
};
}
return {
...skill,
status: matchVersion ? "synced" : "update",
latestVersion,
matchVersion,
};
});
const actionable = candidates.filter((candidate) => candidate.status !== "synced");
if (actionable.length === 0) {
console.log("Nothing to sync.");
return;
}
console.log("");
console.log("To sync");
for (const candidate of actionable) {
console.log(`- ${formatCandidate(candidate, options.bump)}`);
}
if (options.dryRun) {
console.log("");
console.log(`Dry run: would upload ${actionable.length} skill(s).`);
return;
}
const tags = options.tags
.split(",")
.map((tag) => tag.trim())
.filter(Boolean);
for (const candidate of actionable) {
const version =
candidate.status === "new"
? "1.0.0"
: bumpSemver(candidate.latestVersion, options.bump);
console.log(`Publishing ${candidate.slug}@${version}`);
const files = await listTextFiles(candidate.folder);
await publishSkill({
registry,
token: config.token,
skill: candidate,
files,
version,
changelog: options.changelog,
tags,
});
}
console.log("");
console.log(`Uploaded ${actionable.length} skill(s).`);
}
function parseArgs(argv) {
const options = {
roots: [],
dryRun: false,
bump: "patch",
changelog: "",
tags: "latest",
concurrency: 4,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--dry-run") {
options.dryRun = true;
continue;
}
if (arg === "--all") {
continue;
}
if (arg === "--root") {
const value = argv[index + 1];
if (!value) throw new Error("--root requires a directory");
options.roots.push(path.resolve(value));
index += 1;
continue;
}
if (arg === "--bump") {
const value = argv[index + 1];
if (!["patch", "minor", "major"].includes(value)) {
throw new Error("--bump must be patch, minor, or major");
}
options.bump = value;
index += 1;
continue;
}
if (arg === "--changelog") {
const value = argv[index + 1];
if (value == null) throw new Error("--changelog requires text");
options.changelog = value;
index += 1;
continue;
}
if (arg === "--tags") {
const value = argv[index + 1];
if (value == null) throw new Error("--tags requires a value");
options.tags = value;
index += 1;
continue;
}
if (arg === "--concurrency") {
const value = Number(argv[index + 1]);
if (!Number.isInteger(value) || value < 1 || value > 32) {
throw new Error("--concurrency must be an integer between 1 and 32");
}
options.concurrency = value;
index += 1;
continue;
}
if (arg === "-h" || arg === "--help") {
printUsage();
process.exit(0);
}
throw new Error(`Unknown argument: ${arg}`);
}
return options;
}
function printUsage() {
console.log(`Usage: sync-clawhub.mjs [options]
Options:
--root <dir> Extra skill root (repeatable)
--all Accepted for compatibility
--dry-run Show what would be uploaded
--bump <type> patch | minor | major
--changelog <text> Changelog for updates
--tags <tags> Comma-separated tags
--concurrency <n> Registry check concurrency (1-32)
-h, --help Show help`);
}
async function readClawhubConfig() {
const configPath = getConfigPath();
try {
return JSON.parse(await fs.readFile(configPath, "utf8"));
} catch {
return {};
}
}
function getConfigPath() {
const override =
process.env.CLAWHUB_CONFIG_PATH?.trim() || process.env.CLAWDHUB_CONFIG_PATH?.trim();
if (override) {
return path.resolve(override);
}
const home = os.homedir();
if (process.platform === "darwin") {
const clawhub = path.join(home, "Library", "Application Support", "clawhub", "config.json");
const clawdhub = path.join(home, "Library", "Application Support", "clawdhub", "config.json");
return pathForExistingConfig(clawhub, clawdhub);
}
const xdg = process.env.XDG_CONFIG_HOME;
if (xdg) {
const clawhub = path.join(xdg, "clawhub", "config.json");
const clawdhub = path.join(xdg, "clawdhub", "config.json");
return pathForExistingConfig(clawhub, clawdhub);
}
if (process.platform === "win32" && process.env.APPDATA) {
const clawhub = path.join(process.env.APPDATA, "clawhub", "config.json");
const clawdhub = path.join(process.env.APPDATA, "clawdhub", "config.json");
return pathForExistingConfig(clawhub, clawdhub);
}
const clawhub = path.join(home, ".config", "clawhub", "config.json");
const clawdhub = path.join(home, ".config", "clawdhub", "config.json");
return pathForExistingConfig(clawhub, clawdhub);
}
function pathForExistingConfig(primary, legacy) {
if (existsSync(primary)) return path.resolve(primary);
if (existsSync(legacy)) return path.resolve(legacy);
return path.resolve(primary);
}
async function findSkills(roots) {
const deduped = new Map();
for (const root of roots) {
const folders = await findSkillFolders(root);
for (const folder of folders) {
deduped.set(folder.slug, folder);
}
}
return [...deduped.values()].sort((left, right) => left.slug.localeCompare(right.slug));
}
async function findSkillFolders(root) {
const stat = await safeStat(root);
if (!stat?.isDirectory()) return [];
if (await hasSkillMarker(root)) {
return [buildSkillEntry(root)];
}
const entries = await fs.readdir(root, { withFileTypes: true });
const found = [];
for (const entry of entries) {
if (!entry.isDirectory()) continue;
const folder = path.join(root, entry.name);
if (await hasSkillMarker(folder)) {
found.push(buildSkillEntry(folder));
}
}
return found;
}
function buildSkillEntry(folder) {
const base = path.basename(folder);
return {
folder,
slug: sanitizeSlug(base),
displayName: titleCase(base),
};
}
async function hasSkillMarker(folder) {
return Boolean(
(await safeStat(path.join(folder, "SKILL.md")))?.isFile() ||
(await safeStat(path.join(folder, "skill.md")))?.isFile()
);
}
async function listTextFiles(root) {
const files = [];
async function walk(folder) {
const entries = await fs.readdir(folder, { withFileTypes: true });
for (const entry of entries) {
if (entry.name.startsWith(".")) continue;
if (entry.name === "node_modules") continue;
if (entry.name === ".clawhub" || entry.name === ".clawdhub") continue;
const fullPath = path.join(folder, entry.name);
if (entry.isDirectory()) {
await walk(fullPath);
continue;
}
if (!entry.isFile()) continue;
const relPath = path.relative(root, fullPath).split(path.sep).join("/");
const ext = relPath.split(".").pop()?.toLowerCase() ?? "";
if (!TEXT_EXTENSIONS.has(ext)) continue;
const bytes = await fs.readFile(fullPath);
files.push({ relPath, bytes });
}
}
await walk(root);
files.sort((left, right) => left.relPath.localeCompare(right.relPath));
return files;
}
function buildFingerprint(files) {
const payload = files
.map((file) => `${file.relPath}:${sha256(file.bytes)}`)
.sort((left, right) => left.localeCompare(right))
.join("\n");
return crypto.createHash("sha256").update(payload).digest("hex");
}
function sha256(bytes) {
return crypto.createHash("sha256").update(bytes).digest("hex");
}
async function publishSkill({ registry, token, skill, files, version, changelog, tags }) {
const form = new FormData();
form.set(
"payload",
JSON.stringify({
slug: skill.slug,
displayName: skill.displayName,
version,
changelog,
tags,
acceptLicenseTerms: true,
})
);
for (const file of files) {
form.append("files", new Blob([file.bytes], { type: "text/plain" }), file.relPath);
}
const response = await fetch(`${registry}/api/v1/skills`, {
method: "POST",
headers: {
Accept: "application/json",
Authorization: `Bearer ${token}`,
},
body: form,
});
const text = await response.text();
if (!response.ok) {
throw new Error(text || `Publish failed for ${skill.slug} (HTTP ${response.status})`);
}
const result = text ? JSON.parse(text) : {};
console.log(`OK. Published ${skill.slug}@${version}${result.versionId ? ` (${result.versionId})` : ""}`);
}
async function apiJson(registry, token, requestPath) {
const { status, body } = await apiJsonWithStatus(registry, token, requestPath);
if (status < 200 || status >= 300) {
throw new Error(body?.message || `HTTP ${status}`);
}
return body;
}
async function apiJsonWithStatus(registry, token, requestPath) {
const response = await fetch(`${registry}${requestPath}`, {
headers: {
Accept: "application/json",
Authorization: `Bearer ${token}`,
},
});
const text = await response.text();
let body = null;
try {
body = text ? JSON.parse(text) : null;
} catch {
body = { message: text };
}
return { status: response.status, body };
}
async function mapWithConcurrency(items, limit, fn) {
const results = new Array(items.length);
let cursor = 0;
async function worker() {
while (cursor < items.length) {
const index = cursor;
cursor += 1;
results[index] = await fn(items[index], index);
}
}
const count = Math.min(Math.max(limit, 1), Math.max(items.length, 1));
await Promise.all(Array.from({ length: count }, () => worker()));
return results;
}
function formatCandidate(candidate, bump) {
if (candidate.status === "new") {
return `${candidate.slug} NEW (${candidate.fileCount} files)`;
}
return `${candidate.slug} UPDATE ${candidate.latestVersion} -> ${bumpSemver(
candidate.latestVersion,
bump
)} (${candidate.fileCount} files)`;
}
function bumpSemver(version, bump) {
const match = /^(\d+)\.(\d+)\.(\d+)$/.exec(version ?? "");
if (!match) {
throw new Error(`Invalid semver: ${version}`);
}
const major = Number(match[1]);
const minor = Number(match[2]);
const patch = Number(match[3]);
if (bump === "major") return `${major + 1}.0.0`;
if (bump === "minor") return `${major}.${minor + 1}.0`;
return `${major}.${minor}.${patch + 1}`;
}
function sanitizeSlug(value) {
return value
.trim()
.toLowerCase()
.replace(/[^a-z0-9-]+/g, "-")
.replace(/^-+/, "")
.replace(/-+$/, "")
.replace(/--+/g, "-");
}
function titleCase(value) {
return value
.trim()
.replace(/[-_]+/g, " ")
.replace(/\s+/g, " ")
.replace(/\b\w/g, (char) => char.toUpperCase());
}
async function safeStat(filePath) {
try {
return await fs.stat(filePath);
} catch {
return null;
}
}
main().catch((error) => {
console.error(error instanceof Error ? error.message : String(error));
process.exit(1);
});
+3 -8
View File
@@ -4,13 +4,8 @@ set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
SKILLS_DIR="${ROOT_DIR}/skills"
if command -v clawhub >/dev/null 2>&1; then
CLAWHUB_CMD=(clawhub)
elif command -v npx >/dev/null 2>&1; then
CLAWHUB_CMD=(npx -y clawhub)
else
echo "Error: neither clawhub nor npx is available."
echo "See https://docs.openclaw.ai/zh-CN/tools/clawhub"
if ! command -v node >/dev/null 2>&1; then
echo "Error: node is required."
exit 1
fi
@@ -18,4 +13,4 @@ if [ "$#" -eq 0 ]; then
set -- --all
fi
exec "${CLAWHUB_CMD[@]}" sync --root "${SKILLS_DIR}" "$@"
exec node "${ROOT_DIR}/scripts/sync-clawhub.mjs" --root "${SKILLS_DIR}" "$@"
+70
View File
@@ -0,0 +1,70 @@
#!/usr/bin/env node
import path from "node:path";
import {
ensureManagedPathsClean,
syncSharedSkillPackages,
} from "./lib/shared-skill-packages.mjs";
async function main() {
const options = parseArgs(process.argv.slice(2));
const repoRoot = path.resolve(options.repoRoot);
const result = await syncSharedSkillPackages(repoRoot, {
targets: options.targets,
});
if (options.enforceClean) {
ensureManagedPathsClean(repoRoot, result.managedPaths);
}
console.log(`Synced shared workspace packages into ${result.packageDirs.length} skill script package(s).`);
}
function parseArgs(argv) {
const options = {
repoRoot: process.cwd(),
enforceClean: false,
targets: [],
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--repo-root") {
options.repoRoot = argv[index + 1] ?? options.repoRoot;
index += 1;
continue;
}
if (arg === "--enforce-clean") {
options.enforceClean = true;
continue;
}
if (arg === "--target") {
options.targets.push(argv[index + 1] ?? "");
index += 1;
continue;
}
if (arg === "-h" || arg === "--help") {
printUsage();
process.exit(0);
}
throw new Error(`Unknown argument: ${arg}`);
}
return options;
}
function printUsage() {
console.log(`Usage: sync-shared-skill-packages.mjs [options]
Options:
--repo-root <dir> Repository root (default: current directory)
--target <dir> Sync only one skill directory (can be repeated)
--enforce-clean Fail if managed files change after sync
-h, --help Show help`);
}
main().catch((error) => {
console.error(error instanceof Error ? error.message : String(error));
process.exit(1);
});
@@ -0,0 +1,14 @@
{
"lockfileVersion": 1,
"workspaces": {
"": {
"name": "baoyu-danger-gemini-web-scripts",
"dependencies": {
"baoyu-chrome-cdp": "file:./vendor/baoyu-chrome-cdp",
},
},
},
"packages": {
"baoyu-chrome-cdp": ["baoyu-chrome-cdp@file:vendor/baoyu-chrome-cdp", {}],
}
}
@@ -1,183 +1,76 @@
import { spawn, type ChildProcess } from 'node:child_process';
import fs from 'node:fs';
import { mkdir } from 'node:fs/promises';
import net from 'node:net';
import process from 'node:process';
import {
CdpConnection,
findChromeExecutable as findChromeExecutableBase,
findExistingChromeDebugPort,
getFreePort,
killChrome,
launchChrome as launchChromeBase,
openPageSession,
sleep,
waitForChromeDebugPort,
type PlatformCandidates,
} from 'baoyu-chrome-cdp';
import { Endpoint, Headers } from '../constants.js';
import { logger } from './logger.js';
import { cookie_header, fetch_with_timeout, sleep } from './http.js';
import { cookie_header, fetch_with_timeout } from './http.js';
import { read_cookie_file, type CookieMap, write_cookie_file } from './cookie-file.js';
import { resolveGeminiWebChromeProfileDir, resolveGeminiWebCookiePath } from './paths.js';
type CdpSendOptions = { sessionId?: string; timeoutMs?: number };
const GEMINI_APP_URL = 'https://gemini.google.com/app';
class CdpConnection {
private ws: WebSocket;
private nextId = 0;
private pending = new Map<
number,
{ resolve: (v: unknown) => void; reject: (e: Error) => void; timer: ReturnType<typeof setTimeout> | null }
>();
private constructor(ws: WebSocket) {
this.ws = ws;
this.ws.addEventListener('message', (event) => {
try {
const data = typeof event.data === 'string' ? event.data : new TextDecoder().decode(event.data as ArrayBuffer);
const msg = JSON.parse(data) as { id?: number; result?: unknown; error?: { message?: string } };
if (msg.id) {
const p = this.pending.get(msg.id);
if (p) {
this.pending.delete(msg.id);
if (p.timer) clearTimeout(p.timer);
if (msg.error?.message) p.reject(new Error(msg.error.message));
else p.resolve(msg.result);
}
}
} catch {}
});
this.ws.addEventListener('close', () => {
for (const [id, p] of this.pending.entries()) {
this.pending.delete(id);
if (p.timer) clearTimeout(p.timer);
p.reject(new Error('CDP connection closed.'));
}
});
}
static async connect(url: string, timeoutMs: number): Promise<CdpConnection> {
const ws = new WebSocket(url);
await new Promise<void>((resolve, reject) => {
const t = setTimeout(() => reject(new Error('CDP connection timeout.')), timeoutMs);
ws.addEventListener('open', () => {
clearTimeout(t);
resolve();
});
ws.addEventListener('error', () => {
clearTimeout(t);
reject(new Error('CDP connection failed.'));
});
});
return new CdpConnection(ws);
}
async send<T = unknown>(method: string, params?: Record<string, unknown>, opts?: CdpSendOptions): Promise<T> {
const id = ++this.nextId;
const msg: Record<string, unknown> = { id, method };
if (params) msg.params = params;
if (opts?.sessionId) msg.sessionId = opts.sessionId;
const timeoutMs = opts?.timeoutMs ?? 15_000;
const out = await new Promise<unknown>((resolve, reject) => {
const t =
timeoutMs > 0
? setTimeout(() => {
this.pending.delete(id);
reject(new Error(`CDP timeout: ${method}`));
}, timeoutMs)
: null;
this.pending.set(id, { resolve, reject, timer: t });
this.ws.send(JSON.stringify(msg));
});
return out as T;
}
close(): void {
try {
this.ws.close();
} catch {}
}
}
const CHROME_CANDIDATES_FULL: PlatformCandidates = {
darwin: [
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
'/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta',
'/Applications/Chromium.app/Contents/MacOS/Chromium',
'/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
],
win32: [
'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
'C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe',
'C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe',
],
default: [
'/usr/bin/google-chrome',
'/usr/bin/google-chrome-stable',
'/usr/bin/chromium',
'/usr/bin/chromium-browser',
'/snap/bin/chromium',
'/usr/bin/microsoft-edge',
],
};
async function get_free_port(): Promise<number> {
const fixed = parseInt(process.env.GEMINI_WEB_DEBUG_PORT || '', 10);
if (fixed > 0) return fixed;
return await new Promise((resolve, reject) => {
const srv = net.createServer();
srv.unref();
srv.on('error', reject);
srv.listen(0, '127.0.0.1', () => {
const addr = srv.address();
if (!addr || typeof addr === 'string') {
srv.close(() => reject(new Error('Unable to allocate a free TCP port.')));
return;
}
const port = addr.port;
srv.close((err) => (err ? reject(err) : resolve(port)));
});
});
return await getFreePort('GEMINI_WEB_DEBUG_PORT');
}
function find_chrome_executable(): string | null {
const override = process.env.GEMINI_WEB_CHROME_PATH?.trim();
if (override && fs.existsSync(override)) return override;
const candidates: string[] = [];
switch (process.platform) {
case 'darwin':
candidates.push(
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
'/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta',
'/Applications/Chromium.app/Contents/MacOS/Chromium',
'/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
);
break;
case 'win32':
candidates.push(
'C:\\\\Program Files\\\\Google\\\\Chrome\\\\Application\\\\chrome.exe',
'C:\\\\Program Files (x86)\\\\Google\\\\Chrome\\\\Application\\\\chrome.exe',
'C:\\\\Program Files\\\\Microsoft\\\\Edge\\\\Application\\\\msedge.exe',
'C:\\\\Program Files (x86)\\\\Microsoft\\\\Edge\\\\Application\\\\msedge.exe',
);
break;
default:
candidates.push(
'/usr/bin/google-chrome',
'/usr/bin/google-chrome-stable',
'/usr/bin/chromium',
'/usr/bin/chromium-browser',
'/snap/bin/chromium',
'/usr/bin/microsoft-edge',
);
break;
}
for (const p of candidates) {
if (fs.existsSync(p)) return p;
}
return null;
return findChromeExecutableBase({
candidates: CHROME_CANDIDATES_FULL,
envNames: ['GEMINI_WEB_CHROME_PATH'],
}) ?? null;
}
async function wait_for_chrome_debug_port(port: number, timeoutMs: number): Promise<string> {
const start = Date.now();
while (Date.now() - start < timeoutMs) {
try {
const res = await fetch_with_timeout(`http://127.0.0.1:${port}/json/version`, { timeout_ms: 5_000 });
if (!res.ok) throw new Error(`status=${res.status}`);
const j = (await res.json()) as { webSocketDebuggerUrl?: string };
if (j.webSocketDebuggerUrl) return j.webSocketDebuggerUrl;
} catch {}
await sleep(200);
}
throw new Error('Chrome debug port not ready');
async function find_existing_chrome_debug_port(profileDir: string): Promise<number | null> {
return await findExistingChromeDebugPort({ profileDir });
}
async function launch_chrome(profileDir: string, port: number): Promise<ChildProcess> {
const chrome = find_chrome_executable();
if (!chrome) throw new Error('Chrome executable not found.');
async function launch_chrome(profileDir: string, port: number) {
const chromePath = find_chrome_executable();
if (!chromePath) throw new Error('Chrome executable not found.');
const args = [
`--remote-debugging-port=${port}`,
`--user-data-dir=${profileDir}`,
'--no-first-run',
'--no-default-browser-check',
'--disable-popup-blocking',
'https://gemini.google.com/app',
];
return spawn(chrome, args, { stdio: 'ignore' });
return await launchChromeBase({
chromePath,
profileDir,
port,
url: GEMINI_APP_URL,
extraArgs: ['--disable-popup-blocking'],
});
}
async function is_gemini_session_ready(cookies: CookieMap, verbose: boolean): Promise<boolean> {
@@ -209,27 +102,33 @@ async function fetch_google_cookies_via_cdp(
timeoutMs: number,
verbose: boolean,
): Promise<CookieMap> {
await mkdir(profileDir, { recursive: true });
const port = await get_free_port();
const chrome = await launch_chrome(profileDir, port);
const existingPort = await find_existing_chrome_debug_port(profileDir);
const reusing = existingPort !== null;
const port = existingPort ?? await get_free_port();
const chrome = reusing ? null : await launch_chrome(profileDir, port);
let cdp: CdpConnection | null = null;
let targetId: string | null = null;
try {
const wsUrl = await wait_for_chrome_debug_port(port, 30_000);
const wsUrl = await waitForChromeDebugPort(port, 30_000, { includeLastError: true });
cdp = await CdpConnection.connect(wsUrl, 15_000);
const { targetId } = await cdp.send<{ targetId: string }>('Target.createTarget', {
url: 'https://gemini.google.com/app',
newWindow: true,
});
const { sessionId } = await cdp.send<{ sessionId: string }>('Target.attachToTarget', { targetId, flatten: true });
await cdp.send('Network.enable', {}, { sessionId });
if (verbose) {
logger.info('Chrome opened. If needed, complete Google login in the window. Waiting for a valid Gemini session...');
logger.info(reusing
? `Reusing existing Chrome on port ${port}. Waiting for a valid Gemini session...`
: 'Chrome opened. If needed, complete Google login in the window. Waiting for a valid Gemini session...');
}
const page = await openPageSession({
cdp,
reusing,
url: GEMINI_APP_URL,
matchTarget: (target) => target.type === 'page' && target.url.includes('gemini.google.com'),
enableNetwork: true,
});
const { sessionId } = page;
targetId = page.targetId;
const start = Date.now();
let last: CookieMap = {};
@@ -240,14 +139,14 @@ async function fetch_google_cookies_via_cdp(
{ sessionId, timeoutMs: 10_000 },
);
const m: CookieMap = {};
for (const c of cookies) {
if (c?.name && typeof c.value === 'string') m[c.name] = c.value;
const cookieMap: CookieMap = {};
for (const cookie of cookies) {
if (cookie?.name && typeof cookie.value === 'string') cookieMap[cookie.name] = cookie.value;
}
last = m;
if (await is_gemini_session_ready(m, verbose)) {
return m;
last = cookieMap;
if (await is_gemini_session_ready(cookieMap, verbose)) {
return cookieMap;
}
await sleep(1000);
@@ -256,22 +155,19 @@ async function fetch_google_cookies_via_cdp(
throw new Error(`Timed out waiting for a valid Gemini session. Last keys: ${Object.keys(last).join(', ')}`);
} finally {
if (cdp) {
try {
await cdp.send('Browser.close', {}, { timeoutMs: 5_000 });
} catch {}
if (reusing && targetId) {
try {
await cdp.send('Target.closeTarget', { targetId }, { timeoutMs: 5_000 });
} catch {}
} else {
try {
await cdp.send('Browser.close', {}, { timeoutMs: 5_000 });
} catch {}
}
cdp.close();
}
try {
chrome.kill('SIGTERM');
} catch {}
setTimeout(() => {
if (!chrome.killed) {
try {
chrome.kill('SIGKILL');
} catch {}
}
}, 2_000).unref?.();
if (chrome) killChrome(chrome);
}
}
@@ -286,8 +182,8 @@ export async function load_browser_cookies(domain_name: string = '', verbose: bo
const cookies = await fetch_google_cookies_via_cdp(profileDir, 120_000, verbose);
const filtered: CookieMap = {};
for (const [k, v] of Object.entries(cookies)) {
if (typeof v === 'string' && v.length > 0) filtered[k] = v;
for (const [key, value] of Object.entries(cookies)) {
if (typeof value === 'string' && value.length > 0) filtered[key] = value;
}
await write_cookie_file(filtered, resolveGeminiWebCookiePath(), 'cdp');
@@ -0,0 +1,8 @@
{
"name": "baoyu-danger-gemini-web-scripts",
"private": true,
"type": "module",
"dependencies": {
"baoyu-chrome-cdp": "file:./vendor/baoyu-chrome-cdp"
}
}
@@ -0,0 +1,9 @@
{
"name": "baoyu-chrome-cdp",
"private": true,
"version": "0.1.0",
"type": "module",
"exports": {
".": "./src/index.ts"
}
}
@@ -0,0 +1,408 @@
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import process from "node:process";
export type PlatformCandidates = {
darwin?: string[];
win32?: string[];
default: string[];
};
type PendingRequest = {
resolve: (value: unknown) => void;
reject: (error: Error) => void;
timer: ReturnType<typeof setTimeout> | null;
};
type CdpSendOptions = {
sessionId?: string;
timeoutMs?: number;
};
type FetchJsonOptions = {
timeoutMs?: number;
};
type FindChromeExecutableOptions = {
candidates: PlatformCandidates;
envNames?: string[];
};
type ResolveSharedChromeProfileDirOptions = {
envNames?: string[];
appDataDirName?: string;
profileDirName?: string;
wslWindowsHome?: string | null;
};
type FindExistingChromeDebugPortOptions = {
profileDir: string;
timeoutMs?: number;
};
type LaunchChromeOptions = {
chromePath: string;
profileDir: string;
port: number;
url?: string;
headless?: boolean;
extraArgs?: string[];
};
type ChromeTargetInfo = {
targetId: string;
url: string;
type: string;
};
type OpenPageSessionOptions = {
cdp: CdpConnection;
reusing: boolean;
url: string;
matchTarget: (target: ChromeTargetInfo) => boolean;
enablePage?: boolean;
enableRuntime?: boolean;
enableDom?: boolean;
enableNetwork?: boolean;
activateTarget?: boolean;
};
export type PageSession = {
sessionId: string;
targetId: string;
};
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export async function getFreePort(fixedEnvName?: string): Promise<number> {
const fixed = fixedEnvName ? Number.parseInt(process.env[fixedEnvName] ?? "", 10) : NaN;
if (Number.isInteger(fixed) && fixed > 0) return fixed;
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Unable to allocate a free TCP port.")));
return;
}
const port = address.port;
server.close((err) => {
if (err) reject(err);
else resolve(port);
});
});
});
}
export function findChromeExecutable(options: FindChromeExecutableOptions): string | undefined {
for (const envName of options.envNames ?? []) {
const override = process.env[envName]?.trim();
if (override && fs.existsSync(override)) return override;
}
const candidates = process.platform === "darwin"
? options.candidates.darwin ?? options.candidates.default
: process.platform === "win32"
? options.candidates.win32 ?? options.candidates.default
: options.candidates.default;
for (const candidate of candidates) {
if (fs.existsSync(candidate)) return candidate;
}
return undefined;
}
export function resolveSharedChromeProfileDir(options: ResolveSharedChromeProfileDirOptions = {}): string {
for (const envName of options.envNames ?? []) {
const override = process.env[envName]?.trim();
if (override) return path.resolve(override);
}
const appDataDirName = options.appDataDirName ?? "baoyu-skills";
const profileDirName = options.profileDirName ?? "chrome-profile";
if (options.wslWindowsHome) {
return path.join(options.wslWindowsHome, ".local", "share", appDataDirName, profileDirName);
}
const base = process.platform === "darwin"
? path.join(os.homedir(), "Library", "Application Support")
: process.platform === "win32"
? (process.env.APPDATA ?? path.join(os.homedir(), "AppData", "Roaming"))
: (process.env.XDG_DATA_HOME ?? path.join(os.homedir(), ".local", "share"));
return path.join(base, appDataDirName, profileDirName);
}
async function fetchWithTimeout(url: string, timeoutMs?: number): Promise<Response> {
if (!timeoutMs || timeoutMs <= 0) return await fetch(url, { redirect: "follow" });
const ctl = new AbortController();
const timer = setTimeout(() => ctl.abort(), timeoutMs);
try {
return await fetch(url, { redirect: "follow", signal: ctl.signal });
} finally {
clearTimeout(timer);
}
}
async function fetchJson<T = unknown>(url: string, options: FetchJsonOptions = {}): Promise<T> {
const response = await fetchWithTimeout(url, options.timeoutMs);
if (!response.ok) {
throw new Error(`Request failed: ${response.status} ${response.statusText}`);
}
return await response.json() as T;
}
async function isDebugPortReady(port: number, timeoutMs = 3_000): Promise<boolean> {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
`http://127.0.0.1:${port}/json/version`,
{ timeoutMs }
);
return !!version.webSocketDebuggerUrl;
} catch {
return false;
}
}
export async function findExistingChromeDebugPort(options: FindExistingChromeDebugPortOptions): Promise<number | null> {
const timeoutMs = options.timeoutMs ?? 3_000;
const portFile = path.join(options.profileDir, "DevToolsActivePort");
try {
const content = fs.readFileSync(portFile, "utf-8");
const [portLine] = content.split(/\r?\n/);
const port = Number.parseInt(portLine?.trim() ?? "", 10);
if (port > 0 && await isDebugPortReady(port, timeoutMs)) return port;
} catch {}
if (process.platform === "win32") return null;
try {
const result = spawnSync("ps", ["aux"], { encoding: "utf-8", timeout: 5_000 });
if (result.status !== 0 || !result.stdout) return null;
const lines = result.stdout
.split("\n")
.filter((line) => line.includes(options.profileDir) && line.includes("--remote-debugging-port="));
for (const line of lines) {
const portMatch = line.match(/--remote-debugging-port=(\d+)/);
const port = Number.parseInt(portMatch?.[1] ?? "", 10);
if (port > 0 && await isDebugPortReady(port, timeoutMs)) return port;
}
} catch {}
return null;
}
export async function waitForChromeDebugPort(
port: number,
timeoutMs: number,
options?: { includeLastError?: boolean }
): Promise<string> {
const start = Date.now();
let lastError: unknown = null;
while (Date.now() - start < timeoutMs) {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
`http://127.0.0.1:${port}/json/version`,
{ timeoutMs: 5_000 }
);
if (version.webSocketDebuggerUrl) return version.webSocketDebuggerUrl;
lastError = new Error("Missing webSocketDebuggerUrl");
} catch (error) {
lastError = error;
}
await sleep(200);
}
if (options?.includeLastError && lastError) {
throw new Error(
`Chrome debug port not ready: ${lastError instanceof Error ? lastError.message : String(lastError)}`
);
}
throw new Error("Chrome debug port not ready");
}
export class CdpConnection {
private ws: WebSocket;
private nextId = 0;
private pending = new Map<number, PendingRequest>();
private eventHandlers = new Map<string, Set<(params: unknown) => void>>();
private defaultTimeoutMs: number;
private constructor(ws: WebSocket, defaultTimeoutMs = 15_000) {
this.ws = ws;
this.defaultTimeoutMs = defaultTimeoutMs;
this.ws.addEventListener("message", (event) => {
try {
const data = typeof event.data === "string"
? event.data
: new TextDecoder().decode(event.data as ArrayBuffer);
const msg = JSON.parse(data) as {
id?: number;
method?: string;
params?: unknown;
result?: unknown;
error?: { message?: string };
};
if (msg.method) {
const handlers = this.eventHandlers.get(msg.method);
if (handlers) {
handlers.forEach((handler) => handler(msg.params));
}
}
if (msg.id) {
const pending = this.pending.get(msg.id);
if (pending) {
this.pending.delete(msg.id);
if (pending.timer) clearTimeout(pending.timer);
if (msg.error?.message) pending.reject(new Error(msg.error.message));
else pending.resolve(msg.result);
}
}
} catch {}
});
this.ws.addEventListener("close", () => {
for (const [id, pending] of this.pending.entries()) {
this.pending.delete(id);
if (pending.timer) clearTimeout(pending.timer);
pending.reject(new Error("CDP connection closed."));
}
});
}
static async connect(
url: string,
timeoutMs: number,
options?: { defaultTimeoutMs?: number }
): Promise<CdpConnection> {
const ws = new WebSocket(url);
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error("CDP connection timeout.")), timeoutMs);
ws.addEventListener("open", () => {
clearTimeout(timer);
resolve();
});
ws.addEventListener("error", () => {
clearTimeout(timer);
reject(new Error("CDP connection failed."));
});
});
return new CdpConnection(ws, options?.defaultTimeoutMs ?? 15_000);
}
on(method: string, handler: (params: unknown) => void): void {
if (!this.eventHandlers.has(method)) {
this.eventHandlers.set(method, new Set());
}
this.eventHandlers.get(method)?.add(handler);
}
off(method: string, handler: (params: unknown) => void): void {
this.eventHandlers.get(method)?.delete(handler);
}
async send<T = unknown>(method: string, params?: Record<string, unknown>, options?: CdpSendOptions): Promise<T> {
const id = ++this.nextId;
const message: Record<string, unknown> = { id, method };
if (params) message.params = params;
if (options?.sessionId) message.sessionId = options.sessionId;
const timeoutMs = options?.timeoutMs ?? this.defaultTimeoutMs;
const result = await new Promise<unknown>((resolve, reject) => {
const timer = timeoutMs > 0
? setTimeout(() => {
this.pending.delete(id);
reject(new Error(`CDP timeout: ${method}`));
}, timeoutMs)
: null;
this.pending.set(id, { resolve, reject, timer });
this.ws.send(JSON.stringify(message));
});
return result as T;
}
close(): void {
try {
this.ws.close();
} catch {}
}
}
export async function launchChrome(options: LaunchChromeOptions): Promise<ChildProcess> {
await fs.promises.mkdir(options.profileDir, { recursive: true });
const args = [
`--remote-debugging-port=${options.port}`,
`--user-data-dir=${options.profileDir}`,
"--no-first-run",
"--no-default-browser-check",
...(options.extraArgs ?? []),
];
if (options.headless) args.push("--headless=new");
if (options.url) args.push(options.url);
return spawn(options.chromePath, args, { stdio: "ignore" });
}
export function killChrome(chrome: ChildProcess): void {
try {
chrome.kill("SIGTERM");
} catch {}
setTimeout(() => {
if (!chrome.killed) {
try {
chrome.kill("SIGKILL");
} catch {}
}
}, 2_000).unref?.();
}
export async function openPageSession(options: OpenPageSessionOptions): Promise<PageSession> {
let targetId: string;
if (options.reusing) {
const created = await options.cdp.send<{ targetId: string }>("Target.createTarget", { url: options.url });
targetId = created.targetId;
} else {
const targets = await options.cdp.send<{ targetInfos: ChromeTargetInfo[] }>("Target.getTargets");
const existing = targets.targetInfos.find(options.matchTarget);
if (existing) {
targetId = existing.targetId;
} else {
const created = await options.cdp.send<{ targetId: string }>("Target.createTarget", { url: options.url });
targetId = created.targetId;
}
}
const { sessionId } = await options.cdp.send<{ sessionId: string }>(
"Target.attachToTarget",
{ targetId, flatten: true }
);
if (options.activateTarget ?? true) {
await options.cdp.send("Target.activateTarget", { targetId });
}
if (options.enablePage) await options.cdp.send("Page.enable", {}, { sessionId });
if (options.enableRuntime) await options.cdp.send("Runtime.enable", {}, { sessionId });
if (options.enableDom) await options.cdp.send("DOM.enable", {}, { sessionId });
if (options.enableNetwork) await options.cdp.send("Network.enable", {}, { sessionId });
return { sessionId, targetId };
}
@@ -0,0 +1,14 @@
{
"lockfileVersion": 1,
"workspaces": {
"": {
"name": "baoyu-danger-x-to-markdown-scripts",
"dependencies": {
"baoyu-chrome-cdp": "file:./vendor/baoyu-chrome-cdp",
},
},
},
"packages": {
"baoyu-chrome-cdp": ["baoyu-chrome-cdp@file:vendor/baoyu-chrome-cdp", {}],
}
}
@@ -1,7 +1,16 @@
import { spawn, type ChildProcess } from "node:child_process";
import fs from "node:fs";
import { mkdir } from "node:fs/promises";
import net from "node:net";
import {
CdpConnection,
findChromeExecutable as findChromeExecutableBase,
findExistingChromeDebugPort,
getFreePort,
killChrome,
launchChrome as launchChromeBase,
openPageSession,
sleep,
waitForChromeDebugPort,
type PlatformCandidates,
} from "baoyu-chrome-cdp";
import process from "node:process";
import { read_cookie_file, write_cookie_file } from "./cookie-file.js";
@@ -9,201 +18,47 @@ import { resolveXToMarkdownCookiePath } from "./paths.js";
import { X_COOKIE_NAMES, X_REQUIRED_COOKIES, X_LOGIN_URL, X_USER_DATA_DIR } from "./constants.js";
import type { CookieLike } from "./types.js";
type CdpSendOptions = { sessionId?: string; timeoutMs?: number };
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function fetchWithTimeout(
url: string,
init: RequestInit & { timeoutMs?: number } = {}
): Promise<Response> {
const { timeoutMs, ...rest } = init;
if (!timeoutMs || timeoutMs <= 0) return fetch(url, rest);
const ctl = new AbortController();
const t = setTimeout(() => ctl.abort(), timeoutMs);
try {
return await fetch(url, { ...rest, signal: ctl.signal });
} finally {
clearTimeout(t);
}
}
class CdpConnection {
private ws: WebSocket;
private nextId = 0;
private pending = new Map<
number,
{ resolve: (v: unknown) => void; reject: (e: Error) => void; timer: ReturnType<typeof setTimeout> | null }
>();
private constructor(ws: WebSocket) {
this.ws = ws;
this.ws.addEventListener("message", (event) => {
try {
const data =
typeof event.data === "string"
? event.data
: new TextDecoder().decode(event.data as ArrayBuffer);
const msg = JSON.parse(data) as { id?: number; result?: unknown; error?: { message?: string } };
if (msg.id) {
const p = this.pending.get(msg.id);
if (p) {
this.pending.delete(msg.id);
if (p.timer) clearTimeout(p.timer);
if (msg.error?.message) p.reject(new Error(msg.error.message));
else p.resolve(msg.result);
}
}
} catch {}
});
this.ws.addEventListener("close", () => {
for (const [id, p] of this.pending.entries()) {
this.pending.delete(id);
if (p.timer) clearTimeout(p.timer);
p.reject(new Error("CDP connection closed."));
}
});
}
static async connect(url: string, timeoutMs: number): Promise<CdpConnection> {
const ws = new WebSocket(url);
await new Promise<void>((resolve, reject) => {
const t = setTimeout(() => reject(new Error("CDP connection timeout.")), timeoutMs);
ws.addEventListener("open", () => {
clearTimeout(t);
resolve();
});
ws.addEventListener("error", () => {
clearTimeout(t);
reject(new Error("CDP connection failed."));
});
});
return new CdpConnection(ws);
}
async send<T = unknown>(
method: string,
params?: Record<string, unknown>,
opts?: CdpSendOptions
): Promise<T> {
const id = ++this.nextId;
const msg: Record<string, unknown> = { id, method };
if (params) msg.params = params;
if (opts?.sessionId) msg.sessionId = opts.sessionId;
const timeoutMs = opts?.timeoutMs ?? 15_000;
const out = await new Promise<unknown>((resolve, reject) => {
const t =
timeoutMs > 0
? setTimeout(() => {
this.pending.delete(id);
reject(new Error(`CDP timeout: ${method}`));
}, timeoutMs)
: null;
this.pending.set(id, { resolve, reject, timer: t });
this.ws.send(JSON.stringify(msg));
});
return out as T;
}
close(): void {
try {
this.ws.close();
} catch {}
}
}
async function getFreePort(): Promise<number> {
const fixed = parseInt(process.env.X_DEBUG_PORT || "", 10);
if (fixed > 0) return fixed;
return await new Promise((resolve, reject) => {
const srv = net.createServer();
srv.unref();
srv.on("error", reject);
srv.listen(0, "127.0.0.1", () => {
const addr = srv.address();
if (!addr || typeof addr === "string") {
srv.close(() => reject(new Error("Unable to allocate a free TCP port.")));
return;
}
const port = addr.port;
srv.close((err) => (err ? reject(err) : resolve(port)));
});
});
}
const CHROME_CANDIDATES_FULL: PlatformCandidates = {
darwin: [
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
"/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta",
"/Applications/Chromium.app/Contents/MacOS/Chromium",
"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
],
win32: [
"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
"C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
"C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe",
"C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
],
default: [
"/usr/bin/google-chrome",
"/usr/bin/google-chrome-stable",
"/usr/bin/chromium",
"/usr/bin/chromium-browser",
"/snap/bin/chromium",
"/usr/bin/microsoft-edge",
],
};
function findChromeExecutable(): string | null {
const override = process.env.X_CHROME_PATH?.trim();
if (override && fs.existsSync(override)) return override;
const candidates: string[] = [];
switch (process.platform) {
case "darwin":
candidates.push(
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
"/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta",
"/Applications/Chromium.app/Contents/MacOS/Chromium",
"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"
);
break;
case "win32":
candidates.push(
"C:\\\\Program Files\\\\Google\\\\Chrome\\\\Application\\\\chrome.exe",
"C:\\\\Program Files (x86)\\\\Google\\\\Chrome\\\\Application\\\\chrome.exe",
"C:\\\\Program Files\\\\Microsoft\\\\Edge\\\\Application\\\\msedge.exe",
"C:\\\\Program Files (x86)\\\\Microsoft\\\\Edge\\\\Application\\\\msedge.exe"
);
break;
default:
candidates.push(
"/usr/bin/google-chrome",
"/usr/bin/google-chrome-stable",
"/usr/bin/chromium",
"/usr/bin/chromium-browser",
"/snap/bin/chromium",
"/usr/bin/microsoft-edge"
);
break;
}
for (const p of candidates) {
if (fs.existsSync(p)) return p;
}
return null;
return findChromeExecutableBase({
candidates: CHROME_CANDIDATES_FULL,
envNames: ["X_CHROME_PATH"],
}) ?? null;
}
async function waitForChromeDebugPort(port: number, timeoutMs: number): Promise<string> {
const start = Date.now();
while (Date.now() - start < timeoutMs) {
try {
const res = await fetchWithTimeout(`http://127.0.0.1:${port}/json/version`, { timeoutMs: 5_000 });
if (!res.ok) throw new Error(`status=${res.status}`);
const j = (await res.json()) as { webSocketDebuggerUrl?: string };
if (j.webSocketDebuggerUrl) return j.webSocketDebuggerUrl;
} catch {}
await sleep(200);
}
throw new Error("Chrome debug port not ready");
}
async function launchChrome(profileDir: string, port: number): Promise<ChildProcess> {
const chrome = findChromeExecutable();
if (!chrome) throw new Error("Chrome executable not found.");
const args = [
`--remote-debugging-port=${port}`,
`--user-data-dir=${profileDir}`,
"--no-first-run",
"--no-default-browser-check",
"--disable-popup-blocking",
X_LOGIN_URL,
];
return spawn(chrome, args, { stdio: "ignore" });
async function launchChrome(profileDir: string, port: number) {
const chromePath = findChromeExecutable();
if (!chromePath) throw new Error("Chrome executable not found.");
return await launchChromeBase({
chromePath,
profileDir,
port,
url: X_LOGIN_URL,
extraArgs: ["--disable-popup-blocking"],
});
}
async function fetchXCookiesViaCdp(
@@ -212,25 +67,33 @@ async function fetchXCookiesViaCdp(
verbose: boolean,
log?: (message: string) => void
): Promise<Record<string, string>> {
await mkdir(profileDir, { recursive: true });
const port = await getFreePort();
const chrome = await launchChrome(profileDir, port);
const existingPort = await findExistingChromeDebugPort({ profileDir });
const reusing = existingPort !== null;
const port = existingPort ?? await getFreePort("X_DEBUG_PORT");
const chrome = reusing ? null : await launchChrome(profileDir, port);
let cdp: CdpConnection | null = null;
let targetId: string | null = null;
try {
const wsUrl = await waitForChromeDebugPort(port, 30_000);
const wsUrl = await waitForChromeDebugPort(port, 30_000, { includeLastError: true });
cdp = await CdpConnection.connect(wsUrl, 15_000);
const { targetId } = await cdp.send<{ targetId: string }>("Target.createTarget", {
const page = await openPageSession({
cdp,
reusing,
url: X_LOGIN_URL,
newWindow: true,
matchTarget: (target) => target.type === "page" && (
target.url.includes("x.com") || target.url.includes("twitter.com")
),
enableNetwork: true,
});
const { sessionId } = await cdp.send<{ sessionId: string }>("Target.attachToTarget", { targetId, flatten: true });
await cdp.send("Network.enable", {}, { sessionId });
const { sessionId } = page;
targetId = page.targetId;
if (verbose) {
log?.("[x-cookies] Chrome opened. If needed, complete X login in the window. Waiting for cookies...");
log?.(reusing
? `[x-cookies] Reusing existing Chrome on port ${port}. Waiting for cookies...`
: "[x-cookies] Chrome opened. If needed, complete X login in the window. Waiting for cookies...");
}
const start = Date.now();
@@ -255,22 +118,19 @@ async function fetchXCookiesViaCdp(
throw new Error(`Timed out waiting for X cookies. Last keys: ${Object.keys(last).join(", ")}`);
} finally {
if (cdp) {
try {
await cdp.send("Browser.close", {}, { timeoutMs: 5_000 });
} catch {}
if (reusing && targetId) {
try {
await cdp.send("Target.closeTarget", { targetId }, { timeoutMs: 5_000 });
} catch {}
} else {
try {
await cdp.send("Browser.close", {}, { timeoutMs: 5_000 });
} catch {}
}
cdp.close();
}
try {
chrome.kill("SIGTERM");
} catch {}
setTimeout(() => {
if (!chrome.killed) {
try {
chrome.kill("SIGKILL");
} catch {}
}
}, 2_000).unref?.();
if (chrome) killChrome(chrome);
}
}
@@ -0,0 +1,8 @@
{
"name": "baoyu-danger-x-to-markdown-scripts",
"private": true,
"type": "module",
"dependencies": {
"baoyu-chrome-cdp": "file:./vendor/baoyu-chrome-cdp"
}
}
@@ -0,0 +1,9 @@
{
"name": "baoyu-chrome-cdp",
"private": true,
"version": "0.1.0",
"type": "module",
"exports": {
".": "./src/index.ts"
}
}
@@ -0,0 +1,408 @@
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import process from "node:process";
export type PlatformCandidates = {
darwin?: string[];
win32?: string[];
default: string[];
};
type PendingRequest = {
resolve: (value: unknown) => void;
reject: (error: Error) => void;
timer: ReturnType<typeof setTimeout> | null;
};
type CdpSendOptions = {
sessionId?: string;
timeoutMs?: number;
};
type FetchJsonOptions = {
timeoutMs?: number;
};
type FindChromeExecutableOptions = {
candidates: PlatformCandidates;
envNames?: string[];
};
type ResolveSharedChromeProfileDirOptions = {
envNames?: string[];
appDataDirName?: string;
profileDirName?: string;
wslWindowsHome?: string | null;
};
type FindExistingChromeDebugPortOptions = {
profileDir: string;
timeoutMs?: number;
};
type LaunchChromeOptions = {
chromePath: string;
profileDir: string;
port: number;
url?: string;
headless?: boolean;
extraArgs?: string[];
};
type ChromeTargetInfo = {
targetId: string;
url: string;
type: string;
};
type OpenPageSessionOptions = {
cdp: CdpConnection;
reusing: boolean;
url: string;
matchTarget: (target: ChromeTargetInfo) => boolean;
enablePage?: boolean;
enableRuntime?: boolean;
enableDom?: boolean;
enableNetwork?: boolean;
activateTarget?: boolean;
};
export type PageSession = {
sessionId: string;
targetId: string;
};
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export async function getFreePort(fixedEnvName?: string): Promise<number> {
const fixed = fixedEnvName ? Number.parseInt(process.env[fixedEnvName] ?? "", 10) : NaN;
if (Number.isInteger(fixed) && fixed > 0) return fixed;
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Unable to allocate a free TCP port.")));
return;
}
const port = address.port;
server.close((err) => {
if (err) reject(err);
else resolve(port);
});
});
});
}
export function findChromeExecutable(options: FindChromeExecutableOptions): string | undefined {
for (const envName of options.envNames ?? []) {
const override = process.env[envName]?.trim();
if (override && fs.existsSync(override)) return override;
}
const candidates = process.platform === "darwin"
? options.candidates.darwin ?? options.candidates.default
: process.platform === "win32"
? options.candidates.win32 ?? options.candidates.default
: options.candidates.default;
for (const candidate of candidates) {
if (fs.existsSync(candidate)) return candidate;
}
return undefined;
}
export function resolveSharedChromeProfileDir(options: ResolveSharedChromeProfileDirOptions = {}): string {
for (const envName of options.envNames ?? []) {
const override = process.env[envName]?.trim();
if (override) return path.resolve(override);
}
const appDataDirName = options.appDataDirName ?? "baoyu-skills";
const profileDirName = options.profileDirName ?? "chrome-profile";
if (options.wslWindowsHome) {
return path.join(options.wslWindowsHome, ".local", "share", appDataDirName, profileDirName);
}
const base = process.platform === "darwin"
? path.join(os.homedir(), "Library", "Application Support")
: process.platform === "win32"
? (process.env.APPDATA ?? path.join(os.homedir(), "AppData", "Roaming"))
: (process.env.XDG_DATA_HOME ?? path.join(os.homedir(), ".local", "share"));
return path.join(base, appDataDirName, profileDirName);
}
async function fetchWithTimeout(url: string, timeoutMs?: number): Promise<Response> {
if (!timeoutMs || timeoutMs <= 0) return await fetch(url, { redirect: "follow" });
const ctl = new AbortController();
const timer = setTimeout(() => ctl.abort(), timeoutMs);
try {
return await fetch(url, { redirect: "follow", signal: ctl.signal });
} finally {
clearTimeout(timer);
}
}
async function fetchJson<T = unknown>(url: string, options: FetchJsonOptions = {}): Promise<T> {
const response = await fetchWithTimeout(url, options.timeoutMs);
if (!response.ok) {
throw new Error(`Request failed: ${response.status} ${response.statusText}`);
}
return await response.json() as T;
}
async function isDebugPortReady(port: number, timeoutMs = 3_000): Promise<boolean> {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
`http://127.0.0.1:${port}/json/version`,
{ timeoutMs }
);
return !!version.webSocketDebuggerUrl;
} catch {
return false;
}
}
export async function findExistingChromeDebugPort(options: FindExistingChromeDebugPortOptions): Promise<number | null> {
const timeoutMs = options.timeoutMs ?? 3_000;
const portFile = path.join(options.profileDir, "DevToolsActivePort");
try {
const content = fs.readFileSync(portFile, "utf-8");
const [portLine] = content.split(/\r?\n/);
const port = Number.parseInt(portLine?.trim() ?? "", 10);
if (port > 0 && await isDebugPortReady(port, timeoutMs)) return port;
} catch {}
if (process.platform === "win32") return null;
try {
const result = spawnSync("ps", ["aux"], { encoding: "utf-8", timeout: 5_000 });
if (result.status !== 0 || !result.stdout) return null;
const lines = result.stdout
.split("\n")
.filter((line) => line.includes(options.profileDir) && line.includes("--remote-debugging-port="));
for (const line of lines) {
const portMatch = line.match(/--remote-debugging-port=(\d+)/);
const port = Number.parseInt(portMatch?.[1] ?? "", 10);
if (port > 0 && await isDebugPortReady(port, timeoutMs)) return port;
}
} catch {}
return null;
}
export async function waitForChromeDebugPort(
port: number,
timeoutMs: number,
options?: { includeLastError?: boolean }
): Promise<string> {
const start = Date.now();
let lastError: unknown = null;
while (Date.now() - start < timeoutMs) {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
`http://127.0.0.1:${port}/json/version`,
{ timeoutMs: 5_000 }
);
if (version.webSocketDebuggerUrl) return version.webSocketDebuggerUrl;
lastError = new Error("Missing webSocketDebuggerUrl");
} catch (error) {
lastError = error;
}
await sleep(200);
}
if (options?.includeLastError && lastError) {
throw new Error(
`Chrome debug port not ready: ${lastError instanceof Error ? lastError.message : String(lastError)}`
);
}
throw new Error("Chrome debug port not ready");
}
export class CdpConnection {
private ws: WebSocket;
private nextId = 0;
private pending = new Map<number, PendingRequest>();
private eventHandlers = new Map<string, Set<(params: unknown) => void>>();
private defaultTimeoutMs: number;
private constructor(ws: WebSocket, defaultTimeoutMs = 15_000) {
this.ws = ws;
this.defaultTimeoutMs = defaultTimeoutMs;
this.ws.addEventListener("message", (event) => {
try {
const data = typeof event.data === "string"
? event.data
: new TextDecoder().decode(event.data as ArrayBuffer);
const msg = JSON.parse(data) as {
id?: number;
method?: string;
params?: unknown;
result?: unknown;
error?: { message?: string };
};
if (msg.method) {
const handlers = this.eventHandlers.get(msg.method);
if (handlers) {
handlers.forEach((handler) => handler(msg.params));
}
}
if (msg.id) {
const pending = this.pending.get(msg.id);
if (pending) {
this.pending.delete(msg.id);
if (pending.timer) clearTimeout(pending.timer);
if (msg.error?.message) pending.reject(new Error(msg.error.message));
else pending.resolve(msg.result);
}
}
} catch {}
});
this.ws.addEventListener("close", () => {
for (const [id, pending] of this.pending.entries()) {
this.pending.delete(id);
if (pending.timer) clearTimeout(pending.timer);
pending.reject(new Error("CDP connection closed."));
}
});
}
static async connect(
url: string,
timeoutMs: number,
options?: { defaultTimeoutMs?: number }
): Promise<CdpConnection> {
const ws = new WebSocket(url);
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error("CDP connection timeout.")), timeoutMs);
ws.addEventListener("open", () => {
clearTimeout(timer);
resolve();
});
ws.addEventListener("error", () => {
clearTimeout(timer);
reject(new Error("CDP connection failed."));
});
});
return new CdpConnection(ws, options?.defaultTimeoutMs ?? 15_000);
}
on(method: string, handler: (params: unknown) => void): void {
if (!this.eventHandlers.has(method)) {
this.eventHandlers.set(method, new Set());
}
this.eventHandlers.get(method)?.add(handler);
}
off(method: string, handler: (params: unknown) => void): void {
this.eventHandlers.get(method)?.delete(handler);
}
async send<T = unknown>(method: string, params?: Record<string, unknown>, options?: CdpSendOptions): Promise<T> {
const id = ++this.nextId;
const message: Record<string, unknown> = { id, method };
if (params) message.params = params;
if (options?.sessionId) message.sessionId = options.sessionId;
const timeoutMs = options?.timeoutMs ?? this.defaultTimeoutMs;
const result = await new Promise<unknown>((resolve, reject) => {
const timer = timeoutMs > 0
? setTimeout(() => {
this.pending.delete(id);
reject(new Error(`CDP timeout: ${method}`));
}, timeoutMs)
: null;
this.pending.set(id, { resolve, reject, timer });
this.ws.send(JSON.stringify(message));
});
return result as T;
}
close(): void {
try {
this.ws.close();
} catch {}
}
}
export async function launchChrome(options: LaunchChromeOptions): Promise<ChildProcess> {
await fs.promises.mkdir(options.profileDir, { recursive: true });
const args = [
`--remote-debugging-port=${options.port}`,
`--user-data-dir=${options.profileDir}`,
"--no-first-run",
"--no-default-browser-check",
...(options.extraArgs ?? []),
];
if (options.headless) args.push("--headless=new");
if (options.url) args.push(options.url);
return spawn(options.chromePath, args, { stdio: "ignore" });
}
export function killChrome(chrome: ChildProcess): void {
try {
chrome.kill("SIGTERM");
} catch {}
setTimeout(() => {
if (!chrome.killed) {
try {
chrome.kill("SIGKILL");
} catch {}
}
}, 2_000).unref?.();
}
export async function openPageSession(options: OpenPageSessionOptions): Promise<PageSession> {
let targetId: string;
if (options.reusing) {
const created = await options.cdp.send<{ targetId: string }>("Target.createTarget", { url: options.url });
targetId = created.targetId;
} else {
const targets = await options.cdp.send<{ targetInfos: ChromeTargetInfo[] }>("Target.getTargets");
const existing = targets.targetInfos.find(options.matchTarget);
if (existing) {
targetId = existing.targetId;
} else {
const created = await options.cdp.send<{ targetId: string }>("Target.createTarget", { url: options.url });
targetId = created.targetId;
}
}
const { sessionId } = await options.cdp.send<{ sessionId: string }>(
"Target.attachToTarget",
{ targetId, flatten: true }
);
if (options.activateTarget ?? true) {
await options.cdp.send("Target.activateTarget", { targetId });
}
if (options.enablePage) await options.cdp.send("Page.enable", {}, { sessionId });
if (options.enableRuntime) await options.cdp.send("Runtime.enable", {}, { sessionId });
if (options.enableDom) await options.cdp.send("DOM.enable", {}, { sessionId });
if (options.enableNetwork) await options.cdp.send("Network.enable", {}, { sessionId });
return { sessionId, targetId };
}
+7 -2
View File
@@ -17,6 +17,7 @@ Two dimensions: **layout** (information structure) × **style** (visual aestheti
/baoyu-infographic path/to/content.md
/baoyu-infographic path/to/content.md --layout hierarchical-layers --style technical-schematic
/baoyu-infographic path/to/content.md --aspect portrait --lang zh
/baoyu-infographic path/to/content.md --aspect 3:4
/baoyu-infographic # then paste content
```
@@ -26,7 +27,7 @@ Two dimensions: **layout** (information structure) × **style** (visual aestheti
|--------|--------|
| `--layout` | 21 options (see Layout Gallery), default: bento-grid |
| `--style` | 20 options (see Style Gallery), default: craft-handmade |
| `--aspect` | landscape (16:9), portrait (9:16), square (1:1) |
| `--aspect` | Named: landscape (16:9), portrait (9:16), square (1:1). Custom: any W:H ratio (e.g., 3:4, 4:3, 2.35:1) |
| `--lang` | en, zh, ja, etc. |
## Layout Gallery
@@ -220,7 +221,7 @@ Use **single AskUserQuestion call** with multiple questions to confirm all optio
| Question | When | Options |
|----------|------|---------|
| **Combination** | Always | 3+ layout×style combos with rationale |
| **Aspect** | Always | landscape (16:9), portrait (9:16), square (1:1) |
| **Aspect** | Always | Named presets (landscape/portrait/square) or custom W:H ratio (e.g., 3:4, 4:3, 2.35:1) |
| **Language** | Only if source ≠ user language | Language for text content |
**Important**: Do NOT split into separate AskUserQuestion calls. Combine all applicable questions into one call.
@@ -236,6 +237,10 @@ Combine:
4. Structured content from Step 2
5. All text in confirmed language
**Aspect ratio resolution** for `{{ASPECT_RATIO}}`:
- Named presets → ratio string: landscape→`16:9`, portrait→`9:16`, square→`1:1`
- Custom W:H ratios → use as-is (e.g., `3:4`, `4:3`, `2.35:1`)
### Step 6: Generate Image
1. Select available image generation skill (ask user if multiple)
+108 -1
View File
@@ -95,9 +95,115 @@ chrome_profile_path: /path/to/chrome/profile
**Value priority**:
1. CLI arguments
2. Frontmatter
3. EXTEND.md
3. EXTEND.md (account-level → global-level)
4. Skill defaults
## Multi-Account Support
EXTEND.md supports managing multiple WeChat Official Accounts. When `accounts:` block is present, each account can have its own credentials, Chrome profile, and default settings.
**Compatibility rules**:
| Condition | Mode | Behavior |
|-----------|------|----------|
| No `accounts` block | Single-account | Current behavior, unchanged |
| `accounts` with 1 entry | Single-account | Auto-select, no prompt |
| `accounts` with 2+ entries | Multi-account | Prompt to select before publishing |
| `accounts` with `default: true` | Multi-account | Pre-select default, user can switch |
**Multi-account EXTEND.md example**:
```md
default_theme: default
default_color: blue
accounts:
- name: 宝玉的技术分享
alias: baoyu
default: true
default_publish_method: api
default_author: 宝玉
need_open_comment: 1
only_fans_can_comment: 0
app_id: your_wechat_app_id
app_secret: your_wechat_app_secret
- name: AI工具集
alias: ai-tools
default_publish_method: browser
default_author: AI工具集
need_open_comment: 1
only_fans_can_comment: 0
```
**Per-account keys** (can be set per-account or globally as fallback):
`default_publish_method`, `default_author`, `need_open_comment`, `only_fans_can_comment`, `app_id`, `app_secret`, `chrome_profile_path`
**Global-only keys** (always shared across accounts):
`default_theme`, `default_color`
### Account Selection (Step 0.5)
Insert between Step 0 and Step 1 in the Article Posting Workflow:
```
if no accounts block:
→ single-account mode (current behavior)
elif accounts.length == 1:
→ auto-select the only account
elif --account <alias> CLI arg:
→ select matching account
elif one account has default: true:
→ pre-select, show: "Using account: <name> (--account to switch)"
else:
→ prompt user:
"Multiple WeChat accounts configured:
1) <name1> (<alias1>)
2) <name2> (<alias2>)
Select account [1-N]:"
```
### Credential Resolution (API Method)
For a selected account with alias `{alias}`:
1. `app_id` / `app_secret` inline in EXTEND.md account block
2. Env var `WECHAT_{ALIAS}_APP_ID` / `WECHAT_{ALIAS}_APP_SECRET` (alias uppercased, hyphens → underscores)
3. `.baoyu-skills/.env` with prefixed key `WECHAT_{ALIAS}_APP_ID`
4. `~/.baoyu-skills/.env` with prefixed key
5. Fallback to unprefixed `WECHAT_APP_ID` / `WECHAT_APP_SECRET`
**.env multi-account example**:
```bash
# Account: baoyu
WECHAT_BAOYU_APP_ID=your_wechat_app_id
WECHAT_BAOYU_APP_SECRET=your_wechat_app_secret
# Account: ai-tools
WECHAT_AI_TOOLS_APP_ID=your_ai_tools_wechat_app_id
WECHAT_AI_TOOLS_APP_SECRET=your_ai_tools_wechat_app_secret
```
### Chrome Profile (Browser Method)
Each account uses an isolated Chrome profile for independent login sessions:
| Source | Path |
|--------|------|
| Account `chrome_profile_path` in EXTEND.md | Use as-is |
| Auto-generated from alias | `{shared_profile_parent}/wechat-{alias}/` |
| Single-account fallback | Shared default profile (current behavior) |
### CLI `--account` Argument
All publishing scripts accept `--account <alias>`:
```bash
${BUN_X} {baseDir}/scripts/wechat-api.ts <file> --theme default --account ai-tools
${BUN_X} {baseDir}/scripts/wechat-article.ts --markdown <file> --theme default --account baoyu
${BUN_X} {baseDir}/scripts/wechat-browser.ts --markdown <file> --images ./photos/ --account baoyu
```
## Pre-flight Check (Optional)
Before first use, suggest running the environment check. User can skip if they prefer.
@@ -139,6 +245,7 @@ Copy this checklist and check off items as you complete them:
```
Publishing Progress:
- [ ] Step 0: Load preferences (EXTEND.md)
- [ ] Step 0.5: Resolve account (multi-account only)
- [ ] Step 1: Determine input type
- [ ] Step 2: Select method and configure credentials
- [ ] Step 3: Resolve theme/color and validate metadata
@@ -152,6 +152,8 @@ options:
## EXTEND.md Template
### Single Account (Default)
```md
default_theme: [default/grace/simple/modern]
default_color: [preset name, hex, or empty for theme default]
@@ -162,6 +164,40 @@ only_fans_can_comment: [1/0]
chrome_profile_path:
```
### Multi-Account
```md
default_theme: [default/grace/simple/modern]
default_color: [preset name, hex, or empty for theme default]
accounts:
- name: [display name]
alias: [short key, e.g. "baoyu"]
default: true
default_publish_method: [api/browser]
default_author: [author name]
need_open_comment: [1/0]
only_fans_can_comment: [1/0]
app_id: [WeChat App ID, optional]
app_secret: [WeChat App Secret, optional]
- name: [second account name]
alias: [short key, e.g. "ai-tools"]
default_publish_method: [api/browser]
default_author: [author name]
need_open_comment: [1/0]
only_fans_can_comment: [1/0]
```
## Adding More Accounts Later
After initial setup, users can add accounts by editing EXTEND.md:
1. Add an `accounts:` block with list items
2. Move per-account settings (author, publish method, comments) into each account entry
3. Keep global settings (theme, color) at the top level
4. Each account needs a unique `alias` (used for CLI `--account` arg and Chrome profile naming)
5. Set `default: true` on the primary account
## Modifying Preferences Later
Users can edit EXTEND.md directly or delete it to trigger setup again.
@@ -0,0 +1,14 @@
{
"lockfileVersion": 1,
"workspaces": {
"": {
"name": "baoyu-post-to-wechat-scripts",
"dependencies": {
"baoyu-chrome-cdp": "file:./vendor/baoyu-chrome-cdp",
},
},
},
"packages": {
"baoyu-chrome-cdp": ["baoyu-chrome-cdp@file:vendor/baoyu-chrome-cdp", {}],
}
}
+158 -212
View File
@@ -1,172 +1,89 @@
import { spawn } from 'node:child_process';
import fs from 'node:fs';
import { mkdir } from 'node:fs/promises';
import net from 'node:net';
import os from 'node:os';
import { execSync, type ChildProcess } from 'node:child_process';
import path from 'node:path';
import process from 'node:process';
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
import {
CdpConnection,
findChromeExecutable as findChromeExecutableBase,
findExistingChromeDebugPort as findExistingChromeDebugPortBase,
getFreePort as getFreePortBase,
launchChrome as launchChromeBase,
resolveSharedChromeProfileDir,
sleep,
waitForChromeDebugPort,
type PlatformCandidates,
} from 'baoyu-chrome-cdp';
export { CdpConnection, sleep, waitForChromeDebugPort };
const CHROME_CANDIDATES_FULL: PlatformCandidates = {
darwin: [
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
'/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta',
'/Applications/Chromium.app/Contents/MacOS/Chromium',
'/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
],
win32: [
'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
'C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe',
'C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe',
],
default: [
'/usr/bin/google-chrome',
'/usr/bin/google-chrome-stable',
'/usr/bin/chromium',
'/usr/bin/chromium-browser',
'/snap/bin/chromium',
'/usr/bin/microsoft-edge',
],
};
let wslHome: string | null | undefined;
function getWslWindowsHome(): string | null {
if (wslHome !== undefined) return wslHome;
if (!process.env.WSL_DISTRO_NAME) {
wslHome = null;
return null;
}
try {
const raw = execSync('cmd.exe /C "echo %USERPROFILE%"', {
encoding: 'utf-8',
timeout: 5_000,
}).trim().replace(/\r/g, '');
wslHome = execSync(`wslpath -u "${raw}"`, {
encoding: 'utf-8',
timeout: 5_000,
}).trim() || null;
} catch {
wslHome = null;
}
return wslHome;
}
export async function getFreePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (!address || typeof address === 'string') {
server.close(() => reject(new Error('Unable to allocate a free TCP port.')));
return;
}
const port = address.port;
server.close((err) => {
if (err) reject(err);
else resolve(port);
});
});
return await getFreePortBase('WECHAT_BROWSER_DEBUG_PORT');
}
export function findChromeExecutable(chromePathOverride?: string): string | undefined {
if (chromePathOverride?.trim()) return chromePathOverride.trim();
return findChromeExecutableBase({
candidates: CHROME_CANDIDATES_FULL,
envNames: ['WECHAT_BROWSER_CHROME_PATH'],
});
}
export function findChromeExecutable(): string | undefined {
const override = process.env.WECHAT_BROWSER_CHROME_PATH?.trim();
if (override && fs.existsSync(override)) return override;
const candidates: string[] = [];
switch (process.platform) {
case 'darwin':
candidates.push(
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
'/Applications/Chromium.app/Contents/MacOS/Chromium',
);
break;
case 'win32':
candidates.push(
'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
);
break;
default:
candidates.push('/usr/bin/google-chrome', '/usr/bin/chromium');
break;
}
for (const p of candidates) {
if (fs.existsSync(p)) return p;
}
return undefined;
}
export function getDefaultProfileDir(): string {
const override = process.env.BAOYU_CHROME_PROFILE_DIR?.trim();
if (override) return path.resolve(override);
const base = process.platform === 'darwin'
? path.join(os.homedir(), 'Library', 'Application Support')
: process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
return path.join(base, 'baoyu-skills', 'chrome-profile');
return resolveSharedChromeProfileDir({
envNames: ['BAOYU_CHROME_PROFILE_DIR', 'WECHAT_BROWSER_PROFILE_DIR'],
wslWindowsHome: getWslWindowsHome(),
});
}
async function fetchJson<T = unknown>(url: string): Promise<T> {
const res = await fetch(url, { redirect: 'follow' });
if (!res.ok) throw new Error(`Request failed: ${res.status} ${res.statusText}`);
return (await res.json()) as T;
}
async function waitForChromeDebugPort(port: number, timeoutMs: number): Promise<string> {
const start = Date.now();
let lastError: unknown = null;
while (Date.now() - start < timeoutMs) {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(`http://127.0.0.1:${port}/json/version`);
if (version.webSocketDebuggerUrl) return version.webSocketDebuggerUrl;
lastError = new Error('Missing webSocketDebuggerUrl');
} catch (error) {
lastError = error;
}
await sleep(200);
}
throw new Error(`Chrome debug port not ready: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
}
export class CdpConnection {
private ws: WebSocket;
private nextId = 0;
private pending = new Map<number, { resolve: (v: unknown) => void; reject: (e: Error) => void; timer: ReturnType<typeof setTimeout> | null }>();
private eventHandlers = new Map<string, Set<(params: unknown) => void>>();
private constructor(ws: WebSocket) {
this.ws = ws;
this.ws.addEventListener('message', (event) => {
try {
const data = typeof event.data === 'string' ? event.data : new TextDecoder().decode(event.data as ArrayBuffer);
const msg = JSON.parse(data) as { id?: number; method?: string; params?: unknown; result?: unknown; error?: { message?: string } };
if (msg.method) {
const handlers = this.eventHandlers.get(msg.method);
if (handlers) handlers.forEach((h) => h(msg.params));
}
if (msg.id) {
const pending = this.pending.get(msg.id);
if (pending) {
this.pending.delete(msg.id);
if (pending.timer) clearTimeout(pending.timer);
if (msg.error?.message) pending.reject(new Error(msg.error.message));
else pending.resolve(msg.result);
}
}
} catch {}
});
this.ws.addEventListener('close', () => {
for (const [id, pending] of this.pending.entries()) {
this.pending.delete(id);
if (pending.timer) clearTimeout(pending.timer);
pending.reject(new Error('CDP connection closed.'));
}
});
}
static async connect(url: string, timeoutMs: number): Promise<CdpConnection> {
const ws = new WebSocket(url);
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error('CDP connection timeout.')), timeoutMs);
ws.addEventListener('open', () => { clearTimeout(timer); resolve(); });
ws.addEventListener('error', () => { clearTimeout(timer); reject(new Error('CDP connection failed.')); });
});
return new CdpConnection(ws);
}
on(method: string, handler: (params: unknown) => void): void {
if (!this.eventHandlers.has(method)) this.eventHandlers.set(method, new Set());
this.eventHandlers.get(method)!.add(handler);
}
async send<T = unknown>(method: string, params?: Record<string, unknown>, options?: { sessionId?: string; timeoutMs?: number }): Promise<T> {
const id = ++this.nextId;
const message: Record<string, unknown> = { id, method };
if (params) message.params = params;
if (options?.sessionId) message.sessionId = options.sessionId;
const timeoutMs = options?.timeoutMs ?? 15_000;
const result = await new Promise<unknown>((resolve, reject) => {
const timer = timeoutMs > 0 ? setTimeout(() => { this.pending.delete(id); reject(new Error(`CDP timeout: ${method}`)); }, timeoutMs) : null;
this.pending.set(id, { resolve, reject, timer });
this.ws.send(JSON.stringify(message));
});
return result as T;
}
close(): void {
try { this.ws.close(); } catch {}
}
export function getAccountProfileDir(alias: string): string {
const base = getDefaultProfileDir();
return path.join(path.dirname(base), `wechat-${alias}`);
}
export interface ChromeSession {
@@ -177,56 +94,38 @@ export interface ChromeSession {
export async function tryConnectExisting(port: number): Promise<CdpConnection | null> {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(`http://127.0.0.1:${port}/json/version`);
if (version.webSocketDebuggerUrl) {
const cdp = await CdpConnection.connect(version.webSocketDebuggerUrl, 5_000);
return cdp;
}
} catch {}
return null;
const wsUrl = await waitForChromeDebugPort(port, 5_000, { includeLastError: true });
return await CdpConnection.connect(wsUrl, 5_000);
} catch {
return null;
}
}
export async function findExistingChromeDebugPort(): Promise<number | null> {
if (process.platform !== 'darwin' && process.platform !== 'linux') return null;
try {
const { execSync } = await import('node:child_process');
const cmd = process.platform === 'darwin'
? `lsof -nP -iTCP -sTCP:LISTEN 2>/dev/null | grep -i 'google\\|chrome' | awk '{print $9}' | sed 's/.*://'`
: `ss -tlnp 2>/dev/null | grep -i chrome | awk '{print $4}' | sed 's/.*://'`;
const output = execSync(cmd, { encoding: 'utf-8', timeout: 5_000 }).trim();
if (!output) return null;
const ports = output.split('\n').map(p => parseInt(p, 10)).filter(p => !isNaN(p) && p > 0);
for (const port of ports) {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(`http://127.0.0.1:${port}/json/version`);
if (version.webSocketDebuggerUrl) return port;
} catch {}
}
} catch {}
return null;
export async function findExistingChromeDebugPort(profileDir = getDefaultProfileDir()): Promise<number | null> {
return await findExistingChromeDebugPortBase({ profileDir });
}
export async function launchChrome(url: string, profileDir?: string): Promise<{ cdp: CdpConnection; chrome: ReturnType<typeof spawn> }> {
const chromePath = findChromeExecutable();
export async function launchChrome(
url: string,
profileDir?: string,
chromePathOverride?: string,
): Promise<{ cdp: CdpConnection; chrome: ChildProcess }> {
const chromePath = findChromeExecutable(chromePathOverride);
if (!chromePath) throw new Error('Chrome not found. Set WECHAT_BROWSER_CHROME_PATH env var.');
const profile = profileDir ?? getDefaultProfileDir();
await mkdir(profile, { recursive: true });
const port = await getFreePort();
console.log(`[cdp] Launching Chrome (profile: ${profile})`);
const chrome = spawn(chromePath, [
`--remote-debugging-port=${port}`,
`--user-data-dir=${profile}`,
'--no-first-run',
'--no-default-browser-check',
'--disable-blink-features=AutomationControlled',
'--start-maximized',
const chrome = await launchChromeBase({
chromePath,
profileDir: profile,
port,
url,
], { stdio: 'ignore' });
extraArgs: ['--disable-blink-features=AutomationControlled', '--start-maximized'],
});
const wsUrl = await waitForChromeDebugPort(port, 30_000);
const wsUrl = await waitForChromeDebugPort(port, 30_000, { includeLastError: true });
const cdp = await CdpConnection.connect(wsUrl, 30_000);
return { cdp, chrome };
@@ -234,11 +133,14 @@ export async function launchChrome(url: string, profileDir?: string): Promise<{
export async function getPageSession(cdp: CdpConnection, urlPattern: string): Promise<ChromeSession> {
const targets = await cdp.send<{ targetInfos: Array<{ targetId: string; url: string; type: string }> }>('Target.getTargets');
let pageTarget = targets.targetInfos.find((t) => t.type === 'page' && t.url.includes(urlPattern));
const pageTarget = targets.targetInfos.find((target) => target.type === 'page' && target.url.includes(urlPattern));
if (!pageTarget) throw new Error(`Page not found: ${urlPattern}`);
const { sessionId } = await cdp.send<{ sessionId: string }>('Target.attachToTarget', { targetId: pageTarget.targetId, flatten: true });
const { sessionId } = await cdp.send<{ sessionId: string }>('Target.attachToTarget', {
targetId: pageTarget.targetId,
flatten: true,
});
await cdp.send('Page.enable', {}, { sessionId });
await cdp.send('Runtime.enable', {}, { sessionId });
@@ -247,11 +149,20 @@ export async function getPageSession(cdp: CdpConnection, urlPattern: string): Pr
return { cdp, sessionId, targetId: pageTarget.targetId };
}
export async function waitForNewTab(cdp: CdpConnection, initialIds: Set<string>, urlPattern: string, timeoutMs = 30_000): Promise<string> {
export async function waitForNewTab(
cdp: CdpConnection,
initialIds: Set<string>,
urlPattern: string,
timeoutMs = 30_000,
): Promise<string> {
const start = Date.now();
while (Date.now() - start < timeoutMs) {
const targets = await cdp.send<{ targetInfos: Array<{ targetId: string; url: string; type: string }> }>('Target.getTargets');
const newTab = targets.targetInfos.find(t => t.type === 'page' && !initialIds.has(t.targetId) && t.url.includes(urlPattern));
const newTab = targets.targetInfos.find((target) => (
target.type === 'page' &&
!initialIds.has(target.targetId) &&
target.url.includes(urlPattern)
));
if (newTab) return newTab.targetId;
await sleep(500);
}
@@ -259,7 +170,7 @@ export async function waitForNewTab(cdp: CdpConnection, initialIds: Set<string>,
}
export async function clickElement(session: ChromeSession, selector: string): Promise<void> {
const posResult = await session.cdp.send<{ result: { value: string } }>('Runtime.evaluate', {
const position = await session.cdp.send<{ result: { value: string } }>('Runtime.evaluate', {
expression: `
(function() {
const el = document.querySelector('${selector}');
@@ -272,23 +183,46 @@ export async function clickElement(session: ChromeSession, selector: string): Pr
returnByValue: true,
}, { sessionId: session.sessionId });
if (posResult.result.value === 'null') throw new Error(`Element not found: ${selector}`);
const pos = JSON.parse(posResult.result.value);
if (position.result.value === 'null') throw new Error(`Element not found: ${selector}`);
const pos = JSON.parse(position.result.value);
await session.cdp.send('Input.dispatchMouseEvent', { type: 'mousePressed', x: pos.x, y: pos.y, button: 'left', clickCount: 1 }, { sessionId: session.sessionId });
await session.cdp.send('Input.dispatchMouseEvent', {
type: 'mousePressed',
x: pos.x,
y: pos.y,
button: 'left',
clickCount: 1,
}, { sessionId: session.sessionId });
await sleep(50);
await session.cdp.send('Input.dispatchMouseEvent', { type: 'mouseReleased', x: pos.x, y: pos.y, button: 'left', clickCount: 1 }, { sessionId: session.sessionId });
await session.cdp.send('Input.dispatchMouseEvent', {
type: 'mouseReleased',
x: pos.x,
y: pos.y,
button: 'left',
clickCount: 1,
}, { sessionId: session.sessionId });
}
export async function typeText(session: ChromeSession, text: string): Promise<void> {
const lines = text.split('\n');
for (let i = 0; i < lines.length; i++) {
if (lines[i].length > 0) {
await session.cdp.send('Input.insertText', { text: lines[i] }, { sessionId: session.sessionId });
for (let index = 0; index < lines.length; index += 1) {
const line = lines[index];
if (line.length > 0) {
await session.cdp.send('Input.insertText', { text: line }, { sessionId: session.sessionId });
}
if (i < lines.length - 1) {
await session.cdp.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13 }, { sessionId: session.sessionId });
await session.cdp.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13 }, { sessionId: session.sessionId });
if (index < lines.length - 1) {
await session.cdp.send('Input.dispatchKeyEvent', {
type: 'keyDown',
key: 'Enter',
code: 'Enter',
windowsVirtualKeyCode: 13,
}, { sessionId: session.sessionId });
await session.cdp.send('Input.dispatchKeyEvent', {
type: 'keyUp',
key: 'Enter',
code: 'Enter',
windowsVirtualKeyCode: 13,
}, { sessionId: session.sessionId });
}
await sleep(30);
}
@@ -296,8 +230,20 @@ export async function typeText(session: ChromeSession, text: string): Promise<vo
export async function pasteFromClipboard(session: ChromeSession): Promise<void> {
const modifiers = process.platform === 'darwin' ? 4 : 2;
await session.cdp.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'v', code: 'KeyV', modifiers, windowsVirtualKeyCode: 86 }, { sessionId: session.sessionId });
await session.cdp.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'v', code: 'KeyV', modifiers, windowsVirtualKeyCode: 86 }, { sessionId: session.sessionId });
await session.cdp.send('Input.dispatchKeyEvent', {
type: 'keyDown',
key: 'v',
code: 'KeyV',
modifiers,
windowsVirtualKeyCode: 86,
}, { sessionId: session.sessionId });
await session.cdp.send('Input.dispatchKeyEvent', {
type: 'keyUp',
key: 'v',
code: 'KeyV',
modifiers,
windowsVirtualKeyCode: 86,
}, { sessionId: session.sessionId });
}
export async function evaluate<T = unknown>(session: ChromeSession, expression: string): Promise<T> {
@@ -0,0 +1,8 @@
{
"name": "baoyu-post-to-wechat-scripts",
"private": true,
"type": "module",
"dependencies": {
"baoyu-chrome-cdp": "file:./vendor/baoyu-chrome-cdp"
}
}
@@ -0,0 +1,9 @@
{
"name": "baoyu-chrome-cdp",
"private": true,
"version": "0.1.0",
"type": "module",
"exports": {
".": "./src/index.ts"
}
}
@@ -0,0 +1,408 @@
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import process from "node:process";
export type PlatformCandidates = {
darwin?: string[];
win32?: string[];
default: string[];
};
type PendingRequest = {
resolve: (value: unknown) => void;
reject: (error: Error) => void;
timer: ReturnType<typeof setTimeout> | null;
};
type CdpSendOptions = {
sessionId?: string;
timeoutMs?: number;
};
type FetchJsonOptions = {
timeoutMs?: number;
};
type FindChromeExecutableOptions = {
candidates: PlatformCandidates;
envNames?: string[];
};
type ResolveSharedChromeProfileDirOptions = {
envNames?: string[];
appDataDirName?: string;
profileDirName?: string;
wslWindowsHome?: string | null;
};
type FindExistingChromeDebugPortOptions = {
profileDir: string;
timeoutMs?: number;
};
type LaunchChromeOptions = {
chromePath: string;
profileDir: string;
port: number;
url?: string;
headless?: boolean;
extraArgs?: string[];
};
type ChromeTargetInfo = {
targetId: string;
url: string;
type: string;
};
type OpenPageSessionOptions = {
cdp: CdpConnection;
reusing: boolean;
url: string;
matchTarget: (target: ChromeTargetInfo) => boolean;
enablePage?: boolean;
enableRuntime?: boolean;
enableDom?: boolean;
enableNetwork?: boolean;
activateTarget?: boolean;
};
export type PageSession = {
sessionId: string;
targetId: string;
};
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export async function getFreePort(fixedEnvName?: string): Promise<number> {
const fixed = fixedEnvName ? Number.parseInt(process.env[fixedEnvName] ?? "", 10) : NaN;
if (Number.isInteger(fixed) && fixed > 0) return fixed;
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Unable to allocate a free TCP port.")));
return;
}
const port = address.port;
server.close((err) => {
if (err) reject(err);
else resolve(port);
});
});
});
}
export function findChromeExecutable(options: FindChromeExecutableOptions): string | undefined {
for (const envName of options.envNames ?? []) {
const override = process.env[envName]?.trim();
if (override && fs.existsSync(override)) return override;
}
const candidates = process.platform === "darwin"
? options.candidates.darwin ?? options.candidates.default
: process.platform === "win32"
? options.candidates.win32 ?? options.candidates.default
: options.candidates.default;
for (const candidate of candidates) {
if (fs.existsSync(candidate)) return candidate;
}
return undefined;
}
export function resolveSharedChromeProfileDir(options: ResolveSharedChromeProfileDirOptions = {}): string {
for (const envName of options.envNames ?? []) {
const override = process.env[envName]?.trim();
if (override) return path.resolve(override);
}
const appDataDirName = options.appDataDirName ?? "baoyu-skills";
const profileDirName = options.profileDirName ?? "chrome-profile";
if (options.wslWindowsHome) {
return path.join(options.wslWindowsHome, ".local", "share", appDataDirName, profileDirName);
}
const base = process.platform === "darwin"
? path.join(os.homedir(), "Library", "Application Support")
: process.platform === "win32"
? (process.env.APPDATA ?? path.join(os.homedir(), "AppData", "Roaming"))
: (process.env.XDG_DATA_HOME ?? path.join(os.homedir(), ".local", "share"));
return path.join(base, appDataDirName, profileDirName);
}
async function fetchWithTimeout(url: string, timeoutMs?: number): Promise<Response> {
if (!timeoutMs || timeoutMs <= 0) return await fetch(url, { redirect: "follow" });
const ctl = new AbortController();
const timer = setTimeout(() => ctl.abort(), timeoutMs);
try {
return await fetch(url, { redirect: "follow", signal: ctl.signal });
} finally {
clearTimeout(timer);
}
}
async function fetchJson<T = unknown>(url: string, options: FetchJsonOptions = {}): Promise<T> {
const response = await fetchWithTimeout(url, options.timeoutMs);
if (!response.ok) {
throw new Error(`Request failed: ${response.status} ${response.statusText}`);
}
return await response.json() as T;
}
async function isDebugPortReady(port: number, timeoutMs = 3_000): Promise<boolean> {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
`http://127.0.0.1:${port}/json/version`,
{ timeoutMs }
);
return !!version.webSocketDebuggerUrl;
} catch {
return false;
}
}
export async function findExistingChromeDebugPort(options: FindExistingChromeDebugPortOptions): Promise<number | null> {
const timeoutMs = options.timeoutMs ?? 3_000;
const portFile = path.join(options.profileDir, "DevToolsActivePort");
try {
const content = fs.readFileSync(portFile, "utf-8");
const [portLine] = content.split(/\r?\n/);
const port = Number.parseInt(portLine?.trim() ?? "", 10);
if (port > 0 && await isDebugPortReady(port, timeoutMs)) return port;
} catch {}
if (process.platform === "win32") return null;
try {
const result = spawnSync("ps", ["aux"], { encoding: "utf-8", timeout: 5_000 });
if (result.status !== 0 || !result.stdout) return null;
const lines = result.stdout
.split("\n")
.filter((line) => line.includes(options.profileDir) && line.includes("--remote-debugging-port="));
for (const line of lines) {
const portMatch = line.match(/--remote-debugging-port=(\d+)/);
const port = Number.parseInt(portMatch?.[1] ?? "", 10);
if (port > 0 && await isDebugPortReady(port, timeoutMs)) return port;
}
} catch {}
return null;
}
export async function waitForChromeDebugPort(
port: number,
timeoutMs: number,
options?: { includeLastError?: boolean }
): Promise<string> {
const start = Date.now();
let lastError: unknown = null;
while (Date.now() - start < timeoutMs) {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
`http://127.0.0.1:${port}/json/version`,
{ timeoutMs: 5_000 }
);
if (version.webSocketDebuggerUrl) return version.webSocketDebuggerUrl;
lastError = new Error("Missing webSocketDebuggerUrl");
} catch (error) {
lastError = error;
}
await sleep(200);
}
if (options?.includeLastError && lastError) {
throw new Error(
`Chrome debug port not ready: ${lastError instanceof Error ? lastError.message : String(lastError)}`
);
}
throw new Error("Chrome debug port not ready");
}
export class CdpConnection {
private ws: WebSocket;
private nextId = 0;
private pending = new Map<number, PendingRequest>();
private eventHandlers = new Map<string, Set<(params: unknown) => void>>();
private defaultTimeoutMs: number;
private constructor(ws: WebSocket, defaultTimeoutMs = 15_000) {
this.ws = ws;
this.defaultTimeoutMs = defaultTimeoutMs;
this.ws.addEventListener("message", (event) => {
try {
const data = typeof event.data === "string"
? event.data
: new TextDecoder().decode(event.data as ArrayBuffer);
const msg = JSON.parse(data) as {
id?: number;
method?: string;
params?: unknown;
result?: unknown;
error?: { message?: string };
};
if (msg.method) {
const handlers = this.eventHandlers.get(msg.method);
if (handlers) {
handlers.forEach((handler) => handler(msg.params));
}
}
if (msg.id) {
const pending = this.pending.get(msg.id);
if (pending) {
this.pending.delete(msg.id);
if (pending.timer) clearTimeout(pending.timer);
if (msg.error?.message) pending.reject(new Error(msg.error.message));
else pending.resolve(msg.result);
}
}
} catch {}
});
this.ws.addEventListener("close", () => {
for (const [id, pending] of this.pending.entries()) {
this.pending.delete(id);
if (pending.timer) clearTimeout(pending.timer);
pending.reject(new Error("CDP connection closed."));
}
});
}
static async connect(
url: string,
timeoutMs: number,
options?: { defaultTimeoutMs?: number }
): Promise<CdpConnection> {
const ws = new WebSocket(url);
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error("CDP connection timeout.")), timeoutMs);
ws.addEventListener("open", () => {
clearTimeout(timer);
resolve();
});
ws.addEventListener("error", () => {
clearTimeout(timer);
reject(new Error("CDP connection failed."));
});
});
return new CdpConnection(ws, options?.defaultTimeoutMs ?? 15_000);
}
on(method: string, handler: (params: unknown) => void): void {
if (!this.eventHandlers.has(method)) {
this.eventHandlers.set(method, new Set());
}
this.eventHandlers.get(method)?.add(handler);
}
off(method: string, handler: (params: unknown) => void): void {
this.eventHandlers.get(method)?.delete(handler);
}
async send<T = unknown>(method: string, params?: Record<string, unknown>, options?: CdpSendOptions): Promise<T> {
const id = ++this.nextId;
const message: Record<string, unknown> = { id, method };
if (params) message.params = params;
if (options?.sessionId) message.sessionId = options.sessionId;
const timeoutMs = options?.timeoutMs ?? this.defaultTimeoutMs;
const result = await new Promise<unknown>((resolve, reject) => {
const timer = timeoutMs > 0
? setTimeout(() => {
this.pending.delete(id);
reject(new Error(`CDP timeout: ${method}`));
}, timeoutMs)
: null;
this.pending.set(id, { resolve, reject, timer });
this.ws.send(JSON.stringify(message));
});
return result as T;
}
close(): void {
try {
this.ws.close();
} catch {}
}
}
export async function launchChrome(options: LaunchChromeOptions): Promise<ChildProcess> {
await fs.promises.mkdir(options.profileDir, { recursive: true });
const args = [
`--remote-debugging-port=${options.port}`,
`--user-data-dir=${options.profileDir}`,
"--no-first-run",
"--no-default-browser-check",
...(options.extraArgs ?? []),
];
if (options.headless) args.push("--headless=new");
if (options.url) args.push(options.url);
return spawn(options.chromePath, args, { stdio: "ignore" });
}
export function killChrome(chrome: ChildProcess): void {
try {
chrome.kill("SIGTERM");
} catch {}
setTimeout(() => {
if (!chrome.killed) {
try {
chrome.kill("SIGKILL");
} catch {}
}
}, 2_000).unref?.();
}
export async function openPageSession(options: OpenPageSessionOptions): Promise<PageSession> {
let targetId: string;
if (options.reusing) {
const created = await options.cdp.send<{ targetId: string }>("Target.createTarget", { url: options.url });
targetId = created.targetId;
} else {
const targets = await options.cdp.send<{ targetInfos: ChromeTargetInfo[] }>("Target.getTargets");
const existing = targets.targetInfos.find(options.matchTarget);
if (existing) {
targetId = existing.targetId;
} else {
const created = await options.cdp.send<{ targetId: string }>("Target.createTarget", { url: options.url });
targetId = created.targetId;
}
}
const { sessionId } = await options.cdp.send<{ sessionId: string }>(
"Target.attachToTarget",
{ targetId, flatten: true }
);
if (options.activateTarget ?? true) {
await options.cdp.send("Target.activateTarget", { targetId });
}
if (options.enablePage) await options.cdp.send("Page.enable", {}, { sessionId });
if (options.enableRuntime) await options.cdp.send("Runtime.enable", {}, { sessionId });
if (options.enableDom) await options.cdp.send("DOM.enable", {}, { sessionId });
if (options.enableNetwork) await options.cdp.send("Network.enable", {}, { sessionId });
return { sessionId, targetId };
}
@@ -1,13 +1,8 @@
import fs from "node:fs";
import path from "node:path";
import os from "node:os";
import { spawnSync } from "node:child_process";
import { fileURLToPath } from "node:url";
interface WechatConfig {
appId: string;
appSecret: string;
}
import { loadWechatExtendConfig, resolveAccount, loadCredentials } from "./wechat-extend-config.ts";
interface AccessTokenResponse {
access_token?: string;
@@ -38,53 +33,14 @@ interface ArticleOptions {
thumbMediaId: string;
articleType: ArticleType;
imageMediaIds?: string[];
needOpenComment?: number;
onlyFansCanComment?: number;
}
const TOKEN_URL = "https://api.weixin.qq.com/cgi-bin/token";
const UPLOAD_URL = "https://api.weixin.qq.com/cgi-bin/material/add_material";
const DRAFT_URL = "https://api.weixin.qq.com/cgi-bin/draft/add";
function loadEnvFile(envPath: string): Record<string, string> {
const env: Record<string, string> = {};
if (!fs.existsSync(envPath)) return env;
const content = fs.readFileSync(envPath, "utf-8");
for (const line of content.split("\n")) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith("#")) continue;
const eqIdx = trimmed.indexOf("=");
if (eqIdx > 0) {
const key = trimmed.slice(0, eqIdx).trim();
let value = trimmed.slice(eqIdx + 1).trim();
if ((value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))) {
value = value.slice(1, -1);
}
env[key] = value;
}
}
return env;
}
function loadConfig(): WechatConfig {
const cwdEnvPath = path.join(process.cwd(), ".baoyu-skills", ".env");
const homeEnvPath = path.join(os.homedir(), ".baoyu-skills", ".env");
const cwdEnv = loadEnvFile(cwdEnvPath);
const homeEnv = loadEnvFile(homeEnvPath);
const appId = process.env.WECHAT_APP_ID || cwdEnv.WECHAT_APP_ID || homeEnv.WECHAT_APP_ID;
const appSecret = process.env.WECHAT_APP_SECRET || cwdEnv.WECHAT_APP_SECRET || homeEnv.WECHAT_APP_SECRET;
if (!appId || !appSecret) {
throw new Error(
"Missing WECHAT_APP_ID or WECHAT_APP_SECRET.\n" +
"Set via environment variables or in .baoyu-skills/.env file."
);
}
return { appId, appSecret };
}
async function fetchAccessToken(appId: string, appSecret: string): Promise<string> {
const url = `${TOKEN_URL}?grant_type=client_credential&appid=${appId}&secret=${appSecret}`;
@@ -241,6 +197,9 @@ async function publishToDraft(
let article: Record<string, unknown>;
const noc = options.needOpenComment ?? 1;
const ofcc = options.onlyFansCanComment ?? 0;
if (options.articleType === "newspic") {
if (!options.imageMediaIds || options.imageMediaIds.length === 0) {
throw new Error("newspic requires at least one image");
@@ -249,8 +208,8 @@ async function publishToDraft(
article_type: "newspic",
title: options.title,
content: options.content,
need_open_comment: 1,
only_fans_can_comment: 0,
need_open_comment: noc,
only_fans_can_comment: ofcc,
image_info: {
image_list: options.imageMediaIds.map(id => ({ image_media_id: id })),
},
@@ -262,8 +221,8 @@ async function publishToDraft(
title: options.title,
content: options.content,
thumb_media_id: options.thumbMediaId,
need_open_comment: 1,
only_fans_can_comment: 0,
need_open_comment: noc,
only_fans_can_comment: ofcc,
};
if (options.author) article.author = options.author;
if (options.digest) article.digest = options.digest;
@@ -368,6 +327,7 @@ Options:
--theme <name> Theme name for markdown (default, grace, simple, modern). Default: default
--color <name|hex> Primary color (blue, green, vermilion, etc. or hex)
--cover <path> Cover image path (local or URL)
--account <alias> Select account by alias (for multi-account setups)
--no-cite Disable bottom citations for ordinary external links in markdown mode
--dry-run Parse and render only, don't publish
--help Show this help
@@ -412,6 +372,7 @@ interface CliArgs {
theme: string;
color?: string;
cover?: string;
account?: string;
citeStatus: boolean;
dryRun: boolean;
}
@@ -449,6 +410,8 @@ function parseArgs(argv: string[]): CliArgs {
args.color = argv[++i];
} else if (arg === "--cover" && argv[i + 1]) {
args.cover = argv[++i];
} else if (arg === "--account" && argv[i + 1]) {
args.account = argv[++i];
} else if (arg === "--cite") {
args.citeStatus = true;
} else if (arg === "--no-cite") {
@@ -550,6 +513,12 @@ async function main(): Promise<void> {
if (digest) console.error(`[wechat-api] Digest: ${digest.slice(0, 50)}...`);
console.error(`[wechat-api] Type: ${args.articleType}`);
const extConfig = loadWechatExtendConfig();
const resolved = resolveAccount(extConfig, args.account);
if (resolved.name) console.error(`[wechat-api] Account: ${resolved.name} (${resolved.alias})`);
if (!author && resolved.default_author) author = resolved.default_author;
if (args.dryRun) {
console.log(JSON.stringify({
articleType: args.articleType,
@@ -558,13 +527,14 @@ async function main(): Promise<void> {
digest: digest || undefined,
htmlPath,
contentLength: htmlContent.length,
account: resolved.alias || undefined,
}, null, 2));
return;
}
const config = loadConfig();
const creds = loadCredentials(resolved);
console.error("[wechat-api] Fetching access token...");
const accessToken = await fetchAccessToken(config.appId, config.appSecret);
const accessToken = await fetchAccessToken(creds.appId, creds.appSecret);
console.error("[wechat-api] Uploading images...");
const { html: processedHtml, firstMediaId, allMediaIds } = await uploadImagesInHtml(
@@ -617,6 +587,8 @@ async function main(): Promise<void> {
thumbMediaId,
articleType: args.articleType,
imageMediaIds: args.articleType === "newspic" ? allMediaIds : undefined,
needOpenComment: resolved.need_open_comment,
onlyFansCanComment: resolved.only_fans_can_comment,
}, accessToken);
console.log(JSON.stringify({
@@ -3,7 +3,8 @@ import path from 'node:path';
import { spawnSync } from 'node:child_process';
import process from 'node:process';
import { fileURLToPath } from 'node:url';
import { launchChrome, tryConnectExisting, findExistingChromeDebugPort, getPageSession, waitForNewTab, clickElement, typeText, evaluate, sleep, type ChromeSession, type CdpConnection } from './cdp.ts';
import { launchChrome, tryConnectExisting, findExistingChromeDebugPort, getPageSession, waitForNewTab, clickElement, typeText, evaluate, sleep, getAccountProfileDir, type ChromeSession, type CdpConnection } from './cdp.ts';
import { loadWechatExtendConfig, resolveAccount } from './wechat-extend-config.ts';
const WECHAT_URL = 'https://mp.weixin.qq.com/';
@@ -690,6 +691,7 @@ Options:
--image <path> Content image, can repeat (only with --content)
--submit Save as draft
--profile <dir> Chrome profile directory
--account <alias> Select account by alias (for multi-account setups)
--cdp-port <port> Connect to existing Chrome debug port instead of launching new instance
Examples:
@@ -725,6 +727,7 @@ async function main(): Promise<void> {
let submit = false;
let profileDir: string | undefined;
let cdpPort: number | undefined;
let accountAlias: string | undefined;
for (let i = 0; i < args.length; i++) {
const arg = args[i]!;
@@ -741,9 +744,20 @@ async function main(): Promise<void> {
else if (arg === '--image' && args[i + 1]) images.push(args[++i]!);
else if (arg === '--submit') submit = true;
else if (arg === '--profile' && args[i + 1]) profileDir = args[++i];
else if (arg === '--account' && args[i + 1]) accountAlias = args[++i];
else if (arg === '--cdp-port' && args[i + 1]) cdpPort = parseInt(args[++i]!, 10);
}
const extConfig = loadWechatExtendConfig();
const resolved = resolveAccount(extConfig, accountAlias);
if (resolved.name) console.log(`[wechat] Account: ${resolved.name} (${resolved.alias})`);
if (!author && resolved.default_author) author = resolved.default_author;
if (!profileDir && resolved.alias) {
profileDir = resolved.chrome_profile_path || getAccountProfileDir(resolved.alias);
}
if (!markdownFile && !htmlFile && !title) { console.error('Error: --title is required (or use --markdown/--html)'); process.exit(1); }
if (!markdownFile && !htmlFile && !content) { console.error('Error: --content, --html, or --markdown is required'); process.exit(1); }
@@ -1,11 +1,18 @@
import { execSync, spawn } from 'node:child_process';
import fs from 'node:fs';
import { mkdir, readdir } from 'node:fs/promises';
import net from 'node:net';
import os from 'node:os';
import { readdir } from 'node:fs/promises';
import path from 'node:path';
import process from 'node:process';
import {
CdpConnection,
findChromeExecutable,
getDefaultProfileDir,
getAccountProfileDir,
launchChrome,
sleep,
} from './cdp.ts';
import { loadWechatExtendConfig, resolveAccount } from './wechat-extend-config.ts';
const WECHAT_URL = 'https://mp.weixin.qq.com/';
interface MarkdownMeta {
@@ -104,195 +111,6 @@ async function loadImagesFromDir(dir: string): Promise<string[]> {
return images;
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function getFreePort(): Promise<number> {
const fixed = parseInt(process.env.WECHAT_BROWSER_DEBUG_PORT || '', 10);
if (fixed > 0) return fixed;
return new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (!address || typeof address === 'string') {
server.close(() => reject(new Error('Unable to allocate a free TCP port.')));
return;
}
const port = address.port;
server.close((err) => {
if (err) reject(err);
else resolve(port);
});
});
});
}
function findChromeExecutable(): string | undefined {
const override = process.env.WECHAT_BROWSER_CHROME_PATH?.trim();
if (override && fs.existsSync(override)) return override;
const candidates: string[] = [];
switch (process.platform) {
case 'darwin':
candidates.push(
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
'/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta',
'/Applications/Chromium.app/Contents/MacOS/Chromium',
'/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
);
break;
case 'win32':
candidates.push(
'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
'C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe',
'C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe',
);
break;
default:
candidates.push(
'/usr/bin/google-chrome',
'/usr/bin/google-chrome-stable',
'/usr/bin/chromium',
'/usr/bin/chromium-browser',
'/snap/bin/chromium',
'/usr/bin/microsoft-edge',
);
break;
}
for (const p of candidates) {
if (fs.existsSync(p)) return p;
}
return undefined;
}
let _wslHome: string | null | undefined;
function getWslWindowsHome(): string | null {
if (_wslHome !== undefined) return _wslHome;
if (!process.env.WSL_DISTRO_NAME) { _wslHome = null; return null; }
try {
const raw = execSync('cmd.exe /C "echo %USERPROFILE%"', { encoding: 'utf-8', timeout: 5000 }).trim().replace(/\r/g, '');
_wslHome = execSync(`wslpath -u "${raw}"`, { encoding: 'utf-8', timeout: 5000 }).trim() || null;
} catch { _wslHome = null; }
return _wslHome;
}
function getDefaultProfileDir(): string {
const override = process.env.BAOYU_CHROME_PROFILE_DIR?.trim() || process.env.WECHAT_BROWSER_PROFILE_DIR?.trim();
if (override) return path.resolve(override);
const wslHome = getWslWindowsHome();
if (wslHome) return path.join(wslHome, '.local', 'share', 'baoyu-skills', 'chrome-profile');
const base = process.platform === 'darwin'
? path.join(os.homedir(), 'Library', 'Application Support')
: process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
return path.join(base, 'baoyu-skills', 'chrome-profile');
}
async function fetchJson<T = unknown>(url: string): Promise<T> {
const res = await fetch(url, { redirect: 'follow' });
if (!res.ok) throw new Error(`Request failed: ${res.status} ${res.statusText}`);
return (await res.json()) as T;
}
async function waitForChromeDebugPort(port: number, timeoutMs: number): Promise<string> {
const start = Date.now();
let lastError: unknown = null;
while (Date.now() - start < timeoutMs) {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(`http://127.0.0.1:${port}/json/version`);
if (version.webSocketDebuggerUrl) return version.webSocketDebuggerUrl;
lastError = new Error('Missing webSocketDebuggerUrl');
} catch (error) {
lastError = error;
}
await sleep(200);
}
throw new Error(`Chrome debug port not ready: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
}
class CdpConnection {
private ws: WebSocket;
private nextId = 0;
private pending = new Map<number, { resolve: (v: unknown) => void; reject: (e: Error) => void; timer: ReturnType<typeof setTimeout> | null }>();
private eventHandlers = new Map<string, Set<(params: unknown) => void>>();
private constructor(ws: WebSocket) {
this.ws = ws;
this.ws.addEventListener('message', (event) => {
try {
const data = typeof event.data === 'string' ? event.data : new TextDecoder().decode(event.data as ArrayBuffer);
const msg = JSON.parse(data) as { id?: number; method?: string; params?: unknown; result?: unknown; error?: { message?: string } };
if (msg.method) {
const handlers = this.eventHandlers.get(msg.method);
if (handlers) handlers.forEach((h) => h(msg.params));
}
if (msg.id) {
const pending = this.pending.get(msg.id);
if (pending) {
this.pending.delete(msg.id);
if (pending.timer) clearTimeout(pending.timer);
if (msg.error?.message) pending.reject(new Error(msg.error.message));
else pending.resolve(msg.result);
}
}
} catch {}
});
this.ws.addEventListener('close', () => {
for (const [id, pending] of this.pending.entries()) {
this.pending.delete(id);
if (pending.timer) clearTimeout(pending.timer);
pending.reject(new Error('CDP connection closed.'));
}
});
}
static async connect(url: string, timeoutMs: number): Promise<CdpConnection> {
const ws = new WebSocket(url);
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error('CDP connection timeout.')), timeoutMs);
ws.addEventListener('open', () => { clearTimeout(timer); resolve(); });
ws.addEventListener('error', () => { clearTimeout(timer); reject(new Error('CDP connection failed.')); });
});
return new CdpConnection(ws);
}
on(method: string, handler: (params: unknown) => void): void {
if (!this.eventHandlers.has(method)) this.eventHandlers.set(method, new Set());
this.eventHandlers.get(method)!.add(handler);
}
async send<T = unknown>(method: string, params?: Record<string, unknown>, options?: { sessionId?: string; timeoutMs?: number }): Promise<T> {
const id = ++this.nextId;
const message: Record<string, unknown> = { id, method };
if (params) message.params = params;
if (options?.sessionId) message.sessionId = options.sessionId;
const timeoutMs = options?.timeoutMs ?? 15_000;
const result = await new Promise<unknown>((resolve, reject) => {
const timer = timeoutMs > 0 ? setTimeout(() => { this.pending.delete(id); reject(new Error(`CDP timeout: ${method}`)); }, timeoutMs) : null;
this.pending.set(id, { resolve, reject, timer });
this.ws.send(JSON.stringify(message));
});
return result as T;
}
close(): void {
try { this.ws.close(); } catch {}
}
}
interface WeChatBrowserOptions {
title?: string;
content?: string;
@@ -348,29 +166,18 @@ export async function postToWeChat(options: WeChatBrowserOptions): Promise<void>
if (!fs.existsSync(img)) throw new Error(`Image not found: ${img}`);
}
const chromePath = options.chromePath ?? findChromeExecutable();
const chromePath = findChromeExecutable(options.chromePath);
if (!chromePath) throw new Error('Chrome not found. Set WECHAT_BROWSER_CHROME_PATH env var.');
await mkdir(profileDir, { recursive: true });
const port = await getFreePort();
console.log(`[wechat-browser] Launching Chrome (profile: ${profileDir})`);
const chrome = spawn(chromePath, [
`--remote-debugging-port=${port}`,
`--user-data-dir=${profileDir}`,
'--no-first-run',
'--no-default-browser-check',
'--disable-blink-features=AutomationControlled',
'--start-maximized',
WECHAT_URL,
], { stdio: 'ignore' });
const launched = await launchChrome(WECHAT_URL, profileDir, chromePath);
const chrome = launched.chrome;
let cdp: CdpConnection | null = null;
try {
const wsUrl = await waitForChromeDebugPort(port, 30_000);
cdp = await CdpConnection.connect(wsUrl, 30_000);
cdp = launched.cdp;
const targets = await cdp.send<{ targetInfos: Array<{ targetId: string; url: string; type: string }> }>('Target.getTargets');
let pageTarget = targets.targetInfos.find((t) => t.type === 'page' && t.url.includes('mp.weixin.qq.com'));
@@ -859,6 +666,7 @@ Options:
--image <path> Add image (can be repeated)
--submit Save as draft (default: preview only)
--profile <dir> Chrome profile directory
--account <alias> Select account by alias (for multi-account setups)
--help Show this help
Examples:
@@ -880,6 +688,7 @@ async function main(): Promise<void> {
let content: string | undefined;
let markdownFile: string | undefined;
let imagesDir: string | undefined;
let accountAlias: string | undefined;
for (let i = 0; i < args.length; i++) {
const arg = args[i]!;
@@ -897,9 +706,19 @@ async function main(): Promise<void> {
submit = true;
} else if (arg === '--profile' && args[i + 1]) {
profileDir = args[++i];
} else if (arg === '--account' && args[i + 1]) {
accountAlias = args[++i];
}
}
const extConfig = loadWechatExtendConfig();
const resolved = resolveAccount(extConfig, accountAlias);
if (resolved.name) console.log(`[wechat-browser] Account: ${resolved.name} (${resolved.alias})`);
if (!profileDir && resolved.alias) {
profileDir = resolved.chrome_profile_path || getAccountProfileDir(resolved.alias);
}
if (!markdownFile && !title) {
console.error('Error: --title or --markdown is required');
process.exit(1);
@@ -0,0 +1,245 @@
import fs from "node:fs";
import path from "node:path";
import os from "node:os";
export interface WechatAccount {
name: string;
alias: string;
default?: boolean;
default_publish_method?: string;
default_author?: string;
need_open_comment?: number;
only_fans_can_comment?: number;
app_id?: string;
app_secret?: string;
chrome_profile_path?: string;
}
export interface WechatExtendConfig {
default_theme?: string;
default_color?: string;
default_publish_method?: string;
default_author?: string;
need_open_comment?: number;
only_fans_can_comment?: number;
chrome_profile_path?: string;
accounts?: WechatAccount[];
}
export interface ResolvedAccount {
name?: string;
alias?: string;
default_publish_method?: string;
default_author?: string;
need_open_comment: number;
only_fans_can_comment: number;
app_id?: string;
app_secret?: string;
chrome_profile_path?: string;
}
function stripQuotes(s: string): string {
return s.replace(/^['"]|['"]$/g, "");
}
function toBool01(v: string): number {
return v === "1" || v === "true" ? 1 : 0;
}
function parseWechatExtend(content: string): WechatExtendConfig {
const config: WechatExtendConfig = {};
const lines = content.split("\n");
let inAccounts = false;
let current: Record<string, string> | null = null;
const rawAccounts: Record<string, string>[] = [];
for (const raw of lines) {
const trimmed = raw.trim();
if (!trimmed || trimmed.startsWith("#")) continue;
if (trimmed === "accounts:") {
inAccounts = true;
continue;
}
if (inAccounts) {
const listMatch = raw.match(/^\s+-\s+(.+)$/);
if (listMatch) {
if (current) rawAccounts.push(current);
current = {};
const kv = listMatch[1]!;
const ci = kv.indexOf(":");
if (ci > 0) {
current[kv.slice(0, ci).trim()] = stripQuotes(kv.slice(ci + 1).trim());
}
continue;
}
if (current && /^\s{2,}/.test(raw) && !trimmed.startsWith("-")) {
const ci = trimmed.indexOf(":");
if (ci > 0) {
current[trimmed.slice(0, ci).trim()] = stripQuotes(trimmed.slice(ci + 1).trim());
}
continue;
}
if (!/^\s/.test(raw)) {
if (current) rawAccounts.push(current);
current = null;
inAccounts = false;
} else {
continue;
}
}
const ci = trimmed.indexOf(":");
if (ci < 0) continue;
const key = trimmed.slice(0, ci).trim();
const val = stripQuotes(trimmed.slice(ci + 1).trim());
if (val === "null" || val === "") continue;
switch (key) {
case "default_theme": config.default_theme = val; break;
case "default_color": config.default_color = val; break;
case "default_publish_method": config.default_publish_method = val; break;
case "default_author": config.default_author = val; break;
case "need_open_comment": config.need_open_comment = toBool01(val); break;
case "only_fans_can_comment": config.only_fans_can_comment = toBool01(val); break;
case "chrome_profile_path": config.chrome_profile_path = val; break;
}
}
if (current) rawAccounts.push(current);
if (rawAccounts.length > 0) {
config.accounts = rawAccounts.map(a => ({
name: a.name || "",
alias: a.alias || "",
default: a.default === "true" || a.default === "1",
default_publish_method: a.default_publish_method || undefined,
default_author: a.default_author || undefined,
need_open_comment: a.need_open_comment ? toBool01(a.need_open_comment) : undefined,
only_fans_can_comment: a.only_fans_can_comment ? toBool01(a.only_fans_can_comment) : undefined,
app_id: a.app_id || undefined,
app_secret: a.app_secret || undefined,
chrome_profile_path: a.chrome_profile_path || undefined,
}));
}
return config;
}
export function loadWechatExtendConfig(): WechatExtendConfig {
const paths = [
path.join(process.cwd(), ".baoyu-skills", "baoyu-post-to-wechat", "EXTEND.md"),
path.join(
process.env.XDG_CONFIG_HOME || path.join(os.homedir(), ".config"),
"baoyu-skills", "baoyu-post-to-wechat", "EXTEND.md"
),
path.join(os.homedir(), ".baoyu-skills", "baoyu-post-to-wechat", "EXTEND.md"),
];
for (const p of paths) {
try {
const content = fs.readFileSync(p, "utf-8");
return parseWechatExtend(content);
} catch {
continue;
}
}
return {};
}
function selectAccount(config: WechatExtendConfig, alias?: string): WechatAccount | undefined {
if (!config.accounts || config.accounts.length === 0) return undefined;
if (alias) return config.accounts.find(a => a.alias === alias);
if (config.accounts.length === 1) return config.accounts[0];
return config.accounts.find(a => a.default);
}
export function resolveAccount(config: WechatExtendConfig, alias?: string): ResolvedAccount {
const acct = selectAccount(config, alias);
return {
name: acct?.name,
alias: acct?.alias,
default_publish_method: acct?.default_publish_method ?? config.default_publish_method,
default_author: acct?.default_author ?? config.default_author,
need_open_comment: acct?.need_open_comment ?? config.need_open_comment ?? 1,
only_fans_can_comment: acct?.only_fans_can_comment ?? config.only_fans_can_comment ?? 0,
app_id: acct?.app_id,
app_secret: acct?.app_secret,
chrome_profile_path: acct?.chrome_profile_path ?? config.chrome_profile_path,
};
}
function loadEnvFile(envPath: string): Record<string, string> {
const env: Record<string, string> = {};
if (!fs.existsSync(envPath)) return env;
const content = fs.readFileSync(envPath, "utf-8");
for (const line of content.split("\n")) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith("#")) continue;
const eqIdx = trimmed.indexOf("=");
if (eqIdx > 0) {
const key = trimmed.slice(0, eqIdx).trim();
let value = trimmed.slice(eqIdx + 1).trim();
if ((value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))) {
value = value.slice(1, -1);
}
env[key] = value;
}
}
return env;
}
function aliasToEnvKey(alias: string): string {
return alias.toUpperCase().replace(/-/g, "_");
}
export function loadCredentials(account?: ResolvedAccount): { appId: string; appSecret: string } {
if (account?.app_id && account?.app_secret) {
return { appId: account.app_id, appSecret: account.app_secret };
}
const cwdEnvPath = path.join(process.cwd(), ".baoyu-skills", ".env");
const homeEnvPath = path.join(os.homedir(), ".baoyu-skills", ".env");
const cwdEnv = loadEnvFile(cwdEnvPath);
const homeEnv = loadEnvFile(homeEnvPath);
const prefix = account?.alias ? `WECHAT_${aliasToEnvKey(account.alias)}_` : "";
let appId = "";
let appSecret = "";
if (prefix) {
appId = process.env[`${prefix}APP_ID`]
|| cwdEnv[`${prefix}APP_ID`]
|| homeEnv[`${prefix}APP_ID`]
|| "";
appSecret = process.env[`${prefix}APP_SECRET`]
|| cwdEnv[`${prefix}APP_SECRET`]
|| homeEnv[`${prefix}APP_SECRET`]
|| "";
}
if (!appId) {
appId = process.env.WECHAT_APP_ID || cwdEnv.WECHAT_APP_ID || homeEnv.WECHAT_APP_ID || "";
}
if (!appSecret) {
appSecret = process.env.WECHAT_APP_SECRET || cwdEnv.WECHAT_APP_SECRET || homeEnv.WECHAT_APP_SECRET || "";
}
if (!appId || !appSecret) {
const hint = account?.alias ? ` (account: ${account.alias})` : "";
throw new Error(
`Missing WECHAT_APP_ID or WECHAT_APP_SECRET${hint}.\n` +
"Set via EXTEND.md account config, environment variables, or .baoyu-skills/.env file."
);
}
return { appId, appSecret };
}
export function listAccounts(config: WechatExtendConfig): string[] {
return (config.accounts || []).map(a => a.alias);
}
@@ -0,0 +1,31 @@
{
"lockfileVersion": 1,
"workspaces": {
"": {
"name": "baoyu-post-to-weibo-scripts",
"dependencies": {
"baoyu-chrome-cdp": "file:./vendor/baoyu-chrome-cdp",
"front-matter": "^4.0.2",
"highlight.js": "^11.11.1",
"marked": "^15.0.6",
},
},
},
"packages": {
"argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="],
"baoyu-chrome-cdp": ["baoyu-chrome-cdp@file:vendor/baoyu-chrome-cdp", {}],
"esprima": ["esprima@4.0.1", "", { "bin": { "esparse": "./bin/esparse.js", "esvalidate": "./bin/esvalidate.js" } }, "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A=="],
"front-matter": ["front-matter@4.0.2", "", { "dependencies": { "js-yaml": "^3.13.1" } }, "sha512-I8ZuJ/qG92NWX8i5x1Y8qyj3vizhXS31OxjKDu3LKP+7/qBgfIKValiZIEwoVoJKUHlhWtYrktkxV1XsX+pPlg=="],
"highlight.js": ["highlight.js@11.11.1", "", {}, "sha512-Xwwo44whKBVCYoliBQwaPvtd/2tYFkRQtXDWj1nackaV2JPXx3L0+Jvd8/qCJ2p+ML0/XVkJ2q+Mr+UVdpJK5w=="],
"js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="],
"marked": ["marked@15.0.12", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA=="],
"sprintf-js": ["sprintf-js@1.0.3", "", {}, "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="],
}
}
@@ -446,7 +446,9 @@ Options:
}
}
await main().catch((err) => {
console.error(`Error: ${err instanceof Error ? err.message : String(err)}`);
process.exit(1);
});
if (import.meta.main ?? (process.argv[1] && path.resolve(process.argv[1]) === path.resolve(import.meta.filename ?? ''))) {
await main().catch((err) => {
console.error(`Error: ${err instanceof Error ? err.message : String(err)}`);
process.exit(1);
});
}
@@ -0,0 +1,11 @@
{
"name": "baoyu-post-to-weibo-scripts",
"private": true,
"type": "module",
"dependencies": {
"baoyu-chrome-cdp": "file:./vendor/baoyu-chrome-cdp",
"front-matter": "^4.0.2",
"highlight.js": "^11.11.1",
"marked": "^15.0.6"
}
}
@@ -0,0 +1,9 @@
{
"name": "baoyu-chrome-cdp",
"private": true,
"version": "0.1.0",
"type": "module",
"exports": {
".": "./src/index.ts"
}
}
@@ -0,0 +1,408 @@
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import process from "node:process";
export type PlatformCandidates = {
darwin?: string[];
win32?: string[];
default: string[];
};
type PendingRequest = {
resolve: (value: unknown) => void;
reject: (error: Error) => void;
timer: ReturnType<typeof setTimeout> | null;
};
type CdpSendOptions = {
sessionId?: string;
timeoutMs?: number;
};
type FetchJsonOptions = {
timeoutMs?: number;
};
type FindChromeExecutableOptions = {
candidates: PlatformCandidates;
envNames?: string[];
};
type ResolveSharedChromeProfileDirOptions = {
envNames?: string[];
appDataDirName?: string;
profileDirName?: string;
wslWindowsHome?: string | null;
};
type FindExistingChromeDebugPortOptions = {
profileDir: string;
timeoutMs?: number;
};
type LaunchChromeOptions = {
chromePath: string;
profileDir: string;
port: number;
url?: string;
headless?: boolean;
extraArgs?: string[];
};
type ChromeTargetInfo = {
targetId: string;
url: string;
type: string;
};
type OpenPageSessionOptions = {
cdp: CdpConnection;
reusing: boolean;
url: string;
matchTarget: (target: ChromeTargetInfo) => boolean;
enablePage?: boolean;
enableRuntime?: boolean;
enableDom?: boolean;
enableNetwork?: boolean;
activateTarget?: boolean;
};
export type PageSession = {
sessionId: string;
targetId: string;
};
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export async function getFreePort(fixedEnvName?: string): Promise<number> {
const fixed = fixedEnvName ? Number.parseInt(process.env[fixedEnvName] ?? "", 10) : NaN;
if (Number.isInteger(fixed) && fixed > 0) return fixed;
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Unable to allocate a free TCP port.")));
return;
}
const port = address.port;
server.close((err) => {
if (err) reject(err);
else resolve(port);
});
});
});
}
export function findChromeExecutable(options: FindChromeExecutableOptions): string | undefined {
for (const envName of options.envNames ?? []) {
const override = process.env[envName]?.trim();
if (override && fs.existsSync(override)) return override;
}
const candidates = process.platform === "darwin"
? options.candidates.darwin ?? options.candidates.default
: process.platform === "win32"
? options.candidates.win32 ?? options.candidates.default
: options.candidates.default;
for (const candidate of candidates) {
if (fs.existsSync(candidate)) return candidate;
}
return undefined;
}
export function resolveSharedChromeProfileDir(options: ResolveSharedChromeProfileDirOptions = {}): string {
for (const envName of options.envNames ?? []) {
const override = process.env[envName]?.trim();
if (override) return path.resolve(override);
}
const appDataDirName = options.appDataDirName ?? "baoyu-skills";
const profileDirName = options.profileDirName ?? "chrome-profile";
if (options.wslWindowsHome) {
return path.join(options.wslWindowsHome, ".local", "share", appDataDirName, profileDirName);
}
const base = process.platform === "darwin"
? path.join(os.homedir(), "Library", "Application Support")
: process.platform === "win32"
? (process.env.APPDATA ?? path.join(os.homedir(), "AppData", "Roaming"))
: (process.env.XDG_DATA_HOME ?? path.join(os.homedir(), ".local", "share"));
return path.join(base, appDataDirName, profileDirName);
}
async function fetchWithTimeout(url: string, timeoutMs?: number): Promise<Response> {
if (!timeoutMs || timeoutMs <= 0) return await fetch(url, { redirect: "follow" });
const ctl = new AbortController();
const timer = setTimeout(() => ctl.abort(), timeoutMs);
try {
return await fetch(url, { redirect: "follow", signal: ctl.signal });
} finally {
clearTimeout(timer);
}
}
async function fetchJson<T = unknown>(url: string, options: FetchJsonOptions = {}): Promise<T> {
const response = await fetchWithTimeout(url, options.timeoutMs);
if (!response.ok) {
throw new Error(`Request failed: ${response.status} ${response.statusText}`);
}
return await response.json() as T;
}
async function isDebugPortReady(port: number, timeoutMs = 3_000): Promise<boolean> {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
`http://127.0.0.1:${port}/json/version`,
{ timeoutMs }
);
return !!version.webSocketDebuggerUrl;
} catch {
return false;
}
}
export async function findExistingChromeDebugPort(options: FindExistingChromeDebugPortOptions): Promise<number | null> {
const timeoutMs = options.timeoutMs ?? 3_000;
const portFile = path.join(options.profileDir, "DevToolsActivePort");
try {
const content = fs.readFileSync(portFile, "utf-8");
const [portLine] = content.split(/\r?\n/);
const port = Number.parseInt(portLine?.trim() ?? "", 10);
if (port > 0 && await isDebugPortReady(port, timeoutMs)) return port;
} catch {}
if (process.platform === "win32") return null;
try {
const result = spawnSync("ps", ["aux"], { encoding: "utf-8", timeout: 5_000 });
if (result.status !== 0 || !result.stdout) return null;
const lines = result.stdout
.split("\n")
.filter((line) => line.includes(options.profileDir) && line.includes("--remote-debugging-port="));
for (const line of lines) {
const portMatch = line.match(/--remote-debugging-port=(\d+)/);
const port = Number.parseInt(portMatch?.[1] ?? "", 10);
if (port > 0 && await isDebugPortReady(port, timeoutMs)) return port;
}
} catch {}
return null;
}
export async function waitForChromeDebugPort(
port: number,
timeoutMs: number,
options?: { includeLastError?: boolean }
): Promise<string> {
const start = Date.now();
let lastError: unknown = null;
while (Date.now() - start < timeoutMs) {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
`http://127.0.0.1:${port}/json/version`,
{ timeoutMs: 5_000 }
);
if (version.webSocketDebuggerUrl) return version.webSocketDebuggerUrl;
lastError = new Error("Missing webSocketDebuggerUrl");
} catch (error) {
lastError = error;
}
await sleep(200);
}
if (options?.includeLastError && lastError) {
throw new Error(
`Chrome debug port not ready: ${lastError instanceof Error ? lastError.message : String(lastError)}`
);
}
throw new Error("Chrome debug port not ready");
}
export class CdpConnection {
private ws: WebSocket;
private nextId = 0;
private pending = new Map<number, PendingRequest>();
private eventHandlers = new Map<string, Set<(params: unknown) => void>>();
private defaultTimeoutMs: number;
private constructor(ws: WebSocket, defaultTimeoutMs = 15_000) {
this.ws = ws;
this.defaultTimeoutMs = defaultTimeoutMs;
this.ws.addEventListener("message", (event) => {
try {
const data = typeof event.data === "string"
? event.data
: new TextDecoder().decode(event.data as ArrayBuffer);
const msg = JSON.parse(data) as {
id?: number;
method?: string;
params?: unknown;
result?: unknown;
error?: { message?: string };
};
if (msg.method) {
const handlers = this.eventHandlers.get(msg.method);
if (handlers) {
handlers.forEach((handler) => handler(msg.params));
}
}
if (msg.id) {
const pending = this.pending.get(msg.id);
if (pending) {
this.pending.delete(msg.id);
if (pending.timer) clearTimeout(pending.timer);
if (msg.error?.message) pending.reject(new Error(msg.error.message));
else pending.resolve(msg.result);
}
}
} catch {}
});
this.ws.addEventListener("close", () => {
for (const [id, pending] of this.pending.entries()) {
this.pending.delete(id);
if (pending.timer) clearTimeout(pending.timer);
pending.reject(new Error("CDP connection closed."));
}
});
}
static async connect(
url: string,
timeoutMs: number,
options?: { defaultTimeoutMs?: number }
): Promise<CdpConnection> {
const ws = new WebSocket(url);
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error("CDP connection timeout.")), timeoutMs);
ws.addEventListener("open", () => {
clearTimeout(timer);
resolve();
});
ws.addEventListener("error", () => {
clearTimeout(timer);
reject(new Error("CDP connection failed."));
});
});
return new CdpConnection(ws, options?.defaultTimeoutMs ?? 15_000);
}
on(method: string, handler: (params: unknown) => void): void {
if (!this.eventHandlers.has(method)) {
this.eventHandlers.set(method, new Set());
}
this.eventHandlers.get(method)?.add(handler);
}
off(method: string, handler: (params: unknown) => void): void {
this.eventHandlers.get(method)?.delete(handler);
}
async send<T = unknown>(method: string, params?: Record<string, unknown>, options?: CdpSendOptions): Promise<T> {
const id = ++this.nextId;
const message: Record<string, unknown> = { id, method };
if (params) message.params = params;
if (options?.sessionId) message.sessionId = options.sessionId;
const timeoutMs = options?.timeoutMs ?? this.defaultTimeoutMs;
const result = await new Promise<unknown>((resolve, reject) => {
const timer = timeoutMs > 0
? setTimeout(() => {
this.pending.delete(id);
reject(new Error(`CDP timeout: ${method}`));
}, timeoutMs)
: null;
this.pending.set(id, { resolve, reject, timer });
this.ws.send(JSON.stringify(message));
});
return result as T;
}
close(): void {
try {
this.ws.close();
} catch {}
}
}
export async function launchChrome(options: LaunchChromeOptions): Promise<ChildProcess> {
await fs.promises.mkdir(options.profileDir, { recursive: true });
const args = [
`--remote-debugging-port=${options.port}`,
`--user-data-dir=${options.profileDir}`,
"--no-first-run",
"--no-default-browser-check",
...(options.extraArgs ?? []),
];
if (options.headless) args.push("--headless=new");
if (options.url) args.push(options.url);
return spawn(options.chromePath, args, { stdio: "ignore" });
}
export function killChrome(chrome: ChildProcess): void {
try {
chrome.kill("SIGTERM");
} catch {}
setTimeout(() => {
if (!chrome.killed) {
try {
chrome.kill("SIGKILL");
} catch {}
}
}, 2_000).unref?.();
}
export async function openPageSession(options: OpenPageSessionOptions): Promise<PageSession> {
let targetId: string;
if (options.reusing) {
const created = await options.cdp.send<{ targetId: string }>("Target.createTarget", { url: options.url });
targetId = created.targetId;
} else {
const targets = await options.cdp.send<{ targetInfos: ChromeTargetInfo[] }>("Target.getTargets");
const existing = targets.targetInfos.find(options.matchTarget);
if (existing) {
targetId = existing.targetId;
} else {
const created = await options.cdp.send<{ targetId: string }>("Target.createTarget", { url: options.url });
targetId = created.targetId;
}
}
const { sessionId } = await options.cdp.send<{ sessionId: string }>(
"Target.attachToTarget",
{ targetId, flatten: true }
);
if (options.activateTarget ?? true) {
await options.cdp.send("Target.activateTarget", { targetId });
}
if (options.enablePage) await options.cdp.send("Page.enable", {}, { sessionId });
if (options.enableRuntime) await options.cdp.send("Runtime.enable", {}, { sessionId });
if (options.enableDom) await options.cdp.send("DOM.enable", {}, { sessionId });
if (options.enableNetwork) await options.cdp.send("Network.enable", {}, { sessionId });
return { sessionId, targetId };
}
@@ -1,9 +1,7 @@
import { spawn } from 'node:child_process';
import fs from 'node:fs';
import { mkdir, writeFile } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import process from 'node:process';
import {
CdpConnection,
copyHtmlToClipboard,
@@ -11,7 +9,7 @@ import {
findChromeExecutable,
findExistingChromeDebugPort,
getDefaultProfileDir,
getFreePort,
launchChrome,
pasteFromClipboard,
sleep,
waitForChromeDebugPort,
@@ -74,36 +72,17 @@ export async function publishArticle(options: ArticleOptions): Promise<void> {
await mkdir(profileDir, { recursive: true });
// Try reusing an existing Chrome instance with the same profile
const existingPort = findExistingChromeDebugPort(profileDir);
const existingPort = await findExistingChromeDebugPort(profileDir);
let port: number;
let launched = false;
if (existingPort) {
console.log(`[weibo-article] Found existing Chrome on port ${existingPort}, reusing...`);
port = existingPort;
} else {
const chromePath = options.chromePath ?? findChromeExecutable();
const chromePath = findChromeExecutable(options.chromePath);
if (!chromePath) throw new Error('Chrome not found. Set WEIBO_BROWSER_CHROME_PATH env var.');
port = await getFreePort();
console.log(`[weibo-article] Launching Chrome...`);
const chromeArgs = [
`--remote-debugging-port=${port}`,
`--user-data-dir=${profileDir}`,
'--no-first-run',
'--no-default-browser-check',
'--disable-blink-features=AutomationControlled',
'--start-maximized',
WEIBO_ARTICLE_URL,
];
if (process.platform === 'darwin') {
const appPath = chromePath.replace(/\/Contents\/MacOS\/Google Chrome$/, '');
spawn('open', ['-na', appPath, '--args', ...chromeArgs], { stdio: 'ignore' });
} else {
spawn(chromePath, chromeArgs, { stdio: 'ignore' });
}
launched = true;
port = await launchChrome(WEIBO_ARTICLE_URL, profileDir, chromePath);
}
let cdp: CdpConnection | null = null;
@@ -1,4 +1,3 @@
import { spawn } from 'node:child_process';
import fs from 'node:fs';
import { mkdir } from 'node:fs/promises';
import path from 'node:path';
@@ -8,8 +7,8 @@ import {
findChromeExecutable,
findExistingChromeDebugPort,
getDefaultProfileDir,
getFreePort,
killChromeByProfile,
launchChrome as launchWeiboChrome,
sleep,
waitForChromeDebugPort,
} from './weibo-utils.js';
@@ -37,32 +36,11 @@ export async function postToWeibo(options: WeiboPostOptions): Promise<void> {
await mkdir(profileDir, { recursive: true });
const chromePath = options.chromePath ?? findChromeExecutable();
const chromePath = findChromeExecutable(options.chromePath);
if (!chromePath) throw new Error('Chrome not found. Set WEIBO_BROWSER_CHROME_PATH env var.');
const launchChrome = async (): Promise<number> => {
const port = await getFreePort();
console.log(`[weibo-post] Launching Chrome (profile: ${profileDir})`);
const chromeArgs = [
`--remote-debugging-port=${port}`,
`--user-data-dir=${profileDir}`,
'--no-first-run',
'--no-default-browser-check',
'--disable-blink-features=AutomationControlled',
'--start-maximized',
WEIBO_HOME_URL,
];
if (process.platform === 'darwin') {
const appPath = chromePath.replace(/\/Contents\/MacOS\/Google Chrome$/, '');
spawn('open', ['-na', appPath, '--args', ...chromeArgs], { stdio: 'ignore' });
} else {
spawn(chromePath, chromeArgs, { stdio: 'ignore' });
}
return port;
};
let port: number;
const existingPort = findExistingChromeDebugPort(profileDir);
const existingPort = await findExistingChromeDebugPort(profileDir);
if (existingPort) {
console.log(`[weibo-post] Found existing Chrome on port ${existingPort}, checking health...`);
@@ -77,10 +55,10 @@ export async function postToWeibo(options: WeiboPostOptions): Promise<void> {
console.log('[weibo-post] Existing Chrome unresponsive, restarting...');
killChromeByProfile(profileDir);
await sleep(2000);
port = await launchChrome();
port = await launchWeiboChrome(WEIBO_HOME_URL, profileDir, chromePath);
}
} else {
port = await launchChrome();
port = await launchWeiboChrome(WEIBO_HOME_URL, profileDir, chromePath);
}
let cdp: CdpConnection | null = null;
+69 -160
View File
@@ -1,12 +1,23 @@
import { spawnSync } from 'node:child_process';
import fs from 'node:fs';
import net from 'node:net';
import os from 'node:os';
import { execSync, spawnSync } from 'node:child_process';
import path from 'node:path';
import process from 'node:process';
import { fileURLToPath } from 'node:url';
export const CHROME_CANDIDATES = {
import {
CdpConnection,
findChromeExecutable as findChromeExecutableBase,
findExistingChromeDebugPort as findExistingChromeDebugPortBase,
getFreePort as getFreePortBase,
launchChrome as launchChromeBase,
resolveSharedChromeProfileDir,
sleep,
waitForChromeDebugPort,
type PlatformCandidates,
} from 'baoyu-chrome-cdp';
export { CdpConnection, sleep, waitForChromeDebugPort };
export const CHROME_CANDIDATES: PlatformCandidates = {
darwin: [
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
@@ -23,183 +34,81 @@ export const CHROME_CANDIDATES = {
],
};
export function findChromeExecutable(): string | undefined {
const override = process.env.WEIBO_BROWSER_CHROME_PATH?.trim();
if (override && fs.existsSync(override)) return override;
const candidates = process.platform === 'darwin'
? CHROME_CANDIDATES.darwin
: process.platform === 'win32'
? CHROME_CANDIDATES.win32
: CHROME_CANDIDATES.default;
for (const candidate of candidates) {
if (fs.existsSync(candidate)) return candidate;
let wslHome: string | null | undefined;
function getWslWindowsHome(): string | null {
if (wslHome !== undefined) return wslHome;
if (!process.env.WSL_DISTRO_NAME) {
wslHome = null;
return null;
}
return undefined;
try {
const raw = execSync('cmd.exe /C "echo %USERPROFILE%"', {
encoding: 'utf-8',
timeout: 5_000,
}).trim().replace(/\r/g, '');
wslHome = execSync(`wslpath -u "${raw}"`, {
encoding: 'utf-8',
timeout: 5_000,
}).trim() || null;
} catch {
wslHome = null;
}
return wslHome;
}
export function findExistingChromeDebugPort(profileDir: string): number | null {
try {
const result = spawnSync('ps', ['aux'], { encoding: 'utf-8', timeout: 5000 });
if (result.status !== 0 || !result.stdout) return null;
const lines = result.stdout.split('\n');
for (const line of lines) {
if (!line.includes('--remote-debugging-port=') || !line.includes(profileDir)) continue;
const portMatch = line.match(/--remote-debugging-port=(\d+)/);
if (portMatch) return Number(portMatch[1]);
}
} catch {}
return null;
export function findChromeExecutable(chromePathOverride?: string): string | undefined {
if (chromePathOverride?.trim()) return chromePathOverride.trim();
return findChromeExecutableBase({
candidates: CHROME_CANDIDATES,
envNames: ['WEIBO_BROWSER_CHROME_PATH'],
});
}
export async function findExistingChromeDebugPort(profileDir: string): Promise<number | null> {
return await findExistingChromeDebugPortBase({ profileDir });
}
export function killChromeByProfile(profileDir: string): void {
try {
const result = spawnSync('ps', ['aux'], { encoding: 'utf-8', timeout: 5000 });
const result = spawnSync('ps', ['aux'], { encoding: 'utf-8', timeout: 5_000 });
if (result.status !== 0 || !result.stdout) return;
for (const line of result.stdout.split('\n')) {
if (!line.includes(profileDir) || !line.includes('--remote-debugging-port=')) continue;
const pidMatch = line.trim().split(/\s+/)[1];
if (pidMatch) {
try { process.kill(Number(pidMatch), 'SIGTERM'); } catch {}
const pid = line.trim().split(/\s+/)[1];
if (pid) {
try {
process.kill(Number(pid), 'SIGTERM');
} catch {}
}
}
} catch {}
}
export function getDefaultProfileDir(): string {
const override = process.env.BAOYU_CHROME_PROFILE_DIR?.trim() || process.env.WEIBO_BROWSER_PROFILE_DIR?.trim();
if (override) return path.resolve(override);
const base = process.platform === 'darwin'
? path.join(os.homedir(), 'Library', 'Application Support')
: process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
return path.join(base, 'baoyu-skills', 'chrome-profile');
}
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export async function getFreePort(): Promise<number> {
const fixed = parseInt(process.env.WEIBO_BROWSER_DEBUG_PORT || '', 10);
if (fixed > 0) return fixed;
return new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (!address || typeof address === 'string') {
server.close(() => reject(new Error('Unable to allocate a free TCP port.')));
return;
}
const port = address.port;
server.close((err) => {
if (err) reject(err);
else resolve(port);
});
});
return resolveSharedChromeProfileDir({
envNames: ['BAOYU_CHROME_PROFILE_DIR', 'WEIBO_BROWSER_PROFILE_DIR'],
wslWindowsHome: getWslWindowsHome(),
});
}
async function fetchJson<T = unknown>(url: string): Promise<T> {
const res = await fetch(url, { redirect: 'follow' });
if (!res.ok) throw new Error(`Request failed: ${res.status} ${res.statusText}`);
return (await res.json()) as T;
export async function getFreePort(): Promise<number> {
return await getFreePortBase('WEIBO_BROWSER_DEBUG_PORT');
}
export async function waitForChromeDebugPort(port: number, timeoutMs: number): Promise<string> {
const start = Date.now();
let lastError: unknown = null;
export async function launchChrome(url: string, profileDir: string, chromePathOverride?: string): Promise<number> {
const chromePath = findChromeExecutable(chromePathOverride);
if (!chromePath) throw new Error('Chrome not found. Set WEIBO_BROWSER_CHROME_PATH env var.');
while (Date.now() - start < timeoutMs) {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(`http://127.0.0.1:${port}/json/version`);
if (version.webSocketDebuggerUrl) return version.webSocketDebuggerUrl;
lastError = new Error('Missing webSocketDebuggerUrl');
} catch (error) {
lastError = error;
}
await sleep(200);
}
throw new Error(`Chrome debug port not ready: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
}
type PendingRequest = {
resolve: (value: unknown) => void;
reject: (error: Error) => void;
timer: ReturnType<typeof setTimeout> | null;
};
export class CdpConnection {
private ws: WebSocket;
private nextId = 0;
private pending = new Map<number, PendingRequest>();
private defaultTimeoutMs: number;
private constructor(ws: WebSocket, options?: { defaultTimeoutMs?: number }) {
this.ws = ws;
this.defaultTimeoutMs = options?.defaultTimeoutMs ?? 15_000;
this.ws.addEventListener('message', (event) => {
try {
const data = typeof event.data === 'string' ? event.data : new TextDecoder().decode(event.data as ArrayBuffer);
const msg = JSON.parse(data) as { id?: number; method?: string; params?: unknown; result?: unknown; error?: { message?: string } };
if (msg.id) {
const pending = this.pending.get(msg.id);
if (pending) {
this.pending.delete(msg.id);
if (pending.timer) clearTimeout(pending.timer);
if (msg.error?.message) pending.reject(new Error(msg.error.message));
else pending.resolve(msg.result);
}
}
} catch {}
});
this.ws.addEventListener('close', () => {
for (const [id, pending] of this.pending.entries()) {
this.pending.delete(id);
if (pending.timer) clearTimeout(pending.timer);
pending.reject(new Error('CDP connection closed.'));
}
});
}
static async connect(url: string, timeoutMs: number, options?: { defaultTimeoutMs?: number }): Promise<CdpConnection> {
const ws = new WebSocket(url);
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error('CDP connection timeout.')), timeoutMs);
ws.addEventListener('open', () => { clearTimeout(timer); resolve(); });
ws.addEventListener('error', () => { clearTimeout(timer); reject(new Error('CDP connection failed.')); });
});
return new CdpConnection(ws, options);
}
async send<T = unknown>(method: string, params?: Record<string, unknown>, options?: { sessionId?: string; timeoutMs?: number }): Promise<T> {
const id = ++this.nextId;
const message: Record<string, unknown> = { id, method };
if (params) message.params = params;
if (options?.sessionId) message.sessionId = options.sessionId;
const timeoutMs = options?.timeoutMs ?? this.defaultTimeoutMs;
const result = await new Promise<unknown>((resolve, reject) => {
const timer = timeoutMs > 0
? setTimeout(() => { this.pending.delete(id); reject(new Error(`CDP timeout: ${method}`)); }, timeoutMs)
: null;
this.pending.set(id, { resolve, reject, timer });
this.ws.send(JSON.stringify(message));
});
return result as T;
}
close(): void {
try { this.ws.close(); } catch {}
}
const port = await getFreePort();
console.log(`[weibo-cdp] Launching Chrome (profile: ${profileDir})`);
await launchChromeBase({
chromePath,
profileDir,
port,
url,
extraArgs: ['--disable-blink-features=AutomationControlled', '--start-maximized'],
});
return port;
}
export function getScriptDir(): string {
+3
View File
@@ -4,6 +4,7 @@
"": {
"name": "baoyu-post-to-x-scripts",
"dependencies": {
"baoyu-chrome-cdp": "file:./vendor/baoyu-chrome-cdp",
"front-matter": "^4.0.2",
"highlight.js": "^11.11.1",
"marked": "^15.0.6",
@@ -27,6 +28,8 @@
"bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="],
"baoyu-chrome-cdp": ["baoyu-chrome-cdp@file:vendor/baoyu-chrome-cdp", {}],
"character-entities": ["character-entities@2.0.2", "", {}, "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ=="],
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
@@ -3,6 +3,7 @@
"private": true,
"type": "module",
"dependencies": {
"baoyu-chrome-cdp": "file:./vendor/baoyu-chrome-cdp",
"front-matter": "^4.0.2",
"highlight.js": "^11.11.1",
"marked": "^15.0.6",
@@ -0,0 +1,9 @@
{
"name": "baoyu-chrome-cdp",
"private": true,
"version": "0.1.0",
"type": "module",
"exports": {
".": "./src/index.ts"
}
}
@@ -0,0 +1,408 @@
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import process from "node:process";
export type PlatformCandidates = {
darwin?: string[];
win32?: string[];
default: string[];
};
type PendingRequest = {
resolve: (value: unknown) => void;
reject: (error: Error) => void;
timer: ReturnType<typeof setTimeout> | null;
};
type CdpSendOptions = {
sessionId?: string;
timeoutMs?: number;
};
type FetchJsonOptions = {
timeoutMs?: number;
};
type FindChromeExecutableOptions = {
candidates: PlatformCandidates;
envNames?: string[];
};
type ResolveSharedChromeProfileDirOptions = {
envNames?: string[];
appDataDirName?: string;
profileDirName?: string;
wslWindowsHome?: string | null;
};
type FindExistingChromeDebugPortOptions = {
profileDir: string;
timeoutMs?: number;
};
type LaunchChromeOptions = {
chromePath: string;
profileDir: string;
port: number;
url?: string;
headless?: boolean;
extraArgs?: string[];
};
type ChromeTargetInfo = {
targetId: string;
url: string;
type: string;
};
type OpenPageSessionOptions = {
cdp: CdpConnection;
reusing: boolean;
url: string;
matchTarget: (target: ChromeTargetInfo) => boolean;
enablePage?: boolean;
enableRuntime?: boolean;
enableDom?: boolean;
enableNetwork?: boolean;
activateTarget?: boolean;
};
export type PageSession = {
sessionId: string;
targetId: string;
};
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export async function getFreePort(fixedEnvName?: string): Promise<number> {
const fixed = fixedEnvName ? Number.parseInt(process.env[fixedEnvName] ?? "", 10) : NaN;
if (Number.isInteger(fixed) && fixed > 0) return fixed;
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Unable to allocate a free TCP port.")));
return;
}
const port = address.port;
server.close((err) => {
if (err) reject(err);
else resolve(port);
});
});
});
}
export function findChromeExecutable(options: FindChromeExecutableOptions): string | undefined {
for (const envName of options.envNames ?? []) {
const override = process.env[envName]?.trim();
if (override && fs.existsSync(override)) return override;
}
const candidates = process.platform === "darwin"
? options.candidates.darwin ?? options.candidates.default
: process.platform === "win32"
? options.candidates.win32 ?? options.candidates.default
: options.candidates.default;
for (const candidate of candidates) {
if (fs.existsSync(candidate)) return candidate;
}
return undefined;
}
export function resolveSharedChromeProfileDir(options: ResolveSharedChromeProfileDirOptions = {}): string {
for (const envName of options.envNames ?? []) {
const override = process.env[envName]?.trim();
if (override) return path.resolve(override);
}
const appDataDirName = options.appDataDirName ?? "baoyu-skills";
const profileDirName = options.profileDirName ?? "chrome-profile";
if (options.wslWindowsHome) {
return path.join(options.wslWindowsHome, ".local", "share", appDataDirName, profileDirName);
}
const base = process.platform === "darwin"
? path.join(os.homedir(), "Library", "Application Support")
: process.platform === "win32"
? (process.env.APPDATA ?? path.join(os.homedir(), "AppData", "Roaming"))
: (process.env.XDG_DATA_HOME ?? path.join(os.homedir(), ".local", "share"));
return path.join(base, appDataDirName, profileDirName);
}
async function fetchWithTimeout(url: string, timeoutMs?: number): Promise<Response> {
if (!timeoutMs || timeoutMs <= 0) return await fetch(url, { redirect: "follow" });
const ctl = new AbortController();
const timer = setTimeout(() => ctl.abort(), timeoutMs);
try {
return await fetch(url, { redirect: "follow", signal: ctl.signal });
} finally {
clearTimeout(timer);
}
}
async function fetchJson<T = unknown>(url: string, options: FetchJsonOptions = {}): Promise<T> {
const response = await fetchWithTimeout(url, options.timeoutMs);
if (!response.ok) {
throw new Error(`Request failed: ${response.status} ${response.statusText}`);
}
return await response.json() as T;
}
async function isDebugPortReady(port: number, timeoutMs = 3_000): Promise<boolean> {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
`http://127.0.0.1:${port}/json/version`,
{ timeoutMs }
);
return !!version.webSocketDebuggerUrl;
} catch {
return false;
}
}
export async function findExistingChromeDebugPort(options: FindExistingChromeDebugPortOptions): Promise<number | null> {
const timeoutMs = options.timeoutMs ?? 3_000;
const portFile = path.join(options.profileDir, "DevToolsActivePort");
try {
const content = fs.readFileSync(portFile, "utf-8");
const [portLine] = content.split(/\r?\n/);
const port = Number.parseInt(portLine?.trim() ?? "", 10);
if (port > 0 && await isDebugPortReady(port, timeoutMs)) return port;
} catch {}
if (process.platform === "win32") return null;
try {
const result = spawnSync("ps", ["aux"], { encoding: "utf-8", timeout: 5_000 });
if (result.status !== 0 || !result.stdout) return null;
const lines = result.stdout
.split("\n")
.filter((line) => line.includes(options.profileDir) && line.includes("--remote-debugging-port="));
for (const line of lines) {
const portMatch = line.match(/--remote-debugging-port=(\d+)/);
const port = Number.parseInt(portMatch?.[1] ?? "", 10);
if (port > 0 && await isDebugPortReady(port, timeoutMs)) return port;
}
} catch {}
return null;
}
export async function waitForChromeDebugPort(
port: number,
timeoutMs: number,
options?: { includeLastError?: boolean }
): Promise<string> {
const start = Date.now();
let lastError: unknown = null;
while (Date.now() - start < timeoutMs) {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
`http://127.0.0.1:${port}/json/version`,
{ timeoutMs: 5_000 }
);
if (version.webSocketDebuggerUrl) return version.webSocketDebuggerUrl;
lastError = new Error("Missing webSocketDebuggerUrl");
} catch (error) {
lastError = error;
}
await sleep(200);
}
if (options?.includeLastError && lastError) {
throw new Error(
`Chrome debug port not ready: ${lastError instanceof Error ? lastError.message : String(lastError)}`
);
}
throw new Error("Chrome debug port not ready");
}
export class CdpConnection {
private ws: WebSocket;
private nextId = 0;
private pending = new Map<number, PendingRequest>();
private eventHandlers = new Map<string, Set<(params: unknown) => void>>();
private defaultTimeoutMs: number;
private constructor(ws: WebSocket, defaultTimeoutMs = 15_000) {
this.ws = ws;
this.defaultTimeoutMs = defaultTimeoutMs;
this.ws.addEventListener("message", (event) => {
try {
const data = typeof event.data === "string"
? event.data
: new TextDecoder().decode(event.data as ArrayBuffer);
const msg = JSON.parse(data) as {
id?: number;
method?: string;
params?: unknown;
result?: unknown;
error?: { message?: string };
};
if (msg.method) {
const handlers = this.eventHandlers.get(msg.method);
if (handlers) {
handlers.forEach((handler) => handler(msg.params));
}
}
if (msg.id) {
const pending = this.pending.get(msg.id);
if (pending) {
this.pending.delete(msg.id);
if (pending.timer) clearTimeout(pending.timer);
if (msg.error?.message) pending.reject(new Error(msg.error.message));
else pending.resolve(msg.result);
}
}
} catch {}
});
this.ws.addEventListener("close", () => {
for (const [id, pending] of this.pending.entries()) {
this.pending.delete(id);
if (pending.timer) clearTimeout(pending.timer);
pending.reject(new Error("CDP connection closed."));
}
});
}
static async connect(
url: string,
timeoutMs: number,
options?: { defaultTimeoutMs?: number }
): Promise<CdpConnection> {
const ws = new WebSocket(url);
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error("CDP connection timeout.")), timeoutMs);
ws.addEventListener("open", () => {
clearTimeout(timer);
resolve();
});
ws.addEventListener("error", () => {
clearTimeout(timer);
reject(new Error("CDP connection failed."));
});
});
return new CdpConnection(ws, options?.defaultTimeoutMs ?? 15_000);
}
on(method: string, handler: (params: unknown) => void): void {
if (!this.eventHandlers.has(method)) {
this.eventHandlers.set(method, new Set());
}
this.eventHandlers.get(method)?.add(handler);
}
off(method: string, handler: (params: unknown) => void): void {
this.eventHandlers.get(method)?.delete(handler);
}
async send<T = unknown>(method: string, params?: Record<string, unknown>, options?: CdpSendOptions): Promise<T> {
const id = ++this.nextId;
const message: Record<string, unknown> = { id, method };
if (params) message.params = params;
if (options?.sessionId) message.sessionId = options.sessionId;
const timeoutMs = options?.timeoutMs ?? this.defaultTimeoutMs;
const result = await new Promise<unknown>((resolve, reject) => {
const timer = timeoutMs > 0
? setTimeout(() => {
this.pending.delete(id);
reject(new Error(`CDP timeout: ${method}`));
}, timeoutMs)
: null;
this.pending.set(id, { resolve, reject, timer });
this.ws.send(JSON.stringify(message));
});
return result as T;
}
close(): void {
try {
this.ws.close();
} catch {}
}
}
export async function launchChrome(options: LaunchChromeOptions): Promise<ChildProcess> {
await fs.promises.mkdir(options.profileDir, { recursive: true });
const args = [
`--remote-debugging-port=${options.port}`,
`--user-data-dir=${options.profileDir}`,
"--no-first-run",
"--no-default-browser-check",
...(options.extraArgs ?? []),
];
if (options.headless) args.push("--headless=new");
if (options.url) args.push(options.url);
return spawn(options.chromePath, args, { stdio: "ignore" });
}
export function killChrome(chrome: ChildProcess): void {
try {
chrome.kill("SIGTERM");
} catch {}
setTimeout(() => {
if (!chrome.killed) {
try {
chrome.kill("SIGKILL");
} catch {}
}
}, 2_000).unref?.();
}
export async function openPageSession(options: OpenPageSessionOptions): Promise<PageSession> {
let targetId: string;
if (options.reusing) {
const created = await options.cdp.send<{ targetId: string }>("Target.createTarget", { url: options.url });
targetId = created.targetId;
} else {
const targets = await options.cdp.send<{ targetInfos: ChromeTargetInfo[] }>("Target.getTargets");
const existing = targets.targetInfos.find(options.matchTarget);
if (existing) {
targetId = existing.targetId;
} else {
const created = await options.cdp.send<{ targetId: string }>("Target.createTarget", { url: options.url });
targetId = created.targetId;
}
}
const { sessionId } = await options.cdp.send<{ sessionId: string }>(
"Target.attachToTarget",
{ targetId, flatten: true }
);
if (options.activateTarget ?? true) {
await options.cdp.send("Target.activateTarget", { targetId });
}
if (options.enablePage) await options.cdp.send("Page.enable", {}, { sessionId });
if (options.enableRuntime) await options.cdp.send("Runtime.enable", {}, { sessionId });
if (options.enableDom) await options.cdp.send("DOM.enable", {}, { sessionId });
if (options.enableNetwork) await options.cdp.send("Network.enable", {}, { sessionId });
return { sessionId, targetId };
}
+19 -58
View File
@@ -1,9 +1,7 @@
import { spawn } from 'node:child_process';
import fs from 'node:fs';
import { mkdir, writeFile } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import process from 'node:process';
import { parseMarkdown } from './md-to-html.js';
import {
@@ -11,9 +9,10 @@ import {
CdpConnection,
copyHtmlToClipboard,
copyImageToClipboard,
findChromeExecutable,
findExistingChromeDebugPort,
getDefaultProfileDir,
getFreePort,
launchChrome,
openPageSession,
pasteFromClipboard,
sleep,
waitForChromeDebugPort,
@@ -61,25 +60,6 @@ interface ArticleOptions {
chromePath?: string;
}
async function findExistingDebugPort(profileDir: string): Promise<number | null> {
const portFile = path.join(profileDir, 'DevToolsActivePort');
if (!fs.existsSync(portFile)) return null;
try {
const content = fs.readFileSync(portFile, 'utf-8').trim();
if (!content) return null;
const [portLine] = content.split(/\r?\n/);
const port = Number(portLine);
if (!Number.isFinite(port) || port <= 0) return null;
// Verify the port is actually active.
await waitForChromeDebugPort(port, 1500, { includeLastError: true });
return port;
} catch {
return null;
}
}
export async function publishArticle(options: ArticleOptions): Promise<void> {
const { markdownPath, submit = false, profileDir = getDefaultProfileDir() } = options;
@@ -98,32 +78,17 @@ export async function publishArticle(options: ArticleOptions): Promise<void> {
await writeFile(htmlPath, parsed.html, 'utf-8');
console.log(`[x-article] HTML saved to: ${htmlPath}`);
const chromePath = options.chromePath ?? findChromeExecutable(CHROME_CANDIDATES_BASIC);
if (!chromePath) throw new Error('Chrome not found');
await mkdir(profileDir, { recursive: true });
const existingPort = await findExistingDebugPort(profileDir);
const port = existingPort ?? await getFreePort();
const existingPort = await findExistingChromeDebugPort(profileDir);
const reusing = existingPort !== null;
let port = existingPort ?? 0;
if (existingPort) {
if (reusing) {
console.log(`[x-article] Reusing existing Chrome instance on port ${port}`);
} else {
console.log(`[x-article] Launching Chrome...`);
const chromeArgs = [
`--remote-debugging-port=${port}`,
`--user-data-dir=${profileDir}`,
'--no-first-run',
'--no-default-browser-check',
'--disable-blink-features=AutomationControlled',
'--start-maximized',
X_ARTICLES_URL,
];
if (process.platform === 'darwin') {
const appPath = chromePath.replace(/\/Contents\/MacOS\/Google Chrome$/, '');
spawn('open', ['-na', appPath, '--args', ...chromeArgs], { stdio: 'ignore' });
} else {
spawn(chromePath, chromeArgs, { stdio: 'ignore' });
}
const launched = await launchChrome(X_ARTICLES_URL, profileDir, CHROME_CANDIDATES_BASIC, options.chromePath);
port = launched.port;
}
let cdp: CdpConnection | null = null;
@@ -132,20 +97,16 @@ export async function publishArticle(options: ArticleOptions): Promise<void> {
const wsUrl = await waitForChromeDebugPort(port, 30_000, { includeLastError: true });
cdp = await CdpConnection.connect(wsUrl, 30_000, { defaultTimeoutMs: 60_000 });
// Get page target
const targets = await cdp.send<{ targetInfos: Array<{ targetId: string; url: string; type: string }> }>('Target.getTargets');
let pageTarget = targets.targetInfos.find((t) => t.type === 'page' && t.url.startsWith(X_ARTICLES_URL));
if (!pageTarget) {
const { targetId } = await cdp.send<{ targetId: string }>('Target.createTarget', { url: X_ARTICLES_URL });
pageTarget = { targetId, url: X_ARTICLES_URL, type: 'page' };
}
const { sessionId } = await cdp.send<{ sessionId: string }>('Target.attachToTarget', { targetId: pageTarget.targetId, flatten: true });
await cdp.send('Page.enable', {}, { sessionId });
await cdp.send('Runtime.enable', {}, { sessionId });
await cdp.send('DOM.enable', {}, { sessionId });
const page = await openPageSession({
cdp,
reusing,
url: X_ARTICLES_URL,
matchTarget: (target) => target.type === 'page' && target.url.startsWith(X_ARTICLES_URL),
enablePage: true,
enableRuntime: true,
enableDom: true,
});
const { sessionId } = page;
console.log('[x-article] Waiting for articles page...');
await sleep(1000);
+26 -30
View File
@@ -1,4 +1,3 @@
import { spawn } from 'node:child_process';
import fs from 'node:fs';
import { mkdir } from 'node:fs/promises';
import process from 'node:process';
@@ -6,9 +5,10 @@ import {
CHROME_CANDIDATES_FULL,
CdpConnection,
copyImageToClipboard,
findChromeExecutable,
findExistingChromeDebugPort,
getDefaultProfileDir,
getFreePort,
launchChrome,
openPageSession,
pasteFromClipboard,
sleep,
waitForChromeDebugPort,
@@ -28,23 +28,20 @@ interface XBrowserOptions {
export async function postToX(options: XBrowserOptions): Promise<void> {
const { text, images = [], submit = false, timeoutMs = 120_000, profileDir = getDefaultProfileDir() } = options;
const chromePath = options.chromePath ?? findChromeExecutable(CHROME_CANDIDATES_FULL);
if (!chromePath) throw new Error('Chrome not found. Set X_BROWSER_CHROME_PATH env var.');
await mkdir(profileDir, { recursive: true });
const port = await getFreePort();
console.log(`[x-browser] Launching Chrome (profile: ${profileDir})`);
const existingPort = await findExistingChromeDebugPort(profileDir);
const reusing = existingPort !== null;
let port = existingPort ?? 0;
let chrome: Awaited<ReturnType<typeof launchChrome>>['chrome'] | null = null;
if (!reusing) {
const launched = await launchChrome(X_COMPOSE_URL, profileDir, CHROME_CANDIDATES_FULL, options.chromePath);
port = launched.port;
chrome = launched.chrome;
}
const chrome = spawn(chromePath, [
`--remote-debugging-port=${port}`,
`--user-data-dir=${profileDir}`,
'--no-first-run',
'--no-default-browser-check',
'--disable-blink-features=AutomationControlled',
'--start-maximized',
X_COMPOSE_URL,
], { stdio: 'ignore' });
if (reusing) console.log(`[x-browser] Reusing existing Chrome on port ${port}`);
else console.log(`[x-browser] Launching Chrome (profile: ${profileDir})`);
let cdp: CdpConnection | null = null;
@@ -52,18 +49,15 @@ export async function postToX(options: XBrowserOptions): Promise<void> {
const wsUrl = await waitForChromeDebugPort(port, 30_000, { includeLastError: true });
cdp = await CdpConnection.connect(wsUrl, 30_000, { defaultTimeoutMs: 15_000 });
const targets = await cdp.send<{ targetInfos: Array<{ targetId: string; url: string; type: string }> }>('Target.getTargets');
let pageTarget = targets.targetInfos.find((t) => t.type === 'page' && t.url.includes('x.com'));
if (!pageTarget) {
const { targetId } = await cdp.send<{ targetId: string }>('Target.createTarget', { url: X_COMPOSE_URL });
pageTarget = { targetId, url: X_COMPOSE_URL, type: 'page' };
}
const { sessionId } = await cdp.send<{ sessionId: string }>('Target.attachToTarget', { targetId: pageTarget.targetId, flatten: true });
await cdp.send('Page.enable', {}, { sessionId });
await cdp.send('Runtime.enable', {}, { sessionId });
const page = await openPageSession({
cdp,
reusing,
url: X_COMPOSE_URL,
matchTarget: (target) => target.type === 'page' && target.url.includes('x.com'),
enablePage: true,
enableRuntime: true,
});
const { sessionId } = page;
await cdp.send('Input.setIgnoreInputEvents', { ignore: false }, { sessionId });
console.log('[x-browser] Waiting for X editor...');
@@ -193,7 +187,9 @@ export async function postToX(options: XBrowserOptions): Promise<void> {
if (cdp) {
cdp.close();
}
chrome.unref();
if (chrome) {
chrome.unref();
}
}
}
+32 -35
View File
@@ -1,12 +1,13 @@
import { spawn } from 'node:child_process';
import { mkdir } from 'node:fs/promises';
import process from 'node:process';
import {
CHROME_CANDIDATES_FULL,
CdpConnection,
findChromeExecutable,
findExistingChromeDebugPort,
getDefaultProfileDir,
getFreePort,
killChrome,
launchChrome,
openPageSession,
sleep,
waitForChromeDebugPort,
} from './x-utils.js';
@@ -31,43 +32,39 @@ interface QuoteOptions {
export async function quotePost(options: QuoteOptions): Promise<void> {
const { tweetUrl, comment, submit = false, timeoutMs = 120_000, profileDir = getDefaultProfileDir() } = options;
const chromePath = options.chromePath ?? findChromeExecutable(CHROME_CANDIDATES_FULL);
if (!chromePath) throw new Error('Chrome not found. Set X_BROWSER_CHROME_PATH env var.');
await mkdir(profileDir, { recursive: true });
const port = await getFreePort();
console.log(`[x-quote] Launching Chrome (profile: ${profileDir})`);
const existingPort = await findExistingChromeDebugPort(profileDir);
const reusing = existingPort !== null;
let port = existingPort ?? 0;
console.log(`[x-quote] Opening tweet: ${tweetUrl}`);
let chrome: Awaited<ReturnType<typeof launchChrome>>['chrome'] | null = null;
if (!reusing) {
const launched = await launchChrome(tweetUrl, profileDir, CHROME_CANDIDATES_FULL, options.chromePath);
port = launched.port;
chrome = launched.chrome;
}
const chrome = spawn(chromePath, [
`--remote-debugging-port=${port}`,
`--user-data-dir=${profileDir}`,
'--no-first-run',
'--no-default-browser-check',
'--disable-blink-features=AutomationControlled',
'--start-maximized',
tweetUrl,
], { stdio: 'ignore' });
if (reusing) console.log(`[x-quote] Reusing existing Chrome on port ${port}`);
else console.log(`[x-quote] Launching Chrome (profile: ${profileDir})`);
let cdp: CdpConnection | null = null;
let targetId: string | null = null;
try {
const wsUrl = await waitForChromeDebugPort(port, 30_000, { includeLastError: true });
cdp = await CdpConnection.connect(wsUrl, 30_000, { defaultTimeoutMs: 15_000 });
const targets = await cdp.send<{ targetInfos: Array<{ targetId: string; url: string; type: string }> }>('Target.getTargets');
let pageTarget = targets.targetInfos.find((t) => t.type === 'page' && t.url.includes('x.com'));
if (!pageTarget) {
const { targetId } = await cdp.send<{ targetId: string }>('Target.createTarget', { url: tweetUrl });
pageTarget = { targetId, url: tweetUrl, type: 'page' };
}
const { sessionId } = await cdp.send<{ sessionId: string }>('Target.attachToTarget', { targetId: pageTarget.targetId, flatten: true });
await cdp.send('Page.enable', {}, { sessionId });
await cdp.send('Runtime.enable', {}, { sessionId });
const page = await openPageSession({
cdp,
reusing,
url: tweetUrl,
matchTarget: (target) => target.type === 'page' && target.url.includes('x.com'),
enablePage: true,
enableRuntime: true,
});
const { sessionId } = page;
targetId = page.targetId;
console.log('[x-quote] Waiting for tweet to load...');
await sleep(3000);
@@ -176,14 +173,14 @@ export async function quotePost(options: QuoteOptions): Promise<void> {
}
} finally {
if (cdp) {
try { await cdp.send('Browser.close', {}, { timeoutMs: 5_000 }); } catch {}
if (reusing && targetId) {
try { await cdp.send('Target.closeTarget', { targetId }, { timeoutMs: 5_000 }); } catch {}
} else if (!reusing) {
try { await cdp.send('Browser.close', {}, { timeoutMs: 5_000 }); } catch {}
}
cdp.close();
}
setTimeout(() => {
if (!chrome.killed) try { chrome.kill('SIGKILL'); } catch {}
}, 2_000).unref?.();
try { chrome.kill('SIGTERM'); } catch {}
if (chrome) killChrome(chrome);
}
}
+44 -164
View File
@@ -1,16 +1,24 @@
import { execSync, spawnSync } from 'node:child_process';
import fs from 'node:fs';
import net from 'node:net';
import os from 'node:os';
import path from 'node:path';
import process from 'node:process';
import { fileURLToPath } from 'node:url';
export type PlatformCandidates = {
darwin?: string[];
win32?: string[];
default: string[];
};
import {
CdpConnection,
findChromeExecutable as findChromeExecutableBase,
findExistingChromeDebugPort as findExistingChromeDebugPortBase,
getFreePort as getFreePortBase,
killChrome,
launchChrome as launchChromeBase,
openPageSession,
resolveSharedChromeProfileDir,
sleep,
waitForChromeDebugPort,
type PlatformCandidates,
} from 'baoyu-chrome-cdp';
export { CdpConnection, killChrome, openPageSession, sleep, waitForChromeDebugPort };
export type { PlatformCandidates } from 'baoyu-chrome-cdp';
export const CHROME_CANDIDATES_BASIC: PlatformCandidates = {
darwin: [
@@ -53,20 +61,11 @@ export const CHROME_CANDIDATES_FULL: PlatformCandidates = {
],
};
function getCandidatesForPlatform(candidates: PlatformCandidates): string[] {
if (process.platform === 'darwin' && candidates.darwin?.length) return candidates.darwin;
if (process.platform === 'win32' && candidates.win32?.length) return candidates.win32;
return candidates.default;
}
export function findChromeExecutable(candidates: PlatformCandidates): string | undefined {
const override = process.env.X_BROWSER_CHROME_PATH?.trim();
if (override && fs.existsSync(override)) return override;
for (const candidate of getCandidatesForPlatform(candidates)) {
if (fs.existsSync(candidate)) return candidate;
}
return undefined;
return findChromeExecutableBase({
candidates,
envNames: ['X_BROWSER_CHROME_PATH'],
});
}
let _wslHome: string | null | undefined;
@@ -81,158 +80,39 @@ function getWslWindowsHome(): string | null {
}
export function getDefaultProfileDir(): string {
const override = process.env.BAOYU_CHROME_PROFILE_DIR?.trim() || process.env.X_BROWSER_PROFILE_DIR?.trim();
if (override) return path.resolve(override);
const wslHome = getWslWindowsHome();
if (wslHome) return path.join(wslHome, '.local', 'share', 'baoyu-skills', 'chrome-profile');
const base = process.platform === 'darwin'
? path.join(os.homedir(), 'Library', 'Application Support')
: process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share');
return path.join(base, 'baoyu-skills', 'chrome-profile');
}
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export async function getFreePort(): Promise<number> {
const fixed = parseInt(process.env.X_BROWSER_DEBUG_PORT || '', 10);
if (fixed > 0) return fixed;
return new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (!address || typeof address === 'string') {
server.close(() => reject(new Error('Unable to allocate a free TCP port.')));
return;
}
const port = address.port;
server.close((err) => {
if (err) reject(err);
else resolve(port);
});
});
return resolveSharedChromeProfileDir({
envNames: ['BAOYU_CHROME_PROFILE_DIR', 'X_BROWSER_PROFILE_DIR'],
wslWindowsHome: getWslWindowsHome(),
});
}
async function fetchJson<T = unknown>(url: string): Promise<T> {
const res = await fetch(url, { redirect: 'follow' });
if (!res.ok) throw new Error(`Request failed: ${res.status} ${res.statusText}`);
return (await res.json()) as T;
export async function getFreePort(): Promise<number> {
return await getFreePortBase('X_BROWSER_DEBUG_PORT');
}
export async function waitForChromeDebugPort(
port: number,
timeoutMs: number,
options?: { includeLastError?: boolean },
): Promise<string> {
const start = Date.now();
let lastError: unknown = null;
while (Date.now() - start < timeoutMs) {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(`http://127.0.0.1:${port}/json/version`);
if (version.webSocketDebuggerUrl) return version.webSocketDebuggerUrl;
lastError = new Error('Missing webSocketDebuggerUrl');
} catch (error) {
lastError = error;
}
await sleep(200);
}
if (options?.includeLastError && lastError) {
throw new Error(`Chrome debug port not ready: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
}
throw new Error('Chrome debug port not ready');
export async function findExistingChromeDebugPort(profileDir: string): Promise<number | null> {
return await findExistingChromeDebugPortBase({ profileDir });
}
type PendingRequest = {
resolve: (value: unknown) => void;
reject: (error: Error) => void;
timer: ReturnType<typeof setTimeout> | null;
};
export async function launchChrome(
url: string,
profileDir: string,
candidates: PlatformCandidates,
chromePathOverride?: string,
): Promise<{ chrome: Awaited<ReturnType<typeof launchChromeBase>>; port: number }> {
const chromePath = chromePathOverride?.trim() || findChromeExecutable(candidates);
if (!chromePath) throw new Error('Chrome not found. Set X_BROWSER_CHROME_PATH env var.');
export class CdpConnection {
private ws: WebSocket;
private nextId = 0;
private pending = new Map<number, PendingRequest>();
private eventHandlers = new Map<string, Set<(params: unknown) => void>>();
private defaultTimeoutMs: number;
const port = await getFreePort();
const chrome = await launchChromeBase({
chromePath,
profileDir,
port,
url,
extraArgs: ['--start-maximized'],
});
private constructor(ws: WebSocket, options?: { defaultTimeoutMs?: number }) {
this.ws = ws;
this.defaultTimeoutMs = options?.defaultTimeoutMs ?? 15_000;
this.ws.addEventListener('message', (event) => {
try {
const data = typeof event.data === 'string' ? event.data : new TextDecoder().decode(event.data as ArrayBuffer);
const msg = JSON.parse(data) as { id?: number; method?: string; params?: unknown; result?: unknown; error?: { message?: string } };
if (msg.method) {
const handlers = this.eventHandlers.get(msg.method);
if (handlers) handlers.forEach((h) => h(msg.params));
}
if (msg.id) {
const pending = this.pending.get(msg.id);
if (pending) {
this.pending.delete(msg.id);
if (pending.timer) clearTimeout(pending.timer);
if (msg.error?.message) pending.reject(new Error(msg.error.message));
else pending.resolve(msg.result);
}
}
} catch {}
});
this.ws.addEventListener('close', () => {
for (const [id, pending] of this.pending.entries()) {
this.pending.delete(id);
if (pending.timer) clearTimeout(pending.timer);
pending.reject(new Error('CDP connection closed.'));
}
});
}
static async connect(url: string, timeoutMs: number, options?: { defaultTimeoutMs?: number }): Promise<CdpConnection> {
const ws = new WebSocket(url);
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error('CDP connection timeout.')), timeoutMs);
ws.addEventListener('open', () => { clearTimeout(timer); resolve(); });
ws.addEventListener('error', () => { clearTimeout(timer); reject(new Error('CDP connection failed.')); });
});
return new CdpConnection(ws, options);
}
on(method: string, handler: (params: unknown) => void): void {
if (!this.eventHandlers.has(method)) this.eventHandlers.set(method, new Set());
this.eventHandlers.get(method)!.add(handler);
}
async send<T = unknown>(method: string, params?: Record<string, unknown>, options?: { sessionId?: string; timeoutMs?: number }): Promise<T> {
const id = ++this.nextId;
const message: Record<string, unknown> = { id, method };
if (params) message.params = params;
if (options?.sessionId) message.sessionId = options.sessionId;
const timeoutMs = options?.timeoutMs ?? this.defaultTimeoutMs;
const result = await new Promise<unknown>((resolve, reject) => {
const timer = timeoutMs > 0
? setTimeout(() => { this.pending.delete(id); reject(new Error(`CDP timeout: ${method}`)); }, timeoutMs)
: null;
this.pending.set(id, { resolve, reject, timer });
this.ws.send(JSON.stringify(message));
});
return result as T;
}
close(): void {
try { this.ws.close(); } catch {}
}
return { chrome, port };
}
export function getScriptDir(): string {
+31 -37
View File
@@ -1,4 +1,3 @@
import { spawn } from 'node:child_process';
import fs from 'node:fs';
import { mkdir } from 'node:fs/promises';
import path from 'node:path';
@@ -6,9 +5,11 @@ import process from 'node:process';
import {
CHROME_CANDIDATES_FULL,
CdpConnection,
findChromeExecutable,
findExistingChromeDebugPort,
getDefaultProfileDir,
getFreePort,
killChrome,
launchChrome,
openPageSession,
sleep,
waitForChromeDebugPort,
} from './x-utils.js';
@@ -27,9 +28,6 @@ interface XVideoOptions {
export async function postVideoToX(options: XVideoOptions): Promise<void> {
const { text, videoPath, submit = false, timeoutMs = 120_000, profileDir = getDefaultProfileDir() } = options;
const chromePath = options.chromePath ?? findChromeExecutable(CHROME_CANDIDATES_FULL);
if (!chromePath) throw new Error('Chrome not found. Set X_BROWSER_CHROME_PATH env var.');
if (!fs.existsSync(videoPath)) throw new Error(`Video not found: ${videoPath}`);
const absVideoPath = path.resolve(videoPath);
@@ -37,38 +35,37 @@ export async function postVideoToX(options: XVideoOptions): Promise<void> {
await mkdir(profileDir, { recursive: true });
const port = await getFreePort();
console.log(`[x-video] Launching Chrome (profile: ${profileDir})`);
const existingPort = await findExistingChromeDebugPort(profileDir);
const reusing = existingPort !== null;
let port = existingPort ?? 0;
let chrome: Awaited<ReturnType<typeof launchChrome>>['chrome'] | null = null;
if (!reusing) {
const launched = await launchChrome(X_COMPOSE_URL, profileDir, CHROME_CANDIDATES_FULL, options.chromePath);
port = launched.port;
chrome = launched.chrome;
}
const chrome = spawn(chromePath, [
`--remote-debugging-port=${port}`,
`--user-data-dir=${profileDir}`,
'--no-first-run',
'--no-default-browser-check',
'--disable-blink-features=AutomationControlled',
'--start-maximized',
X_COMPOSE_URL,
], { stdio: 'ignore' });
if (reusing) console.log(`[x-video] Reusing existing Chrome on port ${port}`);
else console.log(`[x-video] Launching Chrome (profile: ${profileDir})`);
let cdp: CdpConnection | null = null;
let targetId: string | null = null;
try {
const wsUrl = await waitForChromeDebugPort(port, 30_000, { includeLastError: true });
cdp = await CdpConnection.connect(wsUrl, 30_000, { defaultTimeoutMs: 30_000 });
const targets = await cdp.send<{ targetInfos: Array<{ targetId: string; url: string; type: string }> }>('Target.getTargets');
let pageTarget = targets.targetInfos.find((t) => t.type === 'page' && t.url.includes('x.com'));
if (!pageTarget) {
const { targetId } = await cdp.send<{ targetId: string }>('Target.createTarget', { url: X_COMPOSE_URL });
pageTarget = { targetId, url: X_COMPOSE_URL, type: 'page' };
}
const { sessionId } = await cdp.send<{ sessionId: string }>('Target.attachToTarget', { targetId: pageTarget.targetId, flatten: true });
await cdp.send('Page.enable', {}, { sessionId });
await cdp.send('Runtime.enable', {}, { sessionId });
await cdp.send('DOM.enable', {}, { sessionId });
const page = await openPageSession({
cdp,
reusing,
url: X_COMPOSE_URL,
matchTarget: (target) => target.type === 'page' && target.url.includes('x.com'),
enablePage: true,
enableRuntime: true,
enableDom: true,
});
const { sessionId } = page;
targetId = page.targetId;
await cdp.send('Input.setIgnoreInputEvents', { ignore: false }, { sessionId });
console.log('[x-video] Waiting for X editor...');
@@ -183,15 +180,12 @@ export async function postVideoToX(options: XVideoOptions): Promise<void> {
}
} finally {
if (cdp) {
if (reusing && submit && targetId) {
try { await cdp.send('Target.closeTarget', { targetId }, { timeoutMs: 5_000 }); } catch {}
}
cdp.close();
}
// Don't kill Chrome in preview mode, let user review
if (submit) {
setTimeout(() => {
if (!chrome.killed) try { chrome.kill('SIGKILL'); } catch {}
}, 2_000).unref?.();
try { chrome.kill('SIGTERM'); } catch {}
}
if (chrome && submit) killChrome(chrome);
}
}
+3 -3
View File
@@ -188,7 +188,7 @@ Before translating chunks:
- Splits at markdown block boundaries to preserve structure
- If a single block exceeds the threshold, falls back to line splitting, then word splitting
4. **Assemble translation prompt**:
- Main agent reads `01-analysis.md` (if exists) and assembles shared context using Part 1 of [references/subagent-prompt-template.md](references/subagent-prompt-template.md) — inlining content background, merged glossary, and comprehension challenges
- Main agent reads `01-analysis.md` (if exists) and assembles shared context using Part 1 of [references/subagent-prompt-template.md](references/subagent-prompt-template.md) — inlining the resolved style preset (from `--style` flag, EXTEND.md `style` setting, or default `storytelling`), content background, merged glossary, and comprehension challenges
- Save as `02-prompt.md` in the output directory (shared context only, no task instructions)
5. **Draft translation via subagents** (if Agent tool available):
- Spawn one subagent **per chunk**, all in parallel (Part 2 of the template)
@@ -215,7 +215,7 @@ Before translating chunks:
- **Image-language awareness**: Preserve image references exactly during translation, but after the translation is complete, review referenced images and check whether their likely main text language still matches the translated article language
- **Frontmatter transformation**: If the source has YAML frontmatter, preserve it in the translation with these changes: (1) Rename metadata fields that describe the *source* article — `url``sourceUrl`, `title``sourceTitle`, `description``sourceDescription`, `author``sourceAuthor`, `date``sourceDate`, and any similar origin-metadata fields — by adding a `source` prefix (camelCase). (2) Translate the values of text fields (title, description, etc.) and add them as new top-level fields. (3) Keep other fields (tags, categories, custom fields) as-is, translating their values where appropriate
- **Respect original**: Maintain original meaning and intent; do not add, remove, or editorialize — but sentence structure and imagery may be adapted freely to serve the meaning
- **Translator's notes**: For terms, concepts, or cultural references that target readers may not understand — due to jargon, cultural gaps, or domain-specific knowledge — add a concise explanatory note in parentheses immediately after the term. The note should explain *what it means* in plain language, not just provide the English original. Format: `译文(English original,通俗解释)`. Calibrate annotation depth to the target audience: general readers need more notes than technical readers. Only add notes where genuinely needed; do not over-annotate obvious terms.
- **Translator's notes**: For terms, concepts, or cultural references that target readers may not understand — due to jargon, cultural gaps, or domain-specific knowledge — add a concise explanatory note in parentheses immediately after the term. The note should explain *what it means* in plain language, not just provide the English original. Format: `译文(English original,通俗解释)`. Calibrate annotation depth to the target audience: general readers need more notes than technical readers. For short texts (< 5 sentences), further reduce annotations — only annotate non-common terms that the target audience is unlikely to know; skip terms that are widely recognized or self-explanatory in context. Only add notes where genuinely needed; do not over-annotate obvious terms.
#### Quick Mode
@@ -224,7 +224,7 @@ Translate directly → save to `translation.md`. No analysis file, but still app
#### Normal Mode
1. **Analyze**`01-analysis.md` (domain, tone, audience, terminology, reader comprehension challenges, figurative language & metaphor mapping)
2. **Assemble prompt**`02-prompt.md` (translation instructions with inlined context)
2. **Assemble prompt**`02-prompt.md` (translation instructions with inlined style preset, content background, glossary, and comprehension challenges)
3. **Translate** (following `02-prompt.md`) → `translation.md`
After completion, prompt user: "Translation saved. To further review and polish, reply **继续润色** or **refine**."
@@ -121,7 +121,7 @@ Implicit assumptions: [unstated premises]
## Step 2: Assemble Translation Prompt
Main agent reads `01-analysis.md` and assembles a complete translation prompt using [references/subagent-prompt-template.md](subagent-prompt-template.md). Inline content background, merged glossary, and comprehension challenges into the prompt. Save to `02-prompt.md`.
Main agent reads `01-analysis.md` and assembles a complete translation prompt using [references/subagent-prompt-template.md](subagent-prompt-template.md). Inline the resolved style preset (from `--style` flag, EXTEND.md `style` setting, or default `storytelling`), content background, merged glossary, and comprehension challenges into the prompt. Save to `02-prompt.md`.
This prompt is used by the subagent (chunked) or by the main agent itself (non-chunked).
+30 -6
View File
@@ -1,7 +1,7 @@
---
name: baoyu-url-to-markdown
description: Fetch any URL and convert to markdown using Chrome CDP. Saves the rendered HTML snapshot alongside the markdown, and automatically falls back to the pre-Defuddle HTML-to-Markdown pipeline when Defuddle fails. Supports two modes - auto-capture on page load, or wait for user signal (for pages requiring login). Use when user wants to save a webpage as markdown.
version: 1.56.1
description: Fetch any URL and convert to markdown using Chrome CDP. Saves the rendered HTML snapshot alongside the markdown, uses an upgraded Defuddle pipeline with better web-component handling and YouTube transcript extraction, and automatically falls back to the pre-Defuddle HTML-to-Markdown pipeline when needed. If local browser capture fails entirely, it can fall back to the hosted defuddle.md API. Supports two modes - auto-capture on page load, or wait for user signal (for pages requiring login). Use when user wants to save a webpage as markdown.
version: 1.58.1
metadata:
openclaw:
homepage: https://github.com/JimLiu/baoyu-skills#baoyu-url-to-markdown
@@ -29,7 +29,10 @@ Fetches any URL via Chrome CDP, saves the rendered HTML snapshot, and converts i
| Script | Purpose |
|--------|---------|
| `scripts/main.ts` | CLI entry point for URL fetching |
| `scripts/html-to-markdown.ts` | Defuddle-first conversion with automatic legacy fallback |
| `scripts/html-to-markdown.ts` | Markdown conversion entry point and converter selection |
| `scripts/defuddle-converter.ts` | Defuddle-based conversion |
| `scripts/legacy-converter.ts` | Pre-Defuddle legacy extraction and markdown conversion |
| `scripts/markdown-conversion-shared.ts` | Shared metadata parsing and markdown document helpers |
## Preferences (EXTEND.md)
@@ -115,7 +118,10 @@ Full reference: [references/config/first-time-setup.md](references/config/first-
- Two capture modes: auto or wait-for-user
- Save rendered HTML as a sibling `-captured.html` file
- Clean markdown output with metadata
- Defuddle-first markdown conversion with automatic fallback to the pre-Defuddle extractor from git history
- Upgraded Defuddle-first markdown conversion with automatic fallback to the pre-Defuddle extractor from git history
- Materializes shadow DOM content before conversion so web-component pages survive serialization better
- YouTube pages can include transcript/caption text in the markdown when YouTube exposes a caption track
- If local browser capture fails completely, can fall back to `defuddle.md/<url>` and still save markdown
- Handles login-required pages via wait mode
- Download images and videos to local directories
@@ -168,7 +174,10 @@ Each run saves two files side by side:
- Markdown: YAML front matter with `url`, `title`, `description`, `author`, `published`, optional `coverImage`, and `captured_at`, followed by converted markdown content
- HTML snapshot: `*-captured.html`, containing the rendered page HTML captured from Chrome
When Defuddle or page metadata provides a language hint, the markdown front matter also includes `language`.
The HTML snapshot is saved before any markdown media localization, so it stays a faithful capture of the page DOM used for conversion.
If the hosted `defuddle.md` API fallback is used, markdown is still saved, but there is no local `-captured.html` snapshot for that run.
## Output Directory
@@ -193,13 +202,16 @@ When `--download-media` is enabled:
Conversion order:
1. Try Defuddle first
2. If Defuddle throws, cannot load, returns obviously incomplete markdown, or captures lower-quality content than the legacy pipeline, automatically fall back to the pre-Defuddle extractor
3. The fallback path uses the older Readability/selector/Next.js-data based HTML-to-Markdown implementation recovered from git history
2. For rich pages such as YouTube, prefer Defuddle's extractor-specific output (including transcripts when available) instead of replacing it with the legacy pipeline
3. If Defuddle throws, cannot load, returns obviously incomplete markdown, or captures lower-quality content than the legacy pipeline, automatically fall back to the pre-Defuddle extractor
4. If the entire local browser capture flow fails before markdown can be produced, try the hosted `https://defuddle.md/<url>` API and save its markdown output directly
5. The legacy fallback path uses the older Readability/selector/Next.js-data based HTML-to-Markdown implementation recovered from git history
CLI output will show:
- `Converter: defuddle` when Defuddle succeeds
- `Converter: legacy:...` plus `Fallback used: ...` when fallback was needed
- `Converter: defuddle-api` when local browser capture failed and the hosted API was used instead
## Media Download Workflow
@@ -232,6 +244,18 @@ Based on `download_media` setting in EXTEND.md:
**Troubleshooting**: Chrome not found → set `URL_CHROME_PATH`. Timeout → increase `--timeout`. Complex pages → try `--wait` mode. If markdown quality is poor, inspect the saved `-captured.html` and check whether the run logged a legacy fallback.
### YouTube Notes
- The upgraded Defuddle path uses async extractors, so YouTube pages can include transcript text directly in the markdown body.
- Transcript availability depends on YouTube exposing a caption track. Videos with captions disabled, restricted playback, or blocked regional access may still produce description-only output.
- If the page needs time to finish loading descriptions, chapters, or player metadata, prefer `--wait` and capture after the watch page is fully hydrated.
### Hosted API Fallback
- The hosted fallback endpoint is `https://defuddle.md/<url>`. In shell form: `curl https://defuddle.md/stephango.com`
- Use it only when the local Chrome/CDP capture path fails outright. The local path still has higher fidelity because it can save the captured HTML and handle authenticated pages.
- The hosted API already returns Markdown with YAML frontmatter, so save that response as-is and then apply the normal media-localization step if requested.
## Extension Support
Custom configurations via EXTEND.md. See **Preferences** section for paths and supported options.
@@ -5,7 +5,8 @@
"name": "baoyu-url-to-markdown-scripts",
"dependencies": {
"@mozilla/readability": "^0.6.0",
"defuddle": "^0.10.0",
"baoyu-chrome-cdp": "file:./vendor/baoyu-chrome-cdp",
"defuddle": "^0.12.0",
"jsdom": "^24.1.3",
"linkedom": "^0.18.12",
"turndown": "^7.2.2",
@@ -36,6 +37,8 @@
"asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="],
"baoyu-chrome-cdp": ["baoyu-chrome-cdp@file:vendor/baoyu-chrome-cdp", {}],
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
"call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="],
@@ -58,7 +61,7 @@
"decimal.js": ["decimal.js@10.6.0", "", {}, "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg=="],
"defuddle": ["defuddle@0.10.0", "", { "dependencies": { "commander": "^12.1.0" }, "optionalDependencies": { "mathml-to-latex": "^1.5.0", "temml": "^0.13.1", "turndown": "^7.2.0" }, "peerDependencies": { "jsdom": "^24.0.0" }, "bin": { "defuddle": "dist/cli.js" } }, "sha512-a43juTtHv6Vs4+sxvahVLM5NxoyDsarO1Ag3UxLORI4Fo/nsNFwzDxuQBvosKVGTIRxCwN/mfnWAzNXmQfieqw=="],
"defuddle": ["defuddle@0.12.0", "", { "dependencies": { "commander": "^12.1.0" }, "optionalDependencies": { "mathml-to-latex": "^1.5.0", "temml": "^0.13.1", "turndown": "^7.2.0" }, "peerDependencies": { "jsdom": "^24.0.0" }, "bin": { "defuddle": "dist/cli.js" } }, "sha512-Y/WgyGKBxwxFir+hWNth4nmWDDDb8BzQi3qASS2NWYPXsKU42Ku49/3M5yFYefnRef9prynnmasfnXjk99EWgA=="],
"delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="],
+116 -227
View File
@@ -1,209 +1,84 @@
import { spawn, type ChildProcess } from "node:child_process";
import fs from "node:fs";
import { mkdir } from "node:fs/promises";
import net from "node:net";
import process from "node:process";
import {
CdpConnection,
findChromeExecutable as findChromeExecutableBase,
findExistingChromeDebugPort,
getFreePort,
killChrome,
launchChrome as launchChromeBase,
sleep,
waitForChromeDebugPort,
type PlatformCandidates,
} from 'baoyu-chrome-cdp';
import { resolveUrlToMarkdownChromeProfileDir } from "./paths.js";
import { CDP_CONNECT_TIMEOUT_MS, NETWORK_IDLE_TIMEOUT_MS } from "./constants.js";
import { resolveUrlToMarkdownChromeProfileDir } from './paths.js';
import { NETWORK_IDLE_TIMEOUT_MS } from './constants.js';
type CdpSendOptions = { sessionId?: string; timeoutMs?: number };
const CHROME_CANDIDATES_FULL: PlatformCandidates = {
darwin: [
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
'/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta',
'/Applications/Chromium.app/Contents/MacOS/Chromium',
'/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
],
win32: [
'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
'C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe',
'C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe',
],
default: [
'/usr/bin/google-chrome',
'/usr/bin/google-chrome-stable',
'/usr/bin/chromium',
'/usr/bin/chromium-browser',
'/snap/bin/chromium',
'/usr/bin/microsoft-edge',
],
};
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export { CdpConnection, getFreePort, killChrome, sleep, waitForChromeDebugPort };
async function fetchWithTimeout(url: string, init: RequestInit & { timeoutMs?: number } = {}): Promise<Response> {
const { timeoutMs, ...rest } = init;
if (!timeoutMs || timeoutMs <= 0) return fetch(url, rest);
const ctl = new AbortController();
const t = setTimeout(() => ctl.abort(), timeoutMs);
try {
return await fetch(url, { ...rest, signal: ctl.signal });
} finally {
clearTimeout(t);
}
}
export class CdpConnection {
private ws: WebSocket;
private nextId = 0;
private pending = new Map<number, { resolve: (v: unknown) => void; reject: (e: Error) => void; timer: ReturnType<typeof setTimeout> | null }>();
private eventHandlers = new Map<string, Set<(params: unknown) => void>>();
private constructor(ws: WebSocket) {
this.ws = ws;
this.ws.addEventListener("message", (event) => {
try {
const data = typeof event.data === "string" ? event.data : new TextDecoder().decode(event.data as ArrayBuffer);
const msg = JSON.parse(data) as { id?: number; method?: string; params?: unknown; result?: unknown; error?: { message?: string } };
if (msg.id) {
const p = this.pending.get(msg.id);
if (p) {
this.pending.delete(msg.id);
if (p.timer) clearTimeout(p.timer);
if (msg.error?.message) p.reject(new Error(msg.error.message));
else p.resolve(msg.result);
}
} else if (msg.method) {
const handlers = this.eventHandlers.get(msg.method);
if (handlers) {
for (const h of handlers) h(msg.params);
}
}
} catch {}
});
this.ws.addEventListener("close", () => {
for (const [id, p] of this.pending.entries()) {
this.pending.delete(id);
if (p.timer) clearTimeout(p.timer);
p.reject(new Error("CDP connection closed."));
}
});
}
static async connect(url: string, timeoutMs: number): Promise<CdpConnection> {
const ws = new WebSocket(url);
await new Promise<void>((resolve, reject) => {
const t = setTimeout(() => reject(new Error("CDP connection timeout.")), timeoutMs);
ws.addEventListener("open", () => { clearTimeout(t); resolve(); });
ws.addEventListener("error", () => { clearTimeout(t); reject(new Error("CDP connection failed.")); });
});
return new CdpConnection(ws);
}
on(event: string, handler: (params: unknown) => void): void {
let handlers = this.eventHandlers.get(event);
if (!handlers) {
handlers = new Set();
this.eventHandlers.set(event, handlers);
}
handlers.add(handler);
}
off(event: string, handler: (params: unknown) => void): void {
this.eventHandlers.get(event)?.delete(handler);
}
async send<T = unknown>(method: string, params?: Record<string, unknown>, opts?: CdpSendOptions): Promise<T> {
const id = ++this.nextId;
const msg: Record<string, unknown> = { id, method };
if (params) msg.params = params;
if (opts?.sessionId) msg.sessionId = opts.sessionId;
const timeoutMs = opts?.timeoutMs ?? 15_000;
const out = await new Promise<unknown>((resolve, reject) => {
const t = timeoutMs > 0 ? setTimeout(() => { this.pending.delete(id); reject(new Error(`CDP timeout: ${method}`)); }, timeoutMs) : null;
this.pending.set(id, { resolve, reject, timer: t });
this.ws.send(JSON.stringify(msg));
});
return out as T;
}
close(): void {
try { this.ws.close(); } catch {}
}
}
export async function getFreePort(): Promise<number> {
return await new Promise((resolve, reject) => {
const srv = net.createServer();
srv.unref();
srv.on("error", reject);
srv.listen(0, "127.0.0.1", () => {
const addr = srv.address();
if (!addr || typeof addr === "string") {
srv.close(() => reject(new Error("Unable to allocate a free TCP port.")));
return;
}
const port = addr.port;
srv.close((err) => (err ? reject(err) : resolve(port)));
});
export async function findExistingChromePort(): Promise<number | null> {
return await findExistingChromeDebugPort({
profileDir: resolveUrlToMarkdownChromeProfileDir(),
});
}
export function findChromeExecutable(): string | null {
const override = process.env.URL_CHROME_PATH?.trim();
if (override && fs.existsSync(override)) return override;
const candidates: string[] = [];
switch (process.platform) {
case "darwin":
candidates.push(
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
"/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta",
"/Applications/Chromium.app/Contents/MacOS/Chromium",
"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"
);
break;
case "win32":
candidates.push(
"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
"C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
"C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe",
"C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe"
);
break;
default:
candidates.push(
"/usr/bin/google-chrome",
"/usr/bin/google-chrome-stable",
"/usr/bin/chromium",
"/usr/bin/chromium-browser",
"/snap/bin/chromium",
"/usr/bin/microsoft-edge"
);
break;
}
for (const p of candidates) {
if (fs.existsSync(p)) return p;
}
return null;
return findChromeExecutableBase({
candidates: CHROME_CANDIDATES_FULL,
envNames: ['URL_CHROME_PATH'],
}) ?? null;
}
export async function waitForChromeDebugPort(port: number, timeoutMs: number): Promise<string> {
const start = Date.now();
while (Date.now() - start < timeoutMs) {
try {
const res = await fetchWithTimeout(`http://127.0.0.1:${port}/json/version`, { timeoutMs: 5_000 });
if (!res.ok) throw new Error(`status=${res.status}`);
const j = (await res.json()) as { webSocketDebuggerUrl?: string };
if (j.webSocketDebuggerUrl) return j.webSocketDebuggerUrl;
} catch {}
await sleep(200);
}
throw new Error("Chrome debug port not ready");
export async function launchChrome(url: string, port: number, headless = false) {
const chromePath = findChromeExecutable();
if (!chromePath) throw new Error('Chrome executable not found. Install Chrome or set URL_CHROME_PATH env.');
return await launchChromeBase({
chromePath,
profileDir: resolveUrlToMarkdownChromeProfileDir(),
port,
url,
headless,
extraArgs: ['--disable-popup-blocking'],
});
}
export async function launchChrome(url: string, port: number, headless: boolean = false): Promise<ChildProcess> {
const chrome = findChromeExecutable();
if (!chrome) throw new Error("Chrome executable not found. Install Chrome or set URL_CHROME_PATH env.");
const profileDir = resolveUrlToMarkdownChromeProfileDir();
await mkdir(profileDir, { recursive: true });
const args = [
`--remote-debugging-port=${port}`,
`--user-data-dir=${profileDir}`,
"--no-first-run",
"--no-default-browser-check",
"--disable-popup-blocking",
];
if (headless) args.push("--headless=new");
args.push(url);
return spawn(chrome, args, { stdio: "ignore" });
}
export async function waitForNetworkIdle(cdp: CdpConnection, sessionId: string, timeoutMs: number = NETWORK_IDLE_TIMEOUT_MS): Promise<void> {
export async function waitForNetworkIdle(
cdp: CdpConnection,
sessionId: string,
timeoutMs: number = NETWORK_IDLE_TIMEOUT_MS,
): Promise<void> {
return new Promise((resolve) => {
let timer: ReturnType<typeof setTimeout> | null = null;
let pending = 0;
const cleanup = () => {
if (timer) clearTimeout(timer);
cdp.off("Network.requestWillBeSent", onRequest);
cdp.off("Network.loadingFinished", onFinish);
cdp.off("Network.loadingFailed", onFinish);
cdp.off('Network.requestWillBeSent', onRequest);
cdp.off('Network.loadingFinished', onFinish);
cdp.off('Network.loadingFailed', onFinish);
};
const done = () => { cleanup(); resolve(); };
const resetTimer = () => {
@@ -212,79 +87,93 @@ export async function waitForNetworkIdle(cdp: CdpConnection, sessionId: string,
};
const onRequest = () => { pending++; resetTimer(); };
const onFinish = () => { pending = Math.max(0, pending - 1); if (pending <= 2) resetTimer(); };
cdp.on("Network.requestWillBeSent", onRequest);
cdp.on("Network.loadingFinished", onFinish);
cdp.on("Network.loadingFailed", onFinish);
cdp.on('Network.requestWillBeSent', onRequest);
cdp.on('Network.loadingFinished', onFinish);
cdp.on('Network.loadingFailed', onFinish);
resetTimer();
});
}
export async function waitForPageLoad(cdp: CdpConnection, sessionId: string, timeoutMs: number = 30_000): Promise<void> {
return new Promise((resolve, reject) => {
export async function waitForPageLoad(
cdp: CdpConnection,
sessionId: string,
timeoutMs: number = 30_000,
): Promise<void> {
void sessionId;
return new Promise((resolve) => {
const timer = setTimeout(() => {
cdp.off("Page.loadEventFired", handler);
cdp.off('Page.loadEventFired', handler);
resolve();
}, timeoutMs);
const handler = () => {
clearTimeout(timer);
cdp.off("Page.loadEventFired", handler);
cdp.off('Page.loadEventFired', handler);
resolve();
};
cdp.on("Page.loadEventFired", handler);
cdp.on('Page.loadEventFired', handler);
});
}
export async function createTargetAndAttach(cdp: CdpConnection, url: string): Promise<{ targetId: string; sessionId: string }> {
const { targetId } = await cdp.send<{ targetId: string }>("Target.createTarget", { url });
const { sessionId } = await cdp.send<{ sessionId: string }>("Target.attachToTarget", { targetId, flatten: true });
await cdp.send("Network.enable", {}, { sessionId });
await cdp.send("Page.enable", {}, { sessionId });
export async function createTargetAndAttach(
cdp: CdpConnection,
url: string,
): Promise<{ targetId: string; sessionId: string }> {
const { targetId } = await cdp.send<{ targetId: string }>('Target.createTarget', { url });
const { sessionId } = await cdp.send<{ sessionId: string }>('Target.attachToTarget', { targetId, flatten: true });
await cdp.send('Network.enable', {}, { sessionId });
await cdp.send('Page.enable', {}, { sessionId });
return { targetId, sessionId };
}
export async function navigateAndWait(cdp: CdpConnection, sessionId: string, url: string, timeoutMs: number): Promise<void> {
export async function navigateAndWait(
cdp: CdpConnection,
sessionId: string,
url: string,
timeoutMs: number,
): Promise<void> {
const loadPromise = new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error("Page load timeout")), timeoutMs);
const timer = setTimeout(() => reject(new Error('Page load timeout')), timeoutMs);
const handler = (params: unknown) => {
const p = params as { name?: string };
if (p.name === "load" || p.name === "DOMContentLoaded") {
const event = params as { name?: string };
if (event.name === 'load' || event.name === 'DOMContentLoaded') {
clearTimeout(timer);
cdp.off("Page.lifecycleEvent", handler);
cdp.off('Page.lifecycleEvent', handler);
resolve();
}
};
cdp.on("Page.lifecycleEvent", handler);
cdp.on('Page.lifecycleEvent', handler);
});
await cdp.send("Page.navigate", { url }, { sessionId });
await cdp.send('Page.navigate', { url }, { sessionId });
await loadPromise;
}
export async function evaluateScript<T>(cdp: CdpConnection, sessionId: string, expression: string, timeoutMs: number = 30_000): Promise<T> {
const result = await cdp.send<{ result: { value?: T; type?: string; description?: string } }>(
"Runtime.evaluate",
export async function evaluateScript<T>(
cdp: CdpConnection,
sessionId: string,
expression: string,
timeoutMs: number = 30_000,
): Promise<T> {
const result = await cdp.send<{ result: { value?: T } }>(
'Runtime.evaluate',
{ expression, returnByValue: true, awaitPromise: true },
{ sessionId, timeoutMs }
{ sessionId, timeoutMs },
);
return result.result.value as T;
}
export async function autoScroll(cdp: CdpConnection, sessionId: string, steps: number = 8, waitMs: number = 600): Promise<void> {
let lastHeight = await evaluateScript<number>(cdp, sessionId, "document.body.scrollHeight");
export async function autoScroll(
cdp: CdpConnection,
sessionId: string,
steps: number = 8,
waitMs: number = 600,
): Promise<void> {
let lastHeight = await evaluateScript<number>(cdp, sessionId, 'document.body.scrollHeight');
for (let i = 0; i < steps; i++) {
await evaluateScript<void>(cdp, sessionId, "window.scrollTo(0, document.body.scrollHeight)");
await evaluateScript<void>(cdp, sessionId, 'window.scrollTo(0, document.body.scrollHeight)');
await sleep(waitMs);
const newHeight = await evaluateScript<number>(cdp, sessionId, "document.body.scrollHeight");
const newHeight = await evaluateScript<number>(cdp, sessionId, 'document.body.scrollHeight');
if (newHeight === lastHeight) break;
lastHeight = newHeight;
}
await evaluateScript<void>(cdp, sessionId, "window.scrollTo(0, 0)");
}
export function killChrome(chrome: ChildProcess): void {
try { chrome.kill("SIGTERM"); } catch {}
setTimeout(() => {
if (!chrome.killed) {
try { chrome.kill("SIGKILL"); } catch {}
}
}, 2_000).unref?.();
await evaluateScript<void>(cdp, sessionId, 'window.scrollTo(0, 0)');
}
@@ -0,0 +1,58 @@
import { JSDOM, VirtualConsole } from "jsdom";
import { Defuddle } from "defuddle/node";
import {
type ConversionResult,
type PageMetadata,
isMarkdownUsable,
normalizeMarkdown,
pickString,
} from "./markdown-conversion-shared.js";
export async function tryDefuddleConversion(
html: string,
url: string,
baseMetadata: PageMetadata
): Promise<{ ok: true; result: ConversionResult } | { ok: false; reason: string }> {
try {
const virtualConsole = new VirtualConsole();
virtualConsole.on("jsdomError", (error: Error & { type?: string }) => {
if (error.type === "css parsing" || /Could not parse CSS stylesheet/i.test(error.message)) {
return;
}
console.warn(`[url-to-markdown] jsdom: ${error.message}`);
});
const dom = new JSDOM(html, { url, virtualConsole });
const result = await Defuddle(dom, url, { markdown: true });
const markdown = normalizeMarkdown(result.content || "");
if (!isMarkdownUsable(markdown, html)) {
return { ok: false, reason: "Defuddle returned empty or incomplete markdown" };
}
return {
ok: true,
result: {
metadata: {
...baseMetadata,
title: pickString(result.title, baseMetadata.title) ?? "",
description: pickString(result.description, baseMetadata.description) ?? undefined,
author: pickString(result.author, baseMetadata.author) ?? undefined,
published: pickString(result.published, baseMetadata.published) ?? undefined,
coverImage: pickString(result.image, baseMetadata.coverImage) ?? undefined,
language: pickString(result.language, baseMetadata.language) ?? undefined,
},
markdown,
rawHtml: html,
conversionMethod: "defuddle",
variables: result.variables,
},
};
} catch (error) {
return {
ok: false,
reason: error instanceof Error ? error.message : String(error),
};
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,629 @@
import { Readability } from "@mozilla/readability";
import TurndownService from "turndown";
import { gfm } from "turndown-plugin-gfm";
import {
type AnyRecord,
type ConversionResult,
type PageMetadata,
GOOD_CONTENT_LENGTH,
MIN_CONTENT_LENGTH,
extractPublishedTime,
extractTextFromHtml,
extractTitle,
normalizeMarkdown,
parseDocument,
pickString,
sanitizeHtml,
} from "./markdown-conversion-shared.js";
interface ExtractionCandidate {
title: string | null;
byline: string | null;
excerpt: string | null;
published: string | null;
html: string | null;
textContent: string;
method: string;
}
const CONTENT_SELECTORS = [
"article",
"main article",
"[role='main'] article",
"[itemprop='articleBody']",
".article-content",
".article-body",
".post-content",
".entry-content",
".story-body",
"main",
"[role='main']",
"#content",
".content",
];
const REMOVE_SELECTORS = [
"script",
"style",
"noscript",
"template",
"iframe",
"svg",
"path",
"nav",
"aside",
"footer",
"header",
"form",
".advertisement",
".ads",
".social-share",
".related-articles",
".comments",
".newsletter",
".cookie-banner",
".cookie-consent",
"[role='navigation']",
"[aria-label*='cookie' i]",
];
const NEXT_DATA_CONTENT_PATHS = [
"props.pageProps.content.body",
"props.pageProps.article.body",
"props.pageProps.article.content",
"props.pageProps.post.body",
"props.pageProps.post.content",
"props.pageProps.data.body",
"props.pageProps.story.body.content",
];
const LOW_QUALITY_MARKERS = [
/Join The Conversation/i,
/One Community\. Many Voices/i,
/Read our community guidelines/i,
/Create a free account to share your thoughts/i,
/Become a Forbes Member/i,
/Subscribe to trusted journalism/i,
/\bComments\b/i,
];
function generateExcerpt(excerpt: string | null, textContent: string | null): string | null {
if (excerpt) return excerpt;
if (!textContent) return null;
const trimmed = textContent.trim();
if (!trimmed) return null;
return trimmed.length > 200 ? `${trimmed.slice(0, 200)}...` : trimmed;
}
function parseJsonLdItem(item: AnyRecord): ExtractionCandidate | null {
const type = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"];
if (typeof type !== "string" || !["Article", "NewsArticle", "BlogPosting", "WebPage", "ReportageNewsArticle"].includes(type)) {
return null;
}
const rawContent =
(typeof item.articleBody === "string" && item.articleBody) ||
(typeof item.text === "string" && item.text) ||
(typeof item.description === "string" && item.description) ||
null;
if (!rawContent) return null;
const content = rawContent.trim();
const htmlLike = /<\/?[a-z][\s\S]*>/i.test(content);
const textContent = htmlLike ? extractTextFromHtml(content) : content;
if (textContent.length < MIN_CONTENT_LENGTH) return null;
return {
title: pickString(item.headline, item.name),
byline: extractAuthorFromJsonLd(item.author),
excerpt: pickString(item.description),
published: pickString(item.datePublished, item.dateCreated),
html: htmlLike ? content : null,
textContent,
method: "json-ld",
};
}
function extractAuthorFromJsonLd(authorData: unknown): string | null {
if (typeof authorData === "string") return authorData;
if (!authorData || typeof authorData !== "object") return null;
if (Array.isArray(authorData)) {
const names = authorData
.map((author) => extractAuthorFromJsonLd(author))
.filter((name): name is string => Boolean(name));
return names.length > 0 ? names.join(", ") : null;
}
const author = authorData as AnyRecord;
return typeof author.name === "string" ? author.name : null;
}
function flattenJsonLdItems(data: unknown): AnyRecord[] {
if (!data || typeof data !== "object") return [];
if (Array.isArray(data)) return data.flatMap(flattenJsonLdItems);
const item = data as AnyRecord;
if (Array.isArray(item["@graph"])) {
return (item["@graph"] as unknown[]).flatMap(flattenJsonLdItems);
}
return [item];
}
function tryJsonLdExtraction(document: Document): ExtractionCandidate | null {
const scripts = document.querySelectorAll("script[type='application/ld+json']");
for (const script of scripts) {
try {
const data = JSON.parse(script.textContent ?? "");
for (const item of flattenJsonLdItems(data)) {
const extracted = parseJsonLdItem(item);
if (extracted) return extracted;
}
} catch {
// Ignore malformed blocks.
}
}
return null;
}
function getByPath(value: unknown, path: string): unknown {
let current = value;
for (const part of path.split(".")) {
if (!current || typeof current !== "object") return undefined;
current = (current as AnyRecord)[part];
}
return current;
}
function isContentBlockArray(value: unknown): value is AnyRecord[] {
if (!Array.isArray(value) || value.length === 0) return false;
return value.slice(0, 5).some((item) => {
if (!item || typeof item !== "object") return false;
const obj = item as AnyRecord;
return "type" in obj || "text" in obj || "textHtml" in obj || "content" in obj;
});
}
function extractTextFromContentBlocks(blocks: AnyRecord[]): string {
const parts: string[] = [];
function pushParagraph(text: string): void {
const trimmed = text.trim();
if (!trimmed) return;
parts.push(trimmed, "\n\n");
}
function walk(node: unknown): void {
if (!node || typeof node !== "object") return;
const block = node as AnyRecord;
if (typeof block.text === "string") {
pushParagraph(block.text);
return;
}
if (typeof block.textHtml === "string") {
pushParagraph(extractTextFromHtml(block.textHtml));
return;
}
if (Array.isArray(block.items)) {
for (const item of block.items) {
if (item && typeof item === "object") {
const text = pickString((item as AnyRecord).text);
if (text) parts.push(`- ${text}\n`);
}
}
parts.push("\n");
}
if (Array.isArray(block.components)) {
for (const component of block.components) {
walk(component);
}
}
if (Array.isArray(block.content)) {
for (const child of block.content) {
walk(child);
}
}
}
for (const block of blocks) {
walk(block);
}
return parts.join("").replace(/\n{3,}/g, "\n\n").trim();
}
function tryStringBodyExtraction(
content: string,
meta: AnyRecord,
document: Document,
method: string
): ExtractionCandidate | null {
if (!content || content.length < MIN_CONTENT_LENGTH) return null;
const isHtml = /<\/?[a-z][\s\S]*>/i.test(content);
const html = isHtml ? sanitizeHtml(content) : null;
const textContent = isHtml ? extractTextFromHtml(html) : content.trim();
if (textContent.length < MIN_CONTENT_LENGTH) return null;
return {
title: pickString(meta.headline, meta.title, extractTitle(document)),
byline: pickString(meta.byline, meta.author),
excerpt: pickString(meta.description, meta.excerpt, generateExcerpt(null, textContent)),
published: pickString(meta.datePublished, meta.publishedAt, extractPublishedTime(document)),
html,
textContent,
method,
};
}
function tryNextDataExtraction(document: Document): ExtractionCandidate | null {
try {
const script = document.querySelector("script#__NEXT_DATA__");
if (!script?.textContent) return null;
const data = JSON.parse(script.textContent) as AnyRecord;
const pageProps = (getByPath(data, "props.pageProps") ?? {}) as AnyRecord;
for (const path of NEXT_DATA_CONTENT_PATHS) {
const value = getByPath(data, path);
if (typeof value === "string") {
const parentPath = path.split(".").slice(0, -1).join(".");
const parent = (getByPath(data, parentPath) ?? {}) as AnyRecord;
const meta = {
...pageProps,
...parent,
title: parent.title ?? (pageProps.title as string | undefined),
};
const candidate = tryStringBodyExtraction(value, meta, document, "next-data");
if (candidate) return candidate;
}
if (isContentBlockArray(value)) {
const textContent = extractTextFromContentBlocks(value);
if (textContent.length < MIN_CONTENT_LENGTH) continue;
return {
title: pickString(
getByPath(data, "props.pageProps.content.headline"),
getByPath(data, "props.pageProps.article.headline"),
getByPath(data, "props.pageProps.article.title"),
getByPath(data, "props.pageProps.post.title"),
pageProps.title,
extractTitle(document)
),
byline: pickString(
getByPath(data, "props.pageProps.author.name"),
getByPath(data, "props.pageProps.article.author.name")
),
excerpt: pickString(
getByPath(data, "props.pageProps.content.description"),
getByPath(data, "props.pageProps.article.description"),
pageProps.description,
generateExcerpt(null, textContent)
),
published: pickString(
getByPath(data, "props.pageProps.content.datePublished"),
getByPath(data, "props.pageProps.article.datePublished"),
getByPath(data, "props.pageProps.publishedAt"),
extractPublishedTime(document)
),
html: null,
textContent,
method: "next-data",
};
}
}
} catch {
return null;
}
return null;
}
function buildReadabilityCandidate(
article: ReturnType<Readability["parse"]>,
document: Document,
method: string
): ExtractionCandidate | null {
const textContent = article?.textContent?.trim() ?? "";
if (textContent.length < MIN_CONTENT_LENGTH) return null;
return {
title: pickString(article?.title, extractTitle(document)),
byline: pickString((article as { byline?: string } | null)?.byline),
excerpt: pickString(article?.excerpt, generateExcerpt(null, textContent)),
published: pickString((article as { publishedTime?: string } | null)?.publishedTime, extractPublishedTime(document)),
html: article?.content ? sanitizeHtml(article.content) : null,
textContent,
method,
};
}
function tryReadability(document: Document): ExtractionCandidate | null {
try {
const strictClone = document.cloneNode(true) as Document;
const strictResult = buildReadabilityCandidate(
new Readability(strictClone).parse(),
document,
"readability"
);
if (strictResult) return strictResult;
const relaxedClone = document.cloneNode(true) as Document;
return buildReadabilityCandidate(
new Readability(relaxedClone, { charThreshold: 120 }).parse(),
document,
"readability-relaxed"
);
} catch {
return null;
}
}
function trySelectorExtraction(document: Document): ExtractionCandidate | null {
for (const selector of CONTENT_SELECTORS) {
const element = document.querySelector(selector);
if (!element) continue;
const clone = element.cloneNode(true) as Element;
for (const removeSelector of REMOVE_SELECTORS) {
for (const node of clone.querySelectorAll(removeSelector)) {
node.remove();
}
}
const html = sanitizeHtml(clone.innerHTML);
const textContent = extractTextFromHtml(html);
if (textContent.length < MIN_CONTENT_LENGTH) continue;
return {
title: extractTitle(document),
byline: null,
excerpt: generateExcerpt(null, textContent),
published: extractPublishedTime(document),
html,
textContent,
method: `selector:${selector}`,
};
}
return null;
}
function tryBodyExtraction(document: Document): ExtractionCandidate | null {
const body = document.body;
if (!body) return null;
const clone = body.cloneNode(true) as Element;
for (const removeSelector of REMOVE_SELECTORS) {
for (const node of clone.querySelectorAll(removeSelector)) {
node.remove();
}
}
const html = sanitizeHtml(clone.innerHTML);
const textContent = extractTextFromHtml(html);
if (!textContent) return null;
return {
title: extractTitle(document),
byline: null,
excerpt: generateExcerpt(null, textContent),
published: extractPublishedTime(document),
html,
textContent,
method: "body-fallback",
};
}
function pickBestCandidate(candidates: ExtractionCandidate[]): ExtractionCandidate | null {
if (candidates.length === 0) return null;
const methodOrder = [
"readability",
"readability-relaxed",
"next-data",
"json-ld",
"selector:",
"body-fallback",
];
function methodRank(method: string): number {
const idx = methodOrder.findIndex((entry) =>
entry.endsWith(":") ? method.startsWith(entry) : method === entry
);
return idx === -1 ? methodOrder.length : idx;
}
const ranked = [...candidates].sort((a, b) => {
const rankA = methodRank(a.method);
const rankB = methodRank(b.method);
if (rankA !== rankB) return rankA - rankB;
return (b.textContent.length ?? 0) - (a.textContent.length ?? 0);
});
for (const candidate of ranked) {
if (candidate.textContent.length >= GOOD_CONTENT_LENGTH) {
return candidate;
}
}
for (const candidate of ranked) {
if (candidate.textContent.length >= MIN_CONTENT_LENGTH) {
return candidate;
}
}
return ranked[0];
}
function extractFromHtml(html: string): ExtractionCandidate | null {
const document = parseDocument(html);
const readabilityCandidate = tryReadability(document);
const nextDataCandidate = tryNextDataExtraction(document);
const jsonLdCandidate = tryJsonLdExtraction(document);
const selectorCandidate = trySelectorExtraction(document);
const bodyCandidate = tryBodyExtraction(document);
const candidates = [
readabilityCandidate,
nextDataCandidate,
jsonLdCandidate,
selectorCandidate,
bodyCandidate,
].filter((candidate): candidate is ExtractionCandidate => Boolean(candidate));
const winner = pickBestCandidate(candidates);
if (!winner) return null;
return {
...winner,
title: winner.title ?? extractTitle(document),
published: winner.published ?? extractPublishedTime(document),
excerpt: winner.excerpt ?? generateExcerpt(null, winner.textContent),
};
}
const turndown = new TurndownService({
headingStyle: "atx",
hr: "---",
bulletListMarker: "-",
codeBlockStyle: "fenced",
emDelimiter: "*",
strongDelimiter: "**",
linkStyle: "inlined",
});
turndown.use(gfm);
turndown.remove(["script", "style", "iframe", "noscript", "template", "svg", "path"]);
turndown.addRule("collapseFigure", {
filter: "figure",
replacement(content) {
return `\n\n${content.trim()}\n\n`;
},
});
turndown.addRule("dropInvisibleAnchors", {
filter(node) {
return node.nodeName === "A" && !(node as Element).textContent?.trim();
},
replacement() {
return "";
},
});
function convertHtmlToMarkdown(html: string): string {
if (!html || !html.trim()) return "";
try {
const sanitized = sanitizeHtml(html);
return turndown.turndown(sanitized);
} catch {
return "";
}
}
function fallbackPlainText(html: string): string {
const document = parseDocument(html);
for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) {
for (const el of document.querySelectorAll(selector)) {
el.remove();
}
}
const text = document.body?.textContent ?? document.documentElement?.textContent ?? "";
return normalizeMarkdown(text.replace(/\s+/g, " "));
}
function countBylines(markdown: string): number {
return (markdown.match(/(^|\n)By\s+/g) || []).length;
}
function countUsefulParagraphs(markdown: string): number {
const paragraphs = normalizeMarkdown(markdown).split(/\n{2,}/);
let count = 0;
for (const paragraph of paragraphs) {
const trimmed = paragraph.trim();
if (!trimmed) continue;
if (/^!?\[[^\]]*\]\([^)]+\)$/.test(trimmed)) continue;
if (/^#{1,6}\s+/.test(trimmed)) continue;
if ((trimmed.match(/\b[\p{L}\p{N}']+\b/gu) || []).length < 8) continue;
count++;
}
return count;
}
function countMarkerHits(markdown: string, markers: RegExp[]): number {
let hits = 0;
for (const marker of markers) {
if (marker.test(markdown)) hits++;
}
return hits;
}
export function scoreMarkdownQuality(markdown: string): number {
const normalized = normalizeMarkdown(markdown);
const wordCount = (normalized.match(/\b[\p{L}\p{N}']+\b/gu) || []).length;
const usefulParagraphs = countUsefulParagraphs(normalized);
const headingCount = (normalized.match(/^#{1,6}\s+/gm) || []).length;
const markerHits = countMarkerHits(normalized, LOW_QUALITY_MARKERS);
const bylineCount = countBylines(normalized);
const staffCount = (normalized.match(/\bForbes Staff\b/gi) || []).length;
return (
Math.min(wordCount, 4000) +
usefulParagraphs * 40 +
headingCount * 10 -
markerHits * 180 -
Math.max(0, bylineCount - 1) * 120 -
Math.max(0, staffCount - 1) * 80
);
}
export function shouldCompareWithLegacy(markdown: string): boolean {
const normalized = normalizeMarkdown(markdown);
return (
countMarkerHits(normalized, LOW_QUALITY_MARKERS) > 0 ||
countBylines(normalized) > 1 ||
countUsefulParagraphs(normalized) < 6
);
}
export function convertWithLegacyExtractor(html: string, baseMetadata: PageMetadata): ConversionResult {
const extracted = extractFromHtml(html);
let markdown = extracted?.html ? convertHtmlToMarkdown(extracted.html) : "";
if (!markdown.trim()) {
markdown = extracted?.textContent?.trim() || fallbackPlainText(html);
}
return {
metadata: {
...baseMetadata,
title: pickString(extracted?.title, baseMetadata.title) ?? "",
description: pickString(extracted?.excerpt, baseMetadata.description) ?? undefined,
author: pickString(extracted?.byline, baseMetadata.author) ?? undefined,
published: pickString(extracted?.published, baseMetadata.published) ?? undefined,
},
markdown: normalizeMarkdown(markdown),
rawHtml: html,
conversionMethod: extracted ? `legacy:${extracted.method}` : "legacy:plain-text",
};
}
+129 -26
View File
@@ -3,7 +3,7 @@ import { writeFile, mkdir, access } from "node:fs/promises";
import path from "node:path";
import process from "node:process";
import { CdpConnection, getFreePort, launchChrome, waitForChromeDebugPort, waitForNetworkIdle, waitForPageLoad, autoScroll, evaluateScript, killChrome } from "./cdp.js";
import { CdpConnection, getFreePort, findExistingChromePort, launchChrome, waitForChromeDebugPort, waitForNetworkIdle, waitForPageLoad, autoScroll, evaluateScript, killChrome } from "./cdp.js";
import { absolutizeUrlsScript, extractContent, createMarkdownDocument, type ConversionResult } from "./html-to-markdown.js";
import { localizeMarkdownMedia, countRemoteMedia } from "./media-localizer.js";
import { resolveUrlToMarkdownDataDir } from "./paths.js";
@@ -75,6 +75,55 @@ function deriveHtmlSnapshotPath(markdownPath: string): string {
return path.join(parsed.dir, `${basename}-captured.html`);
}
function extractTitleFromMarkdownDocument(document: string): string {
const normalized = document.replace(/\r\n/g, "\n");
const frontmatterMatch = normalized.match(/^---\n([\s\S]*?)\n---\n?/);
if (frontmatterMatch) {
const titleLine = frontmatterMatch[1]
.split("\n")
.find((line) => /^title:\s*/i.test(line));
if (titleLine) {
const rawValue = titleLine.replace(/^title:\s*/i, "").trim();
const unquoted = rawValue
.replace(/^"(.*)"$/, "$1")
.replace(/^'(.*)'$/, "$1")
.replace(/\\"/g, '"');
if (unquoted) return unquoted;
}
}
const headingMatch = normalized.match(/^#\s+(.+)$/m);
return headingMatch?.[1]?.trim() ?? "";
}
function buildDefuddleApiUrl(targetUrl: string): string {
return `https://defuddle.md/${encodeURIComponent(targetUrl)}`;
}
async function fetchDefuddleApiMarkdown(targetUrl: string): Promise<{ markdown: string; title: string }> {
const apiUrl = buildDefuddleApiUrl(targetUrl);
const response = await fetch(apiUrl, {
headers: {
accept: "text/markdown,text/plain;q=0.9,*/*;q=0.1",
},
});
if (!response.ok) {
throw new Error(`defuddle.md returned ${response.status} ${response.statusText}`);
}
const markdown = (await response.text()).replace(/\r\n/g, "\n").trim();
if (!markdown) {
throw new Error("defuddle.md returned empty markdown");
}
return {
markdown,
title: extractTitleFromMarkdownDocument(markdown),
};
}
async function generateOutputPath(url: string, title: string, outputDir?: string): Promise<string> {
const domain = new URL(url).hostname.replace(/^www\./, "");
const slug = generateSlug(title, url);
@@ -98,21 +147,37 @@ async function waitForUserSignal(): Promise<void> {
}
async function captureUrl(args: Args): Promise<ConversionResult> {
const port = await getFreePort();
const chrome = await launchChrome(args.url, port, false);
const existingPort = await findExistingChromePort();
const reusing = existingPort !== null;
const port = existingPort ?? await getFreePort();
const chrome = reusing ? null : await launchChrome(args.url, port, false);
if (reusing) console.log(`Reusing existing Chrome on port ${port}`);
let cdp: CdpConnection | null = null;
let targetId: string | null = null;
try {
const wsUrl = await waitForChromeDebugPort(port, 30_000);
cdp = await CdpConnection.connect(wsUrl, CDP_CONNECT_TIMEOUT_MS);
const targets = await cdp.send<{ targetInfos: Array<{ targetId: string; type: string; url: string }> }>("Target.getTargets");
const pageTarget = targets.targetInfos.find(t => t.type === "page" && t.url.startsWith("http"));
if (!pageTarget) throw new Error("No page target found");
const { sessionId } = await cdp.send<{ sessionId: string }>("Target.attachToTarget", { targetId: pageTarget.targetId, flatten: true });
await cdp.send("Network.enable", {}, { sessionId });
await cdp.send("Page.enable", {}, { sessionId });
let sessionId: string;
if (reusing) {
const created = await cdp.send<{ targetId: string }>("Target.createTarget", { url: args.url });
targetId = created.targetId;
const attached = await cdp.send<{ sessionId: string }>("Target.attachToTarget", { targetId, flatten: true });
sessionId = attached.sessionId;
await cdp.send("Network.enable", {}, { sessionId });
await cdp.send("Page.enable", {}, { sessionId });
} else {
const targets = await cdp.send<{ targetInfos: Array<{ targetId: string; type: string; url: string }> }>("Target.getTargets");
const pageTarget = targets.targetInfos.find(t => t.type === "page" && t.url.startsWith("http"));
if (!pageTarget) throw new Error("No page target found");
targetId = pageTarget.targetId;
const attached = await cdp.send<{ sessionId: string }>("Target.attachToTarget", { targetId, flatten: true });
sessionId = attached.sessionId;
await cdp.send("Network.enable", {}, { sessionId });
await cdp.send("Page.enable", {}, { sessionId });
}
if (args.wait) {
await waitForUserSignal();
@@ -136,11 +201,18 @@ async function captureUrl(args: Args): Promise<ConversionResult> {
return await extractContent(html, args.url);
} finally {
if (cdp) {
try { await cdp.send("Browser.close", {}, { timeoutMs: 5_000 }); } catch {}
cdp.close();
if (reusing) {
if (cdp && targetId) {
try { await cdp.send("Target.closeTarget", { targetId }, { timeoutMs: 5_000 }); } catch {}
}
if (cdp) cdp.close();
} else {
if (cdp) {
try { await cdp.send("Browser.close", {}, { timeoutMs: 5_000 }); } catch {}
cdp.close();
}
if (chrome) killChrome(chrome);
}
killChrome(chrome);
}
}
@@ -169,14 +241,41 @@ async function main(): Promise<void> {
console.log(`Fetching: ${args.url}`);
console.log(`Mode: ${args.wait ? "wait" : "auto"}`);
const result = await captureUrl(args);
const outputPath = args.output || await generateOutputPath(args.url, result.metadata.title, args.outputDir);
const outputDir = path.dirname(outputPath);
const htmlSnapshotPath = deriveHtmlSnapshotPath(outputPath);
await mkdir(outputDir, { recursive: true });
await writeFile(htmlSnapshotPath, result.rawHtml, "utf-8");
let outputPath: string;
let htmlSnapshotPath: string | null = null;
let document: string;
let conversionMethod: string;
let fallbackReason: string | undefined;
let document = createMarkdownDocument(result);
try {
const result = await captureUrl(args);
outputPath = args.output || await generateOutputPath(args.url, result.metadata.title, args.outputDir);
const outputDir = path.dirname(outputPath);
htmlSnapshotPath = deriveHtmlSnapshotPath(outputPath);
await mkdir(outputDir, { recursive: true });
await writeFile(htmlSnapshotPath, result.rawHtml, "utf-8");
document = createMarkdownDocument(result);
conversionMethod = result.conversionMethod;
fallbackReason = result.fallbackReason;
} catch (error) {
const primaryError = error instanceof Error ? error.message : String(error);
console.warn(`Primary capture failed: ${primaryError}`);
console.warn("Trying defuddle.md API fallback...");
try {
const remoteResult = await fetchDefuddleApiMarkdown(args.url);
outputPath = args.output || await generateOutputPath(args.url, remoteResult.title, args.outputDir);
await mkdir(path.dirname(outputPath), { recursive: true });
document = remoteResult.markdown;
conversionMethod = "defuddle-api";
fallbackReason = `Local browser capture failed: ${primaryError}`;
} catch (remoteError) {
const remoteMessage = remoteError instanceof Error ? remoteError.message : String(remoteError);
throw new Error(`Local browser capture failed (${primaryError}); defuddle.md fallback failed (${remoteMessage})`);
}
}
if (args.downloadMedia) {
const mediaResult = await localizeMarkdownMedia(document, {
@@ -197,11 +296,15 @@ async function main(): Promise<void> {
await writeFile(outputPath, document, "utf-8");
console.log(`Saved: ${outputPath}`);
console.log(`Saved HTML: ${htmlSnapshotPath}`);
console.log(`Title: ${result.metadata.title || "(no title)"}`);
console.log(`Converter: ${result.conversionMethod}`);
if (result.fallbackReason) {
console.warn(`Fallback used: ${result.fallbackReason}`);
if (htmlSnapshotPath) {
console.log(`Saved HTML: ${htmlSnapshotPath}`);
} else {
console.log("Saved HTML: unavailable (defuddle.md fallback)");
}
console.log(`Title: ${extractTitleFromMarkdownDocument(document) || "(no title)"}`);
console.log(`Converter: ${conversionMethod}`);
if (fallbackReason) {
console.warn(`Fallback used: ${fallbackReason}`);
}
}
@@ -0,0 +1,305 @@
import { parseHTML } from "linkedom";
export interface PageMetadata {
url: string;
title: string;
description?: string;
author?: string;
published?: string;
coverImage?: string;
language?: string;
captured_at: string;
}
export interface ConversionResult {
metadata: PageMetadata;
markdown: string;
rawHtml: string;
conversionMethod: string;
fallbackReason?: string;
variables?: Record<string, string>;
}
export type AnyRecord = Record<string, unknown>;
export const MIN_CONTENT_LENGTH = 120;
export const GOOD_CONTENT_LENGTH = 900;
const PUBLISHED_TIME_SELECTORS = [
"meta[property='article:published_time']",
"meta[name='pubdate']",
"meta[name='publishdate']",
"meta[name='date']",
"time[datetime]",
];
const ARTICLE_TYPES = new Set([
"Article",
"NewsArticle",
"BlogPosting",
"WebPage",
"ReportageNewsArticle",
]);
export function pickString(...values: unknown[]): string | null {
for (const value of values) {
if (typeof value === "string") {
const trimmed = value.trim();
if (trimmed) return trimmed;
}
}
return null;
}
export function normalizeMarkdown(markdown: string): string {
return markdown
.replace(/\r\n/g, "\n")
.replace(/[ \t]+\n/g, "\n")
.replace(/\n{3,}/g, "\n\n")
.trim();
}
export function parseDocument(html: string): Document {
const normalized = /<\s*html[\s>]/i.test(html)
? html
: `<!doctype html><html><body>${html}</body></html>`;
return parseHTML(normalized).document as unknown as Document;
}
export function sanitizeHtml(html: string): string {
const { document } = parseHTML(`<div id="__root">${html}</div>`);
const root = document.querySelector("#__root");
if (!root) return html;
for (const selector of ["script", "style", "iframe", "noscript", "template", "svg", "path"]) {
for (const el of root.querySelectorAll(selector)) {
el.remove();
}
}
return root.innerHTML;
}
export function extractTextFromHtml(html: string): string {
const { document } = parseHTML(`<!doctype html><html><body>${html}</body></html>`);
for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) {
for (const el of document.querySelectorAll(selector)) {
el.remove();
}
}
return document.body?.textContent?.replace(/\s+/g, " ").trim() ?? "";
}
export function getMetaContent(document: Document, names: string[]): string | null {
for (const name of names) {
const element =
document.querySelector(`meta[name="${name}"]`) ??
document.querySelector(`meta[property="${name}"]`);
const content = element?.getAttribute("content");
if (content && content.trim()) return content.trim();
}
return null;
}
function normalizeLanguageTag(value: string | null): string | null {
if (!value) return null;
const trimmed = value.trim();
if (!trimmed) return null;
const primary = trimmed.split(/[,\s;]/, 1)[0]?.trim();
if (!primary) return null;
return primary.replace(/_/g, "-");
}
function flattenJsonLdItems(data: unknown): AnyRecord[] {
if (!data || typeof data !== "object") return [];
if (Array.isArray(data)) return data.flatMap(flattenJsonLdItems);
const item = data as AnyRecord;
if (Array.isArray(item["@graph"])) {
return (item["@graph"] as unknown[]).flatMap(flattenJsonLdItems);
}
return [item];
}
function parseJsonLdScripts(document: Document): AnyRecord[] {
const results: AnyRecord[] = [];
const scripts = document.querySelectorAll("script[type='application/ld+json']");
for (const script of scripts) {
try {
const data = JSON.parse(script.textContent ?? "");
results.push(...flattenJsonLdItems(data));
} catch {
// Ignore malformed blocks.
}
}
return results;
}
function isArticleType(item: AnyRecord): boolean {
const value = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"];
return typeof value === "string" && ARTICLE_TYPES.has(value);
}
function extractAuthorFromJsonLd(authorData: unknown): string | null {
if (typeof authorData === "string") return authorData;
if (!authorData || typeof authorData !== "object") return null;
if (Array.isArray(authorData)) {
const names = authorData
.map((author) => extractAuthorFromJsonLd(author))
.filter((name): name is string => Boolean(name));
return names.length > 0 ? names.join(", ") : null;
}
const author = authorData as AnyRecord;
return typeof author.name === "string" ? author.name : null;
}
function extractPrimaryJsonLdMeta(document: Document): Partial<PageMetadata> {
for (const item of parseJsonLdScripts(document)) {
if (!isArticleType(item)) continue;
return {
title: pickString(item.headline, item.name) ?? undefined,
description: pickString(item.description) ?? undefined,
author: extractAuthorFromJsonLd(item.author) ?? undefined,
published: pickString(item.datePublished, item.dateCreated) ?? undefined,
coverImage:
pickString(
item.image,
(item.image as AnyRecord | undefined)?.url,
(Array.isArray(item.image) ? item.image[0] : undefined) as unknown
) ?? undefined,
};
}
return {};
}
export function extractPublishedTime(document: Document): string | null {
for (const selector of PUBLISHED_TIME_SELECTORS) {
const el = document.querySelector(selector);
if (!el) continue;
const value = el.getAttribute("content") ?? el.getAttribute("datetime");
if (value && value.trim()) return value.trim();
}
return null;
}
export function extractTitle(document: Document): string | null {
const ogTitle = document.querySelector("meta[property='og:title']")?.getAttribute("content");
if (ogTitle && ogTitle.trim()) return ogTitle.trim();
const twitterTitle = document.querySelector("meta[name='twitter:title']")?.getAttribute("content");
if (twitterTitle && twitterTitle.trim()) return twitterTitle.trim();
const title = document.querySelector("title")?.textContent?.trim();
if (title) {
const cleaned = title.split(/\s*[-|–—]\s*/)[0]?.trim();
if (cleaned) return cleaned;
}
const h1 = document.querySelector("h1")?.textContent?.trim();
return h1 || null;
}
export function extractMetadataFromHtml(html: string, url: string, capturedAt: string): PageMetadata {
const document = parseDocument(html);
const jsonLd = extractPrimaryJsonLdMeta(document);
const timeEl = document.querySelector("time[datetime]");
const htmlLang = normalizeLanguageTag(document.documentElement?.getAttribute("lang"));
const metaLanguage = normalizeLanguageTag(
pickString(
getMetaContent(document, ["language", "content-language", "og:locale"]),
document.querySelector("meta[http-equiv='content-language']")?.getAttribute("content")
)
);
return {
url,
title:
pickString(
getMetaContent(document, ["og:title", "twitter:title"]),
jsonLd.title,
document.querySelector("h1")?.textContent,
document.title
) ?? "",
description:
pickString(
getMetaContent(document, ["description", "og:description", "twitter:description"]),
jsonLd.description
) ?? undefined,
author:
pickString(
getMetaContent(document, ["author", "article:author", "twitter:creator"]),
jsonLd.author
) ?? undefined,
published:
pickString(
timeEl?.getAttribute("datetime"),
getMetaContent(document, ["article:published_time", "datePublished", "publishdate", "date"]),
jsonLd.published,
extractPublishedTime(document)
) ?? undefined,
coverImage:
pickString(
getMetaContent(document, ["og:image", "twitter:image", "twitter:image:src"]),
jsonLd.coverImage
) ?? undefined,
language: pickString(htmlLang, metaLanguage) ?? undefined,
captured_at: capturedAt,
};
}
export function isMarkdownUsable(markdown: string, html: string): boolean {
const normalized = normalizeMarkdown(markdown);
if (!normalized) return false;
const htmlTextLength = extractTextFromHtml(html).length;
if (htmlTextLength < MIN_CONTENT_LENGTH) return true;
if (normalized.length >= 80) return true;
return normalized.length >= Math.min(200, Math.floor(htmlTextLength * 0.2));
}
export function isYouTubeUrl(url: string): boolean {
try {
const hostname = new URL(url).hostname.toLowerCase();
return hostname === "youtu.be" || hostname.endsWith(".youtube.com") || hostname === "youtube.com";
} catch {
return false;
}
}
function escapeYamlValue(value: string): string {
return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\r?\n/g, "\\n");
}
export function formatMetadataYaml(meta: PageMetadata): string {
const lines = ["---"];
lines.push(`url: ${meta.url}`);
lines.push(`title: "${escapeYamlValue(meta.title)}"`);
if (meta.description) lines.push(`description: "${escapeYamlValue(meta.description)}"`);
if (meta.author) lines.push(`author: "${escapeYamlValue(meta.author)}"`);
if (meta.published) lines.push(`published: "${escapeYamlValue(meta.published)}"`);
if (meta.coverImage) lines.push(`coverImage: "${escapeYamlValue(meta.coverImage)}"`);
if (meta.language) lines.push(`language: "${escapeYamlValue(meta.language)}"`);
lines.push(`captured_at: "${escapeYamlValue(meta.captured_at)}"`);
lines.push("---");
return lines.join("\n");
}
export function createMarkdownDocument(result: ConversionResult): string {
const yaml = formatMetadataYaml(result.metadata);
const escapedTitle = result.metadata.title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const titleRegex = new RegExp(`^#\\s+${escapedTitle}\\s*(\\n|$)`, "i");
const hasTitle = titleRegex.test(result.markdown.trimStart());
const title = result.metadata.title && !hasTitle ? `\n\n# ${result.metadata.title}\n\n` : "\n\n";
return yaml + title + result.markdown;
}
@@ -4,7 +4,8 @@
"type": "module",
"dependencies": {
"@mozilla/readability": "^0.6.0",
"defuddle": "^0.10.0",
"baoyu-chrome-cdp": "file:./vendor/baoyu-chrome-cdp",
"defuddle": "^0.12.0",
"jsdom": "^24.1.3",
"linkedom": "^0.18.12",
"turndown": "^7.2.2",
@@ -0,0 +1,9 @@
{
"name": "baoyu-chrome-cdp",
"private": true,
"version": "0.1.0",
"type": "module",
"exports": {
".": "./src/index.ts"
}
}
@@ -0,0 +1,408 @@
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import process from "node:process";
export type PlatformCandidates = {
darwin?: string[];
win32?: string[];
default: string[];
};
type PendingRequest = {
resolve: (value: unknown) => void;
reject: (error: Error) => void;
timer: ReturnType<typeof setTimeout> | null;
};
type CdpSendOptions = {
sessionId?: string;
timeoutMs?: number;
};
type FetchJsonOptions = {
timeoutMs?: number;
};
type FindChromeExecutableOptions = {
candidates: PlatformCandidates;
envNames?: string[];
};
type ResolveSharedChromeProfileDirOptions = {
envNames?: string[];
appDataDirName?: string;
profileDirName?: string;
wslWindowsHome?: string | null;
};
type FindExistingChromeDebugPortOptions = {
profileDir: string;
timeoutMs?: number;
};
type LaunchChromeOptions = {
chromePath: string;
profileDir: string;
port: number;
url?: string;
headless?: boolean;
extraArgs?: string[];
};
type ChromeTargetInfo = {
targetId: string;
url: string;
type: string;
};
type OpenPageSessionOptions = {
cdp: CdpConnection;
reusing: boolean;
url: string;
matchTarget: (target: ChromeTargetInfo) => boolean;
enablePage?: boolean;
enableRuntime?: boolean;
enableDom?: boolean;
enableNetwork?: boolean;
activateTarget?: boolean;
};
export type PageSession = {
sessionId: string;
targetId: string;
};
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export async function getFreePort(fixedEnvName?: string): Promise<number> {
const fixed = fixedEnvName ? Number.parseInt(process.env[fixedEnvName] ?? "", 10) : NaN;
if (Number.isInteger(fixed) && fixed > 0) return fixed;
return await new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
server.close(() => reject(new Error("Unable to allocate a free TCP port.")));
return;
}
const port = address.port;
server.close((err) => {
if (err) reject(err);
else resolve(port);
});
});
});
}
export function findChromeExecutable(options: FindChromeExecutableOptions): string | undefined {
for (const envName of options.envNames ?? []) {
const override = process.env[envName]?.trim();
if (override && fs.existsSync(override)) return override;
}
const candidates = process.platform === "darwin"
? options.candidates.darwin ?? options.candidates.default
: process.platform === "win32"
? options.candidates.win32 ?? options.candidates.default
: options.candidates.default;
for (const candidate of candidates) {
if (fs.existsSync(candidate)) return candidate;
}
return undefined;
}
export function resolveSharedChromeProfileDir(options: ResolveSharedChromeProfileDirOptions = {}): string {
for (const envName of options.envNames ?? []) {
const override = process.env[envName]?.trim();
if (override) return path.resolve(override);
}
const appDataDirName = options.appDataDirName ?? "baoyu-skills";
const profileDirName = options.profileDirName ?? "chrome-profile";
if (options.wslWindowsHome) {
return path.join(options.wslWindowsHome, ".local", "share", appDataDirName, profileDirName);
}
const base = process.platform === "darwin"
? path.join(os.homedir(), "Library", "Application Support")
: process.platform === "win32"
? (process.env.APPDATA ?? path.join(os.homedir(), "AppData", "Roaming"))
: (process.env.XDG_DATA_HOME ?? path.join(os.homedir(), ".local", "share"));
return path.join(base, appDataDirName, profileDirName);
}
async function fetchWithTimeout(url: string, timeoutMs?: number): Promise<Response> {
if (!timeoutMs || timeoutMs <= 0) return await fetch(url, { redirect: "follow" });
const ctl = new AbortController();
const timer = setTimeout(() => ctl.abort(), timeoutMs);
try {
return await fetch(url, { redirect: "follow", signal: ctl.signal });
} finally {
clearTimeout(timer);
}
}
async function fetchJson<T = unknown>(url: string, options: FetchJsonOptions = {}): Promise<T> {
const response = await fetchWithTimeout(url, options.timeoutMs);
if (!response.ok) {
throw new Error(`Request failed: ${response.status} ${response.statusText}`);
}
return await response.json() as T;
}
async function isDebugPortReady(port: number, timeoutMs = 3_000): Promise<boolean> {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
`http://127.0.0.1:${port}/json/version`,
{ timeoutMs }
);
return !!version.webSocketDebuggerUrl;
} catch {
return false;
}
}
export async function findExistingChromeDebugPort(options: FindExistingChromeDebugPortOptions): Promise<number | null> {
const timeoutMs = options.timeoutMs ?? 3_000;
const portFile = path.join(options.profileDir, "DevToolsActivePort");
try {
const content = fs.readFileSync(portFile, "utf-8");
const [portLine] = content.split(/\r?\n/);
const port = Number.parseInt(portLine?.trim() ?? "", 10);
if (port > 0 && await isDebugPortReady(port, timeoutMs)) return port;
} catch {}
if (process.platform === "win32") return null;
try {
const result = spawnSync("ps", ["aux"], { encoding: "utf-8", timeout: 5_000 });
if (result.status !== 0 || !result.stdout) return null;
const lines = result.stdout
.split("\n")
.filter((line) => line.includes(options.profileDir) && line.includes("--remote-debugging-port="));
for (const line of lines) {
const portMatch = line.match(/--remote-debugging-port=(\d+)/);
const port = Number.parseInt(portMatch?.[1] ?? "", 10);
if (port > 0 && await isDebugPortReady(port, timeoutMs)) return port;
}
} catch {}
return null;
}
export async function waitForChromeDebugPort(
port: number,
timeoutMs: number,
options?: { includeLastError?: boolean }
): Promise<string> {
const start = Date.now();
let lastError: unknown = null;
while (Date.now() - start < timeoutMs) {
try {
const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
`http://127.0.0.1:${port}/json/version`,
{ timeoutMs: 5_000 }
);
if (version.webSocketDebuggerUrl) return version.webSocketDebuggerUrl;
lastError = new Error("Missing webSocketDebuggerUrl");
} catch (error) {
lastError = error;
}
await sleep(200);
}
if (options?.includeLastError && lastError) {
throw new Error(
`Chrome debug port not ready: ${lastError instanceof Error ? lastError.message : String(lastError)}`
);
}
throw new Error("Chrome debug port not ready");
}
export class CdpConnection {
private ws: WebSocket;
private nextId = 0;
private pending = new Map<number, PendingRequest>();
private eventHandlers = new Map<string, Set<(params: unknown) => void>>();
private defaultTimeoutMs: number;
private constructor(ws: WebSocket, defaultTimeoutMs = 15_000) {
this.ws = ws;
this.defaultTimeoutMs = defaultTimeoutMs;
this.ws.addEventListener("message", (event) => {
try {
const data = typeof event.data === "string"
? event.data
: new TextDecoder().decode(event.data as ArrayBuffer);
const msg = JSON.parse(data) as {
id?: number;
method?: string;
params?: unknown;
result?: unknown;
error?: { message?: string };
};
if (msg.method) {
const handlers = this.eventHandlers.get(msg.method);
if (handlers) {
handlers.forEach((handler) => handler(msg.params));
}
}
if (msg.id) {
const pending = this.pending.get(msg.id);
if (pending) {
this.pending.delete(msg.id);
if (pending.timer) clearTimeout(pending.timer);
if (msg.error?.message) pending.reject(new Error(msg.error.message));
else pending.resolve(msg.result);
}
}
} catch {}
});
this.ws.addEventListener("close", () => {
for (const [id, pending] of this.pending.entries()) {
this.pending.delete(id);
if (pending.timer) clearTimeout(pending.timer);
pending.reject(new Error("CDP connection closed."));
}
});
}
static async connect(
url: string,
timeoutMs: number,
options?: { defaultTimeoutMs?: number }
): Promise<CdpConnection> {
const ws = new WebSocket(url);
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error("CDP connection timeout.")), timeoutMs);
ws.addEventListener("open", () => {
clearTimeout(timer);
resolve();
});
ws.addEventListener("error", () => {
clearTimeout(timer);
reject(new Error("CDP connection failed."));
});
});
return new CdpConnection(ws, options?.defaultTimeoutMs ?? 15_000);
}
on(method: string, handler: (params: unknown) => void): void {
if (!this.eventHandlers.has(method)) {
this.eventHandlers.set(method, new Set());
}
this.eventHandlers.get(method)?.add(handler);
}
off(method: string, handler: (params: unknown) => void): void {
this.eventHandlers.get(method)?.delete(handler);
}
async send<T = unknown>(method: string, params?: Record<string, unknown>, options?: CdpSendOptions): Promise<T> {
const id = ++this.nextId;
const message: Record<string, unknown> = { id, method };
if (params) message.params = params;
if (options?.sessionId) message.sessionId = options.sessionId;
const timeoutMs = options?.timeoutMs ?? this.defaultTimeoutMs;
const result = await new Promise<unknown>((resolve, reject) => {
const timer = timeoutMs > 0
? setTimeout(() => {
this.pending.delete(id);
reject(new Error(`CDP timeout: ${method}`));
}, timeoutMs)
: null;
this.pending.set(id, { resolve, reject, timer });
this.ws.send(JSON.stringify(message));
});
return result as T;
}
close(): void {
try {
this.ws.close();
} catch {}
}
}
export async function launchChrome(options: LaunchChromeOptions): Promise<ChildProcess> {
await fs.promises.mkdir(options.profileDir, { recursive: true });
const args = [
`--remote-debugging-port=${options.port}`,
`--user-data-dir=${options.profileDir}`,
"--no-first-run",
"--no-default-browser-check",
...(options.extraArgs ?? []),
];
if (options.headless) args.push("--headless=new");
if (options.url) args.push(options.url);
return spawn(options.chromePath, args, { stdio: "ignore" });
}
export function killChrome(chrome: ChildProcess): void {
try {
chrome.kill("SIGTERM");
} catch {}
setTimeout(() => {
if (!chrome.killed) {
try {
chrome.kill("SIGKILL");
} catch {}
}
}, 2_000).unref?.();
}
export async function openPageSession(options: OpenPageSessionOptions): Promise<PageSession> {
let targetId: string;
if (options.reusing) {
const created = await options.cdp.send<{ targetId: string }>("Target.createTarget", { url: options.url });
targetId = created.targetId;
} else {
const targets = await options.cdp.send<{ targetInfos: ChromeTargetInfo[] }>("Target.getTargets");
const existing = targets.targetInfos.find(options.matchTarget);
if (existing) {
targetId = existing.targetId;
} else {
const created = await options.cdp.send<{ targetId: string }>("Target.createTarget", { url: options.url });
targetId = created.targetId;
}
}
const { sessionId } = await options.cdp.send<{ sessionId: string }>(
"Target.attachToTarget",
{ targetId, flatten: true }
);
if (options.activateTarget ?? true) {
await options.cdp.send("Target.activateTarget", { targetId });
}
if (options.enablePage) await options.cdp.send("Page.enable", {}, { sessionId });
if (options.enableRuntime) await options.cdp.send("Runtime.enable", {}, { sessionId });
if (options.enableDom) await options.cdp.send("DOM.enable", {}, { sessionId });
if (options.enableNetwork) await options.cdp.send("Network.enable", {}, { sessionId });
return { sessionId, targetId };
}